From e9c718e0732477a7f09f3685c8a84f76feeff9c8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:21:58 +0200 Subject: [PATCH 001/196] B2-3d: the route actuation ceiling counts only Forward Ruling 3d A (2026-09-30): the 12/h per-target route actuation ceiling counts only Forward actuations (deploys). Survival actuations (continuity restarts, rollbacks, withdrawals, admitted-route repairs) are recorded for visibility in a separate TargetSupervision.route_survivals log (key 6, newest 12 kept) and are never counted against the ceiling or refused by it. A target whose continuity spent its six restarts can therefore always be rescued by a deploy. Each Survival path keeps its own bound: a continuity's actuations the restart log and its doubling wait, an admitted-route repair the challenge backoff, a deployment's rollback or withdrawal the Forward change it undoes. A candidate whose route proof keeps failing now reloads at most twice the ceiling per window (12 installs, each with its rollback) instead of 12. status prints the survival record beside the ceiling line. TargetSupervision stays at v1 with the added slot: no installed Idunn has written the record (B2 is merged, not installed). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 231 ++++++++++++++++++++++++++++++++----------- src/drivers.rs | 8 +- 2 files changed, 175 insertions(+), 64 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index da57968..f2e78e4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -73,9 +73,17 @@ const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; /// Forward route actuations (fragment write, firewall, `nginx -t`, reload, /// private-mount validation) one target may perform per sliding window. A /// healthy target performs none; a legitimate deployment performs one or two. -/// Survival actuations are counted against it but never refused. +/// Only deployments count: a Survival actuation is recorded beside this +/// ceiling, never counted against it and never refused by it, so a target +/// whose continuity spent its restarts can still be rescued by a deploy. Each +/// Survival path has its own bound instead: a continuity's actuations the +/// restart log, an admitted-route repair the challenge backoff, and a +/// deployment's rollback or withdrawal the Forward change it undoes. const ROUTE_ACTUATION_CEILING: usize = 12; const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; +/// How many Survival route actuation times a target keeps, for `status` +/// only: they are a record, not a meter, and no decision reads them. +const ROUTE_SURVIVAL_RECORD: usize = 12; /// The longest wait between route challenges while proofs keep failing. The /// shortest is the observation max age. const ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS: u64 = 600_000; @@ -934,8 +942,9 @@ impl RouteSupervisionState { } /// What one target has been allowed to do, kept across every generation and -/// present before the first: route actuations and continuity restarts, each a -/// sliding log of the times they happened. Owned by the meter primitives below; +/// present before the first: Forward route actuations and continuity restarts, +/// each a sliding log of the times they happened, and a record of Survival +/// route actuations. Owned by the meter primitives below; /// the route driver only asks, and no generation write carries it. #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( @@ -947,7 +956,8 @@ struct TargetSupervision { schema_version: String, #[cultcache(key = 1)] target: String, - /// The newest `ROUTE_ACTUATION_CEILING` actuation times, ascending. + /// The newest `ROUTE_ACTUATION_CEILING` Forward actuation times, + /// ascending: the only entries the ceiling counts. #[cultcache(key = 2)] route_actuations: Vec, /// The newest `CONTINUITY_RESTART_ATTEMPTS` restart times, ascending. @@ -958,6 +968,10 @@ struct TargetSupervision { continuity_deferred_until: Option, #[cultcache(key = 5)] continuity_deferral_reason: Option, + /// The newest `ROUTE_SURVIVAL_RECORD` Survival actuation times, ascending. + /// Visibility only: never counted, never refused. + #[cultcache(key = 6)] + route_survivals: Vec, } /// The entries of a sliding log still inside `window` at `now`. An entry from @@ -970,8 +984,8 @@ fn live_entries(log: &[u64], now: u64, window: u64) -> Vec { .collect() } -/// Record `now`, keeping the newest `limit` entries. That is exact for "at most -/// `limit` in any window", even after a forced entry pushes past the limit. +/// Record `now`, keeping the newest `limit` entries: all a meter needs to +/// decide "at most `limit` in any window". fn record_entry(log: &mut Vec, now: u64, limit: usize) { clamp_to(log, now); log.push(now); @@ -999,14 +1013,16 @@ impl TargetSupervision { continuity_restarts: Vec::new(), continuity_deferred_until: None, continuity_deferral_reason: None, + route_survivals: Vec::new(), } } - /// Clamp both logs to `now`. Callers that decide from a log write the + /// Clamp every log to `now`. Callers that decide from a log write the /// result back when it changed. fn settle(&mut self, now: u64) { clamp_to(&mut self.route_actuations, now); clamp_to(&mut self.continuity_restarts, now); + clamp_to(&mut self.route_survivals, now); } fn route_used(&self, now: u64) -> usize { @@ -1020,23 +1036,33 @@ impl TargetSupervision { .then(|| live[live.len() - ROUTE_ACTUATION_CEILING] + ROUTE_ACTUATION_WINDOW_MILLIS) } - /// Count one route actuation. Only a `Forward` change can be refused; a - /// `Survival` is always admitted, and counted whenever the ledger can record it. + /// Charge one route actuation. A `Forward` change is counted against the + /// ceiling and refused at it; a `Survival` is only recorded, never + /// counted and never refused. fn charge_route(&mut self, now: u64, kind: RouteActuation) -> Result<(), RouteActuationRefused> { self.settle(now); - if kind == RouteActuation::Forward - && let Some(reopens_at) = self.route_reopens_at(now) - { - return Err(RouteActuationRefused { - target: self.target.clone(), - used: self.route_used(now), - reopens_at_unix_millis: reopens_at, - }); + match kind { + RouteActuation::Forward => { + if let Some(reopens_at) = self.route_reopens_at(now) { + return Err(RouteActuationRefused { + target: self.target.clone(), + used: self.route_used(now), + reopens_at_unix_millis: reopens_at, + }); + } + record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING); + } + RouteActuation::Survival => { + record_entry(&mut self.route_survivals, now, ROUTE_SURVIVAL_RECORD); + } } - record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING); Ok(()) } + fn survivals_in_window(&self, now: u64) -> usize { + live_entries(&self.route_survivals, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() + } + fn restarts_used(&self, now: u64) -> usize { live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len() } @@ -1091,6 +1117,7 @@ impl TargetSupervision { for (log, limit, what) in [ (&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"), (&self.continuity_restarts, CONTINUITY_RESTART_ATTEMPTS, "continuity restart"), + (&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"), ] { ensure!(log.len() <= limit, "{what} log exceeds its bound"); ensure!( @@ -4226,6 +4253,7 @@ fn render_supervision( } if generation.is_none_or(|value| value.route_supervision.is_some()) || !meters.route_actuations.is_empty() + || !meters.route_survivals.is_empty() { lines.push(format!( " route actuations {}/{} in window, reopens-at {}", @@ -4233,6 +4261,15 @@ fn render_supervision( ROUTE_ACTUATION_CEILING, at(meters.route_reopens_at(now)) )); + // A record, not a meter: only the newest are kept, so a saturated + // count says "at least". + let survivals = meters.survivals_in_window(now); + lines.push(format!( + " route survival actuations {}{} in window (uncounted), last-at {}", + if survivals == ROUTE_SURVIVAL_RECORD { "at least " } else { "" }, + survivals, + at(meters.route_survivals.last().copied()) + )); } lines } @@ -4368,7 +4405,7 @@ struct Engine { } /// The route driver's door to the actuation ceiling: every group of host -/// mutations a driver makes for `target` is counted through here. The gate +/// mutations a driver makes for `target` is charged through here. The gate /// owns which charge a request is: a driver asks for the change it is making, /// and the command that owns the transaction decides whether that change is /// deployment (`Forward`, refusable) or the continuity of the admitted @@ -4475,20 +4512,20 @@ impl Engine { } } - /// Count one route actuation against the target's sliding ceiling, in its - /// `TargetSupervision`, before the driver acts. The record is created by - /// the first charge, so a first deployment is metered like any other. A - /// refused `Forward` is the typed `RouteActuationRefused`, and a `Forward` - /// that cannot be recorded is refused too: no charge, no actuation. A - /// `Survival` is never refused, so it is counted when it can be and runs - /// regardless when it cannot: an unwritable store must not stop a rollback, - /// a withdrawal or a repair. + /// Charge one route actuation to the target's `TargetSupervision` before + /// the driver acts. The record is created by the first charge, so a first + /// deployment is metered like any other. A `Forward` is counted against + /// the sliding ceiling; a refused one is the typed `RouteActuationRefused`, + /// and one that cannot be recorded is refused too: no charge, no + /// actuation. A `Survival` is never counted or refused, only recorded when + /// it can be, and runs regardless when it cannot: an unwritable store must + /// not stop a rollback, a withdrawal or a repair. fn charge_route_actuation(&self, target: &str, kind: RouteActuation) -> Result<()> { match self.record_route_actuation(target, kind) { Ok(charged) => charged.map_err(anyhow::Error::new), Err(error) if kind == RouteActuation::Survival => { eprintln!( - "Idunn could not count a survival route actuation of {target}; it proceeds uncounted: {error:#}" + "Idunn could not record a survival route actuation of {target}; it proceeds unrecorded: {error:#}" ); Ok(()) } @@ -12317,6 +12354,8 @@ mod tests { meters(|m| m.route_actuations = (1..=13).collect()).contains("exceeds its bound") ); assert!(meters(|m| m.continuity_restarts = (1..=7).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.route_survivals = (1..=13).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.route_survivals = vec![9, 5]).contains("ascending")); assert!(meters(|m| m.continuity_deferred_until = Some(5)).contains("time or a reason")); assert!(meters(|m| m.continuity_deferral_reason = Some("x".into())).contains("time or a reason")); assert!(meters(|m| m.schema_version = "idunn.target_supervision.v0".into()) @@ -15017,22 +15056,32 @@ mod tests { } #[test] - fn a_survival_actuation_is_counted_past_the_limit_and_never_refused() -> Result<()> { + fn a_survival_actuation_is_recorded_but_never_counted_or_refused() -> Result<()> { let t = 1_000_000_000_u64; let mut meters = TargetSupervision::new("service"); + // Survivals first, past the ceiling: every one is admitted and + // recorded, and none takes a Forward slot. + for offset in 0..20 { + meters.charge_route(t + offset, RouteActuation::Survival)?; + } + assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); + assert_eq!(meters.route_survivals.last(), Some(&(t + 19))); + assert!(meters.route_actuations.is_empty()); + // Twelve Forward changes interleaved with more Survivals: the twelve + // are admitted, and the thirteenth is refused with a reopen time that + // only the oldest Forward entry decides. for offset in 0..12 { - meters.charge_route(t + offset, RouteActuation::Forward)?; - } - meters.charge_route(t + 20, RouteActuation::Survival)?; - assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); - assert_eq!(meters.route_actuations.last(), Some(&(t + 20))); - // The survival took a slot: the forward ceiling is still exact, and - // reopens when the oldest surviving entry leaves. - let refused = meters.charge_route(t + 21, RouteActuation::Forward).unwrap_err(); - assert_eq!(refused.reopens_at_unix_millis, t + 1 + ROUTE_ACTUATION_WINDOW_MILLIS); - for _ in 0..20 { - meters.charge_route(t + 22, RouteActuation::Survival)?; + meters.charge_route(t + 100 + 2 * offset, RouteActuation::Forward)?; + meters.charge_route(t + 101 + 2 * offset, RouteActuation::Survival)?; } + let refused = meters.charge_route(t + 200, RouteActuation::Forward).unwrap_err(); + assert_eq!( + (refused.used, refused.reopens_at_unix_millis), + (ROUTE_ACTUATION_CEILING, t + 100 + ROUTE_ACTUATION_WINDOW_MILLIS) + ); + // A full Forward ceiling still admits and records a Survival. + meters.charge_route(t + 201, RouteActuation::Survival)?; + assert_eq!(meters.route_survivals.last(), Some(&(t + 201))); meters.validate()?; Ok(()) } @@ -15260,6 +15309,21 @@ mod tests { now - 5_000 + ROUTE_ACTUATION_WINDOW_MILLIS )), "{lines}"); assert!(lines.contains("route healthy consecutive-failures 0"), "{lines}"); + assert!( + lines.contains("route survival actuations 0 in window (uncounted), last-at none"), + "{lines}" + ); + meters.route_survivals = vec![now - 7_000, now - 3_000]; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains(&format!( + "route survival actuations 2 in window (uncounted), last-at {}", + now - 3_000 + )), "{lines}"); + meters.route_survivals = (0..ROUTE_SURVIVAL_RECORD as u64).map(|n| now - 100 + n).collect(); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains("route survival actuations at least 12 in window"), "{lines}"); + // Survivals never move the ceiling's count. + assert!(lines.contains("route actuations 12/12 in window"), "{lines}"); // A first deployment has meters and no generation: it shows them alone. let mut first = TargetSupervision::new("fresh"); @@ -17380,10 +17444,11 @@ mod tests { #[test] fn a_full_ledger_still_restores_the_admitted_and_incumbent_route() -> Result<()> { let full = ROUTE_ACTUATION_CEILING; + // The newest recorded Survival, or 0; the Forward ledger stays full. let newest = |world: &EngineFixture| -> Result { let meters = meters_of(world, "service")?; assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); - Ok(*meters.route_actuations.last().unwrap()) + Ok(meters.route_survivals.last().copied().unwrap_or(0)) }; // (i) A post-fence abort withdraws the candidate's membership. @@ -17396,7 +17461,7 @@ mod tests { .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; aborting.step()?; assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); - assert!(newest(&aborting.world)? > before, "the withdrawal was not counted"); + assert!(newest(&aborting.world)? > before, "the withdrawal was not recorded"); // (ii) Supervision repairs a drifted admitted fragment. let supervised = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; @@ -17406,11 +17471,12 @@ mod tests { supervised.drift()?; assert!(supervised.tick()?); assert_eq!(supervised.reloads(), reloads + 1, "the repair did not run"); - assert!(newest(&supervised.world)? > before, "the repair was not counted"); + assert!(newest(&supervised.world)? > before, "the repair was not recorded"); assert!(admitted(&supervised)?.route_supervision.unwrap().degraded_since_unix_millis.is_none()); // (iii) A failed proof at Routing rolls the route back, though the - // install that preceded it took the last free slot. + // install that preceded it took the last free slot. The rollback + // is recorded and leaves the Forward ledger exactly full. let failing = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; failing.stub.set_state("active"); failing.run_to_routing()?; @@ -17419,7 +17485,7 @@ mod tests { let before = now_millis()?; assert!(failing.step().is_err(), "an unanswered stable route was admitted"); assert_eq!(failing.reloads(), 2, "install then rollback"); - assert!(newest(&failing.world)? >= before, "the rollback was not counted"); + assert!(newest(&failing.world)? >= before, "the rollback was not recorded"); assert!(!failing.fragment_path().exists(), "the rollback left the candidate's route"); Ok(()) } @@ -17427,32 +17493,37 @@ mod tests { #[cfg(unix)] #[test] fn a_continuity_restart_is_never_refused_by_a_full_ledger() -> Result<()> { - // Warming: the preflight of a continuity runs and is counted. + // Warming: the preflight of a continuity runs and is recorded, + // and the Forward ledger is left as it was. let warming = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; warming.step()?; assert!(warming.transaction()?.warming.is_some()); warming.ledger(ROUTE_ACTUATION_CEILING)?; - let before = *meters_of(&warming.world, "service")?.route_actuations.last().unwrap(); + let ledger = meters_of(&warming.world, "service")?.route_actuations; warming.step()?; assert!(warming.transaction()?.route_preflight.is_some(), "the preflight was refused"); - assert!(*meters_of(&warming.world, "service")?.route_actuations.last().unwrap() > before); + let after = meters_of(&warming.world, "service")?; + assert_eq!(after.route_survivals.len(), 1, "the preflight was not recorded"); + assert_eq!(after.route_actuations, ledger, "a survival was counted"); - // Routing: its install runs and is counted. + // Routing: its install runs and is recorded, and not counted. let routing = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routing.stub.set_state("active"); routing.run_to_routing()?; routing.ledger(ROUTE_ACTUATION_CEILING)?; - let before = *meters_of(&routing.world, "service")?.route_actuations.last().unwrap(); + let ledger = meters_of(&routing.world, "service")?.route_actuations; routing.step()?; assert!(routing.reloads() >= 1, "the install did not run"); assert!(routing.fragment_path().exists()); - assert!(*meters_of(&routing.world, "service")?.route_actuations.last().unwrap() > before); + let after = meters_of(&routing.world, "service")?; + assert_eq!(after.route_survivals.len(), 1, "the install was not recorded"); + assert_eq!(after.route_actuations, ledger, "a survival was counted"); Ok(()) } #[cfg(unix)] #[test] - fn a_survival_that_cannot_be_counted_runs_and_a_forward_is_refused() -> Result<()> { + fn a_survival_that_cannot_be_recorded_runs_and_a_forward_is_refused() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; let transaction = routed.transaction()?; let preflight = transaction.route_preflight.clone().context("no preflight")?; @@ -17466,7 +17537,7 @@ mod tests { .route .context("no route binding")?, ); - // The ledger cannot be read, so nothing can be counted. + // The ledger cannot be read, so nothing can be recorded. std::fs::write(&routed.world.state_store, b"unreadable")?; let engine = &routed.world.engine; @@ -17482,7 +17553,7 @@ mod tests { &preflight, &engine.route_gate("service", CommandKind::Deploy), )?; - assert_eq!(routed.reloads(), 1, "the uncounted withdrawal did not run"); + assert_eq!(routed.reloads(), 1, "the unrecorded withdrawal did not run"); Ok(()) } @@ -17669,7 +17740,9 @@ mod tests { assert_eq!(state.consecutive_failures, 1); assert!(state.degraded_since_unix_millis.is_some()); assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - assert_eq!(meters_of(&routed.world, "service")?.route_actuations.len(), 2); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_survivals.len(), 2); + assert!(meters.route_actuations.is_empty()); // Tick 2: the route waits, and the dead unit is restarted from a // fresh read, counted in the same write that mints it. @@ -17724,21 +17797,59 @@ mod tests { #[cfg(unix)] #[test] - fn a_candidate_whose_route_proof_fails_reloads_at_most_the_ceiling() -> Result<()> { + fn a_candidate_whose_route_proof_fails_reloads_at_most_twice_the_ceiling() -> Result<()> { let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.stub.hang_up.store(true, Ordering::SeqCst); let mut last = None; - for _ in 0..30 { + for _ in 0..40 { last = Some(routed.step().unwrap_err()); } - // Every install is followed by its rollback and both are counted, so - // twelve reloads spend the window; the rest are refused. - assert_eq!(routed.reloads(), ROUTE_ACTUATION_CEILING); + // Every install is a counted Forward and is followed by one + // recorded rollback, so the rollbacks are bounded by the installs: + // twelve installs spend the window, each reloads twice, and every + // later attempt is refused before it runs anything. + assert_eq!(routed.reloads(), 2 * ROUTE_ACTUATION_CEILING); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_used(now_millis()?), ROUTE_ACTUATION_CEILING); + assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); let last = last.unwrap(); assert!(last.downcast_ref::().is_some(), "{last:#}"); Ok(()) } + + /// The rescue (ruling 3d A): a target whose continuity spent every + /// restart, each charging its preflight and install as Survival, is + /// still deployed: its route changes are the ceiling's first Forward. + #[cfg(unix)] + #[test] + fn a_target_that_spent_its_restarts_can_still_be_deployed() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(routed.transaction()?.warming.is_some()); + set_meters( + &routed.world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + for _ in 0..2 * CONTINUITY_RESTART_ATTEMPTS { + routed.world.engine.charge_route_actuation("service", RouteActuation::Survival)?; + } + let spent = meters_of(&routed.world, "service")?; + assert!(spent.restarts_exhausted(now_millis()?)); + assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_RESTART_ATTEMPTS); + + // Warming's preflight and Routing's install are both admitted. + routed.step()?; + assert!(routed.transaction()?.route_preflight.is_some(), "the preflight was refused"); + routed.run_to_routing()?; + routed.promote()?; + assert!(routed.fragment_path().exists(), "the install was refused"); + assert!(routed.reloads() >= 1); + let after = meters_of(&routed.world, "service")?; + assert_eq!(after.route_actuations.len(), 2, "preflight and install are the Forward"); + Ok(()) + } } } diff --git a/src/drivers.rs b/src/drivers.rs index af18a80..189779b 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5079,10 +5079,10 @@ impl Default for RouteActuators { } } -/// Why a route actuation is being asked for. Only a `Forward` change can be -/// refused: restoring the admitted or incumbent route, and every actuation of a -/// continuity, is survival, not deployment, so a `Survival` is always admitted -/// and counted whenever the ledger can record it. +/// Why a route actuation is being asked for. Only a `Forward` change is +/// counted against the ceiling and can be refused: restoring the admitted or +/// incumbent route, and every actuation of a continuity, is survival, not +/// deployment, so a `Survival` is always admitted and only recorded. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RouteActuation { Forward, From 057bdd5c6c21d97b45a641bab37219933e7c7720 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:39:32 +0200 Subject: [PATCH 002/196] B2-3d: status shows a survival-only record on a generation that no longer routes cargo-mutants on the 3d diff missed `|| -> &&` in render_supervision: no test held meters with only survival entries under an unrouted generation. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index f2e78e4..28185cc 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15334,6 +15334,13 @@ mod tests { // An unrouted, unmetered target shows no route line at all. let lines = render_supervision("plain", Some(&unrouted_generation()?), None, now).join("\n"); assert!(!lines.contains("route actuations"), "{lines}"); + // A generation that no longer routes, whose meters hold only survival + // actuations, still shows the record. + let mut survived = TargetSupervision::new("plain"); + survived.route_survivals = vec![now - 1]; + let lines = + render_supervision("plain", Some(&unrouted_generation()?), Some(&survived), now).join("\n"); + assert!(lines.contains("route survival actuations 1 in window"), "{lines}"); Ok(()) } From 5939abd87ece8f6b8b41947247b45cfe5b2e4a69 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 14:09:24 +0200 Subject: [PATCH 003/196] Route map: B2-3d Soul result; the Survival-bound premise is false Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 9c13941..82b52ed 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -211,6 +211,19 @@ The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map Option D (reset the restart log when a new release is admitted) was not chosen: a new release still inherits its predecessor's restart history. B2 is merged (`46d9c2f`) but not installed; this follow-up (B2-3d) lands before the install. +- **B2-3d status (Self, 2026-09-30).** Built on `hands/b2-3d` (`e9c718e`, `057bdd5`). Soul re-ran `verify.sh` + clean (341) and killed every one of its own mutants on the Forward/Survival split. **But the premise the operator + ruled on is false:** "every Survival path has its own bound" does not hold. + - **F1:** a continuity transaction whose route proof keeps failing retries inside one transaction under the + 60 s resume cap, reloading nginx twice per retry. That is about 120 reloads/h per target, with no end until B5. + - **F2:** a failing post-fence withdrawal reloads about 60/h. `restore` reloads even when the fragment is + already correct. + - **F3:** nothing bounds reloads across the host. + - None of these is new in 3d: Survival was never refused before either. The live Idunn, which predates B2, has + no reload bound at all. What 3d adds is the false claim, at `control_plane.rs:76-81`, plus the admitted + Deploy doubling to 24/h. + - The fix belongs to B5 ("every post-fencing phase ends"), plus a restore/install that skips the reload when + the fragment is already correct. The operator was told on 2026-09-30. - **Q-B2-1 (a):** the continuity restart log moves into `TargetSupervision`; the restart ceiling reads the target's own log; the carry compensator is deleted. - **Q-B2-2: fail at once.** A deploy refused by the route/actuation ceiling before the fence fails immediately, From 6ff7d5a9c77e8ca90632aea70024ad7e15123f96 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:32:38 +0200 Subject: [PATCH 004/196] Route driver: a fragment already correct is neither written nor reloaded `install` and `restore` skip the write, the firewall step and the reload when the fragment on disk already equals what they would write. `restore` still withdraws the endpoint when the prior membership is none, so a withdrawal whose `ufw delete` failed repeats only that step. That skip is sound only while bytes on disk are what nginx last loaded, so every write whose reload fails is now undone: `restore` puts the previous fragment back, and `install` puts the baseline back in both the rollback and the fenced case (it used to retain the candidate for retry, which the retry would now take for live). An install that never loaded needs no reload to roll back. Tests: a retry with an already-correct fragment performs zero reloads (install and a withdrawal with a failing `ufw delete`); a failed reload puts the fragment back so the retry reloads. Two engine tests that counted a reload for withdrawing a never-installed route now count the endpoint withdrawal. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 7 +- src/drivers.rs | 233 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 198 insertions(+), 42 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 28185cc..011c492 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17467,7 +17467,10 @@ mod tests { .engine .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; aborting.step()?; - assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); + // Nothing was installed, so the withdrawal has no fragment to + // remove and no reload to run: it withdraws the endpoint. + assert_eq!(aborting.reloads(), 0); + assert_eq!(aborting.world.route_stubs.count("ufw delete"), 1, "the withdrawal did not run"); assert!(newest(&aborting.world)? > before, "the withdrawal was not recorded"); // (ii) Supervision repairs a drifted admitted fragment. @@ -17560,7 +17563,7 @@ mod tests { &preflight, &engine.route_gate("service", CommandKind::Deploy), )?; - assert_eq!(routed.reloads(), 1, "the unrecorded withdrawal did not run"); + assert_eq!(routed.world.route_stubs.count("ufw delete"), 1, "the unrecorded withdrawal did not run"); Ok(()) } diff --git a/src/drivers.rs b/src/drivers.rs index 189779b..3e1098f 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5332,10 +5332,29 @@ impl NginxRouteDriver { } } + /// Put `prior` back as the route's fragment. Bytes on disk are what nginx + /// last loaded: every write here and in `install` is undone when the + /// reload after it fails. So a fragment that already equals `prior` is + /// live, and is neither written nor reloaded; a retry of a withdrawal + /// whose firewall step failed repeats only that step. fn restore(&self, prior: Option<&[u8]>, gate: &dyn RouteActuationGate) -> Result<()> { + let current = self.current_configuration()?; + let rewrite = current.as_deref() != prior; + if !rewrite && prior.is_some() { + return Ok(()); + } gate.admit(RouteActuation::Survival)?; - self.write_fragment(prior)?; - self.reload()?; + if rewrite { + self.write_fragment(prior)?; + if let Err(reload) = self.reload() { + return match self.write_fragment(current.as_deref()) { + Ok(()) => Err(reload).context("reloading the restored route"), + Err(undo) => Err(reload).context(format!( + "reloading the restored route; putting the previous fragment back also failed: {undo:#}" + )), + }; + } + } // No prior membership means the stable endpoint no longer routes to // anything; its firewall allow goes with the fragment. if prior.is_none() { @@ -5344,21 +5363,6 @@ impl NginxRouteDriver { Ok(()) } - fn fail_after_rollback( - &self, - prior: Option<&[u8]>, - failure: anyhow::Error, - context: &str, - gate: &dyn RouteActuationGate, - ) -> Result { - match self.restore(prior, gate) { - Ok(()) => Err(failure).context(context.to_owned()), - Err(rollback) => Err(failure).context(format!( - "{context}; route rollback also failed: {rollback:#}" - )), - } - } - pub fn preflight( &self, expected: &IdunnExpectedIncarnationRecord, @@ -5431,22 +5435,38 @@ impl NginxRouteDriver { && prior_sha256 == preflight.incumbent_membership_sha256), "route baseline changed after preflight" ); - gate.admit(RouteActuation::Forward)?; - if !candidate_already_written { - atomic_replace(&self.binding.config_path, &rendered)?; + // Bytes on disk are what nginx last loaded (see `restore`), so a + // candidate already written is installed: a retry changes nothing. + if candidate_already_written { + return Ok(sha256_id(&rendered)); } + gate.admit(RouteActuation::Forward)?; + atomic_replace(&self.binding.config_path, &rendered)?; if let Err(error) = self.admit_endpoint().and_then(|()| self.reload()) { - if rollback_allowed { - return self.fail_after_rollback( - preflight.incumbent_configuration.as_deref(), - error, - "candidate route validation or reload failed", - gate, - ); - } - return Err(error).context( - "candidate route reload failed after incumbent route authority was fenced; candidate fragment retained for retry", - ); + // nginx did not load the candidate, so it still serves the + // baseline. The disk is put back to match it, which is the whole + // rollback, and a retry installs and reloads again rather than + // mistaking unloaded bytes for a live route. Past the fence the + // firewall allow stays for that retry; before it, an allow the + // baseline never had goes too. + let context = if rollback_allowed { + "candidate route validation or reload failed" + } else { + "candidate route reload failed after incumbent route authority was fenced; the fragment was put back for retry" + }; + let undo = self.write_fragment(prior.as_deref()).and_then(|()| { + if rollback_allowed && prior.is_none() { + self.withdraw_endpoint() + } else { + Ok(()) + } + }); + return match undo { + Ok(()) => Err(error).context(context), + Err(undo) => Err(error).context(format!( + "{context}; putting the route back also failed: {undo:#}" + )), + }; } ensure!( self.current_configuration()?.as_deref() == Some(rendered.as_slice()), @@ -11817,6 +11837,140 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// Stub host programs that log every call. `systemctl reload` fails while + /// `/systemctl.fail` exists, `ufw delete` while `/ufw.fail` does. + #[cfg(unix)] + fn failable_route_driver( + temp: &Path, + ) -> Result<(NginxRouteDriver, IdunnExpectedIncarnationRecord, PathBuf)> { + use std::os::unix::fs::PermissionsExt; + + let (mut driver, candidate, calls) = stubbed_route_driver(temp)?; + let failable = |name: &str, verb: &str| -> Result { + let path = temp.join(name); + fs::write( + &path, + format!( + "#!/bin/sh\necho \"{name} $*\" >> '{}'\nif [ \"$1\" = {verb} ] && [ -e \"$0.fail\" ]; then exit 1; fi\nexit 0\n", + calls.display() + ), + )?; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755))?; + Ok(path) + }; + driver.systemctl_program = failable("systemctl", "reload")?; + driver.ufw_program = failable("ufw", "delete")?; + Ok((driver, candidate, calls)) + } + + #[cfg(unix)] + fn logged(calls: &Path, prefix: &str) -> usize { + fs::read_to_string(calls) + .unwrap_or_default() + .lines() + .filter(|line| line.starts_with(prefix)) + .count() + } + + /// Admits everything and records what it was asked. + struct Recording(std::cell::RefCell>); + + impl RouteActuationGate for Recording { + fn admit(&self, kind: RouteActuation) -> Result<()> { + self.0.borrow_mut().push(kind); + Ok(()) + } + } + + /// A retry whose fragment is already what it would write reloads nothing: + /// an install already written, and a withdrawal whose firewall step is + /// what failed, repeat only what is still undone. + #[cfg(unix)] + #[test] + fn a_retry_with_an_already_correct_fragment_performs_zero_reloads() -> Result<()> { + let temp = tempfile::tempdir()?; + let (driver, candidate, calls) = failable_route_driver(temp.path())?; + let receipt = preflight_receipt(&driver, &candidate)?; + let gate = Recording(Default::default()); + + driver.install(&candidate, &digest('a'), &receipt, false, &gate)?; + assert_eq!(logged(&calls, "systemctl reload"), 1); + for _ in 0..3 { + driver.install(&candidate, &digest('a'), &receipt, false, &gate)?; + } + assert_eq!(logged(&calls, "systemctl reload"), 1, "an installed candidate reloaded again"); + assert_eq!(logged(&calls, "ufw allow"), 1); + assert_eq!(*gate.0.borrow(), [RouteActuation::Forward], "a no-op was charged"); + + // The withdrawal removes the fragment and reloads, then its firewall + // step fails. Each retry repeats the firewall step and nothing else. + fs::write(temp.path().join("ufw.fail"), b"x")?; + assert!(driver.withdraw_candidate_membership(&receipt, &gate).is_err()); + assert!(!driver.binding.config_path.exists()); + assert_eq!(logged(&calls, "systemctl reload"), 2); + for _ in 0..5 { + assert!(driver.withdraw_candidate_membership(&receipt, &gate).is_err()); + } + assert_eq!(logged(&calls, "systemctl reload"), 2, "a failing withdrawal reloaded again"); + assert_eq!(logged(&calls, "ufw delete"), 6); + fs::remove_file(temp.path().join("ufw.fail"))?; + driver.withdraw_candidate_membership(&receipt, &gate)?; + assert_eq!(logged(&calls, "systemctl reload"), 2); + assert_eq!(logged(&calls, "ufw delete"), 7); + + // A restore to bytes already on disk runs no program and asks nothing. + let rendered = driver.render(&candidate)?; + fs::write(&driver.binding.config_path, &rendered)?; + let (programs, asked) = (logged(&calls, ""), gate.0.borrow().len()); + driver.restore(Some(&rendered), &gate)?; + assert_eq!(logged(&calls, ""), programs); + assert_eq!(gate.0.borrow().len(), asked); + Ok(()) + } + + /// Bytes on disk are what nginx loaded: a write whose reload fails is + /// undone, so the retry writes and reloads again instead of taking the + /// unloaded bytes for a live route. + #[cfg(unix)] + #[test] + fn a_failed_reload_puts_the_fragment_back_so_the_retry_reloads() -> Result<()> { + let temp = tempfile::tempdir()?; + let (driver, candidate, calls) = failable_route_driver(temp.path())?; + let receipt = preflight_receipt(&driver, &candidate)?; + let rendered = driver.render(&candidate)?; + let fail = temp.path().join("systemctl.fail"); + + // Install, fenced (no rollback) and not. + for rollback_allowed in [false, true] { + fs::write(&fail, b"x")?; + assert!( + driver + .install(&candidate, &digest('a'), &receipt, rollback_allowed, &Unmetered) + .is_err() + ); + assert!(!driver.binding.config_path.exists(), "an unloaded candidate stayed on disk"); + fs::remove_file(&fail)?; + let before = logged(&calls, "systemctl reload"); + driver.install(&candidate, &digest('a'), &receipt, rollback_allowed, &Unmetered)?; + assert_eq!(logged(&calls, "systemctl reload"), before + 1, "the retry did not reload"); + assert_eq!(fs::read(&driver.binding.config_path)?, rendered); + if !rollback_allowed { + fs::remove_file(&driver.binding.config_path)?; + } + } + + // Restore: the candidate is live; withdrawing it fails at the reload. + fs::write(&fail, b"x")?; + assert!(driver.withdraw_candidate_membership(&receipt, &Unmetered).is_err()); + assert_eq!(fs::read(&driver.binding.config_path)?, rendered, "the unloaded removal stayed"); + fs::remove_file(&fail)?; + let before = logged(&calls, "systemctl reload"); + driver.withdraw_candidate_membership(&receipt, &Unmetered)?; + assert_eq!(logged(&calls, "systemctl reload"), before + 1, "the retry did not reload"); + assert!(!driver.binding.config_path.exists()); + Ok(()) + } + /// Observation observes: restoring a membership whose fragment is already /// exact touches nothing, and drifted bytes are restored with one reload. #[cfg(unix)] @@ -12014,8 +12168,6 @@ Content-Le".to_vec()), |_| {}), assert!(!config.exists()); assert_eq!(fs::read_dir(&driver.preflight_root)?.count(), 0); - let candidate_bytes = driver.render(&candidate)?; - fs::write(&config, &candidate_bytes)?; let membership_sha256 = driver.install(&candidate, &digest('a'), &preflight, true, &Unmetered)?; assert_eq!( fs::read_to_string(systemctl.with_extension("calls"))?, @@ -12040,15 +12192,16 @@ Content-Le".to_vec()), |_| {}), let next_preflight = driver.preflight(&candidate, &digest('b'), Some(&admitted), &Unmetered)?; assert_eq!(fs::read(&config)?, admitted_bytes); + // The next candidate renders the bytes already live: installing it + // runs nothing, even while a reload would fail. fs::write(systemctl.with_extension("fail"), b"fail\n")?; - assert!( - driver - .install(&candidate, &digest('b'), &next_preflight, false, &Unmetered) - .is_err() - ); - assert_eq!(fs::read(&config)?, admitted_bytes); - fs::remove_file(systemctl.with_extension("fail"))?; driver.install(&candidate, &digest('b'), &next_preflight, false, &Unmetered)?; + assert_eq!(fs::read(&config)?, admitted_bytes); + assert_eq!( + fs::read_to_string(systemctl.with_extension("calls"))?, + "reload\n" + ); + fs::remove_file(systemctl.with_extension("fail"))?; fs::write(&config, b"foreign route\n")?; assert!( driver From fa830c5f6fc900bb05a4bcdcbe310ddfd3bbef47 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:33:26 +0200 Subject: [PATCH 005/196] Route map: B2 + 3d installed on Yggdrasil (a96ad9d) Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 82b52ed..3c94d9a 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -224,6 +224,16 @@ The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map Deploy doubling to 24/h. - The fix belongs to B5 ("every post-fencing phase ends"), plus a restore/install that skips the reload when the fragment is already correct. The operator was told on 2026-09-30. +- **B2 + 3d installed (2026-09-30).** The operator accepted recommendation A: install now, with B5 next. + - Installed at 15:22 CEST: `a96ad9d`, `idunn` sha256 `848c08e5...`; the replaced binary was `01bde676`. + - The migration moved 5 control records to `deployment_transaction.v4` / `admitted_generation.v4`. + - Pre-install backup, taken with Idunn stopped and byte-compared: `/root/idunn-store-backup-20260930T132204Z-pre-route-b2/` + (control.cc `5f879f55...`, history.cc `bafa76ab...`, the other stores, and the old binaries). + - Preflight: 690 transactions, all terminal. They were identical after the install. + - No managed target's PID changed. Raven reattached after 15 s. + - After 10 min: no odin route-continuity rejections (about 95 in the previous 3 h), 0 nginx reloads, and the + Odin post-deploy check `fails=0`. + - B5 and skip-unneeded-reloads are in Hands (`hands/b5`). - **Q-B2-1 (a):** the continuity restart log moves into `TargetSupervision`; the restart ceiling reads the target's own log; the carry compensator is deleted. - **Q-B2-2: fail at once.** A deploy refused by the route/actuation ceiling before the fence fails immediately, From 3f4d7561d9f5d88627835c0fc400a85a03204be0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:40:05 +0200 Subject: [PATCH 006/196] B5: every post-fencing phase ends A deadline resolver runs before a live post-fence transaction's step. Once the phase deadline F0 stamps on entry has passed, or an operator has asked for it with `idunn expire`, the phase is resolved as Q3 rules: - A candidate never issued a write lease (every stateless target, and a stateful one still in Fencing or early Leasing) aborts to the incumbent through the existing post-fencing path. - A continuity always aborts: its candidate is the admitted release, and supervision restarts it under the continuity backoff. A granted lease with no adoption evidence names the failure `lease-not-adopted`. - A deployment whose candidate was issued a lease is reported and left to the operator. That is the open fork: what holds the incumbent after an adopted-lease deployment, and whether a warming candidate's silence is evidence of non-adoption, are not ruled. Resume paths cannot extend a deadline: only `enter_phase` writes it. A record lifted from before deadlines existed is timed once, from the first time the resolver sees it. Lease adoption is recorded with Ready (the candidate's presence, or Odin's Ready correlation, held exactly the granted lease), and the terminal record names `RestartAdmitted` for a continuity whose candidate adopted it. F16: Commit goes on from the topology it admitted instead of returning, so Odin re-stamping every tick no longer sends it round forever. `idunn expire ` writes an `idunn.expiry_request.v1` record and nothing else; the resolver reads it as a passed deadline, and a spent request is deleted. A new control-store document type: an Idunn from before this commit refuses a store holding one. Tests commit Soul's F1 and F2 probes: a continuity whose route proof keeps failing ends at its Routing deadline, and a withdrawal whose `ufw delete` keeps failing reloads once. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 737 ++++++++++++++++++++++++++++++++++++++++- src/deployment_plan.rs | 12 +- 2 files changed, 730 insertions(+), 19 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 011c492..30ab973 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -59,6 +59,7 @@ const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; /// The capability whose declaration makes a target Odin-correlated. pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v1"; +const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1"; /// How many times continuity will restart one target inside one window before /// it concludes the target itself is the problem. More than one because a /// start can fail for a passing reason -- a port still held, a peer not yet up @@ -75,10 +76,12 @@ const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; /// healthy target performs none; a legitimate deployment performs one or two. /// Only deployments count: a Survival actuation is recorded beside this /// ceiling, never counted against it and never refused by it, so a target -/// whose continuity spent its restarts can still be rescued by a deploy. Each -/// Survival path has its own bound instead: a continuity's actuations the -/// restart log, an admitted-route repair the challenge backoff, and a -/// deployment's rollback or withdrawal the Forward change it undoes. +/// whose continuity spent its restarts can still be rescued by a deploy. +/// Survival is bounded elsewhere: a continuity's actuations by the restart log +/// times what one transaction can do before its phase deadline ends it, an +/// admitted-route repair by the challenge backoff, and a withdrawal by the +/// route driver, which does not reload a fragment that is already correct. +/// Nothing bounds reloads across the host. const ROUTE_ACTUATION_CEILING: usize = 12; const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; /// How many Survival route actuation times a target keeps, for `status` @@ -152,6 +155,51 @@ impl DeploymentCommand { } } +/// An operator's request that a live transaction's post-fencing phase end now +/// (`idunn expire`). The CLI only writes this record. The daemon's deadline +/// resolver reads it as a deadline that has passed, and the record is deleted +/// once its transaction no longer has a phase to end. No CLI writes a +/// transaction field. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.expiry_request", schema = "idunn.expiry_request.v1")] +struct ExpiryRequest { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + command_id: String, + #[cultcache(key = 2)] + requested_by: String, + #[cultcache(key = 3)] + requested_at_unix_millis: u64, +} + +impl ExpiryRequest { + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == EXPIRY_REQUEST_SCHEMA, + "expiry request schema is unsupported" + ); + require_id(&self.command_id, "expiry request command id")?; + require_value(&self.requested_by, "expiry requester")?; + ensure!( + self.requested_at_unix_millis > 0, + "expiry request has no request time" + ); + Ok(()) + } +} + +fn expiry_request_envelope(value: &ExpiryRequest, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.command_id, + ExpiryRequest::TYPE, + EXPIRY_REQUEST_SCHEMA, + value, + now, + ) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] #[repr(u8)] @@ -820,6 +868,22 @@ enum AdoptionSource { } impl LeaseAdoptionEvidence { + /// The adoption a stateful candidate's Ready proves: its signed presence + /// held exactly the granted lease. `None` for a stateless candidate. + fn with_ready( + granted_lease_sha256: Option<&str>, + signed_presence_sha256: &str, + source: AdoptionSource, + observed_at_unix_millis: u64, + ) -> Option { + granted_lease_sha256.map(|lease| Self { + write_lease_sha256: lease.to_owned(), + signed_presence_sha256: signed_presence_sha256.to_owned(), + source, + observed_at_unix_millis, + }) + } + /// Whether this evidence is about exactly the lease that was granted. fn names(&self, leasing: &LeasingEvidence) -> bool { leasing.lease_sha256() == Some(self.write_lease_sha256.as_str()) @@ -870,6 +934,17 @@ impl PhaseDeadline { } } +/// How the deadline resolver ends a post-fencing phase. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PhaseEnd { + /// Abort through the post-fencing path. `lease_not_adopted` names a + /// granted lease with no adoption evidence. + Abort { lease_not_adopted: bool }, + /// A stateful deployment whose candidate holds an issued lease: the phase + /// is reported and not resolved. + OperatorRequired, +} + /// How a post-fencing failure recovers. `RestoreIncumbent` is what a /// post-fencing abort has always done and is the default for records that /// predate the field. @@ -1488,6 +1563,46 @@ impl DeploymentTransaction { .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); } + /// A live transaction past the fence that is not already being aborted: + /// the only kind whose phase a deadline, or an operator's expiry, ends. + fn resolves_by_deadline(&self) -> bool { + self.completion.is_none() + && self.pre_fencing_abort.is_none() + && self.post_fencing_abort.is_none() + && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) + } + + /// What an ended post-fencing phase resolves to (Q3). A candidate that was + /// never issued a write lease cannot have written, so the transaction + /// aborts to the incumbent through the post-fencing path, stateless or not. + /// A continuity always aborts: its candidate is the admitted release, and + /// supervision restarts that release under the continuity backoff. A + /// deployment whose candidate was issued a lease may have written under a + /// new state contract, and restoring the incumbent over that is not + /// Idunn's call: it is left for the operator. + fn phase_end(&self) -> PhaseEnd { + let leasing = self.leasing.as_ref(); + let granted = leasing.and_then(LeasingEvidence::lease).is_some(); + let issued = granted || leasing.and_then(LeasingEvidence::prepared_lease).is_some(); + match self.command_kind { + CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, + _ => PhaseEnd::Abort { + lease_not_adopted: granted && self.lease_adoption.is_none(), + }, + } + } + + /// The recovery a completed post-fencing abort names. A continuity whose + /// candidate adopted its lease restarts the admitted release; everything + /// else restores the incumbent. + fn terminal_recovery(&self) -> TerminalRecovery { + if self.command_kind == CommandKind::Continuity && self.lease_adoption.is_some() { + TerminalRecovery::RestartAdmitted + } else { + TerminalRecovery::RestoreIncumbent + } + } + /// Whether this transaction's binding declares stop-then-start. A /// transaction without a plan yet cannot, so it reads as false. fn rollout_stops_incumbent_first(&self) -> bool { @@ -2444,6 +2559,11 @@ enum Command { command_id: String, requested_by: String, }, + Expire { + state_store: PathBuf, + command_id: String, + requested_by: String, + }, Validate { recipe: PathBuf, binding: Option, @@ -2475,6 +2595,11 @@ pub fn run(args: impl Iterator) -> Result<()> { command_id, requested_by, } => cancel(&state_store, &command_id, &requested_by), + Command::Expire { + state_store, + command_id, + requested_by, + } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), } } @@ -2526,6 +2651,7 @@ fn parse(args: impl Iterator) -> Result { "up" => parse_up(args), "status" => parse_status(args), "cancel" => parse_cancel(args), + "expire" => parse_expire(args), "validate" => parse_validate(args), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), @@ -2642,6 +2768,68 @@ fn parse_status(mut args: impl Iterator) -> Result { }) } +fn parse_expire(mut args: impl Iterator) -> Result { + let command_id = args + .next() + .ok_or_else(|| anyhow!("idunn expire requires a command id"))?; + require_id(&command_id, "command id")?; + let mut state_store = RuntimeOptions::default().state_store; + let mut requested_by = env::var("SUDO_USER") + .or_else(|_| env::var("USER")) + .or_else(|_| env::var("USERNAME")) + .unwrap_or_else(|_| "operator".into()); + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--requested-by" => requested_by = string_value(&mut args, &argument)?, + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn expire option {argument:?}"), + } + } + require_value(&requested_by, "expiry requester")?; + Ok(Command::Expire { + state_store, + command_id, + requested_by, + }) +} + +/// Ask the daemon to end a live transaction's post-fencing phase now. This +/// writes a request and nothing else: the daemon's deadline resolver ends the +/// phase exactly as it ends one whose deadline has passed. +fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { + let snapshot = ControlSnapshot::read(store_path)?; + let mut live = snapshot + .transactions + .iter() + .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); + let current = live + .next() + .context("command has no live transaction to expire")?; + ensure!( + live.next().is_none(), + "command has multiple live transactions" + ); + ensure!( + current.value.resolves_by_deadline(), + "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" + ); + let now = now_millis()?; + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: command_id.to_owned(), + requested_by: requested_by.to_owned(), + requested_at_unix_millis: now, + }; + ensure!( + SingleFileMessagePackBackingStore::new(store_path) + .insert_entry_if_absent(expiry_request_envelope(&request, now)?)?, + "an expiry request for this command is already waiting" + ); + println!("{command_id} expiry requested; Idunn ends its phase as an expired deadline"); + Ok(()) +} + fn parse_cancel(mut args: impl Iterator) -> Result { let command_id = args .next() @@ -2787,6 +2975,7 @@ struct ControlSnapshot { transactions: Vec>, admitted: Vec>, targets: Vec>, + expiry_requests: Vec>, } impl ControlSnapshot { @@ -2842,6 +3031,19 @@ impl ControlSnapshot { ); snapshot.targets.push(Stored { envelope, value }); } + ExpiryRequest::TYPE => { + ensure!( + envelope.schema_id.as_deref() == Some(EXPIRY_REQUEST_SCHEMA), + "Idunn control store contains an unsupported expiry request" + ); + let value: ExpiryRequest = decode_record(&envelope)?; + value.validate()?; + ensure!( + envelope.key == value.command_id, + "expiry request key is not its command" + ); + snapshot.expiry_requests.push(Stored { envelope, value }); + } _ => bail!("Idunn control store contains a foreign document"), } } @@ -5172,7 +5374,7 @@ impl Engine { /// own command, so one target brake cannot suspend unrelated continuity. fn resume_one_transaction(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut progressed = false; + let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; let mut candidates = snapshot .transactions .iter() @@ -5212,6 +5414,9 @@ impl Engine { /// transaction would report progress every tick. fn resume_candidate(&self, current: &Stored) -> Result { let id = current.value.transaction_id.as_str(); + if let Some(moved) = self.resolve_phase_deadline(current)? { + return Ok(moved); + } if self .resume_backoff .lock() @@ -5273,6 +5478,8 @@ impl Engine { // finish. self.begin_post_fencing_abort(latest, error)?; } else { + // Resumable only until the phase deadline: the resolver ends + // the phase before the next step once it has passed. self.record_resumable_error(latest, &error)?; } } @@ -6801,8 +7008,15 @@ impl Engine { self.record_gate_wait(current, "candidate is still warming") } CandidateAnswer::Answered { evidence, .. } => { + let adoption = LeaseAdoptionEvidence::with_ready( + lease, + &evidence.canonical_sha256, + AdoptionSource::Direct, + evidence.admitted_at_unix_millis, + ); self.persist_same_phase(current, |next| { next.ready = Some(ReadinessEvidence::RouteProof { evidence }); + next.lease_adoption = adoption; Ok(()) }) } @@ -6831,6 +7045,20 @@ impl Engine { ensure!(semantic_ready, "stored Ready receipt changed meaning"); } else if semantic_ready { let evidence = latest.clone(); + // A Ready stateful correlation carries exactly the current + // lease (the authenticator refuses one that does not). + let adoption = authenticated + .record() + .signed_presence_sha256 + .as_deref() + .and_then(|presence| { + LeaseAdoptionEvidence::with_ready( + current_lease, + presence, + AdoptionSource::OdinTopology, + evidence.admitted_at_unix_millis, + ) + }); let _token = SequenceAdmittedReady { transaction_id: admitted.value.transaction_id.clone(), evidence: evidence.clone(), @@ -6839,6 +7067,7 @@ impl Engine { }; return self.persist_same_phase(&admitted, |next| { next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence }); + next.lease_adoption = adoption; Ok(()) }); } else { @@ -6991,14 +7220,14 @@ impl Engine { .as_ref() .and_then(LeasingEvidence::lease_sha256) .map(str::to_owned); + // A newer sequence admitted here changed the record, and commit + // goes on from that record rather than returning: returning let + // Odin's next re-stamp abort every attempt (F16). let Some((ready_current, authenticated)) = self.admit_latest_topology(current, current_lease_sha256.as_deref())? else { return Ok(()); }; - if ready_current.envelope != current.envelope { - return Ok(()); - } ensure!( is_semantic_ready(&authenticated), "latest Odin observation is not Ready at admission commit" @@ -7071,14 +7300,13 @@ impl Engine { .as_ref() .and_then(LeasingEvidence::lease_sha256) .map(str::to_owned); + // The final proof's own presence makes Odin re-stamp, so this + // admission usually writes. Commit is from the record it wrote. let Some((commit_current, authenticated)) = self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? else { return Ok(()); }; - if commit_current.envelope != ready_current.envelope { - return Ok(()); - } ensure!( is_semantic_ready(&authenticated), "latest Odin observation is not Ready after the final admission challenge" @@ -8329,6 +8557,90 @@ impl Engine { replace_transaction(&self.options.state_store, current, &next) } + /// Every post-fencing phase ends. Before a live transaction past the + /// fence runs its step, a phase whose deadline has passed, or whose + /// command an operator asked to expire, is resolved as `phase_end` says. + /// Returns `Some(moved)` when it decided the tick, `None` to run the step. + /// + /// The deadline is written only by `enter_phase`, so nothing here or in a + /// resume path extends it. A record lifted from before deadlines existed + /// has none; its phase is timed once, from the first time this sees it. + fn resolve_phase_deadline( + &self, + current: &Stored, + ) -> Result> { + let transaction = ¤t.value; + if !transaction.resolves_by_deadline() { + return Ok(None); + } + let now = now_millis()?; + let Some(deadline) = transaction.phase_deadline else { + let mut next = transaction.clone(); + next.enter_phase(transaction.phase, now); + if next.phase_deadline.is_none() { + return Ok(None); + } + replace_transaction(&self.options.state_store, current, &next)?; + return Ok(Some(true)); + }; + let requested = ControlSnapshot::read(&self.options.state_store)? + .expiry_requests + .iter() + .any(|request| request.value.command_id == transaction.command_id); + let cause = if now >= deadline.deadline_at_unix_millis { + format!( + "the {:?} phase passed its deadline at unix ms {}", + transaction.phase, deadline.deadline_at_unix_millis + ) + } else if requested { + format!("the {:?} phase was expired by the operator", transaction.phase) + } else { + return Ok(None); + }; + match transaction.phase_end() { + PhaseEnd::Abort { lease_not_adopted } => { + let error = if lease_not_adopted { + anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") + } else { + anyhow!("{cause}") + }; + self.begin_post_fencing_abort(current, error)?; + Ok(Some(true)) + } + PhaseEnd::OperatorRequired => { + self.report_once( + &format!("deadline:{}", transaction.transaction_id), + format!( + "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", + transaction.transaction_id + ), + ); + Ok(None) + } + } + } + + /// Delete expiry requests whose command has no phase left to end: it was + /// resolved, it finished, or its command is gone. + fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { + let spent = snapshot + .expiry_requests + .iter() + .filter(|request| { + !snapshot.transactions.iter().any(|stored| { + stored.value.command_id == request.value.command_id + && stored.value.resolves_by_deadline() + }) + }) + .map(|request| request.envelope.clone()) + .collect::>(); + if spent.is_empty() { + return Ok(false); + } + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .delete_batch_if_unchanged(&spent) + } + fn record_resumable_error( &self, current: &Stored, @@ -8540,7 +8852,7 @@ impl Engine { next.last_error = Some(abort.error.clone()); next.completion = Some(TransactionCompletion::FailedAfterFencing { error: abort.error.clone(), - recovery: TerminalRecovery::RestoreIncumbent, + recovery: current.value.terminal_recovery(), }); replace_transaction(&self.options.state_store, current, &next) } @@ -9072,6 +9384,7 @@ fn usage() -> &'static str { idunn up [--state-store PATH] [--no-wait]\n\ idunn status [--state-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ + idunn expire [--state-store PATH]\n\ idunn validate --recipe PATH [--binding PATH]\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ @@ -9215,6 +9528,7 @@ mod tests { struct RouteStubs { calls: PathBuf, reload_fails: PathBuf, + ufw_delete_fails: PathBuf, } #[cfg(unix)] @@ -9226,6 +9540,7 @@ mod tests { std::fs::create_dir_all(&dir)?; let calls = dir.join("calls"); let reload_fails = dir.join("reload-fails"); + let ufw_delete_fails = dir.join("ufw-delete-fails"); let program = |name: &str, refuse: &str| -> Result { let path = dir.join(name); std::fs::write( @@ -9248,10 +9563,23 @@ mod tests { reload_fails.display() ), )?, - ufw: program("ufw", "")?, + ufw: program( + "ufw", + &format!( + "if [ \"$1\" = delete ] && [ -e '{}' ]; then exit 1; fi\n", + ufw_delete_fails.display() + ), + )?, preflight_root: root.join("route-preflight"), }; - Ok((actuators, Self { calls, reload_fails })) + Ok(( + actuators, + Self { + calls, + reload_fails, + ufw_delete_fails, + }, + )) } fn refuse_reloads(&self) -> Result<()> { @@ -9259,6 +9587,12 @@ mod tests { Ok(()) } + /// `ufw delete` exits non-zero with an error that is not "non-existent". + fn refuse_ufw_deletes(&self) -> Result<()> { + std::fs::write(&self.ufw_delete_fails, b"x")?; + Ok(()) + } + /// How many logged calls begin with `prefix`, e.g. `systemctl reload`. fn count(&self, prefix: &str) -> usize { std::fs::read_to_string(&self.calls) @@ -12514,6 +12848,61 @@ mod tests { Ok(()) } + /// Q3 as a table: which ended phases abort, which name the unadopted + /// lease, which are left to the operator, and which recovery the terminal + /// record names. + #[test] + fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let lease = write_lease(); + let sha256 = lease.canonical_sha256()?; + let granted = LeasingEvidence::Granted { + lease: lease.clone(), + lease_sha256: sha256.clone(), + }; + let prepared = LeasingEvidence::Prepared { + lease, + lease_sha256: sha256.clone(), + }; + let adoption = LeaseAdoptionEvidence::with_ready( + Some(&sha256), + &sha256_id(b"adopting-presence"), + AdoptionSource::Direct, + 1, + ); + assert!(adoption.is_some()); + assert_eq!( + LeaseAdoptionEvidence::with_ready(None, &sha256_id(b"p"), AdoptionSource::Direct, 1), + None + ); + let abort = |lease_not_adopted| PhaseEnd::Abort { lease_not_adopted }; + use CommandKind::{Continuity, Deploy}; + let cases = [ + (Deploy, None, None, abort(false)), + (Deploy, Some(LeasingEvidence::SkippedStateless), None, abort(false)), + (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), + (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), + (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), + (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), + (Continuity, Some(prepared), None, abort(false)), + (Continuity, Some(granted.clone()), None, abort(true)), + (Continuity, Some(granted), adoption.clone(), abort(false)), + ]; + for (kind, leasing, lease_adoption, expected) in cases { + transaction.command_kind = kind; + transaction.leasing = leasing.clone(); + transaction.lease_adoption = lease_adoption.clone(); + assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); + let recovery = if kind == Continuity && lease_adoption.is_some() { + TerminalRecovery::RestartAdmitted + } else { + TerminalRecovery::RestoreIncumbent + }; + assert_eq!(transaction.terminal_recovery(), recovery, "{kind:?} {lease_adoption:?}"); + } + Ok(()) + } + #[test] fn an_operator_required_recovery_must_say_why() -> Result<()> { let (_, mut failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; @@ -15384,6 +15773,8 @@ mod tests { /// that picked up Idunn's grant reports. lease: Mutex>, reply: Mutex, + /// Runs when the stable listener accepts, before it answers. + on_stable_hit: Mutex>>, } /// How the runtime answers a well-formed challenge. @@ -15540,6 +15931,11 @@ mod tests { &stub.stable_hits }; hits.fetch_add(1, Ordering::SeqCst); + if !candidate + && let Some(hook) = stub.on_stable_hit.lock().unwrap().as_ref() + { + hook(); + } if !stub.hang_up.load(Ordering::SeqCst) { let _ = stub.serve_one(stream); } @@ -15800,6 +16196,7 @@ mod tests { hang_up: AtomicBool::new(false), lease: Mutex::new(None), reply: Mutex::new(Reply::Honest), + on_stable_hit: Mutex::new(None), }); stub.listen(candidate_listener, true); stub.listen(stable_listener, false); @@ -17861,5 +18258,317 @@ mod tests { assert_eq!(after.route_actuations.len(), 2, "preflight and install are the Forward"); Ok(()) } + + // ----------------------------------------------------------------- + // B5: every post-fencing phase ends. + // ----------------------------------------------------------------- + + impl RoutedWorld { + /// One scheduler pass over the transaction, as the daemon runs it: + /// through the deadline resolver, with the resume backoff spent. + fn resume(&self) -> Result<()> { + self.world.engine.resume_backoff.lock().unwrap().clear(); + self.world.engine.resume_one_transaction()?; + Ok(()) + } + + fn resume_until_terminal(&self) -> Result { + for _ in 0..12 { + if latest(&self.world)?.completion.is_some() { + break; + } + self.resume()?; + } + let finished = latest(&self.world)?; + ensure!(finished.completion.is_some(), "never finished: {:?}", finished.last_error); + Ok(finished) + } + } + + /// The live transaction's phase deadline is already behind it. + fn pass_deadline(world: &EngineFixture) -> Result<()> { + let current = resident(world)?; + let mut next = current.value.clone(); + let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, ¤t, &next) + } + + /// Soul's F1 probe, committed. A continuity's route changes are + /// Survival, so no ceiling refuses its install and rollback while its + /// route proof keeps failing. The Routing deadline ends it: no attempt + /// runs after the deadline, and the withdrawal of a rolled-back route + /// reloads nothing. + #[cfg(unix)] + #[test] + fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..3 { + routed.resume()?; + } + // Before the deadline each attempt installs and rolls back, and no + // resume moves the deadline. + assert_eq!(routed.reloads(), 6); + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::Routing); + assert!(waiting.post_fencing_abort.is_none(), "aborted before its deadline"); + assert!(waiting.last_error.is_some()); + assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); + + pass_deadline(&routed.world)?; + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the passed deadline did not end the phase")?; + assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + let finished = routed.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + ), + "{:?}", + finished.completion + ); + assert_eq!(routed.reloads(), 6, "a reload ran after the deadline"); + Ok(()) + } + + /// Soul's F2 probe, committed: a post-fence withdrawal whose + /// `ufw delete` keeps failing reloads once, then repeats only the + /// firewall step. + #[cfg(unix)] + #[test] + fn a_withdrawal_whose_firewall_step_keeps_failing_reloads_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + assert!(routed.fragment_path().exists()); + let installed = routed.reloads(); + routed.world.route_stubs.refuse_ufw_deletes()?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; + for _ in 0..20 { + routed.resume()?; + } + assert_eq!(routed.reloads(), installed + 1, "a failing withdrawal reloaded again"); + assert_eq!(routed.world.route_stubs.count("ufw delete"), 20); + assert!(!routed.fragment_path().exists()); + assert_eq!( + routed.transaction()?.post_fencing_abort.context("no abort")?.route_restoration, + CleanupEvidence::Pending + ); + Ok(()) + } + + /// A record lifted from before deadlines existed is timed once, from + /// the first time the resolver sees it, and never again. + #[test] + fn a_post_fence_record_without_a_deadline_is_timed_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let mut lifted = current.value.clone(); + lifted.phase_deadline = None; + replace_transaction(&routed.world.state_store, ¤t, &lifted)?; + let engine = &routed.world.engine; + + let before = now_millis()?; + assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, Some(true)); + let stamped = routed.transaction()?.phase_deadline.context("the record was not timed")?; + assert_eq!(stamped.phase, DeploymentPhase::Fencing); + assert!(stamped.entered_at_unix_millis >= before); + assert_eq!( + stamped.deadline_at_unix_millis - stamped.entered_at_unix_millis, + u64::from(lifted.plan.as_ref().unwrap().phase_deadlines().fencing_seconds) * 1000 + ); + assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, None); + assert_eq!(routed.transaction()?.phase_deadline, Some(stamped)); + Ok(()) + } + + /// `idunn expire` writes a request and nothing else; the resolver ends + /// the phase from it, and the spent request is deleted. + #[test] + fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { + let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let refused = expire(&early.world.state_store, "continuity-service", "operator") + .expect_err("a transaction before the fence was expired"); + assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); + + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let untouched = resident(&routed.world)?.envelope; + expire(&routed.world.state_store, "continuity-service", "operator")?; + assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!(snapshot.expiry_requests.len(), 1); + assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); + + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the expiry did not end the phase")?; + assert!(abort.error.contains("Fencing phase was expired by the operator"), "{}", abort.error); + routed.resume()?; + assert!( + ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), + "the spent request stayed" + ); + Ok(()) + } + + /// Q3 at the deadline, through the Engine. A continuity whose candidate + /// was granted a lease and never proved it held it fails as + /// `lease-not-adopted`; one whose Ready proved it restarts the admitted + /// release. A deployment whose candidate holds a lease is left to the + /// operator: nothing is aborted and its step still runs. + #[test] + fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { + let granted = |routed: &RoutedWorld| -> Result { + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + Ok(routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned()) + }; + + let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + granted(&unadopted)?; + pass_deadline(&unadopted.world)?; + unadopted.resume()?; + let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; + assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); + assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); + let finished = unadopted.resume_until_terminal()?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + + let adopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + let lease = granted(&adopted)?; + adopted.stub.set_state("active"); + *adopted.stub.lease.lock().unwrap() = Some(lease.clone()); + adopted.step()?; + let ready = adopted.transaction()?; + let Some(ReadinessEvidence::RouteProof { evidence }) = &ready.ready else { + bail!("the candidate was not Ready"); + }; + let adoption = ready.lease_adoption.clone().context("Ready recorded no adoption")?; + assert_eq!(adoption.write_lease_sha256, lease); + assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); + assert_eq!(adoption.source, AdoptionSource::Direct); + adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + pass_deadline(&adopted.world)?; + adopted.resume()?; + let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; + assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + let finished = adopted.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestartAdmitted, + .. + }) + ), + "{:?}", + finished.completion + ); + + let deploy = + build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; + granted(&deploy)?; + pass_deadline(&deploy.world)?; + for _ in 0..3 { + deploy.resume()?; + } + let held = deploy.transaction()?; + assert!(held.post_fencing_abort.is_none(), "a leased deployment was aborted"); + assert_eq!(held.phase, DeploymentPhase::AwaitingReady); + assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); + Ok(()) + } + + /// F16: Commit goes on from the topology it admitted, so Odin + /// re-stamping before the step and again during the final proof does + /// not send it round again. + #[test] + fn commit_completes_while_odin_restamps_every_tick() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + let transaction = routed.transaction()?; + let cursor = transaction.odin_publisher_sequence_cursor; + let store = routed.world.engine.options.odin_correlation_store.clone(); + replace_odin_entry(&store, odin_ready_envelope(&routed.world, &transaction, cursor + 1)?)?; + let later = odin_ready_envelope(&routed.world, &transaction, cursor + 2)?; + *routed.stub.on_stable_hit.lock().unwrap() = Some(Box::new(move || { + replace_odin_entry(&store, later.clone()).expect("Odin re-stamps"); + })); + routed.step()?; + let generation = admitted(&routed)?; + assert_eq!( + generation.latest_odin_observation.context("no Odin receipt")?.publisher_sequence, + cursor + 2, + "commit did not go on from the re-stamped topology" + ); + Ok(()) + } + + fn odin_ready_envelope( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result { + let expected = transaction.expected.as_ref().context("no Expected")?; + Ok(CultCacheEnvelope { + key: crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(world, transaction, sequence, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }) + } + + fn replace_odin_entry(store: &Path, envelope: CultCacheEnvelope) -> Result<()> { + let backing = SingleFileMessagePackBackingStore::new(store); + let current = backing + .pull_all_read_only_snapshot()? + .into_iter() + .find(|entry| entry.key == envelope.key && entry.r#type == envelope.r#type); + ensure!( + backing.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: envelope.r#type.clone(), + key: envelope.key.clone(), + current, + }], + &[envelope], + )?, + "the Odin store changed under the re-stamp" + ); + Ok(()) + } } } diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 5210448..603c9d2 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -467,15 +467,17 @@ pub fn select_dependencies( /// overrides any of them; the resolved values are frozen into the plan. pub const DEFAULT_FENCING_DEADLINE_SECONDS: u32 = 120; pub const DEFAULT_LEASING_DEADLINE_SECONDS: u32 = 120; -// Unsafe for streampixels-service (startup = "create-or-open-after-write-lease"): -// its Postgres migrations run inside AwaitingReady. B5 requires an explicit -// binding value for stateful targets instead of this default. +// A stateful candidate may run migrations inside AwaitingReady (streampixels- +// service runs Postgres's there). When this passes, a deployment whose +// candidate was issued a write lease is left to the operator, not aborted; a +// continuity is aborted and its release restarted. pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300; pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120; pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120; -/// The resolved per-phase deadline durations a plan carries. Nothing reads -/// them at the cut that introduced them. +/// The resolved per-phase deadline durations a plan carries. Entering a +/// post-fencing phase stamps its deadline from them, and the deadline +/// resolver ends the phase once it passes. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct PhaseDeadlines { From 0c80a444f6613345df7a2e755337ab0383daf11d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:46:21 +0200 Subject: [PATCH 007/196] B5: literal control snapshots in tests carry the expiry requests Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 30ab973..336cbc3 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -10740,6 +10740,7 @@ mod tests { schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), }; let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), commands: Vec::new(), transactions: vec![ Stored { @@ -11155,6 +11156,7 @@ mod tests { let one = DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; let two = DeploymentTransaction::new(&command, "ghostlight".into(), 1, None, 100)?; let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), commands: vec![Stored { envelope: command_envelope(&command, 100)?, value: command, @@ -11186,6 +11188,7 @@ mod tests { let unrelated = DeploymentTransaction::new(&unrelated_command, "huginn".into(), 0, None, 101)?; let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), commands: Vec::new(), transactions: [&first, &second, &unrelated] .into_iter() @@ -12704,6 +12707,7 @@ mod tests { generation.odin_publisher_sequence_cursor = 88; let target = generation.target.clone(); let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), commands: Vec::new(), transactions: Vec::new(), admitted: vec![Stored { From 495112565b8d2c9066c330e1d316877d8a40ca6b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:48:18 +0200 Subject: [PATCH 008/196] Watchdog reporting target: Idunn decides incidents, Bifrost delivers First typed Eureka campaign (idunn-watchdog). The typed target and rulings live in Eureka's mind; this doc holds the rationale and design truths. Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-target.md | 80 +++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 docs/watchdog-reporting-target.md diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md new file mode 100644 index 0000000..7f91221 --- /dev/null +++ b/docs/watchdog-reporting-target.md @@ -0,0 +1,80 @@ +# Watchdog reporting: target + +Status: target, Self, 2026-09-30. This is the first campaign whose state is +typed in Eureka's mind (campaign `idunn-watchdog`). Questions, rulings, cut +specs and their verification live there. This document keeps the rationale +and the design truths the campaign produces. The cut map's body facts and +model page will live in `watchdog-reporting-map.md`. + +## Why + +The operator ruled Odin's store-recovery Q-R4 on 2026-09-30 (Odin +`docs/store-recovery-cut.md`, `cdc57bd`). A third set-aside inside ten minutes +stops Odin writing while it keeps serving. The operator added: + +> Yep, and Idunn needs better reporting capabilities so it can serve as +> watchdog. I ought to get a message about this on Discord if it ever happens. + +and, on where delivery lives: + +> Bifrost is indeed intended to be the bridge to external services like Discord + +Today nobody would hear about it. Idunn already detects several conditions an +operator must act on, and for each one it writes a single `eprintln!` line to +journald (`report_once`, `control_plane.rs:4599`): + +- A continuity restart budget is exhausted (6 per hour; `restarts_exhausted`, + decided at `control_plane.rs:5709`). Idunn stops restarting the target, and + the only trace is one log line. +- An admitted route is degraded. It fails its stable-route challenge + (`supervise_admitted_route`, ~5973). The degraded mark is durable, but no + one is told. +- A post-fencing phase deadline expires into `OperatorRequired`. This lives on + B5, which is unmerged on `hands/b5`. The failure record is durable and the + notice is a log line. + +Odin's own store condition (`odin.store_condition.v1`, `contested`) is mapped +but not built. Idunn reads nothing about Odin's store health today. + +The delivery path half exists in Bifrost: + +- `tools/operator-notification.mjs publish-idunn-alarm` builds a + `bifrost.bridge.discord_post_command.v1` `discord-dm` command. +- `tools/cultmesh-bridge-commands.mjs process` turns such commands into DMs, + writing a `discord_post_receipt.v1` for each. + +But no pump runs anywhere, and the bot token file on Yggdrasil is absent +(gamecult-ops `inventory.md` ~740). Idunn has never called the publisher. So +the machine has a detector with no record, and a deliverer that nothing feeds. + +## Design truths this campaign produces + +- **Idunn decides what is an incident.** Idunn owns daemon survival, so the + judgement "an operator must look at this" is its judgement, made where the + condition is decided. An incident is typed state in Idunn's store before any + delivery is attempted. A stderr line is a trace of the record, not the + record itself. +- **Bifrost delivers.** Discord transport, the bot token, the recipient + binding and the delivery receipt belong to Bifrost. Idunn holds no Discord + credential and speaks no Discord API. +- **Survival never waits on delivery.** Bifrost is a managed target like any + other. Its absence, failure or backlog may delay a message. It never delays + or gates an Idunn decision, tick, restart, or deployment. An incident nobody + has delivered stays visible in Idunn's own state and in `idunn status`. +- **One message per incident, not per tick.** A fault that lasts hours + produces one opening notice, not a stream of them. Repeated delivery + attempts for one incident are idempotent at the recipient's end. +- **Nothing sensitive leaves.** A notice names the target, the condition and + the time. It never carries a binding value, a path to a secret, a token, or + a line of configuration. + +## Not this campaign + +- Odin's store-recovery cuts, O2 included. They produce `odin.store_condition` + and are Odin's campaign, still in prose. This campaign consumes the record + once it exists. +- Idunn's B5 phase deadlines. B5 is in prose on `hands/b5`. This campaign + reports its `OperatorRequired` outcome once B5 merges. +- VoidBot's `notify_owner` MCP tool and its separate DM code. +- Bifrost's Persona delivery crossing. +- Eve/TUI lowering of incidents. From 4830af8eb125fda8bea1963bb15c72911c476d60 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:50:55 +0200 Subject: [PATCH 009/196] B5: a continuity over its admitted route reloads nothing while its proof fails The live shape of Soul's F1: a continuity reuses the admitted Expected, so its install and rollback find the admitted fragment already on disk and run no host program, before or after its Routing deadline. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 336cbc3..7e4f281 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -18348,6 +18348,44 @@ mod tests { Ok(()) } + /// The live shape of F1. A continuity reuses the admitted Expected, so + /// its membership is the admitted fragment already on disk: while its + /// proof fails, the install and the rollback write nothing and reload + /// nothing, and neither does the withdrawal after its deadline. + #[cfg(unix)] + #[test] + fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + let binding = next.plan.as_ref().unwrap().parsed_inputs()?.1; + let admitted = NginxRouteDriver::new(binding.route.context("no route binding")?) + .render(next.expected.as_ref().context("no Expected")?)?; + let preflight = next.route_preflight.as_mut().context("no preflight")?; + preflight.incumbent_runtime_instance_id = Some(sha256_id(b"admitted-instance")); + preflight.incumbent_membership_sha256 = Some(sha256_id(&admitted)); + preflight.incumbent_configuration = Some(admitted.clone()); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + std::fs::write(routed.fragment_path(), &admitted)?; + + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..5 { + routed.resume()?; + } + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); + assert!(routed.transaction()?.last_error.is_some(), "the proof did not fail"); + assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); + + pass_deadline(&routed.world)?; + routed.resume_until_terminal()?; + assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); + assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); + Ok(()) + } + /// Soul's F2 probe, committed: a post-fence withdrawal whose /// `ufw delete` keeps failing reloads once, then repeats only the /// firewall step. From 153af6ac402a5356821805d58d70cb5c9cbc36d5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:57:34 +0200 Subject: [PATCH 010/196] Watchdog reporting map: body facts, model page, crossing rationale Imagination step 0b for campaign idunn-watchdog. Forks are admitted in the mind as questions delivery-crossing, paging-conditions, closure-notice. Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-map.md | 148 +++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 docs/watchdog-reporting-map.md diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md new file mode 100644 index 0000000..3b71abf --- /dev/null +++ b/docs/watchdog-reporting-map.md @@ -0,0 +1,148 @@ +# Watchdog reporting: map + +Campaign `idunn-watchdog`. Questions, rulings, cut specs, reports and verdicts +are typed documents in Eureka's mind; this file keeps only body facts, the +model page and rationale. Target rationale: `watchdog-reporting-target.md` +(`4951125`). + +## Body facts + +Source reads are at Idunn `4951125` (main), Idunn `hands/b5` (`0c80a44`, +unmerged, read with `git show`/`git diff main...hands/b5` only), Bifrost +`42728a6`, Odin `cdc57bd`, VoidBot `46d891b`, gamecult-ops `14df7ce`. Host +probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC +(15:51-15:55 CEST). + +### Idunn + +| # | What | Where | Result | +|---|---|---|---| +| I1 | Alert path | `src/control_plane.rs:3798-3812` (`ReportOnce`), `:4599-4610` (`report_once`), `:5114-5138` (`note_fault`, `clear_fault`) | The only operator signal is `eprintln!`. De-dup is `fault_reports: Mutex>` (`:4392`), in memory: an Idunn restart forgets it and reprints. `note_fault` also writes the transaction's `last_error`. | +| I2 | Continuity exhaustion | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | +| I3 | Route degradation | `RouteSupervisionState` `:890-942`; `record_failed_challenge` sets `degraded_since_unix_millis` on the **first** failed challenge, `record_proved_challenge` clears it; reported at `:5975` under `route:` | Durable per incarnation (inside `admitted_generation.v4`); a new generation starts from `default()`. Backoff doubles from the topology max age to a cap. | +| I4 | Other operator-facing faults | `:5645-5651` (`generation-down:`, "holds a dead admitted generation down"), `:6016` (`route-record:`), `note_fault` per transaction | Also stderr plus `last_error`. Not named by the target; listed so the paging-set question sees them. | +| I5 | `OperatorRequired` on main | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | +| I6 | `OperatorRequired` on B5 | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | +| I7 | Store kinds | `:55-61`, `:936-966`; B5 adds `idunn.expiry_request.v1` | `control.cc`: `idunn.deployment_command`, `idunn.deployment_transaction`, `idunn.admitted_generation.v4`, `idunn.target_supervision.v1`. `history.cc` beside it holds terminal transactions (`:3653`), retired one per tick (`:5068-5091`). | +| I8 | Single-writer precedent inside `control.cc` | B5 `ExpiryRequest` doc comment | "The CLI only writes this record. The daemon's deadline resolver reads it ... No CLI writes a transaction field." One writer per kind. | +| I9 | Serve loop | `:5041-5058`, `run_scheduler_tick` `:5098-5110` | 500 ms poll. A tick error is logged, never propagated. | +| I10 | `idunn status` | `render_supervision` `:4215` | Renders restarts used/limit and `degraded-since`; no incident notion. | +| I11 | Idunn already talks CultNet to targets | `src/drivers.rs:5567-5600` `request_runtime_presence_at` | `CultNetMessage::SnapshotRequest { schema_ids, record_keys }` over the route transport (http/tcp/rudp) to a target endpoint. Precedent for reading a typed record out of Odin's catalog. | +| I12 | Idunn reads Odin's file read-only | `src/drivers.rs:5005-5020` (`receive`), unit `--odin-correlation-store /var/lib/gamecult/odin/topology.cc` | Absent file yields `None`; continuity never consults it. | +| I13 | Idunn's published surface | unit `deploy/idunn-yggdrasil.service` comment on `ReadWritePaths` | "the topology store is Idunn's *published* surface, and every managed target must read it". Lives in `/var/lib/gamecult/idunn-projection`, outside Idunn's 0750 root. | +| I14 | Rejected predecessor | `git show 821a8d0^:src/main.rs:2020-2060, 8206-8250` | Before the 2026-09-02 rebuild, Idunn wrote an `IdunnOperatorAlarmRecord` (`alarm_id = alarm::`, one per decision) and ran an operator-supplied shell `--operator-alarm-command` synchronously with a timeout, passing `IDUNN_ALARM_*` env. The rebuild deleted it. | +| I15 | Stale guide lines | `docs/guide.md:47` "escalate to an operator through Bifrost"; `:328` "Idunn is not Bifrost ... Discord delivery, owner DMs" | `:47` describes a capability that does not exist; `:328` stays true. | + +### Yggdrasil + +| # | Command | Result | +|---|---|---| +| Y1 | `systemctl is-active idunn-yggdrasil bifrost bifrost-persona-mouth bifrost-persona-feedback` | all `active` | +| Y2 | `systemctl show idunn-yggdrasil -p User -p ProtectSystem -p ReadWritePaths` | `User=root`, `ProtectSystem=full`. `ReadWritePaths` names no Bifrost path, but `ProtectSystem=full` only makes `/usr`, `/boot`, `/etc` read-only, so root Idunn can physically write `/srv/bifrost` and `/var/lib/gamecult/bifrost`. The `/srv` and `/var/lib` entries are documentation, not a boundary. | +| Y3 | `ls -la /srv/bifrost/env` | `persona-delivery.env` **exists**: `root:root 0600`, 135 bytes, 2026-09-04 02:20. gamecult-ops `inventory.md:741` ("the file is absent") is stale. | +| Y4 | `sudo awk -F= '{print $1, length($2)}' persona-delivery.env` (values not printed) | `BIFROST_DISCORD_BOT_TOKEN` 72 chars, `DISCORD_OWNER_ID` 18 chars. The token and the recipient binding are both already in Bifrost's env. | +| Y5 | `systemctl cat bifrost-persona-mouth` | `User=bifrost-feedback`, `ProtectSystem=strict`, `EnvironmentFile=-/srv/bifrost/env/persona-delivery.env`, `ExecStart=node .../tools/persona-feedback.mjs serve-persona-delivery ...` with signed-request RUDP listener `10.77.0.1:17876` and Starfire permit requester `rudp://10.77.0.2:17877`. | +| Y6 | `systemctl show bifrost-persona-mouth -p ActiveEnterTimestamp -p NRestarts` | Started 2026-08-21 10:51:22 UTC, 0 restarts: **before** the env file was written, so the running process has no token loaded. | +| Y7 | `readlink -f .../persona-feedback/runtime/current`; `grep -c discord-dm .../cultmesh-bridge-commands.mjs` | Current release `cb3239a` (2026-08-08). It ships `cultmesh-bridge-commands.mjs`, `operator-notification.mjs`, `bifrost-bridge.mjs`, but `c063f27` (discord-dm verb, 2026-09-04) is **not** an ancestor: the deployed pump cannot send a DM. `discord-dm` count 0. | +| Y8 | `find /srv/bifrost /var/lib/gamecult/bifrost -name provider-store.cc` | None. The pump's store has never existed on the host. | +| Y9 | `getfacl /var/lib/gamecult/idunn-projection`; `ls -ld /var/lib/gamecult/idunn` | Projection dir `root`, `other::r-x`, default `other::r--`: world-readable. `/var/lib/gamecult/idunn` is `idunn:idunn 0750`; `control.cc` is unreadable to other users. | +| Y10 | `id idunn`; `getent passwd bifrost bifrost-feedback` | `idunn` 986/978; `bifrost` 111/113; `bifrost-feedback` 994/983. No shared group. | + +### Bifrost + +| # | What | Where | Result | +|---|---|---|---| +| BF1 | Alarm publisher | `tools/operator-notification.mjs:41-148` | Opens Bifrost's `.bifrost/provider-store.cc` (repo-relative, `:20`) and `put`s a `bifrost.bridge.discord_post_command.v1` `discord-dm`. The **caller** supplies `recipientId` (`:51-54`). `commandId = idunn-alarm-sha1(alarm JSON)[0..16]`, and the JSON includes `raisedAt`, which defaults to now (`:48`): a re-publish after a restart mints a new id. | +| BF2 | Pump | `tools/cultmesh-bridge-commands.mjs:50-71, 133-221` | One-shot `process`: selects `pending` and `running`, marks `running`, spawns `bifrost-bridge.mjs`, writes `discord_post_receipt.v1` and `completed`/`failed`. `failed` is terminal (no retry). A crash after the Discord POST and before the receipt leaves `running`, which the next run resends. | +| BF3 | Discord send | `tools/bifrost-bridge.mjs:735-770` | Opens the DM channel, posts `content`. No `nonce` is sent (`grep nonce` hits only the Persona permit tools). | +| BF4 | No caller authentication on the command path | BF1, BF2 | Whoever can write the store file can make Bifrost post anything to anyone. | +| BF5 | Authenticated precedent in Bifrost | `tools/persona-discord-delivery.mjs:27-30`, `tools/persona-discord-permit.mjs` | Persona crossing: signed requests, Epiphany-signed permits, "permit request intent persisted before network" in an execution journal. | +| BF6 | History | `git log -S publish-idunn-alarm`: `755649f` (2026-06-07), `c063f27` (2026-09-04) | `c063f27`: "an alarm is a command addressed to the gate, not a document dropped into somebody else's state." | + +### Odin and VoidBot + +| # | What | Where | Result | +|---|---|---|---| +| O1 | `odin.store_condition.v1` | Odin `docs/store-recovery-cut.md:158, 199, 318-325` | Not built. One record, key `topology`, held in Odin's in-memory W, rewritten on change, **flushed with the store** and served by Odin's catalog. While `contested` Odin writes no file, and while `unpersisted` it cannot: in exactly the conditions that matter, `topology.cc` on disk does not carry them. Only the catalog does. | +| O2 | I-F1, I-F2 | Odin `docs/store-recovery-cut.md:496-500` | I-F2 is this campaign's continuity-exhaustion row. I-F1 (hold deploys on a bad condition) is a deployment-brake decision, not reporting. | +| V1 | Peer precedent | VoidBot `packages/core/src/bifrost-discord-command.ts:39-68` | The caller opens Bifrost's store, derives `commandId` from its `idempotencyKey`, optionally spawns the pump itself, and **waits** for the receipt. An unconditional `put` of an existing id overwrites a `completed` command back to `pending`. | +| V2 | VoidBot `notify_owner` | Self's mapper, not re-probed | Not live on Yggdrasil; no Discord-capable VoidBot process runs there. | + +## Model page + +A row per persistent kind this campaign creates or depends on. "New" means the +kind does not exist yet. Names in `code` for new kinds are working names; the +cut spec fixes them. + +| Kind | Identity | Lifecycle | Authority | +|---|---|---|---| +| **Incident record** (new, Idunn `control.cc`, e.g. `idunn.operator_incident.v1`) | Key `::`. Namespace: Idunn's control store, one type. `condition` is a closed enum (`continuity-exhausted`, `route-degraded`, `operator-required`, `odin-store-contested`, ...). `subject` is the target for target conditions and the `transaction_id` for `operator-required`. `opened_at` is written once at opening by Idunn's clock. Injective because of one rule: **at most one open incident per `(condition, subject)`**; the opener checks it before writing. Two different faults on one target are different conditions, so they never collide. Nothing in the key comes from Bifrost or Odin. | **Created** at the site that decides the condition (I2, I3, I6, Odin read) in the same store transaction family as that condition's own write, before any trace line. **Closed** by a write of `closed_at` and `close_reason` on the same record, never a delete: continuity-exhausted closes when the supervision pass sees the admitted workload running or a new generation is admitted, **not** when the window slides (I2 would flap otherwise); route-degraded closes on `record_proved_challenge` or when its generation is replaced; odin-store-contested closes when Odin's catalog stops reporting it; operator-required closes when its transaction leaves the phase, for which **no path exists yet** (I6). **Reopened**: never revived; a recurrence is a new record with a new `opened_at`. **Replayed after an Idunn restart**: the record is durable, so the decision site finds the open incident and writes nothing; `ReportOnce` is not consulted. **Pruned**: a closed record retires to `history.cc` one per tick, like a terminal transaction, after the projection has carried its closure for a retention period (length is a cut detail). After pruning, `idunn status` shows only open incidents. | Idunn's daemon, at the condition's decision site, is the only writer. Forbidden: the CLI (it may request, as `ExpiryRequest` does, never write the record), Bifrost, Odin, any reader of the projection, `ReportOnce`, a repair loop that opens incidents from log text. | +| **Incident projection** (new, derived) | One file in `/var/lib/gamecult/idunn-projection/` beside `topology.cc`, holding the open incidents plus closed ones inside retention, keyed exactly as the record. World-readable (Y9). | Rewritten whenever an incident opens or closes. Idunn restart: regenerated from `control.cc`. Never an input to Idunn. | Derived, notification-only. Idunn writes it; nothing else does. Whether it is signed with Idunn's service identity, as the topology projection is, follows that precedent (cut detail). Forbidden: Idunn reading it back, any Bifrost write. | +| **Stderr trace** (exists) | `report_once` key, in memory | Printed once per process lifetime per fault; lost on restart (I1). | **Demoted**: `ReportOnce` is no longer an owner of "has the operator been told". It de-dups stderr only. | +| **Delivery request / journal entry** (Bifrost store) | Keyed by the Idunn incident key plus the notice kind (`opened`, and `closed` if Q `closure-notice` says so). Injective because the incident key is. Not derived from payload text or time (BF1's `raisedAt` hash is the counter-example). | **Created** by Bifrost when its reader first sees a notice-worthy incident state in the projection. **Revised**: `pending` -> `sent` or `failed`, with an attempt count and last error. **Retried** until sent, with backoff; a create for an existing key writes nothing (V1's unconditional `put` is the counter-example). **Replayed after a Bifrost restart**: journal is durable; a `pending` entry is retried. **Pruned** only after the incident key has left Idunn's projection. | Bifrost's reader only. Forbidden: Idunn (it never writes Bifrost's store, even though Y2 shows it physically could), VoidBot, the operator by hand. Which Bifrost process: see Rationale. | +| **Delivery receipt** (Bifrost, `discord_post_receipt.v1` or the journal's `sent` state) | Same key as the request. | Written once on success with the Discord message id. | Bifrost. **Idunn does nothing with it**: it neither reads it in the daemon nor waits on it (`survival-independent`). `idunn status` does not show delivery state; Bifrost's journal is where a failed delivery is visible. | +| **Recipient binding** | `DISCORD_OWNER_ID` in `/srv/bifrost/env/persona-delivery.env` (Y4). One recipient, the operator. | Operator-supplied; changes by editing that file. | Bifrost (with the operator as supplier). Forbidden: Idunn and every request payload. BF1's caller-supplied `recipientId` is the counter-example and does not survive. | +| **Bot token** | `BIFROST_DISCORD_BOT_TOKEN` in the same file (Y4), the one GameCult bot. | Operator-supplied, root `0600`, loaded by systemd `EnvironmentFile`. A process started before the file changed has the old value (Y6). | Bifrost holds it; the operator supplies it. Forbidden: Idunn, any notice content (`no-sensitive-egress`). | +| **Odin store condition** (input, not built) | `odin.store_condition.v1`, key `topology` (O1). | Owned by Odin's campaign. Rewritten on change; `contested` and `unpersisted` exist only in Odin's memory and catalog, not on disk. | Odin. Idunn reads it through Odin's catalog with a `SnapshotRequest` (I11 precedent), never from `topology.cc`, and never writes it. An unreachable Odin yields no store incident; continuity already covers a dead Odin. | +| **De-dup key** | Not a separate kind: it **is** the incident key. At Idunn: one open incident per `(condition, subject)`. At Bifrost: one journal entry per `(incident key, notice kind)`. At Discord: the journal key hashed into the message `nonce` with `enforce_nonce`, which covers a crash between the POST and the journal write. The Discord de-dup window is short and unverified here; outside it, a crash in that gap can still send twice. | Lives as long as the incident and its journal entry. | Idunn owns the incident half, Bifrost the delivery half. | +| **Delivery-failure signal** | **Empty.** Nothing tells the operator that the watchdog's own delivery is failing. | | Bifrost's journal holds the state; nobody surfaces it. Out of this table's reach; left for the delivery-mechanism cut (see Rationale). | +| **Operator acknowledgement** | **Not created.** No kind lets the operator mark an incident as seen. Nothing in the target asks for one. | | | + +Empty cells, stated plainly: + +- `operator-required` closure: no path on `hands/b5` ends a transaction left to + the operator (I6). The incident can open and never close until B5 names that + action. +- Delivery-failure visibility: no owner. +- Discord `enforce_nonce` window: not verified against Discord's current docs + in this pass. + +## Rationale + +**Idunn publishes and Bifrost reads, not the reverse.** Three crossings were +weighed against the target: + +1. *Idunn writes a command into Bifrost's store* (the `c063f27` shape). Idunn + would write another organ's state; it would carry the recipient (BF1); an + unconditional `put` re-arms a completed command (V1); and the deployed pump + cannot send a DM anyway (Y7). Nothing stops root Idunn from doing it (Y2), + so the boundary would be doctrine alone. +2. *Idunn sends a CultNet command to a Bifrost listener.* Idunn becomes a + delivery client with timeouts and retry state in its tick, which is where + `survival-independent` breaks. The rebuild already deleted the synchronous + version of this (I14). +3. *Idunn writes its incident into its own store and its world-readable + projection; a Bifrost reader derives its own delivery journal.* No write + crosses an owner boundary. Idunn holds no recipient and no token. Bifrost + down means a late message, not a slow tick. This is the shape of the + existing topology projection (I13), which every managed target already + reads. **Recommended.** It reverses `c063f27`'s direction, so it is a + question for the operator, not a default. + +**Which Bifrost process delivers is doctrine, not a fork.** A service earns a +daemon only when isolation protects a named invariant. A notice can wait a +minute, so a systemd timer running a one-shot Bifrost reader under a Bifrost +user, with `EnvironmentFile=/srv/bifrost/env/persona-delivery.env`, is the +smallest body. Folding it into `bifrost-persona-mouth` was rejected: that +process runs the Epiphany-permit-gated Persona crossing, and a watchdog page +that needs Starfire's permit would stop whenever the workstation is off. +Idunn-invoked delivery was deleted once already (I14). + +**DM is the default, not a question.** The operator's words are "I ought to +get a message", and the recipient binding that exists is `DISCORD_OWNER_ID`. + +**Retiring `publish-idunn-alarm` follows from the crossing.** If the crossing +is (3), nothing calls the publisher. The unauthenticated command path (BF4) +then carries no watchdog traffic. Closing it is Bifrost's own business, and +VoidBot's caller is not live on Yggdrasil (V2). If the operator picks (1), that +path must be authenticated before Idunn uses it. + +**Exhaustion closes on health, not on the window.** `restarts_exhausted` turns +false the moment one timestamp leaves the hour (I2). With a still-dead target +that happens several times an hour. Closing on it would mean one incident, and +one notice, per slide. + +**Why `ReportOnce` is demoted rather than extended.** Persisting its map would +make an in-memory de-dup cache the owner of "was the operator told", keyed by +free-form strings, with no closure and no subject. The incident record puts +that decision at the condition's own site, as `record-before-delivery` asks. From 487f4f14b83ccf2145a2a076d59a10d1f6725031 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:58:14 +0200 Subject: [PATCH 011/196] Watchdog target: a closure notice follows an opening notice Operator ruled closure-notice = opening-and-closure (target r2). Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-target.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index 7f91221..962f118 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -61,9 +61,11 @@ the machine has a detector with no record, and a deliverer that nothing feeds. other. Its absence, failure or backlog may delay a message. It never delays or gates an Idunn decision, tick, restart, or deployment. An incident nobody has delivered stays visible in Idunn's own state and in `idunn status`. -- **One message per incident, not per tick.** A fault that lasts hours - produces one opening notice, not a stream of them. Repeated delivery - attempts for one incident are idempotent at the recipient's end. +- **Two messages per incident at most, never one per tick.** A fault that + lasts hours produces one opening notice, not a stream of them. When it + clears, one closure notice follows, and only if the opening notice was sent. + Repeated delivery attempts for either notice are idempotent at the + recipient's end. - **Nothing sensitive leaves.** A notice names the target, the condition and the time. It never carries a binding value, a path to a secret, a token, or a line of configuration. From 436d6445fc217d41275003bbbf0070852f3479a9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:12:18 +0200 Subject: [PATCH 012/196] Watchdog map: cut probes C1-C20; incidents live in their own incidents.cc Imagination's cut-mapping probes and rationale. Self collapses the model page's record and projection rows: control.cc refuses foreign types, so incidents.cc is both the record and what Bifrost reads. Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-map.md | 103 ++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index 3b71abf..cb40b8a 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -68,6 +68,36 @@ probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC | V1 | Peer precedent | VoidBot `packages/core/src/bifrost-discord-command.ts:39-68` | The caller opens Bifrost's store, derives `commandId` from its `idempotencyKey`, optionally spawns the pump itself, and **waits** for the receipt. An unconditional `put` of an existing id overwrites a `completed` command back to `pending`. | | V2 | VoidBot `notify_owner` | Self's mapper, not re-probed | Not live on Yggdrasil; no Discord-capable VoidBot process runs there. | +### Cut-mapping probes (2026-09-30, 14:00-14:10 UTC) + +Pinned heads: Idunn `487f4f14` (origin/main), Bifrost `42728a67`, gamecult-ops +`14df7ce4`, all read with `git ls-remote origin refs/heads/main`. Line numbers +in Idunn are unchanged from `4951125`, because `153af6a` and `487f4f1` touched +docs only. + +| # | What | Where / command | Result | +|---|---|---|---| +| C1 | `control.cc` refuses unknown types | Idunn `src/control_plane.rs:2845` | `_ => bail!("Idunn control store contains a foreign document")`. A new type in `control.cc` makes every older binary unable to read the store at all, so a rollback of the incident cut would take continuity down for every target. | +| C2 | Idunn's store writes take a blocking exclusive lock; reads can skip it | CultLib `packages/cultcache-rs/src/lib.rs:737-785` | `pull_all_read_only_snapshot` reads unlocked. Writes are `fs2::lock_exclusive`, which blocks. A reader that held the lock could stall an Idunn write. | +| C3 | The TS store takes no lock | CultLib `packages/cultcache-ts/src/single-file-messagepack-backing-store.ts` (no `lock` anywhere); present at Bifrost's pinned CultLib `f67f5122` (`pullAll` at `:44`) | A Bifrost reader using `pullAll` cannot block Idunn. | +| C4 | A JS reader decodes an Idunn-written store | `scp ygg:/var/lib/gamecult/idunn-projection/topology.cc`; scratch `probe-read.mjs` with cultcache-ts `pullAll` | 18 envelopes decoded (`gamecult.service_trust_anchor.v1`, ...). No `.lock` file was created beside the copy. | +| C5 | Rust `DatabaseEntry` payload shape in JS | scratch `probe-decode.mjs`, `@msgpack/msgpack` `decode(envelope.payload)` | A **positional array**: index `n` is `#[cultcache(key = n)]`. The Bifrost reader decodes a tuple, so the seam must be pinned by bytes Idunn wrote, not by a hand-built object. | +| C6 | Projection publish mode | Idunn `src/drivers.rs:4385-4404`, `:7670-7685` | Publication is compare-exchange, then `chmod 0644` on the store and its `.lock` sibling. | +| C7 | Continuity's healthy site | Idunn `src/control_plane.rs:5631-5633` | `if observation.is_some() { continue; }`: the admitted workload was observed running this pass. The exhaustion decision is `:5709`, and the fault clears at `:5723`. | +| C8 | Test harness for exhaustion | Idunn `src/control_plane.rs:17659-17689` | Existing tests use `routed_world(Odin::Unreachable, 1, ...)`, `set_meters`, `routed.workload.kill()` and `routed.tick()`. | +| C9 | Idunn has no one-shot or timer workload | `grep -i "oneshot\|\.timer\|OnCalendar" src/` returns nothing; recipe schema `src/deployment.rs:1935-2000` | A target is a resident service with `[service]`, health contract and state slots. Idunn cannot deploy a systemd timer. | +| C10 | No Bifrost binding in current Idunn | `sudo ls /etc/gamecult/idunn/bindings/` on ygg | ghostlight, heimdall, odin, raven-muninn, streampixels-service, streampixels-web. No Bifrost. | +| C11 | Bifrost's old Idunn manifest is residue | `/srv/odin/deploy-manifests/bifrost-persona-feedback` on ygg (2026-09-04); `grep deploy-manifests\|IDUNN_ACTUATOR src/` in Idunn returns nothing | It requires `IDUNN_ACTUATOR=1` / `IDUNN_COMMAND_AUTHORITY=idunn-daemon` env from the pre-rebuild Idunn, which no longer exists. gamecult-ops `runbooks/bifrost-persona-feedback-yggdrasil.md` still describes that path. The persona-feedback `current` has pointed at `cb3239a` since 2026-08-08. | +| C12 | How Idunn itself is installed | ygg `/usr/local/sbin/idunn-a96ad9d-build.sh`; `ls -la /usr/local/bin/idunn` (2026-09-30 13:22) | One hand-written script per commit. It runs `cargo test --lib` and `cargo build --release` in a pinned rust image under `/srv/build/idunn-` and copies the unit. The install is manual. No such script is in any repo. | +| C13 | Legacy alarm consumers in gamecult-ops | `scripts/idunn/notify-idunn-operator-alarm.{ps1,cmd}` (25 + 2 lines); `scripts/idunn/start-idunn-local.ps1:10,134-136` | The only callers of `publish-idunn-alarm`. They pass `--operator-alarm-command` to the deleted pre-rebuild Idunn (I14). No Idunn scheduled task or process runs on Starfire (`Get-ScheduledTask`, `Get-Process`). | +| C14 | `discord-dm` consumers in Bifrost | `grep discord-dm` | `cultmesh-bridge-commands.mjs:152,227` exist only for `operator-notification.mjs` (BF6). `bifrost-bridge.mjs` `discord-dm` (`:48`, `:473-535`) is also used by `docs/bridge.md:156` and `tests/Bifrost.Web.Tests/BridgeCliTests.cs:481`, so it stays. | +| C15 | Bridge CLI contract | Bifrost `tools/bifrost-bridge.mjs:1266-1289`; `tools/persona-discord-delivery.mjs:45` | The bridge refuses to act without `--cultmesh-command-id`. The Persona precedent spawns it with `--receipt-store` under its private state directory and parses stdout JSON. | +| C16 | Crash-recovery precedent | Bifrost `tests/persona-discord-delivery.test.mjs`, "running journal recovers as terminal unknown without a second Discord post" | A `running` execution found at start becomes terminal `unknown` and is never re-posted. | +| C17 | Discord nonce | docs.discord.com/developers/resources/message (fetched 2026-09-30) | `nonce`: integer or string, "up to 25 characters". `enforce_nonce`: "checked for uniqueness in the past few minutes"; on a duplicate from the same author, "that message will be returned and no new message will be created". | +| C18 | Bifrost tests and mutation tooling | Bifrost: no root `package.json`, `node:test` files under `tests/`, `grep stryker` returns nothing | Tests run with `node --test`, with CultLib found through `VOIDBOT_CULTLIB_ROOT`. StrykerJS is not installed. | +| C19 | Idunn verification | `scripts/verify.sh`; eureka `tools/stopgap/ygg-verify.sh` (rust image carries cargo-mutants) | `cargo check` for the Windows GNU target twice, then `cargo test --locked --lib`. | +| C20 | Idunn source mentions no transport | `grep -rin "discord\|bifrost" src/` at `487f4f1` | No hits. This is the negative-grep baseline. | + ## Model page A row per persistent kind this campaign creates or depends on. "New" means the @@ -76,8 +106,7 @@ cut spec fixes them. | Kind | Identity | Lifecycle | Authority | |---|---|---|---| -| **Incident record** (new, Idunn `control.cc`, e.g. `idunn.operator_incident.v1`) | Key `::`. Namespace: Idunn's control store, one type. `condition` is a closed enum (`continuity-exhausted`, `route-degraded`, `operator-required`, `odin-store-contested`, ...). `subject` is the target for target conditions and the `transaction_id` for `operator-required`. `opened_at` is written once at opening by Idunn's clock. Injective because of one rule: **at most one open incident per `(condition, subject)`**; the opener checks it before writing. Two different faults on one target are different conditions, so they never collide. Nothing in the key comes from Bifrost or Odin. | **Created** at the site that decides the condition (I2, I3, I6, Odin read) in the same store transaction family as that condition's own write, before any trace line. **Closed** by a write of `closed_at` and `close_reason` on the same record, never a delete: continuity-exhausted closes when the supervision pass sees the admitted workload running or a new generation is admitted, **not** when the window slides (I2 would flap otherwise); route-degraded closes on `record_proved_challenge` or when its generation is replaced; odin-store-contested closes when Odin's catalog stops reporting it; operator-required closes when its transaction leaves the phase, for which **no path exists yet** (I6). **Reopened**: never revived; a recurrence is a new record with a new `opened_at`. **Replayed after an Idunn restart**: the record is durable, so the decision site finds the open incident and writes nothing; `ReportOnce` is not consulted. **Pruned**: a closed record retires to `history.cc` one per tick, like a terminal transaction, after the projection has carried its closure for a retention period (length is a cut detail). After pruning, `idunn status` shows only open incidents. | Idunn's daemon, at the condition's decision site, is the only writer. Forbidden: the CLI (it may request, as `ExpiryRequest` does, never write the record), Bifrost, Odin, any reader of the projection, `ReportOnce`, a repair loop that opens incidents from log text. | -| **Incident projection** (new, derived) | One file in `/var/lib/gamecult/idunn-projection/` beside `topology.cc`, holding the open incidents plus closed ones inside retention, keyed exactly as the record. World-readable (Y9). | Rewritten whenever an incident opens or closes. Idunn restart: regenerated from `control.cc`. Never an input to Idunn. | Derived, notification-only. Idunn writes it; nothing else does. Whether it is signed with Idunn's service identity, as the topology projection is, follows that precedent (cut detail). Forbidden: Idunn reading it back, any Bifrost write. | +| **Incident record** (new, `idunn.operator_incident.v1`, its own file `/var/lib/gamecult/idunn-projection/incidents.cc`, published 0644 beside `topology.cc`) | Key `::`. Namespace: `incidents.cc`, one type; the file refuses any other. It is never written to `control.cc`, which bails on unknown types, so a rollback would stop continuity (C1). `condition` is a closed enum (`continuity-exhausted` first; `odin-store-contested`, `odin-store-unpersisted` and `operator-required` join when their sources exist). `subject` is the target for target conditions and the `transaction_id` for `operator-required`. `opened_at` is written once by Idunn's clock. Injective because of one rule: **at most one open incident per `(condition, subject)`**, checked by a compare-exchange create. | **Created** at the site that decides the condition, after that decision and before its trace line. **Closed** by writing `closed_at` and `close_reason` on the same record, never by deleting it. Continuity-exhausted closes when the pass sees the workload running (`recovered`) or the target is no longer admitted, **not** when the window slides (I2). Odin conditions close when the catalog stops reporting them. `operator-required` has **no closing path yet** (I6; follow_up `b5-operator-required-no-exit`). **Reopened**: never; a recurrence is a new record with a new `opened_at`. **Replayed after an Idunn restart**: the durable open record makes the site write nothing. **Pruned**: closed more than 7 days ago, it retires to `history.cc`. **Store failure**: reported once as a fault and never returned into the tick (`survival-independent`). | Idunn's daemon, at the decision site, is the only writer. The same file is the record and what Bifrost reads, so no separate projection exists to drift. Forbidden: the CLI (`status` reads only), Bifrost and every other reader, Odin, `ReportOnce`, `control.cc`, a repair loop that opens incidents from log text. | | **Stderr trace** (exists) | `report_once` key, in memory | Printed once per process lifetime per fault; lost on restart (I1). | **Demoted**: `ReportOnce` is no longer an owner of "has the operator been told". It de-dups stderr only. | | **Delivery request / journal entry** (Bifrost store) | Keyed by the Idunn incident key plus the notice kind (`opened`, and `closed` if Q `closure-notice` says so). Injective because the incident key is. Not derived from payload text or time (BF1's `raisedAt` hash is the counter-example). | **Created** by Bifrost when its reader first sees a notice-worthy incident state in the projection. **Revised**: `pending` -> `sent` or `failed`, with an attempt count and last error. **Retried** until sent, with backoff; a create for an existing key writes nothing (V1's unconditional `put` is the counter-example). **Replayed after a Bifrost restart**: journal is durable; a `pending` entry is retried. **Pruned** only after the incident key has left Idunn's projection. | Bifrost's reader only. Forbidden: Idunn (it never writes Bifrost's store, even though Y2 shows it physically could), VoidBot, the operator by hand. Which Bifrost process: see Rationale. | | **Delivery receipt** (Bifrost, `discord_post_receipt.v1` or the journal's `sent` state) | Same key as the request. | Written once on success with the Discord message id. | Bifrost. **Idunn does nothing with it**: it neither reads it in the daemon nor waits on it (`survival-independent`). `idunn status` does not show delivery state; Bifrost's journal is where a failed delivery is visible. | @@ -146,3 +175,73 @@ one notice, per slide. make an in-memory de-dup cache the owner of "was the operator told", keyed by free-form strings, with no closure and no subject. The incident record puts that decision at the condition's own site, as `record-before-delivery` asks. + +**The incident store is its own file, and that file is the projection.** The +model page placed the record in `control.cc` with a separate projection +derived from it. C1 overturns that: an unknown type in `control.cc` makes every +older Idunn binary refuse the whole store, so rolling back the incident cut +would stop continuity for every target. The record therefore lives in its own +CultCache file, `/var/lib/gamecult/idunn-projection/incidents.cc`, written by +Idunn with compare-exchange and published `0644` like `topology.cc` (C6). An +incident carries nothing sensitive (`no-sensitive-egress`), so there is no +private half to keep apart, and the derived copy disappears. What is lost: the +open write is not atomic with any `control.cc` write. That costs nothing +today, because the exhaustion decision writes nothing to `control.cc` (I2), +and later sources are read-only observations. The model page's +"Incident record" and "Incident projection" rows collapse into this one file. +Self owns the edit to the model page. + +**A broken incident store never reaches the tick.** Every incident operation +returns its error to one call site, which traces it once with `report_once` +and moves on. An unreadable store means no incident opens (the stderr trace +still prints); it never changes what continuity decides. The Idunn cut's +survival test pins this with an unwritable store path. + +**Bifrost reads without the lock.** Idunn's writes block on an exclusive lock +(C2). A reader that took the lock could stall a tick. cultcache-ts `pullAll` +takes none (C3, C4), and Idunn's rename is atomic, so the reader only ever sees +a whole snapshot. The reader never opens Idunn's file through a CultMesh node, +which could flush. + +**Delivery at most once, with the nonce as the second guard.** This follows +the Persona precedent (C16): a journal entry found `running` at start becomes +`unknown` and is never re-posted. A bridge exit that failed is retried on a +later run, with the same nonce and `enforce_nonce` (C17). A retry inside +Discord's "past few minutes" returns the original message instead of creating +another. The remaining duplicate needs the bridge to have posted, then exited +non-zero, and the retry to fall outside that window. The attempt cap and +timer period bound it, and it is recorded as a residual risk rather than +engineered away. + +**Who sees a delivery failure: Bifrost, through systemd.** There is one +channel to the operator, and it is the one that failed, so a failure cannot +page over it. The reader exits non-zero while any journal entry is `unknown` +or has exhausted its attempts. The unit then shows in `systemctl --failed` and +in its journal, and the reader's `status` verb lists the entries. Idunn reads +none of this (`survival-independent`). This is a default: the alternatives +either couple Idunn to delivery state or need a second channel that does not +exist. + +**Deployment of the reader has no Idunn path.** Idunn deploys resident +services, and a timer does not fit its recipe model (C9). No Bifrost binding +exists in current Idunn (C10), and the old Bifrost manifest belongs to the +Idunn that was rebuilt away (C11). The ruled one-shot reader can be +hand-installed by a gamecult-ops script and runbook, the same interim footing +as `huginn.service` and `bifrost-persona-mouth`. Otherwise Idunn must first +learn a new workload kind. That choice is an operator question raised in the +ops cut. + +**The reader gets its own release root.** It installs under +`/srv/bifrost/watchdog-notice/releases/-` and does not +repoint `/srv/bifrost/persona-feedback/runtime/current`. Repointing that link +would stage new code under `bifrost-persona-mouth`. With the token file now +present (Y3, Y6), the mouth's next restart would also make the Persona crossing +able to post. That crossing is out of scope and must not change as a side +effect. + +**Cut order.** The Idunn incident cut comes first. It writes the golden +fixture that Bifrost's reader is tested against (C5). The Bifrost subtraction +cut is independent and lands before the reader, so the reader is written +against a tree with one Discord command vocabulary. The gamecult-ops +subtraction is independent too. The ops deploy cut comes last and needs both +behaviour cuts merged. From 202e5a2e109bde2bcb5da2076ebc0e5010e6ee1a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:21:02 +0200 Subject: [PATCH 013/196] Idunn records continuity exhaustion as a durable incident in incidents.cc Idunn's daemon, at the continuity decision site, writes one idunn.operator_incident.v1 record per exhaustion into its own published incidents.cc (--incident-store), never control.cc. The record opens once per (target, condition), closes on an observed running workload (recovered) or on the target leaving the admitted set, and retires to incident-history.cc seven days after closing. A store failure is traced once and never returned into the tick. idunn status prints each target's incidents. ReportOnce is demoted to a stderr trace. Does not build the fixture test until tests/fixtures/idunn.operator_incident.v1.cc lands in the next commit. Co-Authored-By: Claude Sonnet 5.5 --- deploy/idunn-yggdrasil.service | 1 + docs/guide.md | 3 +- src/control_plane.rs | 428 ++++++++++++++++++++++- src/drivers.rs | 2 +- src/incident.rs | 619 +++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 6 files changed, 1038 insertions(+), 16 deletions(-) create mode 100644 src/incident.rs diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index 5474c67..9758127 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -41,6 +41,7 @@ ExecStart=/usr/local/bin/idunn serve \ --source-root /var/lib/gamecult/idunn/sources \ --staging-root /var/lib/gamecult/idunn/staging \ --topology-store /var/lib/gamecult/idunn-projection/topology.cc \ + --incident-store /var/lib/gamecult/idunn-projection/incidents.cc \ --idunn-identity-store /var/lib/gamecult/idunn/idunn-service-identity.cc \ --odin-correlation-store /var/lib/gamecult/odin/topology.cc \ --odin-trust-anchor /etc/gamecult/idunn/odin-topology-anchor.cc \ diff --git a/docs/guide.md b/docs/guide.md index f018670..e48bc85 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -44,7 +44,8 @@ Idunn centralizes that lifecycle work: - record every deployment/restart request and result as typed state; - record release targets, deployment artifacts, state migrations, and rollout results as typed state; -- escalate to an operator through Bifrost when human action is needed. +- record operator incidents in its published `incidents.cc`, which Bifrost reads + and delivers. Idunn never waits on delivery. The desired shape is simple: diff --git a/src/control_plane.rs b/src/control_plane.rs index 28185cc..db1e6b4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -44,6 +44,7 @@ use crate::drivers::{ SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; +use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -2380,6 +2381,9 @@ struct RuntimeOptions { source_root: PathBuf, staging_root: PathBuf, topology_store: PathBuf, + /// Operator incidents, published world-readable beside the topology + /// projection. Not part of the control store: see `incident`. + incident_store: PathBuf, odin_correlation_store: PathBuf, odin_trust_anchor: PathBuf, idunn_identity_store: PathBuf, @@ -2405,6 +2409,7 @@ impl Default for RuntimeOptions { source_root: PathBuf::from("/var/lib/gamecult/idunn/sources"), staging_root: PathBuf::from("/var/lib/gamecult/idunn/staging"), topology_store: PathBuf::from("/var/lib/gamecult/idunn/topology.cc"), + incident_store: PathBuf::from("/var/lib/gamecult/idunn-projection/incidents.cc"), odin_correlation_store: PathBuf::from( "/var/lib/gamecult/odin/idunn-runtime-topology.cc", ), @@ -2437,6 +2442,7 @@ enum Command { }, Status { state_store: PathBuf, + incident_store: PathBuf, command_id: Option, }, Cancel { @@ -2468,8 +2474,9 @@ pub fn run(args: impl Iterator) -> Result<()> { ), Command::Status { state_store, + incident_store, command_id, - } => status(&state_store, command_id.as_deref()), + } => status(&state_store, &incident_store, command_id.as_deref()), Command::Cancel { state_store, command_id, @@ -2543,6 +2550,7 @@ fn parse_serve(mut args: impl Iterator) -> Result { "--source-root" => options.source_root = path_value(&mut args, &argument)?, "--staging-root" => options.staging_root = path_value(&mut args, &argument)?, "--topology-store" => options.topology_store = path_value(&mut args, &argument)?, + "--incident-store" => options.incident_store = path_value(&mut args, &argument)?, "--odin-correlation-store" => { options.odin_correlation_store = path_value(&mut args, &argument)? } @@ -2626,11 +2634,14 @@ fn parse_up(mut args: impl Iterator) -> Result { } fn parse_status(mut args: impl Iterator) -> Result { - let mut state_store = RuntimeOptions::default().state_store; + let defaults = RuntimeOptions::default(); + let mut state_store = defaults.state_store; + let mut incident_store = defaults.incident_store; let mut command_id = None; while let Some(argument) = args.next() { match argument.as_str() { "--state-store" => state_store = path_value(&mut args, &argument)?, + "--incident-store" => incident_store = path_value(&mut args, &argument)?, "--command" => command_id = Some(string_value(&mut args, &argument)?), "--help" | "-h" => bail!(usage()), _ => bail!("unknown Idunn status option {argument:?}"), @@ -2638,6 +2649,7 @@ fn parse_status(mut args: impl Iterator) -> Result { } Ok(Command::Status { state_store, + incident_store, command_id, }) } @@ -3091,7 +3103,7 @@ impl ControlSnapshot { } } -fn decode_record(envelope: &CultCacheEnvelope) -> Result +pub(crate) fn decode_record(envelope: &CultCacheEnvelope) -> Result where T: for<'de> Deserialize<'de> + Serialize, { @@ -3634,7 +3646,7 @@ fn target_supervision_envelope(value: &TargetSupervision, now: u64) -> Result( +pub(crate) fn typed_envelope( key: &str, record_type: &str, schema: &str, @@ -3792,8 +3804,12 @@ fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) &after != before } -/// Says a fault once while it lasts. A scheduler tick that hits the same +/// Says a fault once on stderr while it lasts. A scheduler tick that hits the same /// unreadable file every half second must not print it every half second. +/// +/// It is a trace and owns nothing else: it is in memory, keyed by free-form +/// strings, and an Idunn restart forgets it. Whether the operator is told +/// belongs to the incident record (`incident`), never to this map. #[derive(Default)] struct ReportOnce { last: Mutex>, @@ -4096,7 +4112,7 @@ fn submit( } } -fn status(store_path: &Path, command_id: Option<&str>) -> Result<()> { +fn status(store_path: &Path, incident_store: &Path, command_id: Option<&str>) -> Result<()> { let snapshot = ControlSnapshot::read(store_path)?; // Commands stay resident; their finished transactions do not. Without // history a completed command would report as though it had never run, @@ -4185,18 +4201,32 @@ fn status(store_path: &Path, command_id: Option<&str>) -> Result<()> { // What supervision holds for each target: the union of the admitted // generations and the metered targets, so a target that has only been // deployed to (no generation yet) still shows what it has used. - let now = now_millis()?; - for target in supervised_targets(&snapshot) { - for line in render_supervision( + for line in render_targets(&snapshot, incident_store, now_millis()?) { + println!("{line}"); + } + Ok(()) +} + +/// Every supervised target with its meters and its incidents. An incident +/// store that cannot be read costs the incident lines and nothing else. +fn render_targets(snapshot: &ControlSnapshot, incident_store: &Path, now: u64) -> Vec { + let incidents = IncidentStore::new(incident_store).read(); + let mut lines = Vec::new(); + for target in supervised_targets(snapshot) { + lines.extend(render_supervision( target, snapshot.admitted_for(target).map(|stored| &stored.value), snapshot.supervision_for(target).map(|stored| &stored.value), now, - ) { - println!("{line}"); + )); + if let Ok(records) = &incidents { + lines.extend(render_incidents(records, target, now)); } } - Ok(()) + if let Err(error) = &incidents { + lines.push(format!("incidents unreadable: {}", error.root_cause())); + } + lines } /// Every target supervision holds something for: the admitted generations @@ -4609,6 +4639,57 @@ impl Engine { } } + fn incidents(&self) -> IncidentStore { + IncidentStore::new(&self.options.incident_store) + } + + /// Settle the outcome of one incident operation. A failure never leaves + /// this function: it is traced once, under its operation, and continuity + /// decides exactly as it would have with a healthy store. It names the + /// operation and the root cause and never a path or a value. Incident + /// writes are never progress. + fn record_incident(&self, operation: &str, outcome: Result) { + let key = format!("incident-store:{operation}"); + match outcome { + Ok(_) => self.clear_fault(&key), + Err(error) => self.report_once( + &key, + format!( + "Idunn could not {operation} an operator incident: {}", + error.root_cause() + ), + ), + } + } + + /// Close every open incident whose target is no longer admitted, then + /// retire what has been closed past retention. + fn reconcile_incidents(&self, snapshot: &ControlSnapshot) { + let store = self.incidents(); + let close_unadmitted = || -> Result { + let now = now_millis()?; + let mut closed = false; + for record in store.read()? { + if record.is_open() + && !snapshot + .admitted + .iter() + .any(|stored| stored.value.target == record.subject) + { + closed |= store.close( + record.condition, + &record.subject, + CloseReason::NoLongerAdmitted, + now, + )?; + } + } + Ok(closed) + }; + self.record_incident("close", close_unadmitted()); + self.record_incident("retire", now_millis().and_then(|now| store.retire_closed(now))); + } + fn host_access(&self) -> Result> { let hub = self .host_actuators @@ -5463,6 +5544,7 @@ impl Engine { .lock() .expect("challenge wait mutex") .retain(|target, wait| snapshot.target_stamp(target) == wait.stamp); + self.reconcile_incidents(&snapshot); let mut progressed = false; let mut admitted = snapshot.admitted.iter().collect::>(); admitted.sort_by_key(|stored| stored.value.target.as_str()); @@ -5631,6 +5713,20 @@ impl Engine { } }; if observation.is_some() { + // The admitted workload was seen running: the only evidence + // that ends an exhaustion incident. The restart window sliding + // is not health and never closes one. + self.record_incident( + "close", + now_millis().and_then(|now| { + self.incidents().close( + IncidentCondition::ContinuityExhausted, + ¤t.value.target, + CloseReason::Recovered, + now, + ) + }), + ); continue; } let workload_error = operational_error @@ -5707,6 +5803,16 @@ impl Engine { continue; } if supervision.restarts_exhausted(now) { + // The record comes first and the trace after it: the trace is + // this record's shadow, never its substitute. + self.record_incident( + "open", + self.incidents().open( + IncidentCondition::ContinuityExhausted, + ¤t.value.target, + now, + ), + ); self.report_once( &continuity_key, format!( @@ -9013,7 +9119,7 @@ fn require_selector(value: &str) -> Result<()> { require_id(value, "deployment selector") } -fn require_id(value: &str, label: &str) -> Result<()> { +pub(crate) fn require_id(value: &str, label: &str) -> Result<()> { ensure!( !value.is_empty() && value.len() <= 256 @@ -9070,7 +9176,7 @@ fn usage() -> &'static str { "Idunn deployment, admission, and continuity control plane\n\n\ idunn serve [runtime options] [--host-actuator-bind ADDR]\n\ idunn up [--state-store PATH] [--no-wait]\n\ - idunn status [--state-store PATH] [--command ID]\n\ + idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn validate --recipe PATH [--binding PATH]\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ @@ -9089,6 +9195,7 @@ mod tests { use super::*; use crate::drivers::{SystemdWorkloadObservation, incarnation_key}; + use crate::incident::INCIDENT_RETENTION_MILLIS; /// Fixed clock for the signing fixture; correlations must land inside /// DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS of it to authenticate. @@ -9303,6 +9410,7 @@ mod tests { source_root: root.join("sources"), staging_root: root.join("staging"), topology_store: root.join("topology.cc"), + incident_store: root.join("incidents.cc"), odin_correlation_store: root.join("odin-correlation.cc"), odin_trust_anchor: odin_anchor, idunn_identity_store: idunn, @@ -12760,6 +12868,11 @@ mod tests { }) } + fn revive(&self) { + self.alive + .store(true, std::sync::atomic::Ordering::SeqCst); + } + fn kill(&self) { self.alive .store(false, std::sync::atomic::Ordering::SeqCst); @@ -14498,6 +14611,293 @@ mod tests { Ok(()) } + // ---- operator incidents ---------------------------------------------- + + fn incident_path(world: &EngineFixture) -> PathBuf { + world.root.join("incidents.cc") + } + + fn incidents_of(world: &EngineFixture) -> Result> { + IncidentStore::new(&incident_path(world)).read() + } + + /// The control store never holds an incident: it would stop every older + /// binary that reads it. + fn assert_control_store_holds_no_incident(world: &EngineFixture) -> Result<()> { + ControlSnapshot::read(&world.state_store)?; + for envelope in SingleFileMessagePackBackingStore::new(&world.state_store) + .pull_all_read_only_snapshot()? + { + assert_ne!(envelope.r#type, "idunn.operator_incident"); + } + Ok(()) + } + + /// Six restarts inside the window, the last long enough ago that its wait is + /// over: only the ceiling holds. + fn spend_the_window(world: &EngineFixture) -> Result<()> { + set_meters( + world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + ) + } + + /// One entry has left the window, and the newest restart is recent enough + /// that continuity is still waiting: exhaustion is over, nothing restarts. + fn slide_the_window(world: &EngineFixture) -> Result<()> { + set_meters( + world, + &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 10_000])?, + ) + } + + #[test] + fn an_exhausted_target_opens_one_incident_across_ticks_and_restarts() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + assert!(!incident_path(&world).exists()); + + assert!(!world.engine.supervise_one_admitted_generation()?); + let opened = incidents_of(&world)?; + assert_eq!(opened.len(), 1); + assert_eq!(opened[0].subject, "service"); + assert_eq!(opened[0].condition, IncidentCondition::ContinuityExhausted); + assert!(opened[0].is_open()); + let written = std::fs::read(incident_path(&world))?; + + for _ in 0..3 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + // An Idunn restart forgets its in-memory trace; the durable record + // keeps the decision from being written again. + let restarted = + Engine::open_with_systemd_workload(world.engine.options.clone(), workload.clone())?; + assert!(restarted.fault_reports.lock().unwrap().is_empty()); + assert!(!restarted.supervise_one_admitted_generation()?); + assert_eq!(std::fs::read(incident_path(&world))?, written); + assert_eq!(incidents_of(&world)?, opened); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn an_exhaustion_incident_closes_on_observed_health_not_on_the_window() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let first = incidents_of(&world)?; + assert_eq!(first.len(), 1); + + // The window slides while the workload is still dead: exhaustion is + // over, the target is not healthy, and the incident stays open. + slide_the_window(&world)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(incidents_of(&world)?, first); + + // Seen running, it closes as recovered. + workload.revive(); + assert!(!world.engine.supervise_one_admitted_generation()?); + let closed = incidents_of(&world)?; + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].incident_key, first[0].incident_key); + assert!( + closed[0] + .closed_at_unix_millis + .is_some_and(|at| at >= closed[0].opened_at_unix_millis) + ); + assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + + // A later exhaustion is a new record with a new opening time. + std::thread::sleep(Duration::from_millis(5)); + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let again = incidents_of(&world)?; + assert_eq!(again.len(), 2); + let reopened = again + .iter() + .find(|record| record.is_open()) + .context("no new incident")?; + assert!(reopened.opened_at_unix_millis > first[0].opened_at_unix_millis); + assert_ne!(reopened.incident_key, first[0].incident_key); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn an_incident_whose_target_is_no_longer_admitted_closes() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert!(incidents_of(&world)?[0].is_open()); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone(); + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&[admitted])? + ); + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::NoLongerAdmitted)); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn a_tick_retires_incidents_closed_past_retention() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let store = IncidentStore::new(&incident_path(&world)); + let condition = IncidentCondition::ContinuityExhausted; + let long_ago = now_millis()? - INCIDENT_RETENTION_MILLIS - 10_000; + assert!(store.open(condition, "gone", long_ago)?); + assert!(store.close(condition, "gone", CloseReason::Recovered, long_ago + 1)?); + assert!(store.open(condition, "recent", now_millis()?)?); + assert!(store.close(condition, "recent", CloseReason::Recovered, now_millis()?)?); + + world.engine.supervise_one_admitted_generation()?; + let live = store.read()?; + assert_eq!(live.len(), 1); + assert_eq!(live[0].subject, "recent"); + assert!(world.root.join("incident-history.cc").exists()); + Ok(()) + } + + /// The same exhaustion and recovery sequence, reduced to what continuity + /// decided: each tick's return, and the transactions and restart log it left. + fn continuity_decisions( + world: &EngineFixture, + engine: &Engine, + workload: &SwitchWorkload, + ) -> Result> { + let mut decided = Vec::new(); + let tick = |decided: &mut Vec<(bool, usize, usize)>| -> Result<()> { + let moved = engine.supervise_one_admitted_generation()?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + decided.push(( + moved, + snapshot.transactions.len(), + snapshot.supervision_or_new("service").continuity_restarts.len(), + )); + Ok(()) + }; + workload.kill(); + spend_the_window(world)?; + tick(&mut decided)?; + tick(&mut decided)?; + slide_the_window(world)?; + tick(&mut decided)?; + workload.revive(); + tick(&mut decided)?; + workload.kill(); + spend_the_window(world)?; + tick(&mut decided)?; + Ok(decided) + } + + #[test] + fn an_unwritable_incident_store_changes_nothing_continuity_decides() -> Result<()> { + const CANARY: &str = "CANARY-6b1d-secret-segment"; + let healthy_workload = SwitchWorkload::new(); + let healthy = EngineFixture::with_workload(healthy_workload.clone())?; + admit_incumbent(&healthy)?; + let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; + assert!(incidents_of(&healthy)?.iter().any(|record| record.is_open())); + + // Not permission-dependent: a store path beneath a regular file cannot + // be created for any uid, root included. + let broken_workload = SwitchWorkload::new(); + let mut broken = EngineFixture::with_workload(broken_workload.clone())?; + admit_incumbent(&broken)?; + let blocker = broken.root.join(CANARY); + std::fs::write(&blocker, b"a file, not a directory")?; + let mut options = broken.engine.options.clone(); + options.incident_store = blocker.join("incidents.cc"); + broken.engine = Engine::open_with_systemd_workload(options, broken_workload.clone())?; + + let actual = continuity_decisions(&broken, &broken.engine, &broken_workload)?; + assert_eq!(actual, expected, "a broken incident store changed a continuity decision"); + assert!(!blocker.join("incidents.cc").exists()); + + // The fault is traced once, under its operation, and echoes no path. + let reports = broken.engine.fault_reports.lock().unwrap(); + let faults = reports + .iter() + .filter(|(key, _)| key.starts_with("incident-store:")) + .collect::>(); + assert_eq!( + faults.len(), + 1, + "{:?}", + faults.iter().map(|(key, _)| key).collect::>() + ); + let message = faults[0].1.last.lock().unwrap().clone().context("fault cleared")?; + assert!(message.contains("open an operator incident"), "{message}"); + assert!(!message.contains(CANARY), "{message}"); + drop(reports); + assert_control_store_holds_no_incident(&broken) + } + + #[test] + fn status_renders_incidents_and_survives_an_unreadable_store() -> Result<()> { + const CANARY: &str = "CANARY-90c2-secret-segment"; + let temp = TempDir::new()?; + let now = now_millis()?; + let mut snapshot = ControlSnapshot::default(); + let meters = TargetSupervision::new("service"); + snapshot.targets.push(Stored { + envelope: target_supervision_envelope(&meters, now)?, + value: meters, + }); + + let readable = temp.path().join("incidents.cc"); + IncidentStore::new(&readable).open(IncidentCondition::ContinuityExhausted, "service", now)?; + let lines = render_targets(&snapshot, &readable, now); + assert!(lines[0].starts_with("target service "), "{lines:?}"); + assert_eq!( + lines.last().map(String::as_str), + Some(format!(" incident continuity-exhausted opened-at {now}").as_str()) + ); + + let unreadable = temp.path().join(CANARY); + std::fs::write(&unreadable, b"not a cultcache file")?; + let lines = render_targets(&snapshot, &unreadable, now); + assert!(lines[0].starts_with("target service "), "{lines:?}"); + let last = lines.last().context("no lines")?; + assert!(last.starts_with("incidents unreadable: "), "{last}"); + assert!(!lines.iter().any(|line| line.contains(CANARY)), "{lines:?}"); + Ok(()) + } + + #[test] + fn the_incident_store_is_named_by_serve_and_status() { + let Command::Serve(options) = + parse(["serve", "--incident-store", "/tmp/i.cc"].into_iter().map(str::to_owned)) + .unwrap() + else { + panic!("expected serve") + }; + assert_eq!(options.incident_store, PathBuf::from("/tmp/i.cc")); + let Command::Status { incident_store, .. } = + parse(["status", "--incident-store", "/tmp/j.cc"].into_iter().map(str::to_owned)) + .unwrap() + else { + panic!("expected status") + }; + assert_eq!(incident_store, PathBuf::from("/tmp/j.cc")); + } + #[test] fn continuity_stops_when_the_targets_own_log_has_spent_the_window() -> Result<()> { let workload = SwitchWorkload::new(); diff --git a/src/drivers.rs b/src/drivers.rs index 189779b..eb162e9 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -7667,7 +7667,7 @@ fn ensure_bundle_is_reachable_by_workload( /// gets EACCES. A chmod by hand does not hold, because each publish writes a /// new file. Integrity here comes from the signatures over the records, not /// from the mode, so the published copy is readable. -fn publish_projection_mode(path: &Path) -> Result<()> { +pub(crate) fn publish_projection_mode(path: &Path) -> Result<()> { #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; diff --git a/src/incident.rs b/src/incident.rs new file mode 100644 index 0000000..fa6f1a6 --- /dev/null +++ b/src/incident.rs @@ -0,0 +1,619 @@ +//! Operator incidents: the durable record that a condition needing a human +//! opened, and that it closed. +//! +//! One file, `incidents.cc`, holds one type. Idunn's daemon is its only writer, +//! at the site that decides the condition; it is also the file readers such as +//! Bifrost open, so no second copy exists to drift. It is never `control.cc`, +//! which refuses any type it does not know: a type added there would stop +//! continuity for every target on a rollback. +//! +//! A record carries no free text. A workload's error never enters it, so the +//! published file leaks nothing a reader should not have. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::{Deserialize, Serialize}; + +use crate::control_plane::{decode_record, require_id, typed_envelope}; +use crate::drivers::publish_projection_mode; + +const INCIDENT_SCHEMA: &str = "idunn.operator_incident.v1"; + +/// How long a closed incident stays in the published file before it retires to +/// the incident history. +pub(crate) const INCIDENT_RETENTION_MILLIS: u64 = 7 * 24 * 60 * 60 * 1000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum IncidentCondition { + ContinuityExhausted, +} + +impl IncidentCondition { + /// The spelling in the key and on `idunn status`. Pinned equal to the + /// serialized spelling by `the_incident_fixture_is_what_idunn_reads`. + pub(crate) const fn name(self) -> &'static str { + match self { + Self::ContinuityExhausted => "continuity-exhausted", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum CloseReason { + Recovered, + NoLongerAdmitted, +} + +impl CloseReason { + pub(crate) const fn name(self) -> &'static str { + match self { + Self::Recovered => "recovered", + Self::NoLongerAdmitted => "no-longer-admitted", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.operator_incident", schema = "idunn.operator_incident.v1")] +pub(crate) struct IncidentRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `::`. Injective because at most one + /// incident per `(condition, subject)` is open at a time. + #[cultcache(key = 1)] + pub(crate) incident_key: String, + #[cultcache(key = 2)] + pub(crate) condition: IncidentCondition, + /// The target the condition is about. + #[cultcache(key = 3)] + pub(crate) subject: String, + #[cultcache(key = 4)] + pub(crate) opened_at_unix_millis: u64, + #[cultcache(key = 5)] + pub(crate) closed_at_unix_millis: Option, + #[cultcache(key = 6)] + pub(crate) close_reason: Option, +} + +impl IncidentRecord { + fn opened(condition: IncidentCondition, subject: &str, now: u64) -> Self { + Self { + schema_version: INCIDENT_SCHEMA.to_owned(), + incident_key: format!("{}:{subject}:{now}", condition.name()), + condition, + subject: subject.to_owned(), + opened_at_unix_millis: now, + closed_at_unix_millis: None, + close_reason: None, + } + } + + pub(crate) fn is_open(&self) -> bool { + self.closed_at_unix_millis.is_none() + } + + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == INCIDENT_SCHEMA, + "incident schema is unsupported" + ); + require_id(&self.subject, "incident subject")?; + ensure!( + self.incident_key + == format!( + "{}:{}:{}", + self.condition.name(), + self.subject, + self.opened_at_unix_millis + ), + "incident key is not its condition, subject and opening time" + ); + ensure!( + self.closed_at_unix_millis.is_some() == self.close_reason.is_some(), + "incident has a closing time or a reason but not both" + ); + if let Some(closed_at) = self.closed_at_unix_millis { + ensure!( + closed_at >= self.opened_at_unix_millis, + "incident closes before it opens" + ); + } + Ok(()) + } + + fn envelope(&self, now: u64) -> Result { + self.validate()?; + typed_envelope( + &self.incident_key, + Self::TYPE, + INCIDENT_SCHEMA, + self, + now, + ) + } +} + +struct Stored { + envelope: CultCacheEnvelope, + record: IncidentRecord, +} + +/// The incident file and its history sibling. Every write is a compare-exchange +/// against exactly what was read, and every write is followed by the +/// world-readable publication mode. +pub(crate) struct IncidentStore { + path: PathBuf, +} + +impl IncidentStore { + pub(crate) fn new(path: &Path) -> Self { + Self { + path: path.to_path_buf(), + } + } + + fn history_path(&self) -> PathBuf { + self.path.with_file_name("incident-history.cc") + } + + fn backing(&self) -> SingleFileMessagePackBackingStore { + SingleFileMessagePackBackingStore::new(&self.path) + } + + /// Every record, oldest key first. An absent file is empty. The file + /// refuses any document that is not an operator incident. + pub(crate) fn read(&self) -> Result> { + Ok(self + .read_stored()? + .into_iter() + .map(|stored| stored.record) + .collect()) + } + + fn read_stored(&self) -> Result> { + if !self.path.exists() { + return Ok(Vec::new()); + } + let mut stored = Vec::new(); + for envelope in self + .backing() + .pull_all_read_only_snapshot() + .context("reading the incident store")? + { + ensure!( + envelope.r#type == IncidentRecord::TYPE + && envelope.schema_id.as_deref() == Some(INCIDENT_SCHEMA), + "the incident store holds a document that is not an operator incident" + ); + let record: IncidentRecord = decode_record(&envelope)?; + record.validate()?; + ensure!( + envelope.key == record.incident_key, + "incident store key differs from the incident's identity" + ); + stored.push(Stored { envelope, record }); + } + stored.sort_by(|left, right| left.record.incident_key.cmp(&right.record.incident_key)); + Ok(stored) + } + + fn open_record<'a>( + stored: &'a [Stored], + condition: IncidentCondition, + subject: &str, + ) -> Option<&'a Stored> { + stored.iter().find(|stored| { + stored.record.is_open() + && stored.record.condition == condition + && stored.record.subject == subject + }) + } + + /// Open an incident unless one is already open for `(condition, subject)`. + /// `Ok(true)` means this call wrote it. + pub(crate) fn open( + &self, + condition: IncidentCondition, + subject: &str, + now: u64, + ) -> Result { + let stored = self.read_stored()?; + if Self::open_record(&stored, condition, subject).is_some() { + return Ok(false); + } + let record = IncidentRecord::opened(condition, subject, now); + let written = self.backing().compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IncidentRecord::TYPE.into(), + key: record.incident_key.clone(), + current: None, + }], + &[record.envelope(now)?], + )?; + if written { + publish_projection_mode(&self.path)?; + } + Ok(written) + } + + /// Close the open incident for `(condition, subject)`, if there is one. + /// `Ok(true)` means this call wrote the closure. + pub(crate) fn close( + &self, + condition: IncidentCondition, + subject: &str, + reason: CloseReason, + now: u64, + ) -> Result { + let stored = self.read_stored()?; + let Some(open) = Self::open_record(&stored, condition, subject) else { + return Ok(false); + }; + let mut closed = open.record.clone(); + // A clock stepped back must not close an incident before it opened. + closed.closed_at_unix_millis = Some(now.max(closed.opened_at_unix_millis)); + closed.close_reason = Some(reason); + let written = self.backing().compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IncidentRecord::TYPE.into(), + key: closed.incident_key.clone(), + current: Some(open.envelope.clone()), + }], + &[closed.envelope(now)?], + )?; + if written { + publish_projection_mode(&self.path)?; + } + Ok(written) + } + + /// Move every incident closed more than `INCIDENT_RETENTION_MILLIS` ago to + /// the history file, then delete it here. History first: a crash between + /// the two leaves the record in both, and `insert_entry_if_absent` makes + /// the repeat a no-op. An open incident never moves. + pub(crate) fn retire_closed(&self, now: u64) -> Result { + let retiring = self + .read_stored()? + .into_iter() + .filter(|stored| { + stored.record.closed_at_unix_millis.is_some_and(|closed_at| { + now.saturating_sub(closed_at) > INCIDENT_RETENTION_MILLIS + }) + }) + .map(|stored| stored.envelope) + .collect::>(); + if retiring.is_empty() { + return Ok(false); + } + let history = SingleFileMessagePackBackingStore::new(&self.history_path()); + for envelope in &retiring { + history + .insert_entry_if_absent(envelope.clone()) + .context("archiving a closed incident")?; + } + let deleted = self + .backing() + .delete_batch_if_unchanged(&retiring) + .context("retiring a closed incident")?; + if deleted { + publish_projection_mode(&self.path)?; + } + Ok(deleted) + } +} + +/// The lines `idunn status` prints for one target's incidents: each open one, +/// and each closed one still inside retention. +pub(crate) fn render_incidents(records: &[IncidentRecord], target: &str, now: u64) -> Vec { + records + .iter() + .filter(|record| record.subject == target) + .filter(|record| { + record.closed_at_unix_millis.is_none_or(|closed_at| { + now.saturating_sub(closed_at) <= INCIDENT_RETENTION_MILLIS + }) + }) + .map(|record| { + let mut line = format!( + " incident {} opened-at {}", + record.condition.name(), + record.opened_at_unix_millis + ); + if let (Some(closed_at), Some(reason)) = + (record.closed_at_unix_millis, record.close_reason) + { + line.push_str(&format!(" closed-at {closed_at} {}", reason.name())); + } + line + }) + .collect() +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + + const OPENED: u64 = 1_700_000_000_000; + const CLOSED: u64 = 1_700_000_100_000; + const FIXTURE: &str = "tests/fixtures/idunn.operator_incident.v1.cc"; + + fn fixture_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE) + } + + /// The two records the fixture holds, spelled independently of the store: + /// one open, one closed. + fn fixture_records() -> [IncidentRecord; 2] { + let closed_key = format!("continuity-exhausted:closed-target:{OPENED}"); + let open_key = format!("continuity-exhausted:open-target:{CLOSED}"); + [ + IncidentRecord { + schema_version: "idunn.operator_incident.v1".into(), + incident_key: closed_key, + condition: IncidentCondition::ContinuityExhausted, + subject: "closed-target".into(), + opened_at_unix_millis: OPENED, + closed_at_unix_millis: Some(CLOSED), + close_reason: Some(CloseReason::Recovered), + }, + IncidentRecord { + schema_version: "idunn.operator_incident.v1".into(), + incident_key: open_key, + condition: IncidentCondition::ContinuityExhausted, + subject: "open-target".into(), + opened_at_unix_millis: CLOSED, + closed_at_unix_millis: None, + close_reason: None, + }, + ] + } + + #[test] + fn incident_records_validate_their_key_and_closure() { + let [closed, open] = fixture_records(); + closed.validate().unwrap(); + open.validate().unwrap(); + + let mut wrong_key = open.clone(); + wrong_key.incident_key.push('x'); + assert!(wrong_key.validate().is_err()); + + let mut reason_missing = open.clone(); + reason_missing.closed_at_unix_millis = Some(CLOSED + 1); + assert!(reason_missing.validate().is_err()); + + let mut closing_missing = open.clone(); + closing_missing.close_reason = Some(CloseReason::Recovered); + assert!(closing_missing.validate().is_err()); + + let mut before_open = closed.clone(); + before_open.closed_at_unix_millis = Some(OPENED - 1); + assert!(before_open.validate().is_err()); + + let mut empty_subject = open.clone(); + empty_subject.subject = String::new(); + empty_subject.incident_key = format!("continuity-exhausted::{CLOSED}"); + assert!(empty_subject.validate().is_err()); + + let mut wrong_schema = open; + wrong_schema.schema_version = "idunn.operator_incident.v0".into(); + assert!(wrong_schema.validate().is_err()); + } + + #[test] + fn the_incident_fixture_is_what_idunn_reads() { + if std::env::var_os("IDUNN_WRITE_FIXTURES").is_some() { + let temp = TempDir::new().unwrap(); + let store = IncidentStore::new(&temp.path().join("incidents.cc")); + assert!( + store + .open(IncidentCondition::ContinuityExhausted, "closed-target", OPENED) + .unwrap() + ); + assert!( + store + .close( + IncidentCondition::ContinuityExhausted, + "closed-target", + CloseReason::Recovered, + CLOSED + ) + .unwrap() + ); + assert!( + store + .open(IncidentCondition::ContinuityExhausted, "open-target", CLOSED) + .unwrap() + ); + std::fs::copy(temp.path().join("incidents.cc"), fixture_path()).unwrap(); + } + // A copy, so reading the committed file can never leave a lock beside it. + let temp = TempDir::new().unwrap(); + let copy = temp.path().join("incidents.cc"); + std::fs::copy(fixture_path(), ©).unwrap(); + assert_eq!( + IncidentStore::new(©).read().unwrap(), + fixture_records().to_vec() + ); + // The wire spelling readers match on is the spelling the key and + // status use. + let payload = rmp_serde::to_vec(&IncidentCondition::ContinuityExhausted).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&payload).unwrap(), + IncidentCondition::ContinuityExhausted.name() + ); + let payload = rmp_serde::to_vec(&CloseReason::NoLongerAdmitted).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&payload).unwrap(), + CloseReason::NoLongerAdmitted.name() + ); + } + + #[test] + fn the_store_opens_once_closes_once_and_reopens_as_a_new_record() { + let temp = TempDir::new().unwrap(); + let store = IncidentStore::new(&temp.path().join("incidents.cc")); + let condition = IncidentCondition::ContinuityExhausted; + assert!(store.read().unwrap().is_empty()); + assert!(!store.close(condition, "service", CloseReason::Recovered, OPENED).unwrap()); + + assert!(store.open(condition, "service", OPENED).unwrap()); + assert!(!store.open(condition, "service", OPENED + 5).unwrap()); + assert!(store.open(condition, "other", OPENED).unwrap()); + + assert!(store.close(condition, "service", CloseReason::Recovered, CLOSED).unwrap()); + assert!(!store.close(condition, "service", CloseReason::Recovered, CLOSED).unwrap()); + + assert!(store.open(condition, "service", CLOSED + 1).unwrap()); + let records = store.read().unwrap(); + assert_eq!(records.len(), 3); + let keys = records.iter().map(|r| r.incident_key.as_str()).collect::>(); + assert!(keys.contains(&format!("continuity-exhausted:service:{OPENED}").as_str())); + assert!(keys.contains(&format!("continuity-exhausted:service:{}", CLOSED + 1).as_str())); + + // A clock stepped back closes at the opening time, never before it. + assert!(store.close(condition, "other", CloseReason::NoLongerAdmitted, OPENED - 9).unwrap()); + let other = store.read().unwrap().into_iter().find(|r| r.subject == "other").unwrap(); + assert_eq!(other.closed_at_unix_millis, Some(OPENED)); + assert_eq!(other.close_reason, Some(CloseReason::NoLongerAdmitted)); + } + + #[test] + fn closed_incidents_retire_to_history_after_retention() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + for subject in ["old", "young", "open"] { + assert!(store.open(condition, subject, OPENED).unwrap()); + } + assert!(store.close(condition, "old", CloseReason::Recovered, OPENED + 1).unwrap()); + let young_closed = OPENED + INCIDENT_RETENTION_MILLIS; + assert!(store.close(condition, "young", CloseReason::Recovered, young_closed).unwrap()); + + // Exactly at the boundary nothing has aged out; one millisecond later + // only the older closure has. + let boundary = OPENED + 1 + INCIDENT_RETENTION_MILLIS; + assert!(!store.retire_closed(boundary).unwrap()); + assert!(store.retire_closed(boundary + 1).unwrap()); + assert!(!store.retire_closed(boundary + 1).unwrap()); + + let live = store.read().unwrap(); + let mut subjects = live.iter().map(|r| r.subject.as_str()).collect::>(); + subjects.sort_unstable(); + assert_eq!(subjects, ["open", "young"]); + + let history = SingleFileMessagePackBackingStore::new(&temp.path().join("incident-history.cc")) + .pull_all_read_only_snapshot() + .unwrap(); + assert_eq!(history.len(), 1); + assert_eq!(history[0].key, format!("continuity-exhausted:old:{OPENED}")); + + // Later the younger one retires too; the open one never does. + assert!(store.retire_closed(young_closed + INCIDENT_RETENTION_MILLIS + 1).unwrap()); + let live = store.read().unwrap(); + assert_eq!(live.len(), 1); + assert_eq!(live[0].subject, "open"); + } + + #[test] + fn the_store_refuses_any_document_that_is_not_an_incident() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let foreign = CultCacheEnvelope { + key: "x".into(), + r#type: "idunn.target_supervision".into(), + payload: vec![0x90], + stored_at: "2026-01-01T00:00:00+00:00".into(), + schema_id: Some("idunn.target_supervision.v1".into()), + }; + SingleFileMessagePackBackingStore::new(&path) + .insert_entry_if_absent(foreign) + .unwrap(); + let store = IncidentStore::new(&path); + assert!(store.read().is_err()); + assert!( + store + .open(IncidentCondition::ContinuityExhausted, "service", OPENED) + .is_err() + ); + } + + #[test] + fn status_renders_open_and_recently_closed_incidents_for_their_target() { + let [closed, open] = fixture_records(); + let records = [closed, open]; + let lines = render_incidents(&records, "closed-target", CLOSED + 1); + assert_eq!( + lines, + [format!( + " incident continuity-exhausted opened-at {OPENED} closed-at {CLOSED} recovered" + )] + ); + assert_eq!( + render_incidents(&records, "open-target", CLOSED + 1), + [format!(" incident continuity-exhausted opened-at {CLOSED}")] + ); + assert!(render_incidents(&records, "unrelated", CLOSED + 1).is_empty()); + // A closed incident past retention is history, not status. + assert!( + render_incidents(&records, "closed-target", CLOSED + INCIDENT_RETENTION_MILLIS + 1) + .is_empty() + ); + assert_eq!( + render_incidents(&records, "closed-target", CLOSED + INCIDENT_RETENTION_MILLIS).len(), + 1 + ); + } + + /// Idunn runs with `UMask=027`, so a file it creates lands 0640 unless the + /// store publishes it. The umask is process-wide, so it is held only around + /// each write and only one test at a time may hold it. + #[cfg(unix)] + fn under_service_umask(write: impl FnOnce() -> T) -> T { + static UMASK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _held = UMASK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + // SAFETY: umask only swaps the process file-creation mask. + let previous = unsafe { libc::umask(0o027) }; + let result = write(); + unsafe { libc::umask(previous) }; + result + } + + #[cfg(unix)] + #[test] + fn published_incident_store_is_world_readable() { + use std::os::unix::fs::PermissionsExt; + + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let lock = temp.path().join("incidents.cc.lock"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + let mode = |path: &Path| std::fs::metadata(path).unwrap().permissions().mode() & 0o777; + + assert!(under_service_umask(|| store.open(condition, "service", OPENED)).unwrap()); + assert_eq!((mode(&path), mode(&lock)), (0o644, 0o644), "after open"); + + // Each write replaces the file, and the replacement is published too. + assert!( + under_service_umask(|| store.close(condition, "service", CloseReason::Recovered, CLOSED)) + .unwrap() + ); + assert_eq!((mode(&path), mode(&lock)), (0o644, 0o644), "after close"); + + assert!( + under_service_umask(|| store.retire_closed(CLOSED + INCIDENT_RETENTION_MILLIS + 1)) + .unwrap() + ); + assert_eq!(mode(&path), 0o644, "after retire"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 7b3fd60..b96d835 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,4 +4,5 @@ pub mod deployment_plan; pub mod drivers; pub mod host; pub mod host_actuator; +mod incident; pub mod provisioning; From 7f7528b33e8e5c607c19cc7f8f8a8da4def50fca Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:23:57 +0200 Subject: [PATCH 014/196] Add the idunn.operator_incident.v1 wire fixture One open and one closed record, written by IncidentStore on Yggdrasil under IDUNN_WRITE_FIXTURES=1 and read back by the_incident_fixture_is_what_idunn_reads. Bifrost's reader tests copy this file. Builds. Co-Authored-By: Claude Sonnet 5.5 --- tests/fixtures/idunn.operator_incident.v1.cc | Bin 0 -> 737 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/fixtures/idunn.operator_incident.v1.cc diff --git a/tests/fixtures/idunn.operator_incident.v1.cc b/tests/fixtures/idunn.operator_incident.v1.cc new file mode 100644 index 0000000000000000000000000000000000000000..180c6f0f2ae325984fa3c77bc4f3d193916ac76d GIT binary patch literal 737 zcmbQdDY-PKBsnoTBUP`sB)=$Cugq}b^j(=LrFnUJ`30#(i9o^l%)I2xl+?TupyYNO zQhRZT1GNAV9s_S?R4Ww&tZVp?B$gJJq^4LU=j0csrs$R=7Nw__SQ(ld zAOIfob{ZKN8S5Gv>KdAa7#Ud^npzncXd4&+$s@h^f|qnhY{BO0wJ5GW&%nUgeg0_% z1BCJH-GY@xsmb|esYR(NQ*P>EGr$TM_jzcZHU#^eJWp5P^)!i2!0P4ING^u^^S}WB D;7=HM literal 0 HcmV?d00001 From 6394100a93bcc70416f351a6ce59f63b0c56349e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:51:06 +0200 Subject: [PATCH 015/196] B5: pin the idunn expire verb as the CLI parses and runs it Kills three cargo-mutants survivors: the --requested-by arm, run's dispatch, and expire's own-live-transaction filter. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 7e4f281..247a752 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -18472,6 +18472,47 @@ mod tests { Ok(()) } + /// The `expire` verb as the CLI parses and runs it: the options land + /// where they say, and it finds only a live transaction of its own + /// command. + #[test] + fn idunn_expire_parses_its_options_and_asks_only_about_its_own_live_transaction() + -> Result<()> { + let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); + assert_eq!( + parse(args(&["expire", "up-service", "--requested-by", "alice", "--state-store", "/s/c.cc"]).into_iter())?, + Command::Expire { + state_store: PathBuf::from("/s/c.cc"), + command_id: "up-service".into(), + requested_by: "alice".into(), + } + ); + let help = parse(args(&["expire", "up-service", "--help"]).into_iter()).unwrap_err(); + assert!(format!("{help}").contains("idunn expire "), "{help}"); + assert!(parse(args(&["expire", "up-service", "--force"]).into_iter()).is_err()); + assert!(parse(args(&["expire"]).into_iter()).is_err()); + + // The verb runs: a store with nothing live refuses. + let empty = TempDir::new()?; + let store = empty.path().join("control.cc").display().to_string(); + let refused = run(args(&["expire", "up-service", "--state-store", &store]).into_iter()) + .expect_err("expired a command with no transaction"); + assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); + + // A finished transaction of the command is not a live one. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; + routed.resume_until_terminal()?; + assert!(resident(&routed.world)?.value.is_terminal(), "the record was archived"); + let refused = expire(&routed.world.state_store, "continuity-service", "operator") + .expect_err("expired a finished transaction"); + assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); + Ok(()) + } + /// Q3 at the deadline, through the Engine. A continuity whose candidate /// was granted a lease and never proved it held it fails as /// `lease-not-adopted`; one whose Ready proved it restarts the admitted From 855ccf8247faa8d5d00543291cd1c94614c20cf9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:53:01 +0200 Subject: [PATCH 016/196] Pin the incident retention and the status usage; keep transport names out of src The retention constant and the usage line survived mutation untested. The incident module doc named the delivery service, which the no-transport grep over src/ must not find. Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 1 + src/incident.rs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index db1e6b4..dc4bb63 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14889,6 +14889,7 @@ mod tests { panic!("expected serve") }; assert_eq!(options.incident_store, PathBuf::from("/tmp/i.cc")); + assert!(usage().contains("idunn status [--state-store PATH] [--incident-store PATH]")); let Command::Status { incident_store, .. } = parse(["status", "--incident-store", "/tmp/j.cc"].into_iter().map(str::to_owned)) .unwrap() diff --git a/src/incident.rs b/src/incident.rs index fa6f1a6..cd56c54 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -2,8 +2,8 @@ //! opened, and that it closed. //! //! One file, `incidents.cc`, holds one type. Idunn's daemon is its only writer, -//! at the site that decides the condition; it is also the file readers such as -//! Bifrost open, so no second copy exists to drift. It is never `control.cc`, +//! at the site that decides the condition; it is also the file external +//! readers open, so no second copy exists to drift. It is never `control.cc`, //! which refuses any type it does not know: a type added there would stop //! continuity for every target on a rollback. //! @@ -488,6 +488,8 @@ mod tests { #[test] fn closed_incidents_retire_to_history_after_retention() { + // The ruled retention: seven days. + assert_eq!(INCIDENT_RETENTION_MILLIS, 604_800_000); let temp = TempDir::new().unwrap(); let path = temp.path().join("incidents.cc"); let store = IncidentStore::new(&path); From c06ff2b84caa2d29d7a7483e962d030edd0a97bc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:56:02 +0200 Subject: [PATCH 017/196] B5: the expire test asks beside another target's live deployment A terminal record leaves the live set, so it could not tell the verb's own command filter from a looser one; another target's deployment in flight can. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 247a752..a726080 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -18473,8 +18473,8 @@ mod tests { } /// The `expire` verb as the CLI parses and runs it: the options land - /// where they say, and it finds only a live transaction of its own - /// command. + /// where they say, and it asks only about its own command's live + /// transaction. #[test] fn idunn_expire_parses_its_options_and_asks_only_about_its_own_live_transaction() -> Result<()> { @@ -18499,17 +18499,37 @@ mod tests { .expect_err("expired a command with no transaction"); assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); - // A finished transaction of the command is not a live one. + // Another target's deployment in flight is not this command's. let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed - .world - .engine - .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; - routed.resume_until_terminal()?; - assert!(resident(&routed.world)?.value.is_terminal(), "the record was archived"); - let refused = expire(&routed.world.state_store, "continuity-service", "operator") - .expect_err("expired a finished transaction"); - assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); + let now = now_millis()?; + let other = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-other".into(), + kind: CommandKind::Deploy, + selector: "other".into(), + requested_by: "test".into(), + requested_at_unix_millis: now, + }; + let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; + assert!(SingleFileMessagePackBackingStore::new(&routed.world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: other.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: sealing.transaction_id.clone(), + current: None, + }, + ], + &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], + )?); + expire(&routed.world.state_store, "continuity-service", "operator")?; + let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].value.command_id, "continuity-service"); Ok(()) } From 448f554af57807f8d3a2615b5de3f2d3d00b65e9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:15:38 +0200 Subject: [PATCH 018/196] Watchdog map: Bifrost suite probes C21-C28 Only cultcache-ts was renamed in CultLib; Bifrost's tests and three tools still use the bare name. Production Persona delivery is unaffected: its release bundles CultLib f67f5122. Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-map.md | 50 +++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index cb40b8a..70bcd93 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -94,10 +94,30 @@ docs only. | C15 | Bridge CLI contract | Bifrost `tools/bifrost-bridge.mjs:1266-1289`; `tools/persona-discord-delivery.mjs:45` | The bridge refuses to act without `--cultmesh-command-id`. The Persona precedent spawns it with `--receipt-store` under its private state directory and parses stdout JSON. | | C16 | Crash-recovery precedent | Bifrost `tests/persona-discord-delivery.test.mjs`, "running journal recovers as terminal unknown without a second Discord post" | A `running` execution found at start becomes terminal `unknown` and is never re-posted. | | C17 | Discord nonce | docs.discord.com/developers/resources/message (fetched 2026-09-30) | `nonce`: integer or string, "up to 25 characters". `enforce_nonce`: "checked for uniqueness in the past few minutes"; on a duplicate from the same author, "that message will be returned and no new message will be created". | -| C18 | Bifrost tests and mutation tooling | Bifrost: no root `package.json`, `node:test` files under `tests/`, `grep stryker` returns nothing | Tests run with `node --test`, with CultLib found through `VOIDBOT_CULTLIB_ROOT`. StrykerJS is not installed. | +| C18 | Bifrost tests and mutation tooling | Bifrost: no root `package.json`, `node:test` files under `tests/`, `grep stryker` returns nothing | Tests run with `node --test`. StrykerJS is not installed. How the tests find CultLib: C23 (the tests ignore `VOIDBOT_CULTLIB_ROOT`). | | C19 | Idunn verification | `scripts/verify.sh`; eureka `tools/stopgap/ygg-verify.sh` (rust image carries cargo-mutants) | `cargo check` for the Windows GNU target twice, then `cargo test --locked --lib`. | | C20 | Idunn source mentions no transport | `grep -rin "discord\|bifrost" src/` at `487f4f1` | No hits. This is the negative-grep baseline. | +### Bifrost suite probes (2026-09-30, 15:00-15:30 UTC) + +Pinned heads: Bifrost `42728a67` (origin/main) and `hands/watchdog-bifrost-retire` +`4619939e`; CultLib pin `36ea08d3` (origin/main `016df462`); Epiphany origin/main +`4d1113ef`. Probes ran in a scratch clone, never in `F:\Projects\Bifrost`. Suite +runs went through `ygg-verify.sh` in `node:24.14.1-bookworm`, with CultLib cloned +to `/CultLib`, the sibling of `/src`, then `npm ci` and `npm run build:ts`. Each +file ran alone with the TAP reporter, then the whole glob ran once. + +| # | What | Where / command | Result | +|---|---|---|---| +| C21 | Bare `cultcache-ts` require sites | Bifrost `git grep -nF '("cultcache-ts")'` at `42728a67` | `tests/persona-feedback-cli.test.mjs:15,55,62`, `tests/persona-discord-delivery.test.mjs:33`, `tests/persona-discord-crossing-rust-smoke.test.mjs:13`, `tools/persona-feedback.mjs:114`, `tools/agent-transport.mjs:40`, `tools/governance-threads.mjs:36`, and `tools/operator-notification.mjs:186`, which cut `bifrost-retire-alarm` deletes. `persona-feedback.mjs` calls `loadRuntime()` at module top (`:23`), so every verb fails at import. The bare `cultnet-ts` and `cultmesh-ts` sites resolve. Tools that load `dist/index.js` by path do not depend on the name: `bifrost-crossing-documents.mjs:271`, `bifrost-repository-release-authority.mjs:161`, `cultmesh-bridge-commands.mjs:313-314` and `provider-advertisement.mjs:47-119`. | +| C22 | What CultLib publishes | `git show :packages/{cultcache,cultnet,cultmesh}-ts/package.json` at `36ea08d3` and `016df462`; `git log -S'"@gamecult/cultcache-ts"'` | Only `cultcache-ts` is scoped: `@gamecult/cultcache-ts`, renamed in `8cb3b728` (2026-09-04). `cultnet-ts` and `cultmesh-ts` are still bare and depend on `@gamecult/cultcache-ts ^0.14.0`. The commit message says they "move with it", but only their dependency moved. The directory `packages/cultcache-ts` keeps its name, and every package declares `exports`. A local resolve at `016df462`, anchored at `packages/cultcache-ts/package.json`, gives `@gamecult/cultcache-ts` -> `dist/index.js` and `cultcache-ts` -> `MODULE_NOT_FOUND`. | +| C23 | How Bifrost finds CultLib | C21 sites; `ls .github/workflows` | Tools use `VOIDBOT_CULTLIB_ROOT`, falling back to the sibling `../CultLib`. **Tests hard-code the sibling `../CultLib` and ignore the variable.** Bifrost has no `package.json` and no CultLib dependency. No CI runs the node tests: the only workflow is `publish-container.yml`. | +| C24 | **Production Persona crossing is not broken by the rename** | `ssh ygg`, read-only: `systemctl cat bifrost-persona-mouth bifrost-persona-feedback`; `cat .../runtime/current/release-manifest.txt`; `createRequire(/CultLib/packages/cultcache-ts/package.json).resolve("cultcache-ts")` | Both units run release `cb3239a…-f67f5122…`. The manifest says `cultlib_commit=f67f5122`, node `v24.14.1`. `VOIDBOT_CULTLIB_ROOT` points at the release's own CultLib. There the package is named `cultcache-ts`, `node_modules` holds real `cultcache-ts`, `cultmesh-ts` and `cultnet-ts` directories, and the bare name resolves to the release's `dist/index.js`. `f67f5122` is an ancestor of `8cb3b728`, so it predates the rename. A restart reloads the same pinned bundle. Since the C21 fix, Bifrost needs CultLib `8cb3b728` or later. A release that pairs it with an older CultLib fails at import (follow_up `stale-bifrost-persona-manifest` owns that path). | +| C25 | The Rust fixture tests have no producer | `tests/persona-discord-crossing-rust-smoke.test.mjs:17`, `tests/persona-discord-rudp-cross-language.test.mjs:21`; Epiphany `git log -S persona-discord-crossing-fixture`, `git grep` at `4d1113ef` | Both tests `cargo run --bin epiphany-persona-discord-{crossing,rudp-client}-fixture` in the sibling `../Epiphany`, with `CARGO_TARGET_DIR` hard-coded to `C:\\Users\\Meta\\.cargo-target-codex`. Epiphany deleted both bins in `387afe49` (2026-08-23), and neither exists at `4d1113ef`. So since 2026-08-23 these tests fail on every host, with cargo or without. `persona_discord_crossing.rs` survives as a library module. | +| C26 | Suite at base | ygg-verify Bifrost `42728a67`, CultLib `36ea08d3` at `/CultLib` | Per file (tests/pass/fail/skip): rust-smoke 1/0/1/0 (`spawn cargo ENOENT`); delivery 9/3/6/0 (`MODULE_NOT_FOUND 'cultcache-ts'`); permit 2/2/0/0; rudp 1/0/1/0 (`spawn cargo ENOENT`); feedback-cli 1/0/1/0 (the file fails to load, so its 8 tests are counted as 1); idunn-health 3/3/0/0. Glob total 17/8/9, exit 1, the same as Hands h1. | +| C27 | Suite with the fix | scratch commit `a6770c5` on `42728a67`: the scoped name at the seven live C21 sites, a skip guard on both Rust tests, and the `CARGO_TARGET_DIR` override removed. Same command. | rust-smoke 1/0/0/1 (`# SKIP`); delivery 9/9/0/0; permit 2/2/0/0; rudp 1/0/0/1 (`# SKIP`); feedback-cli 8/8/0/0; idunn-health 3/3/0/0. Glob total 24 tests, 22 pass, 0 fail, 2 skipped, exit 0. `agent-transport.mjs` and `governance-threads.mjs` were checked only by name resolution (C22), because no test loads them. | +| C28 | C3 at the reader's install pin | `git show 36ea08d3:packages/cultcache-ts/src/single-file-messagepack-backing-store.ts` | `pullAll` is at `:43`, and the file has no lock. C3 holds at `36ea08d3`, which is the CultLib that cut `ops-notice-deploy` installs. | + ## Model page A row per persistent kind this campaign creates or depends on. "New" means the @@ -245,3 +265,31 @@ cut is independent and lands before the reader, so the reader is written against a tree with one Discord command vocabulary. The gamecult-ops subtraction is independent too. The ops deploy cut comes last and needs both behaviour cuts merged. + +**The Bifrost suite follows the owner's name, with no fallback.** CultLib +renamed `cultcache-ts` to `@gamecult/cultcache-ts` so that the org owns the +name on a public registry (C22). The consumer changes the string at each live +require site (C21) and keeps the directory anchor, which did not move. A +try-both-names loader was rejected. It would be a local shim that keeps the +old name alive in Bifrost after its owner retired it. It would also let a +release silently pair new Bifrost with a pre-rename CultLib. After the change +that pairing fails loudly at import (C24). Production does not move: its +bundle pins CultLib `f67f5122` (C24). `cultnet-ts` and `cultmesh-ts` stay bare +because that is what CultLib publishes. Whether they should follow the scope +is CultLib's decision, not Bifrost's. + +**The Rust fixture tests skip with a reason; they are not deleted.** Their +producers were deleted from Epiphany on 2026-08-23 (C25), so installing cargo +would not make them pass. Each one skips unless cargo and its named fixture +source are present, and the skip reason names the Epiphany commit that removed +the fixture. Deleting them would throw away the only Rust-authored byte seam +test the Persona crossing has. Restoring a producer belongs to Epiphany and the +Persona crossing, both outside this campaign (follow_up +`bifrost-rust-fixture-producer`). The hard-coded Windows `CARGO_TARGET_DIR` +goes, because it is wrong on every host except one workstation. + +**`bifrost-suite` does not depend on `bifrost-retire-alarm`.** It leaves +`operator-notification.mjs` alone, because that cut deletes the file, so the +two branches merge in either order without conflict. The reader depends on +both. It is written against a tree with one Discord command vocabulary, and it +is verified by a suite that loads. From e00ff5ec40eb859bfdcaa76ef31be421f524627f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:54:06 +0200 Subject: [PATCH 019/196] Route map: Q3-i ruled (issued lease counts as possibly written); Q3-ii open Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 3c94d9a..12e48db 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -590,6 +590,20 @@ have written. (c) trusts a declaration over evidence. Depends on it: B5, the recovery variants of the terminal record, and the replacement for `idunn cancel`. +**Q3 follow-ups from B5 Hands (`hands/b5` `c06ff2b`), 2026-09-30:** +- **Q3-i: does an issued lease with no adoption evidence count as possibly + written?** **RULED yes** by the operator, 2026-09-30. A candidate cannot prove + it holds the lease while warming, and streampixels-service migrates in exactly + that window. So any issued lease on a deploy is treated as possibly adopted. +- **Q3-ii: after an adopted-lease deploy expires, what stops continuity from + restarting the incumbent on state the candidate may have migrated?** OPEN. + The operator on the offered hold-until-next-`idunn up` option, verbatim: "I + don't like any of these options, unless a stateful deploy running out of time + is an extreme edge case, in which case A". It is not extreme: the default + `awaiting_ready` deadline is 300 s (`deployment_plan.rs:474`), so any stateful + release that migrates and then fails to reach Ready lands here. Alternatives + are being put to the operator. + **Q4. The operator's recovery verb after `8ae00a1` is deleted.** **RULED (b) by the operator, 2026-09-29.** From 3fad441917a4b3ac11fdef879b3a66adb15e183e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:56:54 +0200 Subject: [PATCH 020/196] Route map: Q3-ii ruled: snapshot/restore before Ready, roll forward after Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 12e48db..00083a1 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -596,7 +596,23 @@ replacement for `idunn cancel`. it holds the lease while warming, and streampixels-service migrates in exactly that window. So any issued lease on a deploy is treated as possibly adopted. - **Q3-ii: after an adopted-lease deploy expires, what stops continuity from - restarting the incumbent on state the candidate may have migrated?** OPEN. + restarting the incumbent on state the candidate may have migrated?** + **RULED 2026-09-30: D before Ready, E after.** Operator, verbatim: "D before + Ready, E after; this actually closes the hole, and is a nice feature for Idunn + to have". + - **D, before the candidate reports Ready:** after the fence and before the + write lease is issued, Idunn snapshots the target's state through a + per-target snapshot/restore declared in its binding. If the deadline expires + before Ready, Idunn restores the snapshot and brings back the incumbent. A + failed restore is the only terminal `OperatorRequired`. + - **E, after Ready:** the candidate is known-good and may have served traffic, + so an expiry rolls forward. The candidate stays the release supervision keeps + alive, and Idunn completes or retries routing and commit. + - Assumption Soul must falsify: the incumbent cannot write between the fence + and the lease, so the snapshot is exact. + - Map pending (Imagination). + + History, the question as first put: The operator on the offered hold-until-next-`idunn up` option, verbatim: "I don't like any of these options, unless a stateful deploy running out of time is an extreme edge case, in which case A". It is not extreme: the default From 5f232a612019ec41651bdf7a1bf08af210cd7908 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:16:06 +0200 Subject: [PATCH 021/196] Route map: Q3-ii cut map (snapshot before Ready, roll forward after); Q-S1..S5 open Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 541 ++++++++++++++++++++++++++++++++++- 1 file changed, 540 insertions(+), 1 deletion(-) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 00083a1..af13920 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -610,7 +610,8 @@ replacement for `idunn cancel`. alive, and Idunn completes or retries routing and commit. - Assumption Soul must falsify: the incumbent cannot write between the fence and the lease, so the snapshot is exact. - - Map pending (Imagination). + - Mapped: section 4, "Q3-ii cut: snapshot before Ready, roll forward after" (Imagination, 2026-09-30); + it establishes the assumption for Idunn-managed writers and names what breaks it (Q3-ii.2). History, the question as first put: The operator on the offered hold-until-next-`idunn up` option, verbatim: "I @@ -1639,6 +1640,544 @@ admitted provider … Unrouted providers fall back to their Q1 class evidence". - An unadopted lease resolves without an operator. - Commit completes while Odin re-stamps every tick. +### Q3-ii cut: snapshot before Ready, roll forward after + +Imagination, 2026-09-30. Nothing has landed. It maps the Q3-ii ruling, verbatim: "D before Ready, E after; this +actually closes the hole, and is a nice feature for Idunn to have". Q3-i is also ruled: any issued lease counts as +possibly written. + +Anchors: +- Idunn `hands/b5` = `c06ff2b` (`src/` unless named; main's `src/` = `a96ad9d`, so B5 is the only code delta); +- gamecult-ops `0961976`, StreamPixels `a49f7d8`, Heimdall `e8e2832`, Ghostlight and Odin working trees (recipes read only). + +"Probe" means one of: +- a read-only command on Yggdrasil, 2026-09-30 about 16:00 UTC (18:00 CEST); nothing was written to the host; +- a throwaway `postgres:16` container on Starfire, destroyed after the run. + +"Read" means source at those heads. + +#### Q3-ii.1 Body facts + +- **S1. Fence order (read).** The candidate starts in Starting and runs alongside the incumbent through Warming + (`candidate-then-promote`). + - Fencing (`advance_fencing` `:6712-6797`) stops the incumbent and revokes its lease. It does this **only when the + incumbent holds a lease** (`:6719`, `:6733-6750`). The phase then moves to Leasing (`:6796`). + - Leasing's first write is `LeasingEvidence::Prepared` (`:6953-6963`); under Q3-i, Prepared already counts as issued. + - Ready is written in AwaitingReady together with `lease_adoption`: route-proof at `:7018`, Odin-correlated at + `:7069`. Routing only refreshes an existing Odin receipt (`:7118`). So `ready.is_some()` is monotone inside a + transaction, and it is the Ready boundary. +- **S2. An unleased incumbent is not fenced (read).** A stateful candidate replacing an incumbent without a lease + (for example the first deploy after a recipe declares state) leaves the incumbent running until post-commit + cleanup. `incumbent_was_stopped_during_fencing` (`:9058-9066`) derives "stopped" from the incumbent's lease + digest. +- **S3. Candidate writers before the lease (read).** + - **StreamPixels** loads the application, and with it `runPostgresMigrations`, only after `waitForWriteLease` + (`deployment/idunn/runtime-presence.mjs:58-59`). + - **Heimdall** builds its app, which runs `createStore`, and then `ensureSchema`, before any lease check + (`src/index.ts:18`, `src/store/index.ts:16-18`). It also writes `runtime-presence-sequence` into its state root on + every health pulse, lease or not (`src/index.ts:115`). + - Recipes declare CultCache-file slots and validate that writable slots open after the lease + (`deployment.rs:960-969`). **Postgres is declared nowhere.** `StateRecovery` and `MigrationDeclaration` are + parsed and never consumed by the control plane (grep). +- **S4. Out-of-band writers (probe).** + - `pg_stat_activity` on `streampixels` holds one session: pid 799973, which is unit + `idunn-streampixels-service-8c469c…`, the admitted Idunn workload. + - The `heimdall` database is held by the **legacy `heimdall.service`**, which Idunn does not manage (secret-files + map P8). + - No timer or cron touches either database. + - Operator scripts `apps/service/scripts/create-creator-invite.ts` and `grant-operator-role.ts` write + `streampixels` when an operator runs them. +- **S5. Sizes and times (probe).** + + | Store | Size | Snapshot | + |---|---|---| + | `streampixels` database | 9.0 MB | `pg_dump -Fc` 137 ms, 48 KB archive | + | `heimdall` database | 38 MB | `pg_dump -Fc` 1,042 ms, 13.7 MB archive | + | streampixels-service state root | 12 KB | | + | odin state root | 21 MB | | + | ghostlight-world-v2 state root | 204 KB | | + | heimdall state root | 12 KB | | + + Free disk is 1.6 TB. Postgres is 16.15, on local peer auth. +- **S6. State roots do not match their recipes (probe).** + - `ghostlight-world-v2/service/play-turn-v1.cc` (plus its `.lock`) is live and **undeclared** in the Ghostlight + recipe. + - Heimdall's state root holds `provider-health-identity.cc` (a private identity, mode 0600) and + `runtime-presence-sequence`. It does not hold its declared `heimdall.service.cc`; Heimdall is not admitted. + - The installed odin binding's `state_root` is `/var/lib/gamecult/odin`. The template + `idunn/yggdrasil/bindings/odin.toml.in` says `odin-v2`, which does not exist on the host. That is template + drift (ops follow-up). +- **S7. Postgres restore mechanics (probe, container).** + - In-place `pg_restore --clean --if-exists --single-transaction` **leaves tables the candidate created**: the + probe's `sessions` table survived. An in-place restore is not exact. + - A side database restored with `--no-owner --role=` and swapped with two `ALTER DATABASE … RENAME` in one + transaction block swaps atomically and leaves no candidate residue. + - `RENAME` refuses while another session is connected: it waits 5 s, then fails "being accessed by other users". + So the swap terminates the database's sessions first. + - **Fingerprint.** Take the sha256 of the archive's text (`pg_restore -f -`), with `--` comments, blank lines and + the per-dump random `\restrict`/`\unrestrict` lines removed. It equals the same fingerprint of `pg_dump` of the + restored database and of the source. Changing one row changes it. + - Neither live database has database-level ACLs, role settings or extensions (probe). pg_hba is `local all all + peer`, so root reaches Postgres through `runuser -u postgres` with no secret, as the backup producer already does. +- **S8. Supervision is blocked for a live transaction (read).** + - While any transaction of the target is at or past Fencing and not yet Complete, the target's supervision pass + skips it entirely (`:5684-5701`). That includes a running post-fencing abort, whose phase stays below Complete. + - After the terminal write, supervision repairs the Expected projection (`:5745`) and mints continuity (`:5987`). + - So a restore **inside** the abort cannot race continuity. A restore that fails and then goes terminal can: the + next mint restarts the incumbent on half-restored state. +- **S9. The abort path has no deadline (read).** + - `resolves_by_deadline` excludes `post_fencing_abort` (`:1568-1573`), and each abort step retries every tick. + - `route_restoration` retries `ufw delete` forever. B5's own test `:18394` pins 20 attempts and no end. + - The abort order is route, lease, candidate, topology, source (`:8717-8858`). A state restore placed after the + route step would inherit that livelock. +- **S10. Commit and route supervision need a promoted route (read).** + - `AdmittedGeneration::from_transaction` requires Committing, Ready and a routing disposition (`:2370-2418`). + - Route supervision bails without `RoutingEvidence::Promoted` (`:6049-6055`). + - Its repair `restore_admitted_membership` (`:6130`) installs from a membership digest and reloads only when the + fragment differs (`6ff7d5a`). +- **S11. Continuity re-admits stored plans under current rules (read).** `parsed_inputs` runs `binding.admit` + (`deployment_plan.rs:634-643`). A new admission rule must therefore be compile-time only, as secret-files §2 + already records. +- **S12. Sandbox (read, and the installed unit probed identical).** Idunn runs as root with `ProtectSystem=full`. + `/var/lib/gamecult/idunn` and every target's state root are `ReadWritePaths`, so a snapshot directory under + Idunn's root needs no unit change. + +#### Q3-ii.2 The exactness assumption (question 1) + +The assumption is that the incumbent cannot write between the fence and the lease. It holds for Idunn-managed +writers under three conditions. The body breaks two of them today. + +1. **Every live incumbent is stopped at the fence when the candidate is stateful.** Broken by S2. Fix: when + `expected.write_lease_required`, Fencing stops any exact incumbent, leased or not. `FencingEvidence::Revoked` + records `incumbent_stopped: bool`, and `incumbent_was_stopped_during_fencing` reads that field instead of + deriving it from the lease. +2. **The candidate writes nothing before its lease.** + - This is declared and validated for CultCache slots. + - It is not enforced for Postgres. StreamPixels complies. Heimdall does not: its schema DDL runs at start, and its + sequence file is written every pulse (S3). + - The fence cannot cover this. It is a target-contract fault, and the fixes belong in Heimdall (follow-ups H-S1, + H-S2 below). +3. **No out-of-band writer.** Today that means the legacy `heimdall.service` and hand-run StreamPixels operator + scripts (S4). + +**What breaks when it is not exact:** +- A restore silently discards whatever an out-of-band writer committed after the snapshot. +- If the candidate migrated before the fence, the "incumbent's state" captured is already the candidate's. + +**What closes it.** Condition 1 is closed in code. For conditions 2 and 3, the snapshot drivers check the no-writer +precondition at the store. They check at snapshot start, again after the capture, and again immediately before +Leasing writes Prepared: +- a Postgres store requires zero sessions on its database; +- a file store takes every captured file's `.lock` exclusively and non-blocking; a held lock means a live owner. + +A failed check is a typed refusal inside Snapshotting. No lease is issued, so the deadline ends it as a plain abort, +with nothing to restore. + +With the current body, this refuses every Heimdall deploy until H-S1 lands. That is the correct outcome. + +One residual stays open: a writer that connects and disconnects between two checks goes unseen. Only Q-S3 (b) closes +that. + +**Order in the phase machine after this cut:** Fencing (stop every incumbent, revoke) → **Snapshotting** (new) → +Leasing (Prepared = issued) → AwaitingReady (Ready durable = the D/E boundary) → Routing → Committing. + +#### Q3-ii.3 Model page additions + +| Kind | What names it | Life over time | Who decides | +|---|---|---|---| +| **Snapshot declaration** (`[state_snapshot]` in the operator binding) | the binding's target; frozen into the plan through `binding_blob` | Changes only by an ops binding change and a new Deploy plan. An old plan without it keeps running continuity (S11) | gamecult-ops template; Idunn validates shape at parse and coverage at compile | +| **Snapshot evidence** (transaction key 37, `state_snapshot`) | the transaction; one per Deploy past Fencing | Written once, when every store is captured and digested. Read by the restore. Terminal with its transaction | Engine (Snapshotting). Forbidden: CLI, drivers writing the record | +| **Snapshot bytes** (`/var/lib/gamecult/idunn/snapshots///`, 0700 root) | path derived from the transaction id | Rewritten whole on a Snapshotting replay. Deleted by the transaction's own terminal cleanup step, except under a state hold, where they are deleted when the hold is released. No janitor | the transaction that owns the path | +| **State restoration** (`PostFencingAbort.state_restoration`) | the abort | `Skipped` (no lease issued) → `Pending{deadline_at}` → `Complete{proof}` or `Failed{store, kind}` | abort path. Forbidden: supervision, CLI | +| **State hold** (`TargetSupervision.state_hold`, supervision v2) | target | Set in the same CAS as a terminal `OperatorRequired{RestoreFailed}`. Released per Q-S4. Never set by anything else | Engine at the terminal write; release per Q-S4. Forbidden: continuity mint, which only reads it | +| **Roll-forward record** (transaction key 38 `roll_forward` intent; completion `Admitted{recovery: RollForward}`) | the transaction | Set by the resolver after Ready. Terminal at the forced commit | deadline resolver | + +#### Q3-ii.4 Snapshot/restore contract (question 2) + +**Binding shape.** It is typed and closed; there are no command strings. Idunn accepts no imperative input: see the +unit's own comment and `cli_exposes_only_declarative_commands`. + +```toml +[state_snapshot] +snapshot_seconds = 120 # optional; Idunn default 120. The Snapshotting deadline +restore_seconds = 300 # optional; Idunn default 300. The abort's restore deadline + +[[state_snapshot.stores]] +kind = "state-slots" # the recipe's writable slots under workload.state_root (scope: Q-S2) + +[[state_snapshot.stores]] +kind = "postgres" +database = "streampixels" # local socket, peer auth as OS user postgres; no URL, no credential file +``` + +- **Parse** (`deployment.rs`, `OperatorBinding` `:266-287`, new `StateSnapshotBinding`): + - duration bounds as in `DeadlineBinding` (`:734-750`); + - database names are identifiers; + - no duplicate stores; + - `state-slots` requires a `workload.state_root`. +- **Compile-time coverage** goes in `compile_deployment_plan` (`deployment_plan.rs:674-722`), never in `admit()` + (S11): + - a stateful recipe with a `state-slots` store must declare writable file slots; + - the no-declaration case is Q-S1. +- The durations live in the binding table and are frozen by `binding_blob`. `PhaseDeadlines` is untouched, so no + v3 plan changes its `plan_id`. + +**Where the bytes live, and who owns and prunes them.** `snapshots///` under Idunn's own +root: +- slot copies keep their relative path, owner and mode; +- `postgres/.dump` is 0600. + +The **manifest** is not a file. It is the typed snapshot evidence on the transaction: +- per slot: relative path, size, mode, uid, gid, sha256, and whether it existed; +- per database: archive sha256, fingerprint (S7), owner role, and the zero-session check times. + +Pruning: +- the transaction deletes its own directory in a new last cleanup step, on both `PostCommitCleanup` and + `PostFencingAbort`; +- under a state hold it is kept, and deleted on release. + +The path is derived from the transaction id, so a crash leaves nothing a later step cannot find. gamecult-ops adds +`/var/lib/gamecult/idunn/snapshots` to the backup `EXCLUDES`. It duplicates live stores already captured, and it +can hold state-root private identities (S6). + +**Budget.** The write pause (fence stop → lease) grows by the capture time. Measured today: about 0.2 s for +streampixels-service and about 1 s for heimdall (S5). The defaults leave two orders of magnitude of headroom. + +**Secrets.** +- Idunn never opens a service credential file. Postgres is reached by peer auth, so the database URL files stay + unread; this keeps secret-files §1 "Idunn never opens a service credential for reading". +- `pg_dump`/`pg_restore` stderr can quote row data, and `idunn status` prints `last_error` verbatim (secret-files P7). + So no tool stderr enters `last_error`, `report_once` or the journal. The error carries a closed `SnapshotFault` + kind, the exit status, and the stderr's sha256. The full stderr goes to `snapshots/…/stderr-` (0600), which + is pruned with the directory. + +**Idempotence and crash safety.** +- **Snapshotting is repeatable while the fence holds.** A crash before the evidence write means the next tick + deletes the directory and captures again. +- **Restore is decided from observable state, never from a sub-step log.** + - Postgres: if the live fingerprint already equals the recorded one, the store is done. This covers a crash after + the swap. Else, if `__idunn_restore` exists and its fingerprint equals the recorded one, terminate the + database's sessions, swap both names in one transaction, and drop the displaced database. Else drop and recreate + the side database from the archive. + - Slots: write each one by temp file and rename in its own directory; remove a slot file that did not exist at + snapshot time; leave `.lock` files alone. + - Replaying any prefix of these steps converges. + +**How restore proves itself.** +- Postgres: fingerprint equality on the side database **before** the swap, so a bad restore never swaps. After the + swap, the live name resolves to the side database's oid. +- Slots: a re-walk equals the manifest byte for byte, including absences. +- The proof digests go into `state_restoration: Complete{proof}`. + +**No declared snapshot:** Q-S1. + +#### Q3-ii.5 Phase machine (question 3) + +- **Snapshotting** is a new `DeploymentPhase` variant between Fencing and Leasing (`:206-216`, `Ord`-derived). + - `PhaseDeadline::entering` (`:913-934`) gains its arm from `snapshot_seconds`. + - `resolves_by_deadline` (`:1568-1573`) covers it through the existing range. + - It is entered only by a Deploy whose plan declares a snapshot. Continuity and stateless targets go Fencing → + Leasing as today. + - Expiry in Snapshotting → `Abort{restore: false}`. The partial directory is removed by the abort's cleanup. +- **Restore is a step, not a phase.** `PostFencingAbort` (`:1382-1400`) gains `state_restoration` and + `deadline_at_unix_millis`. + - `begin_post_fencing_abort` (`:8694-8712`) stamps the deadline. For a restoring abort that is `restore_seconds`; + for any other abort it is an Idunn default of 300 s (constant). + - **Order becomes lease → candidate → state → route → topology → source.** Restore needs only the candidate + stopped and its lease revoked, and S9's livelock must not hold state hostage. "Never two writers" is about + lease-then-process, and that order is preserved. +- **Every abort ends.** + - On the abort deadline, if `state_restoration` or any step before it is still Pending, the transaction completes + `FailedAfterFencing{OperatorRequired{RestoreFailed}}`. A restore that cannot even stop the candidate is a failed + restore. + - Steps after it (route, topology, source) are marked a new `CleanupEvidence::Abandoned`, reported once. The + transaction completes with its normal recovery. + - Abandoning those steps is safe because supervision owns their repair: Expected repair and demotion (`:5745`, + `:5943-5985`), and the admitted-route restore (`:6130`). + - This ends B5's `ufw delete` loop for every abort, not only D's. +- **The Ready boundary is `transaction.ready.is_some()`** (S1). `PhaseEnd` is rebuilt as follows. The B5 table test + is rewritten to it. + + | Command | Ready durable | Lease issued (Prepared or Granted) | End | + |---|---|---|---| + | Deploy | yes | any | `RollForward` | + | Deploy | no | yes | `Abort{restore: true}` (snapshot evidence required by validation) | + | Deploy | no | no | `Abort{restore: false}` | + | Continuity | any | any | `Abort{restore: false}`, as B5 | + + Continuity keeps B5's arm. Its candidate is the admitted release, it restarts under the restart log, and it never + changes the state contract. That is a default, not a question. +- **E replaces the abort after Ready. It is a forced commit, and it only writes the store.** + - The resolver persists `roll_forward = Some{cause, at}` and, if needed, `enter_phase(Committing)`. + `resolves_by_deadline` excludes a record with `roll_forward`, so it fires once. + - `advance_committing` (`:7206-7392`) with `roll_forward` set skips: + - the final route proof; + - Odin re-admission; + - the workload observation; + - lease currency. + - It then commits the generation from the durable evidence: + - Routing with no route receipt: a routed target gets a new `RoutingEvidence::Unproven{membership_sha256}`, the + digest of the rendered fragment. An unrouted target gets `SkippedUnrouted`. + - Committing with `Promoted`: kept. + - `Unproven` starts `route_supervision` degraded. Supervision (`:6037-6140`) accepts `Unproven`: it repairs with + `restore_admitted_membership`, proves, and on success rewrites the receipt to `Promoted`. + - A candidate that died after Ready is admitted anyway, and continuity restarts it under the restart log. That is + "the release supervision keeps alive", literally. + - **Boundedness.** The forced commit does no actuation. Every later actuation is supervision's: route repair spaced + by the challenge backoff up to `ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS` (`:92`) and reloading only on a fragment + difference; process restarts bounded by the restart log (`:69-72`). No post-fence path retries forever. +- **D and `TargetSupervision`.** + - The restore runs inside the live transaction, so supervision cannot mint over it (S8). + - A `RestoreFailed` terminal sets `state_hold` in the **same CAS** as the completion (a three-envelope CAS like the + mint at `:5987-6031`). + - The mint path checks the hold right after `restarts_exhausted` (`:5916`) and mints nothing, with one report. + - Expected repair may still run; it starts no process. Deploy commands stay admissible (Q-S4). +- **Terminal records.** + - `FailedAfterFencing{recovery}`, where `recovery` is one of: + - `RestoreIncumbent`: D restored, or no lease was issued; + - `RestartAdmitted`: continuity, as B5; + - `OperatorRequired{reason: OperatorRequiredReason::RestoreFailed{store, kind}}`: typed. B5's free string + `reason` has no production writer (watchdog map I5). + - `Admitted{generation_id, recovery: Option}`, with `Some(RollForward{cause})` only there. + - Validation (`:2040-2075`, `:2180-2205`) pins each variant to its completion. +- **Schemas and lifts. One store-version install.** + - `deployment_transaction.v5` carries: + - the Snapshotting variant; + - keys 37 `state_snapshot` and 38 `roll_forward`; + - `FencingEvidence::Revoked.incumbent_stopped`; + - `PostFencingAbort.{state_restoration, deadline_at_unix_millis}`; + - `CleanupEvidence::Abandoned`; + - the completion shapes above. + - The v4 lift (the `read_transaction_record` pattern, `:3659-3704`): + - `incumbent_stopped = incumbent_lease_sha256.is_some()`; + - `state_restoration = Skipped`; + - an in-flight abort's deadline is stamped once on first sight (B5's rule for lifted phase deadlines, + `:8588-8596`); + - `Admitted.recovery = None`. + - History is lifted unchanged. + - `admitted_generation.v5` carries `RoutingEvidence::Unproven`. `target_supervision.v2` carries `state_hold`, lifted + as `None`. + - Every lifted fixture must re-encode canonically: the B2 F1 scar. + - An older Idunn refuses the v5 store, so rollback is the pre-install backup, as for B2. + +#### Q3-ii.6 What B5 keeps, changes and deletes (question 4) + +- **Keeps:** + - the resolver's shape (`resolve_phase_deadline` `:8568-8623`); + - `PhaseDeadline` and `enter_phase` as the only deadline writer; + - `idunn expire` and `idunn.expiry_request.v1`. Before Ready it now restores; after Ready it rolls forward, through + the same resolver; + - lease-adoption evidence, `lease-not-adopted`, and the continuity arms; + - the F16 commit fix and the reload skip. +- **Changes:** + - `phase_end` (`:1583-1596`), `terminal_recovery` (`:1598-1606`) and `resolves_by_deadline` (`:1568-1573`), to the + table above; + - the resolver's match (`:8600-8620`); + - `incumbent_was_stopped_during_fencing` (`:9058-9066`); + - `post_fencing_abort_intent` (`:9081-9110`); + - the abort order (`:8717-8858`). +- **Deletes (the "left running" arm):** + - `PhaseEnd::OperatorRequired` and its doc (`:943-945`); + - the `CommandKind::Deploy if issued` arm (`:1588`); + - the resolver's `OperatorRequired` branch (`:8610-8619`, the `deadline:` report); + - the "left to the operator" comment (`deployment_plan.rs:470-473`); + - the three `OperatorRequired` rows of `an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease` + (`:12887-12889`); the test is rewritten to the Ready table; + - the string-reason test `an_operator_required_recovery_must_say_why` (`:12911-12935`), rewritten for the typed + reason. +- **Recommendation: merge B5 first, after its own Soul pass. This map is a follow-on cut** (Q-S5). + - B5 bounds the live continuity reload loop (B2-3d F1). + - Its left-running arm is no worse than today's resumable-forever post-fence. + - D and E carry two schema bumps and a new driver, and need their own falsification. + - Cost of the gap: a stateful deploy that expires after a lease stays live and reported, as today. `idunn expire` + re-enters that arm (watchdog I6). The StreamPixels-service redeploy in the ship sequence runs under that + behaviour. + +#### Q3-ii.7 Cuts + +Order: SR-1 → SR-2 → SR-3 → SR-4. The build and tests run on Yggdrasil (`ygg-verify.sh`) against the single `idunn` +crate. The build activates no new target. + +- **SR-1. Snapshot drivers (Idunn `drivers.rs`, `deployment.rs`).** No phase-machine change. + - One port: `StateSnapshotPort { capture, check_no_writer, restore, prove }`. + - Two implementations: + - `SlotSnapshotDriver`: files, with `.lock` exclusivity; + - `PostgresSnapshotDriver`: `runuser -u postgres` for `pg_dump -Fc`, `pg_restore`, `psql`; side database, + fingerprint, swap. The program paths are injectable, as the route stubs are. + - The binding table parse and validation. + - The closed `SnapshotFault` kind and the stderr quarantine. +- **SR-2. D (Idunn `control_plane.rs`, `deployment_plan.rs`).** + - Deletes first: B5's arm (Q3-ii.6). + - Then: + - the Fencing change (Q3-ii.2 item 1); + - the Snapshotting phase; + - the restore step, abort deadline, reorder and `Abandoned`; + - the state hold; + - the compile-time coverage rule; + - all three schema bumps and lifts, including E's shapes, so the store moves once. +- **SR-3. E (Idunn `control_plane.rs`).** + - the `RollForward` end and the forced commit; + - `Unproven` in Routing and supervision; + - `Admitted.recovery`. +- **SR-4. Ops (gamecult-ops).** + - `[state_snapshot]` for `streampixels-service` (`state-slots` + `postgres streampixels`), `ghostlight` + (`state-slots`), `odin` (`state-slots`), and the Heimdall template (`state-slots` + `postgres heimdall`); + - the backup exclude; + - fixing the odin template's `state_root` drift (S6). +- **Owning-repo follow-ups (not in this campaign's cuts):** + - **H-S1** Heimdall opens Postgres and runs `ensureSchema` only after its lease. + - **H-S2** Heimdall moves `runtime-presence-sequence` out of its state root; it is runtime identity, not state. + - **G-S1** Ghostlight declares `service/play-turn-v1.cc` as a slot (under Q-S2 (a)). + - The legacy `heimdall.service` retires before Heimdall's first Idunn deploy (secret-files Cut 5). + +#### Q3-ii.8 Authority map + +- **Owner.** + - The deadline resolver owns how an ended phase resolves (restore, roll forward, or plain abort). + - The abort path owns the restore. + - The Engine's Snapshotting step owns the snapshot. +- **Inputs:** + - the plan's frozen binding (`[state_snapshot]`); + - the recipe's slots; + - `ready`, `leasing` and `fencing` evidence; + - the phase and abort deadlines; + - expiry requests; + - the stores themselves, through the port. +- **Outputs:** + - snapshot evidence; + - `state_restoration`; + - the terminal completion with its recovery; + - `state_hold`; + - an admitted generation with `Unproven` routing. +- **Derived state:** + - Q3-i's "possibly written" is derived from `leasing`; + - "stopped at fence" is derived from `incumbent_stopped` and no longer from the lease digest; + - the snapshot directory is cache-only, and its truth is the evidence digests; + - `report_once` text is notification-only. +- **Forbidden writers:** + - supervision writing restore or snapshot state, or minting over a held target; + - the CLI writing any of it (`expire` writes only its request); + - drivers writing records; + - the backup producer restoring anything; + - a route or topology step deciding the recovery. +- **Shared paths:** deadline expiry and `idunn expire` go through one resolver. Direct and Odin-correlated Ready use + one boundary field. D aborts and plain aborts use one abort path with one deadline rule. The forced commit and the + normal commit use one `from_transaction`. +- **Deletion line:** + - B5's `OperatorRequired` left-running arm; + - lease-derived fence-stop inference; + - the unbounded abort step. + +#### Q3-ii.9 Verification + +Each test pins a rule. Each must be killed by the named mutation. + +| Test (behaviour) | Rule pinned | Mutation that must kill it | +|---|---|---| +| a deploy that expires before Ready restores the snapshot, and the incumbent restarts on the pre-lease bytes | D | restore marks Complete without writing; or `PhaseEnd` returns `Abort{restore: false}` | +| continuity mints nothing while the restore is Pending | S8 | widen the supervision skip range's lower bound past the abort | +| a failed restore is terminal `OperatorRequired{RestoreFailed}`, and supervision mints no restart | only terminal operator case; hold | drop the hold check, or set the hold outside the completion CAS | +| a deploy that expires in Routing after Ready is admitted `Unproven`, restores nothing, and supervision later promotes it | E | Ready-side arm returns Abort | +| an expiry in Committing rolls forward from the promoted receipt | E | forced commit rewrites the routing receipt | +| the Ready table: (Deploy, Continuity) × ready × (no lease, Prepared, Granted) | boundary and Q3-i | treat Prepared as unissued | +| `idunn expire` before Ready restores; after Ready it rolls forward | shared path | a separate expire branch | +| the fence stops an unleased incumbent when the candidate is stateful | exactness 1 | revert the `:6719` condition | +| a held slot lock, or a live Postgres session (fake runner), refuses Snapshotting with no lease issued | exactness 2/3 | skip either check, or check only at start | +| crash replay at every snapshot and restore step converges (persist the prefix, rerun) | idempotence | the restore trusts a sub-step flag over observation | +| the restore removes a slot file the candidate created | exact restore | copy-over only | +| an abort whose `ufw delete` fails forever ends at its deadline: state restored first, route `Abandoned`, recovery `RestoreIncumbent` | every abort ends | route before state; or Abandoned allowed on the state step | +| a v4 store (every fixture) lifts to v5, and supervision v1 to v2; each re-encodes canonically | schema | drop a field default | +| a stateful Deploy with no `[state_snapshot]` is refused at compile, and continuity of an old admitted plan still runs | Q-S1 (a); S11 | put the rule in `admit()` | +| a canary in fake `pg_restore` stderr appears in no `last_error`, report or `status` line | secrets | copy stderr into the error | +| Postgres integration, `#[ignore]`, run on Yggdrasil against a scratch cluster or container: side-database restore drops candidate tables; fingerprint mismatch refuses the swap; the swap terminates a live session; a crash between swap and drop replays | S7 | in-place `--clean` restore | + +Then run `cargo mutants --in-diff`. + +**Only a live Yggdrasil rehearsal, with the operator present, can show:** +- `runuser`/`pg_dump` reach the socket under the real unit sandbox; +- the snapshot directory is written under the real `ReadWritePaths`; +- a streampixels-service Deploy from a scratch branch whose candidate migrates and never reports Ready restores + `streampixels`, after which the incumbent serves again. Use a shortened `awaiting_ready_seconds` in a scratch + binding; +- the measured write pause; +- E: a Ready candidate whose stable-route proof is blocked is admitted `Unproven`, and is promoted by supervision once + unblocked; +- the backup run skips `snapshots/`. + +#### Q3-ii.10 Subtraction ledger (estimate) + +| | Production | Tests | +|---|---|---| +| Deleted | about -40 (B5 arm, comment, lease-derived stop inference) | about -30 (rows, string-reason test) | +| SR-1 drivers and binding | about +450 | about +300 (fake runners), +120 (ignored Postgres integration) | +| SR-2 D, lifts, hold, abort deadline | about +450 | about +450 | +| SR-3 E | about +180 | about +250 | +| SR-4 ops | about +20 of binding TOML, 1 backup exclude line | none | + +- **Net:** about +1,050 production and +1,100 test lines in Idunn. It adds two schema bumps plus one supervision + bump, installed as one store move, and no new target, service or dependency (it uses the host's `pg_dump`, + `pg_restore` and `runuser`). +- **What it buys:** the ruled capability, which closes Q3-ii's hole, and an end for every post-fence path, including + B5's abort livelock. +- **What it retires:** + - the operator-only exit for stateful deploys; + - the `b5-operator-required-no-exit` follow-up (watchdog map). + +#### Q3-ii.11 Watchdog interface (question 5) + +The interface is a single point. The `operator-required` incident opens at the terminal CAS that writes +`FailedAfterFencing{OperatorRequired{RestoreFailed}}` and sets `state_hold`; its subject is the transaction id, as +the watchdog map models it. It closes when that hold is released (Q-S4). Nothing else in this cut opens or closes +it. + +#### Q3-ii.12 Operator questions + +- **Q-S1. A stateful target whose binding declares no snapshot.** + - Options: + - (a) Refuse a new stateful Deploy plan at compile, with a typed error. Continuity of already-admitted plans is + unaffected. + - (b) Roll forward even before Ready: no D for that target. + - (c) Keep B5's left-running report. + - **Recommend (a).** D is only a guarantee if it cannot be skipped by omission. (b) quietly reopens the hole this + ruling closed. + - Depends on it: SR-2's compile rule, SR-4's binding list, and whether Heimdall can deploy before H-S1. +- **Q-S2. What `state-slots` captures.** + - Options: + - (a) The recipe's declared writable slots only. The recipe is the authority on what is state. Undeclared files + in the state root are reported once and not restored. + - (b) The whole state-root tree. + - **Recommend (a).** It matches "the recipe declares; Idunn infers nothing". + - Under (b), Heimdall's `runtime-presence-sequence` would be restored backwards, and its presence sequence would + regress (S6). + - Cost of (a): Ghostlight must declare `play-turn-v1.cc` (G-S1). Until then, a failed Ghostlight deploy restores + everything but that file. +- **Q-S3. How hard the fence holds against out-of-band writers.** + - Options: + - (a) Detect and refuse: zero sessions and free locks, checked at snapshot start, after the capture, and + immediately before Prepared. + - (b) Also block: `ALTER DATABASE … ALLOW_CONNECTIONS false` from snapshot to grant. Idunn owns the undo on + grant, abort and crash replay. + - **Recommend (a).** Today's only out-of-band writers are the legacy Heimdall unit, which retires, and hand-run + scripts. (b) adds a database-configuration mutation that Idunn must always undo, and a missed undo locks the + service out of its own database. + - Depends on it: SR-1's scope and the residual in Q3-ii.2. +- **Q-S4. What releases a state hold after a failed restore.** + - Options: + - (a) A successful Deploy commit of that target, **or** `idunn release `. The verb writes a typed request + the daemon consumes, as `idunn expire` does, after the operator has repaired the state by hand from the + retained snapshot. + - (b) A Deploy commit only. + - (c) The verb only. + - **Recommend (a).** A deploy is already an operator act that captures its own snapshot. The verb covers "I fixed + it by hand; bring the incumbent back" without forcing a redeploy. + - Depends on it: SR-2's hold, the retained snapshot's pruning, and the watchdog incident's closing event. +- **Q-S5. Merge order.** + - Options: + - (a) Merge B5 after its Soul pass; SR-1..SR-4 follow as their own cuts. + - (b) Hold B5 until D and E land. + - **Recommend (a)**, for the reasons in Q3-ii.6. Under (a), the StreamPixels-service redeploy in the ship sequence + runs with B5's report-and-wait on a stateful expiry. + ### Odin-authority cuts (Imagination audit, 2026-09-30) Source: the Odin-lifecycle authority audit (anchors Idunn `94c7691`/`583b2a7`, Odin `5c37860`), folded in as From 6854b0f825a54461b4536bb09be4cc3a3ecbf453 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:29:56 +0200 Subject: [PATCH 022/196] Route map: B5 Soul pass 1 (merge after batch 2: lease wedge, reload-skip crash gap) Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index af13920..3429930 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -234,6 +234,26 @@ The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map - After 10 min: no odin route-continuity rejections (about 95 in the previous 3 h), 0 nginx reloads, and the Odin post-deploy check `fails=0`. - B5 and skip-unneeded-reloads are in Hands (`hands/b5`). +- **B5 Soul pass 1 (2026-09-30, on `a96ad9d..c06ff2b`): merge after the batch-2 fixes; no install before them.** + Soul reran verify: 352 passed. Batch 2, now in Hands, fixes: + - **A (high).** A continuity aborted from `Prepared` leaves a lease on disk that `driver.grant` wrote before + `Granted` was durable (`control_plane.rs:6916-6930`, `:9081`). The next fencing refuses, and the target is + wedged. + - **B (medium, a regression from `6ff7d5a`).** The reload skip hides a crash between the write and the reload on + the `restore`/`install` transaction paths, leaving nginx proxying to a dead port. The skip goes back to + supervision only. + - **E.** The resolver's command filter is unpinned. + - **F.** Odin-correlated adoption has no committed test; mutants M6 and M7 survive. + - **J.** A flaky backoff test (`:14200`). + + Recorded: + - **C** (the abort has no deadline) is closed by the Q3-ii cut. + - **D.** There is no host-wide reload bound. 170/h is about a third of the F6 storm, and 4830af8 lowers the real + worst case. + - **G.** A newer Ready can rewrite `lease_adoption`. It names the same lease, so it is harmless. + - **H.** AwaitingReady and Routing keep the F16 shape. It is unreachable at today's heartbeats. + - **I.** An `idunn expire` request written while Idunn is stopped lingers, and there is no verb to withdraw it, + which is a downgrade hazard. - **Q-B2-1 (a):** the continuity restart log moves into `TargetSupervision`; the restart ceiling reads the target's own log; the carry compensator is deleted. - **Q-B2-2: fail at once.** A deploy refused by the route/actuation ceiling before the fence fails immediately, From 4ee70f90bae6eef593bb9742162b98f145770a78 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:33:57 +0200 Subject: [PATCH 023/196] Route driver: transaction paths always write and reload (B5 Soul B) `6ff7d5a` skipped the write and reload in `install` and `restore` when the fragment on disk already equalled what they would write. On a transaction path that hides a crash between a write and its reload: an abort that retries after the crash reloads nothing and then stops the candidate, so nginx keeps proxying to a dead port, and a fenced install skips every retry (and a first deploy's `ufw allow`) until its deadline. The skip on equal bytes goes back to supervision only. What stays: a transaction whose candidate membership is the preflight's baseline (a continuity over its admitted route) has no route change to make or undo, so its install, rollback and withdrawal run nothing. No write of its own can have been cut short. `RoutePreflightReceipt::changes_route` names it. `restore_admitted_membership` now puts back the bytes it replaced when its reload fails, instead of deleting the fragment, so the disk still shows what nginx serves. Tests: a crash between a write and its reload is reloaded on retry (install past the fence and withdrawal, driver level; the abort, engine level); a transaction that does not change the route runs nothing. Soul's F2 probe test is gone with the rule it pinned: a withdrawal whose `ufw delete` keeps failing reloads on every retry again, until the Q3-ii cut gives the abort a deadline. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 48 ++++++------- src/drivers.rs | 157 +++++++++++++++++++++++++------------------ 2 files changed, 111 insertions(+), 94 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index a726080..6662b48 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -78,10 +78,11 @@ const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; /// ceiling, never counted against it and never refused by it, so a target /// whose continuity spent its restarts can still be rescued by a deploy. /// Survival is bounded elsewhere: a continuity's actuations by the restart log -/// times what one transaction can do before its phase deadline ends it, an -/// admitted-route repair by the challenge backoff, and a withdrawal by the -/// route driver, which does not reload a fragment that is already correct. -/// Nothing bounds reloads across the host. +/// times what one transaction can do before its phase deadline ends it (none, +/// over its admitted route), and an admitted-route repair by the challenge +/// backoff. A withdrawal that keeps failing retries, reloading each time, +/// under the resume backoff alone: the abort has no deadline. Nothing bounds +/// reloads across the host. const ROUTE_ACTUATION_CEILING: usize = 12; const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; /// How many Survival route actuation times a target keeps, for `status` @@ -17868,10 +17869,7 @@ mod tests { .engine .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; aborting.step()?; - // Nothing was installed, so the withdrawal has no fragment to - // remove and no reload to run: it withdraws the endpoint. - assert_eq!(aborting.reloads(), 0); - assert_eq!(aborting.world.route_stubs.count("ufw delete"), 1, "the withdrawal did not run"); + assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); assert!(newest(&aborting.world)? > before, "the withdrawal was not recorded"); // (ii) Supervision repairs a drifted admitted fragment. @@ -17964,7 +17962,7 @@ mod tests { &preflight, &engine.route_gate("service", CommandKind::Deploy), )?; - assert_eq!(routed.world.route_stubs.count("ufw delete"), 1, "the unrecorded withdrawal did not run"); + assert_eq!(routed.reloads(), 1, "the unrecorded withdrawal did not run"); Ok(()) } @@ -18191,7 +18189,7 @@ mod tests { #[cfg(unix)] #[test] - fn a_failing_reload_that_deletes_the_fragment_backs_the_route_off() -> Result<()> { + fn a_failing_reload_that_puts_the_drift_back_backs_the_route_off() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.admit()?; routed.world.route_stubs.refuse_reloads()?; @@ -18201,7 +18199,7 @@ mod tests { routed.tick()?; } assert_eq!(routed.reloads() - reloads, 1, "the route was reloaded again and again"); - assert!(!routed.fragment_path().exists()); + assert_eq!(std::fs::read(routed.fragment_path())?, b"drifted bytes\n"); assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); Ok(()) } @@ -18302,8 +18300,7 @@ mod tests { /// Soul's F1 probe, committed. A continuity's route changes are /// Survival, so no ceiling refuses its install and rollback while its /// route proof keeps failing. The Routing deadline ends it: no attempt - /// runs after the deadline, and the withdrawal of a rolled-back route - /// reloads nothing. + /// runs after the deadline. #[cfg(unix)] #[test] fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { @@ -18344,7 +18341,9 @@ mod tests { "{:?}", finished.completion ); - assert_eq!(routed.reloads(), 6, "a reload ran after the deadline"); + // After the deadline no attempt runs; the withdrawal writes and + // reloads once, whatever the rolled-back disk shows. + assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); Ok(()) } @@ -18386,33 +18385,28 @@ mod tests { Ok(()) } - /// Soul's F2 probe, committed: a post-fence withdrawal whose - /// `ufw delete` keeps failing reloads once, then repeats only the - /// firewall step. + /// The abort's withdrawal wrote the baseline back and Idunn died before + /// the reload: the disk already shows the baseline, and the retry still + /// reloads, so nginx does not keep proxying to the stopped candidate. #[cfg(unix)] #[test] - fn a_withdrawal_whose_firewall_step_keeps_failing_reloads_once() -> Result<()> { + fn an_abort_whose_reload_a_crash_cut_off_reloads_on_retry() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote()?; - assert!(routed.fragment_path().exists()); let installed = routed.reloads(); - routed.world.route_stubs.refuse_ufw_deletes()?; routed .world .engine .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; - for _ in 0..20 { - routed.resume()?; - } - assert_eq!(routed.reloads(), installed + 1, "a failing withdrawal reloaded again"); - assert_eq!(routed.world.route_stubs.count("ufw delete"), 20); - assert!(!routed.fragment_path().exists()); + std::fs::remove_file(routed.fragment_path())?; + routed.resume()?; assert_eq!( routed.transaction()?.post_fencing_abort.context("no abort")?.route_restoration, - CleanupEvidence::Pending + CleanupEvidence::Complete ); + assert_eq!(routed.reloads(), installed + 1, "the withdrawal trusted unloaded bytes"); Ok(()) } diff --git a/src/drivers.rs b/src/drivers.rs index 3e1098f..7903f94 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -707,6 +707,14 @@ const ROUTE_HTTP_CHUNK_LINE_BYTES: usize = 128; const ROUTE_HTTP_SNAPSHOT_PATH: &str = "/cultnet/snapshot"; impl RoutePreflightReceipt { + /// Whether the candidate's membership differs from the baseline the + /// preflight found. A continuity over its admitted route does not: its + /// transaction has no route change to make or to undo. + pub fn changes_route(&self) -> bool { + self.incumbent_membership_sha256.as_deref() + != Some(self.candidate_membership_sha256.as_str()) + } + pub fn validate(&self) -> Result<()> { require_driver_id(&self.route_id, "route preflight")?; require_sha256_id( @@ -5332,28 +5340,21 @@ impl NginxRouteDriver { } } - /// Put `prior` back as the route's fragment. Bytes on disk are what nginx - /// last loaded: every write here and in `install` is undone when the - /// reload after it fails. So a fragment that already equals `prior` is - /// live, and is neither written nor reloaded; a retry of a withdrawal - /// whose firewall step failed repeats only that step. + /// Put `prior` back as the route's fragment, and always write and reload: + /// on a transaction path, bytes on disk that already equal `prior` may be + /// a write whose reload a crash cut off. A reload that fails puts the + /// previous fragment back, so the disk shows what nginx still serves. fn restore(&self, prior: Option<&[u8]>, gate: &dyn RouteActuationGate) -> Result<()> { - let current = self.current_configuration()?; - let rewrite = current.as_deref() != prior; - if !rewrite && prior.is_some() { - return Ok(()); - } gate.admit(RouteActuation::Survival)?; - if rewrite { - self.write_fragment(prior)?; - if let Err(reload) = self.reload() { - return match self.write_fragment(current.as_deref()) { - Ok(()) => Err(reload).context("reloading the restored route"), - Err(undo) => Err(reload).context(format!( - "reloading the restored route; putting the previous fragment back also failed: {undo:#}" - )), - }; - } + let current = self.current_configuration()?; + self.write_fragment(prior)?; + if let Err(reload) = self.reload() { + return match self.write_fragment(current.as_deref()) { + Ok(()) => Err(reload).context("reloading the restored route"), + Err(undo) => Err(reload).context(format!( + "reloading the restored route; putting the previous fragment back also failed: {undo:#}" + )), + }; } // No prior membership means the stable endpoint no longer routes to // anything; its firewall allow goes with the fragment. @@ -5435,11 +5436,15 @@ impl NginxRouteDriver { && prior_sha256 == preflight.incumbent_membership_sha256), "route baseline changed after preflight" ); - // Bytes on disk are what nginx last loaded (see `restore`), so a - // candidate already written is installed: a retry changes nothing. - if candidate_already_written { + // A candidate whose membership is the baseline's (a continuity over + // its admitted route) changes nothing, so nothing is written, and no + // crash can have cut a write of this transaction short. + if !preflight.changes_route() { return Ok(sha256_id(&rendered)); } + // Otherwise bytes already on disk prove nothing: a crash may have cut + // off the reload after an earlier attempt's write. So it always + // writes and reloads. gate.admit(RouteActuation::Forward)?; atomic_replace(&self.binding.config_path, &rendered)?; if let Err(error) = self.admit_endpoint().and_then(|()| self.reload()) { @@ -5488,12 +5493,6 @@ impl NginxRouteDriver { Ok(self.current_configuration()?.as_deref() == Some(rendered.as_slice())) } - /// Restore the exact membership owned by an admitted generation. Any - /// different fragment is drift, not a rollback baseline, so it is never - /// restored after a failed reload. A failed reload removes the newly - /// written fragment so the next continuity pass cannot mistake disk bytes - /// for an adopted route. The caller must still challenge the stable - /// listener to prove that nginx workers adopted this membership. /// Put the route back exactly as the candidate found it. /// /// The preflight receipt captured the incumbent's configuration before this @@ -5510,11 +5509,21 @@ impl NginxRouteDriver { preflight.route_id == self.binding.route_id, "route preflight receipt describes another route" ); + if !preflight.changes_route() { + return Ok(()); + } self.restore(preflight.incumbent_configuration.as_deref(), gate) } + /// Restore the exact membership owned by an admitted generation. A failed + /// reload puts back the bytes it replaced, so the disk still shows what + /// nginx serves and the next pass sees the drift again rather than taking + /// unloaded bytes for an adopted route. The caller must still challenge + /// the stable listener to prove that nginx workers adopted this membership. + /// /// A fragment already equal to the admitted membership is left alone: no - /// write, no `nginx -t`, no firewall call, no reload. + /// write, no `nginx -t`, no firewall call, no reload. Supervision is the + /// only path that skips on equal bytes. pub fn restore_admitted_membership( &self, expected: &IdunnExpectedIncarnationRecord, @@ -5531,12 +5540,13 @@ impl NginxRouteDriver { } gate.admit(RouteActuation::Survival)?; self.validate_candidate_in_private_mount(&rendered)?; + let replaced = self.current_configuration()?; atomic_replace(&self.binding.config_path, &rendered)?; if let Err(reload) = self.admit_endpoint().and_then(|()| self.reload()) { - return match self.write_fragment(None) { + return match self.write_fragment(replaced.as_deref()) { Ok(()) => Err(reload).context("reloading the exact admitted route membership"), - Err(cleanup) => Err(reload).context(format!( - "reloading the exact admitted route membership; removing its unproved fragment also failed: {cleanup:#}" + Err(undo) => Err(reload).context(format!( + "reloading the exact admitted route membership; putting the replaced fragment back also failed: {undo:#}" )), }; } @@ -5671,6 +5681,9 @@ impl NginxRouteDriver { self.current_configuration()?.as_deref() == Some(rendered.as_slice()), "candidate route changed before rollback" ); + if !preflight.changes_route() { + return Ok(()); + } self.restore(preflight.incumbent_configuration.as_deref(), gate) } } @@ -11882,49 +11895,57 @@ Content-Le".to_vec()), |_| {}), } } - /// A retry whose fragment is already what it would write reloads nothing: - /// an install already written, and a withdrawal whose firewall step is - /// what failed, repeat only what is still undone. + /// On a transaction path, bytes already on disk prove nothing: a crash may + /// have cut off the reload after the write. The install past the fence and + /// the withdrawal both write and reload whatever the disk shows, and the + /// install admits its firewall allow again. #[cfg(unix)] #[test] - fn a_retry_with_an_already_correct_fragment_performs_zero_reloads() -> Result<()> { + fn a_crash_between_a_write_and_its_reload_is_reloaded_on_retry() -> Result<()> { let temp = tempfile::tempdir()?; let (driver, candidate, calls) = failable_route_driver(temp.path())?; let receipt = preflight_receipt(&driver, &candidate)?; + let rendered = driver.render(&candidate)?; + + // The install wrote its fragment and Idunn died before the reload. + fs::write(&driver.binding.config_path, &rendered)?; + driver.install(&candidate, &digest('a'), &receipt, false, &Unmetered)?; + assert_eq!(logged(&calls, "systemctl reload"), 1, "the unloaded install was not reloaded"); + assert_eq!(logged(&calls, "ufw allow"), 1, "the first deploy's allow was skipped"); + + // The withdrawal removed it and Idunn died before the reload. + fs::remove_file(&driver.binding.config_path)?; + driver.withdraw_candidate_membership(&receipt, &Unmetered)?; + assert_eq!(logged(&calls, "systemctl reload"), 2, "the unloaded withdrawal was not reloaded"); + assert_eq!(logged(&calls, "ufw delete"), 1); + Ok(()) + } + + /// A continuity over its admitted route has the baseline's membership: its + /// install, rollback and withdrawal change nothing and run nothing. + #[cfg(unix)] + #[test] + fn a_transaction_that_does_not_change_the_route_runs_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let (driver, candidate, calls) = failable_route_driver(temp.path())?; + let rendered = driver.render(&candidate)?; + let mut receipt = preflight_receipt(&driver, &candidate)?; + receipt.incumbent_runtime_instance_id = Some(digest('b')); + receipt.incumbent_membership_sha256 = Some(sha256_id(&rendered)); + receipt.incumbent_configuration = Some(rendered.clone()); + assert!(!receipt.changes_route()); + fs::write(&driver.binding.config_path, &rendered)?; let gate = Recording(Default::default()); driver.install(&candidate, &digest('a'), &receipt, false, &gate)?; - assert_eq!(logged(&calls, "systemctl reload"), 1); - for _ in 0..3 { - driver.install(&candidate, &digest('a'), &receipt, false, &gate)?; - } - assert_eq!(logged(&calls, "systemctl reload"), 1, "an installed candidate reloaded again"); - assert_eq!(logged(&calls, "ufw allow"), 1); - assert_eq!(*gate.0.borrow(), [RouteActuation::Forward], "a no-op was charged"); - - // The withdrawal removes the fragment and reloads, then its firewall - // step fails. Each retry repeats the firewall step and nothing else. - fs::write(temp.path().join("ufw.fail"), b"x")?; - assert!(driver.withdraw_candidate_membership(&receipt, &gate).is_err()); - assert!(!driver.binding.config_path.exists()); - assert_eq!(logged(&calls, "systemctl reload"), 2); - for _ in 0..5 { - assert!(driver.withdraw_candidate_membership(&receipt, &gate).is_err()); - } - assert_eq!(logged(&calls, "systemctl reload"), 2, "a failing withdrawal reloaded again"); - assert_eq!(logged(&calls, "ufw delete"), 6); - fs::remove_file(temp.path().join("ufw.fail"))?; + driver.rollback(&candidate, &digest('a'), &receipt, &gate)?; driver.withdraw_candidate_membership(&receipt, &gate)?; - assert_eq!(logged(&calls, "systemctl reload"), 2); - assert_eq!(logged(&calls, "ufw delete"), 7); + assert_eq!(logged(&calls, ""), 0, "an unchanged route ran a program"); + assert!(gate.0.borrow().is_empty(), "an unchanged route was charged"); + assert_eq!(fs::read(&driver.binding.config_path)?, rendered); - // A restore to bytes already on disk runs no program and asks nothing. - let rendered = driver.render(&candidate)?; - fs::write(&driver.binding.config_path, &rendered)?; - let (programs, asked) = (logged(&calls, ""), gate.0.borrow().len()); - driver.restore(Some(&rendered), &gate)?; - assert_eq!(logged(&calls, ""), programs); - assert_eq!(gate.0.borrow().len(), asked); + // The first-route receipt does change it. + assert!(preflight_receipt(&driver, &candidate)?.changes_route()); Ok(()) } @@ -12219,7 +12240,9 @@ Content-Le".to_vec()), |_| {}), .restore_admitted_membership(&candidate, &admitted.membership_sha256, &Unmetered) .is_err() ); - assert!(!config.exists()); + // The failed reload puts back what it replaced: the disk shows what + // nginx still serves. + assert_eq!(fs::read(&config)?, b"foreign route after admission\n"); fs::remove_file(systemctl.with_extension("fail"))?; driver.restore_admitted_membership(&candidate, &admitted.membership_sha256, &Unmetered)?; assert_eq!(fs::read(&config)?, admitted_bytes); From 47067cce08a0501a6faf6f5e37be565158a7b94f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:34:30 +0200 Subject: [PATCH 024/196] B5: an abort withdraws a lease issued as Prepared (B5 Soul A) The physical grant lands before `Granted` is durable. A continuity that died in that window and passed its deadline was aborted from `Prepared`, its abort withdrew only a `Granted` lease, and the lease left on disk made every later fencing of the target refuse "unexpected process write lease". The abort now withdraws any issued lease; revoking an absent one is a no-op. Test: a granted lease on disk behind a `Prepared` record, deadline passed: the abort removes it and the next fencing's revoke succeeds. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 49 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 6662b48..c2812f9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -560,6 +560,15 @@ impl LeasingEvidence { } } + /// The lease Idunn issued, granted or only prepared. A prepared lease may + /// already be on disk: the physical grant runs before `Granted` is durable. + fn issued_lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { + match self { + Self::SkippedStateless => None, + Self::Prepared { lease, .. } | Self::Granted { lease, .. } => Some(lease), + } + } + fn prepared_lease(&self) -> Option<(&IdunnProcessWriteLeaseRecord, &str)> { match self { Self::Prepared { @@ -1584,7 +1593,7 @@ impl DeploymentTransaction { fn phase_end(&self) -> PhaseEnd { let leasing = self.leasing.as_ref(); let granted = leasing.and_then(LeasingEvidence::lease).is_some(); - let issued = granted || leasing.and_then(LeasingEvidence::prepared_lease).is_some(); + let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); match self.command_kind { CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, _ => PhaseEnd::Abort { @@ -8740,11 +8749,14 @@ impl Engine { if abort.lease_withdrawal == CleanupEvidence::Pending { let expected = required(¤t.value.expected, "Expected projection")?; let activation = required(¤t.value.activation, "activation")?; + // A prepared lease is withdrawn too: its physical grant may have + // landed before `Granted` was durable. Revoking an absent lease, + // and withdrawing an absent projection, are no-ops. let lease = current .value .leasing .as_ref() - .and_then(LeasingEvidence::lease) + .and_then(LeasingEvidence::issued_lease) .context("post-fencing abort lost the lease it must withdraw")?; let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; let lease_path = binding @@ -9093,7 +9105,7 @@ fn post_fencing_abort_intent( lease_withdrawal: if transaction .leasing .as_ref() - .and_then(LeasingEvidence::lease) + .and_then(LeasingEvidence::issued_lease) .is_some() { CleanupEvidence::Pending @@ -18606,6 +18618,37 @@ mod tests { Ok(()) } + /// B5 Soul A. The physical grant lands before `Granted` is durable. A + /// continuity that dies in that window and passes its deadline is + /// aborted from `Prepared`; the abort still withdraws the lease on + /// disk, so the next fencing of the target is not refused. + #[test] + fn an_abort_from_a_prepared_lease_withdraws_a_grant_already_on_disk() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + let current = resident(&routed.world)?; + let mut crashed = current.value.clone(); + let Some(LeasingEvidence::Granted { lease, lease_sha256 }) = crashed.leasing.clone() else { + bail!("no granted lease"); + }; + crashed.leasing = Some(LeasingEvidence::Prepared { lease: lease.clone(), lease_sha256 }); + replace_transaction(&routed.world.state_store, ¤t, &crashed)?; + let driver = CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ); + assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); + + pass_deadline(&routed.world)?; + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + routed.resume_until_terminal()?; + assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); + // What the next fencing does to a candidate's lease path. + driver.revoke_exact(None)?; + Ok(()) + } + /// F16: Commit goes on from the topology it admitted, so Odin /// re-stamping before the step and again during the final proof does /// not send it round again. From e0b31eee763062cac4c08bb2893796312d1a3401 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:34:48 +0200 Subject: [PATCH 025/196] B5: pin the resolver's command filter (B5 Soul E) With two live transactions, an expiry request for one leaves the other running. Soul's mutant "any request expires every live transaction" survived the suite. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 64 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index c2812f9..ef2e2ce 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -18539,6 +18539,70 @@ mod tests { Ok(()) } + /// The resolver reads a request as its own transaction's only: with two + /// live transactions, a request for the other leaves this one running. + #[test] + fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let now = now_millis()?; + let other = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-other".into(), + kind: CommandKind::Deploy, + selector: "other".into(), + requested_by: "test".into(), + requested_at_unix_millis: now, + }; + let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: other.command_id.clone(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: other.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: sealing.transaction_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: request.command_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&other, now)?, + transaction_envelope(&sealing, now)?, + expiry_request_envelope(&request, now)?, + ], + )?); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!(snapshot.transactions.iter().filter(|t| !t.value.is_terminal()).count(), 2); + let mine = snapshot + .transactions + .iter() + .find(|t| t.value.command_id == "continuity-service") + .context("no continuity")?; + assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); + let after = ControlSnapshot::read(&routed.world.state_store)?; + let unchanged = after + .transactions + .iter() + .find(|t| t.value.command_id == "continuity-service") + .context("no continuity")?; + assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); + Ok(()) + } + /// Q3 at the deadline, through the Engine. A continuity whose candidate /// was granted a lease and never proved it held it fails as /// `lease-not-adopted`; one whose Ready proved it restarts the admitted From c04f24f518834f69e2703a87a762c1375138ca57 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:35:48 +0200 Subject: [PATCH 026/196] B5: pin lease adoption from Odin's Ready word (B5 Soul F) A stateful candidate Odin reports on is warming until it holds the lease; the world's fixture now says so, and a post-fence warming word lets the grant happen. Odin's Ready carrying the granted lease records the adoption, sourced `OdinTopology` and naming the presence Odin observed. Kills Soul's M6 (the source labelled `Direct`) and M7 (no adoption from Odin). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 117 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index ef2e2ce..defc123 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -12210,6 +12210,20 @@ mod tests { transaction: &DeploymentTransaction, sequence: u64, ready: bool, + ) -> Result> { + signed_correlation_as(world, transaction, sequence, ready, "active", None, b"presence") + } + + /// Odin's correlation about the transaction's incarnation, observing a + /// presence in `state` that holds `lease`, identified by `presence`. + fn signed_correlation_as( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + state: &str, + lease: Option<&str>, + presence: &[u8], ) -> Result> { let expected = transaction.expected.as_ref().context("no Expected")?; let activation = transaction.activation.as_ref().context("no activation")?; @@ -12219,10 +12233,10 @@ mod tests { expected_projection_sha256: expected.canonical_sha256()?, expected: true, current_activation_sha256: Some(activation.canonical_sha256()?), - signed_presence_sha256: Some(sha256_id(b"presence")), - observed_presence_state: Some("active".into()), + signed_presence_sha256: Some(sha256_id(presence)), + observed_presence_state: Some(state.into()), observed_presence_publisher_sequence: Some(sequence), - observed_write_lease_sha256: None, + observed_write_lease_sha256: lease.map(str::to_owned), observed_capabilities: expected .capabilities .iter() @@ -16221,7 +16235,17 @@ mod tests { if phase >= DeploymentPhase::Fencing && provides_odin { // Odin's own first word about the candidate, as for any // Odin-correlated target. - let warming = signed_correlation(&world, &transaction, 4, false)?; + // A stateful candidate is warming, not active, until it holds + // the lease Idunn grants after the fence. + let warming = signed_correlation_as( + &world, + &transaction, + 4, + false, + if stateful { "warming" } else { "active" }, + None, + b"presence", + )?; let authenticated = world.engine.authenticate_topology_bytes( &ControlSnapshot::read(&world.state_store)?, &transaction, @@ -16286,6 +16310,33 @@ mod tests { // valid correlation that says the target is not Ready. let store = &world.engine.options.odin_correlation_store; let poisoned = match odin { + _ if provides_odin && stateful => { + // A new warming word after the fence: what the lease + // grant waits for. + SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( + CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &transaction.target, + &transaction.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation_as( + &world, + &transaction, + 5, + false, + "warming", + None, + b"presence-5", + )?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some( + cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + ), + }, + )?; + Vec::new() + } _ if provides_odin => { odin_reports_ready(&world, &transaction, 5)?; Vec::new() @@ -18713,6 +18764,64 @@ mod tests { Ok(()) } + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word + /// carries the granted lease, and that Ready records the adoption, + /// sourced from Odin's topology and naming the presence Odin observed. + #[test] + fn an_odin_ready_word_records_a_stateful_candidates_lease_adoption() -> Result<()> { + let routed = build_routed_world( + Odin::Unreachable, + 1, + DeploymentPhase::Fencing, + true, + true, + CommandKind::Continuity, + )?; + for _ in 0..8 { + if routed.transaction()?.phase == DeploymentPhase::AwaitingReady { + break; + } + routed.step()?; + } + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady, "{:?}", waiting.last_error); + let lease = waiting + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + assert_eq!(waiting.lease_adoption, None); + + let ready = CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &waiting.target, + &waiting.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation_as( + &routed.world, + &waiting, + waiting.odin_publisher_sequence_cursor + 1, + true, + "active", + Some(&lease), + b"presence-ready", + )?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }; + replace_odin_entry(&routed.world.engine.options.odin_correlation_store, ready)?; + routed.step()?; + let adopted = routed.transaction()?; + assert!(matches!(adopted.ready, Some(ReadinessEvidence::OdinCorrelated { .. }))); + let adoption = adopted.lease_adoption.context("Odin's Ready recorded no adoption")?; + assert_eq!(adoption.source, AdoptionSource::OdinTopology); + assert_eq!(adoption.write_lease_sha256, lease); + assert_eq!(adoption.signed_presence_sha256, sha256_id(b"presence-ready")); + Ok(()) + } + /// F16: Commit goes on from the topology it admitted, so Odin /// re-stamping before the step and again during the final proof does /// not send it round again. From 64da79402a20e1b5e1605d1bae7b7c60d53d3d26 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:35:48 +0200 Subject: [PATCH 027/196] Deterministic backoff test (B5 Soul J) `a_failing_step_is_retried_after_a_backoff_not_every_tick` raced the wall-clock wait against ten ticks and lost on a loaded host (9 of 15 mutant runs). The test now owns the wait: held open, ten ticks retry nothing; once due, the next tick retries. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index defc123..b0a7323 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14234,14 +14234,31 @@ mod tests { world .engine .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let attempts = || workload.0.load(std::sync::atomic::Ordering::SeqCst); + world.engine.run_scheduler_tick()?; + assert_eq!(attempts(), 1, "the step did not run"); + // The wait is set from the wall clock, and a loaded host can spend it + // inside a few ticks. So the test owns the wait: held open, no tick + // retries; once due, the next tick does. + let id = resident(&world)?.value.transaction_id; + let set_due = |due: u64| { + world + .engine + .resume_backoff + .lock() + .unwrap() + .get_mut(&id) + .expect("the failure set no backoff") + .not_before_unix_millis = due; + }; + set_due(now_millis()? + RESUME_BACKOFF_CEILING_MILLIS / 2); for _ in 0..10 { world.engine.run_scheduler_tick()?; } - assert_eq!( - workload.0.load(std::sync::atomic::Ordering::SeqCst), - 1, - "the wedged step was retried inside its backoff" - ); + assert_eq!(attempts(), 1, "the wedged step was retried inside its backoff"); + set_due(1); + world.engine.run_scheduler_tick()?; + assert_eq!(attempts(), 2, "the step was not retried once its backoff was due"); Ok(()) } From 224f006983a39edd507785b358bda38e9652e0e3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:42:42 +0200 Subject: [PATCH 028/196] Route map: Q-S1..Q-S4 ruled (a); Q-S5 open Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 3429930..bc42588 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -2152,6 +2152,8 @@ it. #### Q3-ii.12 Operator questions +**Rulings, operator, 2026-09-30:** "S1: Refuse / S2: Declared state / S3: Detect and refuse / S4: deploy or release". That is **Q-S1 (a), Q-S2 (a), Q-S3 (a), Q-S4 (a)**. Q-S5 was left blank and is re-asked. + - **Q-S1. A stateful target whose binding declares no snapshot.** - Options: - (a) Refuse a new stateful Deploy plan at compile, with a typed error. Continuity of already-admitted plans is From 8b9e8dfe117be24850a10553e64c6e815d4f6c08 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:42:51 +0200 Subject: [PATCH 029/196] Route map: Q-S5 ruled (a), B5 first Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index bc42588..ede571b 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -2152,7 +2152,7 @@ it. #### Q3-ii.12 Operator questions -**Rulings, operator, 2026-09-30:** "S1: Refuse / S2: Declared state / S3: Detect and refuse / S4: deploy or release". That is **Q-S1 (a), Q-S2 (a), Q-S3 (a), Q-S4 (a)**. Q-S5 was left blank and is re-asked. +**Rulings, operator, 2026-09-30:** "S1: Refuse / S2: Declared state / S3: Detect and refuse / S4: deploy or release". That is **Q-S1 (a), Q-S2 (a), Q-S3 (a), Q-S4 (a)**. Q-S5: "B5 first", which is **(a)**: B5 merges after its Soul pass, and SR-1..SR-4 follow as their own cuts. - **Q-S1. A stateful target whose binding declares no snapshot.** - Options: From dcd0aa3f2b1112c25d68105ea878cbc0de037bbe Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:50:45 +0200 Subject: [PATCH 030/196] Secrets map: Heimdall batch 5 status and Soul pass 5 (hold; batch 6 in Hands) Co-Authored-By: Claude Opus 5.5 --- docs/secret-files-cut.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/secret-files-cut.md b/docs/secret-files-cut.md index 5dbb1c2..5bdffab 100644 --- a/docs/secret-files-cut.md +++ b/docs/secret-files-cut.md @@ -13,6 +13,23 @@ secrets into a transcript. The operator's words: "I'll rotate before prod, not t concerned about Anthropic having my oauth; do map the cut that'll ensure it doesn't happen again". +**Heimdall `hands/secret-files` status (Self, 2026-09-30).** +- **Batch 5** (`efd1137..b5d7b6b`) fixed Soul pass 4's four findings: + - F1: command-plane diagnostics are fixed text plus the error's class and code, and every provider body goes + through `readProviderJson`. + - F2: patron support keeps only the Bifrost status. + - F3: the recipe test records the secrets `loadConfig` actually reads. + - F4: `checkSchema` runs on every start. +- **Soul pass 5: hold.** F1-F3 held. Batch 6, in Hands, fixes: + - **S5-1.** `checkSchema` passes a DEFERRABLE key, a `UNIQUE ... INCLUDE`, a same-named non-unique replacement for + the partial unique index, and retyped columns. Each of these breaks the store at runtime. + - **S5-2.** The privilege check demands UPDATE on `audit_events`, which the store only INSERTs into. + - **S5-3 (older than this batch).** `app.ts:1310` writes a live, 300 s redeemable completion code into the audit + event `auth_completion_created`. + - **S5-4 to S5-7.** Test gaps: canary position, the `errorIdentity` filters, pool close, spread-env reads. + - A flaky live-Postgres test. +- Noted, not in scope: **S5-8**, forced RLS with no policy passes the check. + **Rulings and defaults, 2026-09-30 (Self).** - **Q3 A is the operator's ruling:** "I'll rotate before prod". Cuts 1-4 land with the current values; rotation overwrites the credential files later. The token-encryption and signing-key sub-question waits for rotation time. From aef0da34af84f9a83ce4c0cc5729ff7f0d24166e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:51:17 +0200 Subject: [PATCH 031/196] B5: the Odin adoption test steps through sequence admission AwaitingReady admits a new Odin sequence in one step and records Ready from it in the next. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index b0a7323..41fda31 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -18829,9 +18829,19 @@ mod tests { schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), }; replace_odin_entry(&routed.world.engine.options.odin_correlation_store, ready)?; - routed.step()?; + // One step admits the new sequence, the next records Ready from it. + for _ in 0..3 { + if routed.transaction()?.ready.is_some() { + break; + } + routed.step()?; + } let adopted = routed.transaction()?; - assert!(matches!(adopted.ready, Some(ReadinessEvidence::OdinCorrelated { .. }))); + assert!( + matches!(adopted.ready, Some(ReadinessEvidence::OdinCorrelated { .. })), + "{:?}", + adopted.last_error + ); let adoption = adopted.lease_adoption.context("Odin's Ready recorded no adoption")?; assert_eq!(adoption.source, AdoptionSource::OdinTopology); assert_eq!(adoption.write_lease_sha256, lease); From 32cf1f7bf4127a016945fec124b5d6d39086c42d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:56:33 +0200 Subject: [PATCH 032/196] Incident writes never wait; exhaustion closes on health and the window; one retention predicate Every incident write is one nonblocking try_compare_exchange_snapshot of the whole file against the snapshot it was decided on (as drivers.rs does for the write lease). A held lock or a lost race is an error and a no-op for the tick, retried on a later pass; it is also what makes one open incident per (condition, subject) real. The incident .lock is no longer published: readers of incidents.cc take no lock, so only Idunn can hold it. History is written the same way. An exhaustion incident closes as recovered only when the workload is observed running and restarts_exhausted is false (ruling exhaustion-close-rule), so a release that survives one pass after each restart keeps one incident. Reconcile closes only incidents whose condition ends with admission and whose subject is not admitted; its dead accumulator is gone. IncidentRecord owns the retention predicate that retirement and status both use. status writes to a writer and serve's loop is serve_while, so tests reach both. Not verified yet: this commit has not been built. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 271 +++++++++++++++++++++++++++---- src/drivers.rs | 41 ++--- src/incident.rs | 377 ++++++++++++++++++++++++++++++------------- 3 files changed, 528 insertions(+), 161 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index dc4bb63..e663d7a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::env; use std::fs::{self, File, OpenOptions}; +use std::io::Write; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -2476,7 +2477,12 @@ pub fn run(args: impl Iterator) -> Result<()> { state_store, incident_store, command_id, - } => status(&state_store, &incident_store, command_id.as_deref()), + } => status( + &mut std::io::stdout().lock(), + &state_store, + &incident_store, + command_id.as_deref(), + ), Command::Cancel { state_store, command_id, @@ -4112,7 +4118,12 @@ fn submit( } } -fn status(store_path: &Path, incident_store: &Path, command_id: Option<&str>) -> Result<()> { +fn status( + out: &mut impl Write, + store_path: &Path, + incident_store: &Path, + command_id: Option<&str>, +) -> Result<()> { let snapshot = ControlSnapshot::read(store_path)?; // Commands stay resident; their finished transactions do not. Without // history a completed command would report as though it had never run, @@ -4140,10 +4151,11 @@ fn status(store_path: &Path, incident_store: &Path, command_id: Option<&str>) -> ); transactions.sort_by_key(|value| value.ordinal); let (state, detail) = derived_command_status(&transactions); - println!( + writeln!( + out, "{} {} {} {}", command.command_id, command.selector, state, detail - ); + )?; // Naming one command asks about that command, so print what a stuck // transaction is actually waiting on. A gate reason lives in // `last_error` and was never rendered anywhere, which left "Sealing" @@ -4152,49 +4164,53 @@ fn status(store_path: &Path, incident_store: &Path, command_id: Option<&str>) -> // ids are here because they are exactly what a brake release must name. if command_id.is_some() { for transaction in transactions { - println!(" transaction {}", transaction.transaction_id); - println!( + writeln!(out, " transaction {}", transaction.transaction_id)?; + writeln!( + out, " target {} phase {:?}", transaction.target, transaction.phase - ); + )?; if let Some(expected) = &transaction.expected { - println!(" runtime {}", expected.runtime_id); - println!(" release {}", expected.sealed_release_id); + writeln!(out, " runtime {}", expected.runtime_id)?; + writeln!(out, " release {}", expected.sealed_release_id)?; } - println!( + writeln!( + out, " odin publisher cursor {}", transaction.odin_publisher_sequence_cursor - ); + )?; if let Some(evidence) = &transaction.latest_odin_observation { - println!( + writeln!( + out, " latest odin observation sequence {}", evidence.publisher_sequence - ); + )?; } if let Some(reason) = &transaction.last_error { - println!(" waiting on {reason}"); + writeln!(out, " waiting on {reason}")?; } // A Complete transaction that is not terminal still owns its // target through unfinished cleanup. Without this an operator // sees "Complete" and a stale reason and cannot tell what is // holding the target. if let Some(completion) = &transaction.completion { - println!(" completion {completion:?}"); + writeln!(out, " completion {completion:?}")?; } if let Some(abort) = &transaction.pre_fencing_abort { - println!(" pre-fencing abort {abort:?}"); + writeln!(out, " pre-fencing abort {abort:?}")?; } if let Some(abort) = &transaction.post_fencing_abort { - println!(" post-fencing abort {abort:?}"); + writeln!(out, " post-fencing abort {abort:?}")?; } if let Some(cleanup) = &transaction.post_commit_cleanup { - println!(" post-commit cleanup {cleanup:?}"); + writeln!(out, " post-commit cleanup {cleanup:?}")?; } - println!( + writeln!( + out, " terminal {} owns-target {}", transaction.is_terminal(), transaction.blocks_new_target_mutation() - ); + )?; } } } @@ -4202,7 +4218,7 @@ fn status(store_path: &Path, incident_store: &Path, command_id: Option<&str>) -> // generations and the metered targets, so a target that has only been // deployed to (no generation yet) still shows what it has used. for line in render_targets(&snapshot, incident_store, now_millis()?) { - println!("{line}"); + writeln!(out, "{line}")?; } Ok(()) } @@ -4648,7 +4664,7 @@ impl Engine { /// decides exactly as it would have with a healthy store. It names the /// operation and the root cause and never a path or a value. Incident /// writes are never progress. - fn record_incident(&self, operation: &str, outcome: Result) { + fn record_incident(&self, operation: &str, outcome: Result) { let key = format!("incident-store:{operation}"); match outcome { Ok(_) => self.clear_fault(&key), @@ -4662,21 +4678,18 @@ impl Engine { } } - /// Close every open incident whose target is no longer admitted, then - /// retire what has been closed past retention. + /// Close every open incident whose subject is a target that is no longer + /// admitted, then retire what has been closed past retention. fn reconcile_incidents(&self, snapshot: &ControlSnapshot) { let store = self.incidents(); - let close_unadmitted = || -> Result { + let close_unadmitted = || -> Result<()> { let now = now_millis()?; - let mut closed = false; for record in store.read()? { if record.is_open() - && !snapshot - .admitted - .iter() - .any(|stored| stored.value.target == record.subject) + && record.condition.ends_with_admission() + && snapshot.admitted_for(&record.subject).is_none() { - closed |= store.close( + store.close( record.condition, &record.subject, CloseReason::NoLongerAdmitted, @@ -4684,7 +4697,7 @@ impl Engine { )?; } } - Ok(closed) + Ok(()) }; self.record_incident("close", close_unadmitted()); self.record_incident("retire", now_millis().and_then(|now| store.retire_closed(now))); @@ -5121,7 +5134,13 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { fn serve(options: RuntimeOptions) -> Result<()> { let (_lock, engine) = boot(options)?; - loop { + serve_while(&engine, || true) +} + +/// The daemon's loop: tick, sleep, and go again while `again` says so. `serve` +/// says so forever; a test says so a few times. +fn serve_while(engine: &Engine, mut again: impl FnMut() -> bool) -> Result<()> { + while again() { match engine.run_scheduler_tick() { Ok(true) => continue, Ok(false) => {} @@ -5136,6 +5155,7 @@ fn serve(options: RuntimeOptions) -> Result<()> { } thread::sleep(Duration::from_millis(engine.options.poll_millis)); } + Ok(()) } impl Engine { @@ -5714,11 +5734,21 @@ impl Engine { }; if observation.is_some() { // The admitted workload was seen running: the only evidence - // that ends an exhaustion incident. The restart window sliding - // is not health and never closes one. + // that ends an exhaustion incident, once the window no longer holds it. + // The window sliding alone is not health and never closes one. self.record_incident( "close", now_millis().and_then(|now| { + // Running is not enough while the window still holds + // the restarts that exhausted it: a release that + // survives one poll after each restart is still the + // same incident. + if snapshot + .supervision_or_new(¤t.value.target) + .restarts_exhausted(now) + { + return Ok(false); + } self.incidents().close( IncidentCondition::ContinuityExhausted, ¤t.value.target, @@ -14899,6 +14929,177 @@ mod tests { assert_eq!(incident_store, PathBuf::from("/tmp/j.cc")); } + #[test] + fn a_release_that_runs_one_pass_after_each_restart_keeps_one_incident() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + + // Each cycle: the restarts are spent, the release dies, is seen running + // for one pass (its last restart took), and dies again. + for _ in 0..4 { + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1, "a flapping release opened another incident"); + assert!(records[0].is_open(), "running while still exhausted closed the incident"); + } + + // Real recovery: running, and the window no longer holds the restarts. + slide_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn a_target_leaving_admission_closes_only_its_own_incident() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let service = admit_incumbent(&world)?; + let store = IncidentStore::new(&incident_path(&world)); + let condition = IncidentCondition::ContinuityExhausted; + let now = now_millis()?; + for subject in ["service", "neighbour", "gone"] { + assert!(store.open(condition, subject, now)?); + } + + // Two targets are admitted: service, and a neighbour. Only the target + // that is not admitted has its incident closed. + let mut snapshot = ControlSnapshot::read(&world.state_store)?; + let envelope = snapshot + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone(); + snapshot.admitted.push(Stored { + envelope, + value: AdmittedGeneration { + target: "neighbour".into(), + ..service + }, + }); + world.engine.reconcile_incidents(&snapshot); + + let closed = |subject: &str| -> Result> { + Ok(incidents_of(&world)? + .into_iter() + .find(|record| record.subject == subject) + .context("no such incident")? + .close_reason) + }; + assert_eq!(closed("service")?, None); + assert_eq!(closed("neighbour")?, None); + assert_eq!(closed("gone")?, Some(CloseReason::NoLongerAdmitted)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_held_incident_lock_changes_nothing_continuity_decides() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let healthy_workload = SwitchWorkload::new(); + let healthy = EngineFixture::with_workload(healthy_workload.clone())?; + admit_incumbent(&healthy)?; + let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; + + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let path = incident_path(&world); + let actual = run_while_held(hold_lock_of(&path), || { + continuity_decisions(&world, &world.engine, &workload) + })?; + assert_eq!(actual, expected, "a held incident lock changed a continuity decision"); + assert!(!path.exists(), "a contended write landed"); + + // Every refused write is a traced fault naming no path. + { + let reports = world.engine.fault_reports.lock().unwrap(); + let faults = reports + .iter() + .filter(|(key, _)| key.starts_with("incident-store:")) + .map(|(_, report)| report.last.lock().unwrap().clone().unwrap_or_default()) + .collect::>(); + assert!(!faults.is_empty()); + let root = world.root.display().to_string(); + assert!(faults.iter().all(|fault| !fault.contains(&root)), "{faults:?}"); + } + + // The lock is free again: the next pass retries and the incident lands. + world.engine.supervise_one_admitted_generation()?; + assert!(incidents_of(&world)?.iter().any(|record| record.is_open())); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn status_prints_an_open_incident_under_its_target() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let opened = incidents_of(&world)?[0].opened_at_unix_millis; + + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &incident_path(&world), None)?; + let printed = String::from_utf8(printed)?; + let target = printed.find("target service ").context("no target line")?; + let incident = printed + .find(&format!(" incident continuity-exhausted opened-at {opened}")) + .context("status printed no incident")?; + assert!(target < incident, "{printed}"); + Ok(()) + } + + #[test] + fn serve_ticks_until_told_to_stop() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + let mut options = world.engine.options.clone(); + options.poll_millis = 1; + let engine = Engine::open_with_systemd_workload(options, workload.clone())?; + + let mut asked = 0; + serve_while(&engine, || { + asked += 1; + asked <= 3 + })?; + assert_eq!(asked, 4); + assert!(incidents_of(&world)?.iter().any(|record| record.is_open()), "no tick ran"); + Ok(()) + } + + #[test] + fn run_dispatches_serve_and_status() { + let temp = TempDir::new().unwrap(); + let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); + // A control store beneath a regular file: serve's boot cannot make it, + // and a serve that returned at once would say Ok instead. + let file = temp.path().join("not-a-directory"); + std::fs::write(&file, b"a file").unwrap(); + let store = file.join("control.cc"); + assert!( + run(args(&["serve", "--state-store", store.to_str().unwrap()]).into_iter()).is_err() + ); + // A control store that is not one: status reads it and refuses. + let junk = temp.path().join("junk.cc"); + std::fs::write(&junk, b"not a cultcache file").unwrap(); + assert!( + run(args(&["status", "--state-store", junk.to_str().unwrap()]).into_iter()).is_err() + ); + } + #[test] fn continuity_stops_when_the_targets_own_log_has_spent_the_window() -> Result<()> { let workload = SwitchWorkload::new(); diff --git a/src/drivers.rs b/src/drivers.rs index eb162e9..9bf4372 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -7659,29 +7659,34 @@ fn ensure_bundle_is_reachable_by_workload( Ok(()) } +/// One published file: `0644`, because Idunn's `UMask=027` would land it +/// `0640 root:root` and deny its readers. A chmod by hand does not hold, since +/// each publish writes a new file. +pub(crate) fn publish_file_mode(path: &Path) -> Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if path.exists() { + fs::set_permissions(path, fs::Permissions::from_mode(0o644)) + .with_context(|| format!("publishing {}", path.display()))?; + } + } + Ok(()) +} + /// The topology store is Idunn's *published* surface: every managed target /// reads it to verify its own Expected incarnation against the Idunn anchor. /// /// Idunn runs with `UMask=027`, which is right for its private state and wrong /// for this one file -- it lands `0640 root:root`, and a `DynamicUser` workload -/// gets EACCES. A chmod by hand does not hold, because each publish writes a -/// new file. Integrity here comes from the signatures over the records, not -/// from the mode, so the published copy is readable. -pub(crate) fn publish_projection_mode(path: &Path) -> Result<()> { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - // The lock sibling too: CultCache opens it alongside the store, so a - // 0640 lock denies the read just as surely as a 0640 store, and it is - // created fresh under Idunn's umask on every publish. - for path in [path.to_path_buf(), authority_lock_path(path)] { - if path.exists() { - fs::set_permissions(&path, fs::Permissions::from_mode(0o644)) - .with_context(|| format!("publishing {}", path.display()))?; - } - } - } - Ok(()) +/// gets EACCES. Integrity here comes from the signatures over the records, not +/// from the mode, so the published copy is readable. The lock sibling too: +/// CultCache opens it alongside the store, so a 0640 lock denies the read just +/// as surely as a 0640 store, and it is created fresh under Idunn's umask on +/// every publish. +fn publish_projection_mode(path: &Path) -> Result<()> { + publish_file_mode(path)?; + publish_file_mode(&authority_lock_path(path)) } fn build_machine_id(workspace: &Path) -> Result { diff --git a/src/incident.rs b/src/incident.rs index cd56c54..7de86cf 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -12,14 +12,15 @@ use std::path::{Path, PathBuf}; -use anyhow::{Context, Result, ensure}; +use anyhow::{Context, Result, bail, ensure}; use cultcache_rs::{ - CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + TryCompareExchangeSnapshotOutcome, }; use serde::{Deserialize, Serialize}; use crate::control_plane::{decode_record, require_id, typed_envelope}; -use crate::drivers::publish_projection_mode; +use crate::drivers::publish_file_mode; const INCIDENT_SCHEMA: &str = "idunn.operator_incident.v1"; @@ -41,6 +42,15 @@ impl IncidentCondition { Self::ContinuityExhausted => "continuity-exhausted", } } + + /// Whether the subject is an admitted target, so that the target leaving + /// admission ends the condition. A condition about anything else (a + /// transaction, a store) answers false and is closed by its own owner. + pub(crate) const fn ends_with_admission(self) -> bool { + match self { + Self::ContinuityExhausted => true, + } + } } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -98,6 +108,14 @@ impl IncidentRecord { self.closed_at_unix_millis.is_none() } + /// Whether the record still belongs in the published file: open, or closed + /// no more than `INCIDENT_RETENTION_MILLIS` ago. The one predicate behind + /// both retirement and `idunn status`. + pub(crate) fn inside_retention(&self, now: u64) -> bool { + self.closed_at_unix_millis + .is_none_or(|closed_at| now.saturating_sub(closed_at) <= INCIDENT_RETENTION_MILLIS) + } + pub(crate) fn validate(&self) -> Result<()> { ensure!( self.schema_version == INCIDENT_SCHEMA, @@ -139,14 +157,28 @@ impl IncidentRecord { } } -struct Stored { - envelope: CultCacheEnvelope, - record: IncidentRecord, +/// One read of the incident file: its envelopes exactly as stored, and the +/// records decoded from them, in the same order. Every write expects exactly +/// these envelopes, so a write only lands on the file it was decided against. +struct Snapshot { + envelopes: Vec, + records: Vec, } -/// The incident file and its history sibling. Every write is a compare-exchange -/// against exactly what was read, and every write is followed by the -/// world-readable publication mode. +impl Snapshot { + fn open_record(&self, condition: IncidentCondition, subject: &str) -> Option { + self.records.iter().position(|record| { + record.is_open() && record.condition == condition && record.subject == subject + }) + } +} + +/// The incident file and its history sibling. Incidents are written from the +/// scheduler tick, so no write waits: each is one nonblocking compare-exchange +/// of the whole file against the snapshot it was decided on. A lost race or a +/// held lock is an error and a no-op, retried on a later pass. The lock beside +/// the file is not published: the readers of `incidents.cc` take no lock, so +/// only Idunn can ever hold it. pub(crate) struct IncidentStore { path: PathBuf, } @@ -162,57 +194,54 @@ impl IncidentStore { self.path.with_file_name("incident-history.cc") } - fn backing(&self) -> SingleFileMessagePackBackingStore { - SingleFileMessagePackBackingStore::new(&self.path) - } - /// Every record, oldest key first. An absent file is empty. The file /// refuses any document that is not an operator incident. pub(crate) fn read(&self) -> Result> { - Ok(self - .read_stored()? - .into_iter() - .map(|stored| stored.record) - .collect()) + let mut records = self.snapshot()?.records; + records.sort_by(|left, right| left.incident_key.cmp(&right.incident_key)); + Ok(records) } - fn read_stored(&self) -> Result> { - if !self.path.exists() { - return Ok(Vec::new()); - } - let mut stored = Vec::new(); - for envelope in self - .backing() + fn snapshot(&self) -> Result { + let envelopes = SingleFileMessagePackBackingStore::new(&self.path) .pull_all_read_only_snapshot() - .context("reading the incident store")? - { + .context("reading the incident store")?; + let mut records = Vec::new(); + for envelope in &envelopes { ensure!( envelope.r#type == IncidentRecord::TYPE && envelope.schema_id.as_deref() == Some(INCIDENT_SCHEMA), "the incident store holds a document that is not an operator incident" ); - let record: IncidentRecord = decode_record(&envelope)?; + let record: IncidentRecord = decode_record(envelope)?; record.validate()?; ensure!( envelope.key == record.incident_key, "incident store key differs from the incident's identity" ); - stored.push(Stored { envelope, record }); + records.push(record); } - stored.sort_by(|left, right| left.record.incident_key.cmp(&right.record.incident_key)); - Ok(stored) + Ok(Snapshot { envelopes, records }) } - fn open_record<'a>( - stored: &'a [Stored], - condition: IncidentCondition, - subject: &str, - ) -> Option<&'a Stored> { - stored.iter().find(|stored| { - stored.record.is_open() - && stored.record.condition == condition - && stored.record.subject == subject - }) + /// Replace `path`'s whole content with `replacements` if it is still + /// exactly `expected`. Never waits for the lock. + fn exchange( + path: &Path, + expected: &[CultCacheEnvelope], + replacements: &[CultCacheEnvelope], + ) -> Result<()> { + match SingleFileMessagePackBackingStore::new(path) + .try_compare_exchange_snapshot(expected, replacements)? + { + TryCompareExchangeSnapshotOutcome::Exchanged => Ok(()), + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("the incident store changed while it was being written") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + bail!("the incident store is locked by another holder") + } + } } /// Open an incident unless one is already open for `(condition, subject)`. @@ -223,23 +252,24 @@ impl IncidentStore { subject: &str, now: u64, ) -> Result { - let stored = self.read_stored()?; - if Self::open_record(&stored, condition, subject).is_some() { + self.open_on(&self.snapshot()?, condition, subject, now) + } + + fn open_on( + &self, + snapshot: &Snapshot, + condition: IncidentCondition, + subject: &str, + now: u64, + ) -> Result { + if snapshot.open_record(condition, subject).is_some() { return Ok(false); } - let record = IncidentRecord::opened(condition, subject, now); - let written = self.backing().compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IncidentRecord::TYPE.into(), - key: record.incident_key.clone(), - current: None, - }], - &[record.envelope(now)?], - )?; - if written { - publish_projection_mode(&self.path)?; - } - Ok(written) + let mut replacements = snapshot.envelopes.clone(); + replacements.push(IncidentRecord::opened(condition, subject, now).envelope(now)?); + Self::exchange(&self.path, &snapshot.envelopes, &replacements)?; + publish_file_mode(&self.path)?; + Ok(true) } /// Close the open incident for `(condition, subject)`, if there is one. @@ -251,60 +281,56 @@ impl IncidentStore { reason: CloseReason, now: u64, ) -> Result { - let stored = self.read_stored()?; - let Some(open) = Self::open_record(&stored, condition, subject) else { + let snapshot = self.snapshot()?; + let Some(index) = snapshot.open_record(condition, subject) else { return Ok(false); }; - let mut closed = open.record.clone(); + let mut closed = snapshot.records[index].clone(); // A clock stepped back must not close an incident before it opened. closed.closed_at_unix_millis = Some(now.max(closed.opened_at_unix_millis)); closed.close_reason = Some(reason); - let written = self.backing().compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IncidentRecord::TYPE.into(), - key: closed.incident_key.clone(), - current: Some(open.envelope.clone()), - }], - &[closed.envelope(now)?], - )?; - if written { - publish_projection_mode(&self.path)?; - } - Ok(written) + let mut replacements = snapshot.envelopes.clone(); + replacements[index] = closed.envelope(now)?; + Self::exchange(&self.path, &snapshot.envelopes, &replacements)?; + publish_file_mode(&self.path)?; + Ok(true) } - /// Move every incident closed more than `INCIDENT_RETENTION_MILLIS` ago to - /// the history file, then delete it here. History first: a crash between - /// the two leaves the record in both, and `insert_entry_if_absent` makes - /// the repeat a no-op. An open incident never moves. + /// Move every incident that has left retention to the history file, then + /// drop it here. History first: a crash between the two leaves the record + /// in both, and the repeat finds it already archived. An open incident + /// never moves. pub(crate) fn retire_closed(&self, now: u64) -> Result { - let retiring = self - .read_stored()? - .into_iter() - .filter(|stored| { - stored.record.closed_at_unix_millis.is_some_and(|closed_at| { - now.saturating_sub(closed_at) > INCIDENT_RETENTION_MILLIS - }) - }) - .map(|stored| stored.envelope) - .collect::>(); + let snapshot = self.snapshot()?; + let (kept, retiring): (Vec<_>, Vec<_>) = snapshot + .envelopes + .iter() + .zip(&snapshot.records) + .partition(|(_, record)| record.inside_retention(now)); if retiring.is_empty() { return Ok(false); } - let history = SingleFileMessagePackBackingStore::new(&self.history_path()); - for envelope in &retiring { - history - .insert_entry_if_absent(envelope.clone()) - .context("archiving a closed incident")?; + let history_path = self.history_path(); + let archived = SingleFileMessagePackBackingStore::new(&history_path) + .pull_all_read_only_snapshot() + .context("reading the incident history")?; + let mut history = archived.clone(); + history.extend( + retiring + .iter() + .filter(|(envelope, _)| !archived.iter().any(|held| held.key == envelope.key)) + .map(|(envelope, _)| (*envelope).clone()), + ); + if history.len() != archived.len() { + Self::exchange(&history_path, &archived, &history)?; } - let deleted = self - .backing() - .delete_batch_if_unchanged(&retiring) - .context("retiring a closed incident")?; - if deleted { - publish_projection_mode(&self.path)?; - } - Ok(deleted) + let kept = kept + .into_iter() + .map(|(envelope, _)| envelope.clone()) + .collect::>(); + Self::exchange(&self.path, &snapshot.envelopes, &kept)?; + publish_file_mode(&self.path)?; + Ok(true) } } @@ -313,12 +339,7 @@ impl IncidentStore { pub(crate) fn render_incidents(records: &[IncidentRecord], target: &str, now: u64) -> Vec { records .iter() - .filter(|record| record.subject == target) - .filter(|record| { - record.closed_at_unix_millis.is_none_or(|closed_at| { - now.saturating_sub(closed_at) <= INCIDENT_RETENTION_MILLIS - }) - }) + .filter(|record| record.subject == target && record.inside_retention(now)) .map(|record| { let mut line = format!( " incident {} opened-at {}", @@ -336,7 +357,10 @@ pub(crate) fn render_incidents(records: &[IncidentRecord], target: &str, now: u6 } #[cfg(test)] -mod tests { +pub(crate) mod tests { + #[cfg(unix)] + use std::time::{Duration, Instant}; + use tempfile::TempDir; use super::*; @@ -592,7 +616,7 @@ mod tests { #[cfg(unix)] #[test] - fn published_incident_store_is_world_readable() { + fn published_incident_store_is_world_readable_and_its_lock_is_not() { use std::os::unix::fs::PermissionsExt; let temp = TempDir::new().unwrap(); @@ -603,19 +627,156 @@ mod tests { let mode = |path: &Path| std::fs::metadata(path).unwrap().permissions().mode() & 0o777; assert!(under_service_umask(|| store.open(condition, "service", OPENED)).unwrap()); - assert_eq!((mode(&path), mode(&lock)), (0o644, 0o644), "after open"); + assert_eq!(mode(&path), 0o644, "after open"); + assert_eq!(mode(&lock) & 0o007, 0, "the lock is open to others after open"); // Each write replaces the file, and the replacement is published too. assert!( under_service_umask(|| store.close(condition, "service", CloseReason::Recovered, CLOSED)) .unwrap() ); - assert_eq!((mode(&path), mode(&lock)), (0o644, 0o644), "after close"); + assert_eq!(mode(&path), 0o644, "after close"); + assert_eq!(mode(&lock) & 0o007, 0, "the lock is open to others after close"); assert!( under_service_umask(|| store.retire_closed(CLOSED + INCIDENT_RETENTION_MILLIS + 1)) .unwrap() ); assert_eq!(mode(&path), 0o644, "after retire"); + assert_eq!(mode(&lock) & 0o007, 0, "the lock is open to others after retire"); + } + + /// An exclusive lock on a store's lock file, held through another file + /// description. flock conflicts between descriptions, so the holder need + /// not be another process or another uid: the test does not depend on who + /// it runs as. + #[cfg(unix)] + pub(crate) struct HeldLock(#[allow(dead_code)] std::fs::File); + + #[cfg(unix)] + pub(crate) fn hold_lock_of(store: &Path) -> HeldLock { + use std::os::unix::io::AsRawFd; + + let mut name = store.file_name().unwrap().to_owned(); + name.push(".lock"); + let file = std::fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(store.with_file_name(name)) + .unwrap(); + // SAFETY: flock on a descriptor this function owns. + assert_eq!(unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) }, 0); + HeldLock(file) + } + + /// Run `work` while `held` is held, and fail if it waited on it. The + /// holder lets go after three seconds so that work which does wait ends, + /// late, and the test fails instead of hanging. + #[cfg(unix)] + pub(crate) fn run_while_held(held: HeldLock, work: impl FnOnce() -> T) -> T { + let (release, released) = std::sync::mpsc::channel::<()>(); + let holder = std::thread::spawn(move || { + let _ = released.recv_timeout(Duration::from_secs(3)); + drop(held); + }); + let started = Instant::now(); + let result = work(); + let waited = started.elapsed(); + let _ = release.send(()); + holder.join().unwrap(); + assert!(waited < Duration::from_secs(2), "the work waited {waited:?} on a held lock"); + result + } + + #[cfg(unix)] + #[test] + fn no_incident_write_waits_on_a_held_lock_and_each_lands_when_it_is_freed() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + assert!(store.open(condition, "closing", OPENED).unwrap()); + assert!(store.close(condition, "closing", CloseReason::Recovered, OPENED + 1).unwrap()); + assert!(store.open(condition, "open", OPENED).unwrap()); + let before = std::fs::read(&path).unwrap(); + let retired = OPENED + 1 + INCIDENT_RETENTION_MILLIS + 1; + + let outcomes = run_while_held(hold_lock_of(&path), || { + ( + store.open(condition, "new", CLOSED), + store.close(condition, "open", CloseReason::Recovered, CLOSED), + store.retire_closed(retired), + ) + }); + for outcome in [outcomes.0, outcomes.1, outcomes.2] { + assert!(outcome.is_err()); + } + assert_eq!(std::fs::read(&path).unwrap(), before, "a contended write changed the file"); + assert!(!temp.path().join("incident-history.cc").exists()); + + // Nothing was lost by the refusal: the same calls land once it is free. + assert!(store.open(condition, "new", CLOSED).unwrap()); + assert!(store.close(condition, "open", CloseReason::Recovered, CLOSED).unwrap()); + assert!(store.retire_closed(retired).unwrap()); + } + + #[cfg(unix)] + #[test] + fn a_held_history_lock_keeps_the_incident_where_it_is() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let history = temp.path().join("incident-history.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + assert!(store.open(condition, "old", OPENED).unwrap()); + assert!(store.close(condition, "old", CloseReason::Recovered, OPENED + 1).unwrap()); + let before = std::fs::read(&path).unwrap(); + let retired = OPENED + 1 + INCIDENT_RETENTION_MILLIS + 1; + + let outcome = run_while_held(hold_lock_of(&history), || store.retire_closed(retired)); + assert!(outcome.is_err()); + assert_eq!(std::fs::read(&path).unwrap(), before); + assert!(store.retire_closed(retired).unwrap()); + } + + #[test] + fn a_write_decided_on_a_stale_snapshot_does_not_land() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + + // Two writers read the file with no incident open for the subject. + let stale = store.snapshot().unwrap(); + assert!(store.open(condition, "service", OPENED).unwrap()); + + // The slower one must lose: one open incident per subject. + let error = store + .open_on(&stale, condition, "service", OPENED + 1) + .unwrap_err(); + assert!(error.to_string().contains("changed"), "{error}"); + let records = store.read().unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].opened_at_unix_millis, OPENED); + } + + #[test] + fn retiring_after_a_crash_between_history_and_delete_archives_once() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let store = IncidentStore::new(&path); + let condition = IncidentCondition::ContinuityExhausted; + assert!(store.open(condition, "old", OPENED).unwrap()); + assert!(store.close(condition, "old", CloseReason::Recovered, OPENED + 1).unwrap()); + // History already holds the record: the delete never ran. + let envelope = store.snapshot().unwrap().envelopes[0].clone(); + let history = SingleFileMessagePackBackingStore::new(&temp.path().join("incident-history.cc")); + assert!(history.insert_entry_if_absent(envelope).unwrap()); + + assert!(store.retire_closed(OPENED + 1 + INCIDENT_RETENTION_MILLIS + 1).unwrap()); + assert!(store.read().unwrap().is_empty()); + assert_eq!(history.pull_all_read_only_snapshot().unwrap().len(), 1); } } From 9ad7e4209b4f8c4a0474a9932b852b1af17deb81 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 19:01:30 +0200 Subject: [PATCH 033/196] Incident retirement test: the archive-first crash state is the designed one A refused delete after a successful archive leaves the record in both files by design; the test asserted the history file was absent. It now asserts the repeat archives nothing twice. Co-Authored-By: Claude Sonnet 5.5 --- src/incident.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/incident.rs b/src/incident.rs index 7de86cf..65149c0 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -714,12 +714,15 @@ pub(crate) mod tests { assert!(outcome.is_err()); } assert_eq!(std::fs::read(&path).unwrap(), before, "a contended write changed the file"); - assert!(!temp.path().join("incident-history.cc").exists()); // Nothing was lost by the refusal: the same calls land once it is free. + // (Retirement archives first, so its refused delete left the record in + // history too; the repeat archives nothing twice.) assert!(store.open(condition, "new", CLOSED).unwrap()); assert!(store.close(condition, "open", CloseReason::Recovered, CLOSED).unwrap()); assert!(store.retire_closed(retired).unwrap()); + let history = SingleFileMessagePackBackingStore::new(&temp.path().join("incident-history.cc")); + assert_eq!(history.pull_all_read_only_snapshot().unwrap().len(), 1); } #[cfg(unix)] From 3ed5f262baf96fad72079acb6b5e2e2428216639 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 19:59:41 +0200 Subject: [PATCH 034/196] Incident lock tests retry after release; a forked child can hold a released flock A test that held incidents.cc.lock and then released it failed once in a full parallel run: another test's forked child briefly shared the open file description. The follow-up calls are exactly the "retried on a later pass" the store promises, so the tests now retry for up to two seconds instead of asserting the first call after release lands. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 9 +++++++-- src/incident.rs | 22 ++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e663d7a..1c22d7b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15033,8 +15033,13 @@ mod tests { } // The lock is free again: the next pass retries and the incident lands. - world.engine.supervise_one_admitted_generation()?; - assert!(incidents_of(&world)?.iter().any(|record| record.is_open())); + // (A child another test forked while it was held can keep it a moment.) + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !incidents_of(&world)?.iter().any(|record| record.is_open()) { + assert!(std::time::Instant::now() < deadline, "the incident never landed"); + world.engine.supervise_one_admitted_generation()?; + std::thread::sleep(Duration::from_millis(20)); + } assert_control_store_holds_no_incident(&world) } diff --git a/src/incident.rs b/src/incident.rs index 65149c0..95030c1 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -690,6 +690,20 @@ pub(crate) mod tests { result } + /// Retry `attempt` for up to two seconds. A released lock can stay held for + /// a moment by a child another test forked while it was held. + #[cfg(unix)] + fn eventually(mut attempt: impl FnMut() -> Result) -> T { + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match attempt() { + Ok(value) => return value, + Err(error) if Instant::now() >= deadline => panic!("never landed: {error:#}"), + Err(_) => std::thread::sleep(Duration::from_millis(20)), + } + } + } + #[cfg(unix)] #[test] fn no_incident_write_waits_on_a_held_lock_and_each_lands_when_it_is_freed() { @@ -718,9 +732,9 @@ pub(crate) mod tests { // Nothing was lost by the refusal: the same calls land once it is free. // (Retirement archives first, so its refused delete left the record in // history too; the repeat archives nothing twice.) - assert!(store.open(condition, "new", CLOSED).unwrap()); - assert!(store.close(condition, "open", CloseReason::Recovered, CLOSED).unwrap()); - assert!(store.retire_closed(retired).unwrap()); + assert!(eventually(|| store.open(condition, "new", CLOSED))); + assert!(eventually(|| store.close(condition, "open", CloseReason::Recovered, CLOSED))); + assert!(eventually(|| store.retire_closed(retired))); let history = SingleFileMessagePackBackingStore::new(&temp.path().join("incident-history.cc")); assert_eq!(history.pull_all_read_only_snapshot().unwrap().len(), 1); } @@ -741,7 +755,7 @@ pub(crate) mod tests { let outcome = run_while_held(hold_lock_of(&history), || store.retire_closed(retired)); assert!(outcome.is_err()); assert_eq!(std::fs::read(&path).unwrap(), before); - assert!(store.retire_closed(retired).unwrap()); + assert!(eventually(|| store.retire_closed(retired))); } #[test] From 1f7c8e8f427a077a02f92665402bb891a79440a7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:07:49 +0200 Subject: [PATCH 035/196] Route map: B5 Soul pass 2 (merge and install after batch 3 pins); P1 reload-owed window recorded Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index ede571b..baabb18 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -254,6 +254,21 @@ The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map - **H.** AwaitingReady and Routing keep the F16 shape. It is unreachable at today's heartbeats. - **I.** An `idunn expire` request written while Idunn is stopped lingers, and there is no verb to withdraw it, which is a downgrade hazard. +- **B5 batch 2 (`c06ff2b..aef0da3`) and Soul pass 2 (2026-09-30): merge and install after batch 3.** + - A, B, E, F and J hold. Verify: 356 passed. cargo-mutants caught 14 of 14 viable mutants, and Soul's hand + mutants killed 16 of 20. + - Batch 3, in Hands: test pins for MB2 (`changes_route` swapped for "no incumbent"), ME2/ME3 (the resolver filter + by id length or first byte) and MJ2 (the computed backoff unpinned). `restore` withdraws the firewall rule + before reloading when there is no prior route, so a failing `ufw delete` no longer reloads nginx each retry. + - Hands corrected Self's brief: the continuity path's zero reloads came from `6ff7d5a`'s skip, not `4830af8`. The + narrow no-change case (`changes_route() == false`) keeps it without a write, so there is no crash window. + - Recorded, not in B5: + - **P1:** a crash between `atomic_replace` and the reload in supervision's `restore_admitted_membership` + (`drivers.rs:5543-5545`). Idunn has no SIGTERM handler, so a stop in that window leaves nginx on the drift + until something else reloads. This also exists in live `a96ad9d`. Fix with Q3-ii or a durable + "reload owed" marker. + - **The abort revokes before it stops the candidate** (`control_plane.rs:8758` before `:8783`). Q3-ii owns the + abort order. - **Q-B2-1 (a):** the continuity restart log moves into `TargetSupervision`; the restart ceiling reads the target's own log; the carry compensator is deleted. - **Q-B2-2: fail at once.** A deploy refused by the route/actuation ceiling before the fence fails immediately, From f85e74377f4014e0d7e5c9377a2c1055ff8f201b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:08:58 +0200 Subject: [PATCH 036/196] B5 batch 3: firewall first on a withdrawal to nothing; four test pins `restore` with no prior membership now withdraws the stable endpoint's firewall allow before it writes the fragment and reloads. A failing `ufw delete` retries without touching nginx, which bounds the withdrawal reload loop at zero until the Q3-ii cut gives the abort a deadline. A crash between the two steps is safe: a rule already gone reads as withdrawn, and the retry removes the fragment and reloads. Pins from Soul pass 2: - MB2: a redeploy over an existing incumbent route installs, rolls back and withdraws with a reload each (`changes_route` as `is_none()` survived). - ME2/ME3: the resolver's command filter is tested with an id of equal length and the same leading bytes. - MJ2: the backoff test asserts the wait `back_off` writes, one poll after the failure. - Soul's F2 probe returns as a test: a withdrawal whose `ufw delete` keeps failing reloads nginx not at all over 20 retries. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 47 ++++++++++++++++++++++- src/drivers.rs | 91 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 132 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 41fda31..00a7775 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14235,8 +14235,24 @@ mod tests { .engine .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; let attempts = || workload.0.load(std::sync::atomic::Ordering::SeqCst); + let before = now_millis()?; world.engine.run_scheduler_tick()?; + let after = now_millis()?; assert_eq!(attempts(), 1, "the step did not run"); + // The first failure waits one poll interval: the wait back_off wrote. + let poll = world.engine.options.poll_millis; + let written = world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&resident(&world)?.value.transaction_id) + .map(|backoff| backoff.not_before_unix_millis) + .expect("the failure set no backoff"); + assert!( + (before + poll..=after + poll).contains(&written), + "back_off wrote {written}, not one poll ({poll} ms) after the failure" + ); // The wait is set from the wall clock, and a loaded host can spend it // inside a few ticks. So the test owns the wait: held open, no tick // retries; once due, the next tick does. @@ -18490,6 +18506,31 @@ mod tests { Ok(()) } + /// Soul's F2 probe, committed: a post-fence withdrawal whose + /// `ufw delete` keeps failing withdraws the firewall allow first, so its + /// retries reload nginx not at all. + #[cfg(unix)] + #[test] + fn a_withdrawal_whose_firewall_step_keeps_failing_never_reloads() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + let installed = routed.reloads(); + routed.world.route_stubs.refuse_ufw_deletes()?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; + for _ in 0..20 { + routed.resume()?; + } + assert_eq!(routed.reloads(), installed, "a failing withdrawal reloaded nginx"); + assert_eq!(routed.world.route_stubs.count("ufw delete"), 20); + assert!(routed.fragment_path().exists(), "the route went before its firewall rule"); + Ok(()) + } + /// A record lifted from before deadlines existed is timed once, from /// the first time the resolver sees it, and never again. #[test] @@ -18622,9 +18663,13 @@ mod tests { requested_at_unix_millis: now, }; let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; + // The request names a command whose id has this transaction's + // length and first bytes, and differs only in its last. + let near = "continuity-servicx"; + assert_eq!(near.len(), "continuity-service".len()); let request = ExpiryRequest { schema_version: EXPIRY_REQUEST_SCHEMA.into(), - command_id: other.command_id.clone(), + command_id: near.into(), requested_by: "operator".into(), requested_at_unix_millis: now, }; diff --git a/src/drivers.rs b/src/drivers.rs index 7903f94..b8375e1 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5344,8 +5344,16 @@ impl NginxRouteDriver { /// on a transaction path, bytes on disk that already equal `prior` may be /// a write whose reload a crash cut off. A reload that fails puts the /// previous fragment back, so the disk shows what nginx still serves. + /// + /// With no prior membership the stable endpoint routes to nothing, and + /// its firewall allow is withdrawn first: a failing `ufw delete` then + /// retries without touching nginx, and a crash after it is safe, because + /// a rule already gone reads as withdrawn. fn restore(&self, prior: Option<&[u8]>, gate: &dyn RouteActuationGate) -> Result<()> { gate.admit(RouteActuation::Survival)?; + if prior.is_none() { + self.withdraw_endpoint()?; + } let current = self.current_configuration()?; self.write_fragment(prior)?; if let Err(reload) = self.reload() { @@ -5356,11 +5364,6 @@ impl NginxRouteDriver { )), }; } - // No prior membership means the stable endpoint no longer routes to - // anything; its firewall allow goes with the fragment. - if prior.is_none() { - self.withdraw_endpoint()?; - } Ok(()) } @@ -11873,6 +11876,12 @@ Content-Le".to_vec()), |_| {}), }; driver.systemctl_program = failable("systemctl", "reload")?; driver.ufw_program = failable("ufw", "delete")?; + // `/ufw.absent`: the rule is already gone, as ufw reports it. + let ufw = fs::read_to_string(&driver.ufw_program)?.replace( + "exit 0\n", + "if [ \"$1\" = delete ] && [ -e \"$0.absent\" ]; then echo 'Could not delete non-existent rule' >&2; exit 1; fi\nexit 0\n", + ); + fs::write(&driver.ufw_program, ufw)?; Ok((driver, candidate, calls)) } @@ -11921,6 +11930,78 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// A redeploy over an existing route changes it: the candidate differs + /// from the incumbent's fragment, so it installs and reloads, and its + /// rollback and withdrawal put the incumbent back and reload. + #[cfg(unix)] + #[test] + fn a_deploy_over_an_incumbent_route_installs_and_reloads() -> Result<()> { + let temp = tempfile::tempdir()?; + let (driver, candidate, calls) = failable_route_driver(temp.path())?; + let incumbent = b"# incumbent membership\n".to_vec(); + let mut receipt = preflight_receipt(&driver, &candidate)?; + receipt.incumbent_runtime_instance_id = Some(digest('b')); + receipt.incumbent_membership_sha256 = Some(sha256_id(&incumbent)); + receipt.incumbent_configuration = Some(incumbent.clone()); + assert!(receipt.changes_route()); + fs::write(&driver.binding.config_path, &incumbent)?; + + driver.install(&candidate, &digest('a'), &receipt, true, &Unmetered)?; + assert_eq!(logged(&calls, "systemctl reload"), 1, "the redeploy did not install"); + assert_eq!(fs::read(&driver.binding.config_path)?, driver.render(&candidate)?); + driver.rollback(&candidate, &digest('a'), &receipt, &Unmetered)?; + assert_eq!(logged(&calls, "systemctl reload"), 2, "the rollback did not reload"); + assert_eq!(fs::read(&driver.binding.config_path)?, incumbent); + driver.install(&candidate, &digest('a'), &receipt, true, &Unmetered)?; + driver.withdraw_candidate_membership(&receipt, &Unmetered)?; + assert_eq!(logged(&calls, "systemctl reload"), 4, "the withdrawal did not reload"); + assert_eq!(fs::read(&driver.binding.config_path)?, incumbent); + Ok(()) + } + + /// With nothing left to route, the withdrawal takes the firewall allow + /// first. A `ufw delete` that keeps failing retries without reloading + /// nginx; once it goes through, the fragment is removed and reloaded. + #[cfg(unix)] + #[test] + fn a_failing_firewall_withdrawal_retries_without_reloading() -> Result<()> { + let temp = tempfile::tempdir()?; + let (driver, candidate, calls) = failable_route_driver(temp.path())?; + let receipt = preflight_receipt(&driver, &candidate)?; + driver.install(&candidate, &digest('a'), &receipt, false, &Unmetered)?; + let installed = logged(&calls, "systemctl reload"); + + fs::write(temp.path().join("ufw.fail"), b"x")?; + for _ in 0..5 { + assert!(driver.withdraw_candidate_membership(&receipt, &Unmetered).is_err()); + } + assert_eq!(logged(&calls, "systemctl reload"), installed, "a failing ufw delete reloaded nginx"); + assert_eq!(logged(&calls, "ufw delete"), 5); + fs::remove_file(temp.path().join("ufw.fail"))?; + driver.withdraw_candidate_membership(&receipt, &Unmetered)?; + assert_eq!(logged(&calls, "systemctl reload"), installed + 1); + assert!(!driver.binding.config_path.exists()); + Ok(()) + } + + /// A crash after the firewall rule went and before the fragment did: the + /// retry reads the gone rule as withdrawn and finishes the withdrawal. + #[cfg(unix)] + #[test] + fn a_withdrawal_resumed_after_its_firewall_rule_went_finishes() -> Result<()> { + let temp = tempfile::tempdir()?; + let (driver, candidate, calls) = failable_route_driver(temp.path())?; + let receipt = preflight_receipt(&driver, &candidate)?; + driver.install(&candidate, &digest('a'), &receipt, false, &Unmetered)?; + let installed = logged(&calls, "systemctl reload"); + + fs::write(temp.path().join("ufw.absent"), b"x")?; + driver.withdraw_candidate_membership(&receipt, &Unmetered)?; + assert!(!driver.binding.config_path.exists(), "the fragment outlived the withdrawal"); + assert_eq!(logged(&calls, "systemctl reload"), installed + 1, "nginx was not reloaded"); + Ok(()) + } + /// A continuity over its admitted route has the baseline's membership: its /// install, rollback and withdrawal change nothing and run nothing. #[cfg(unix)] From 85964d0f0098ac9b1e582d434b316c9175cbafb0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:35:30 +0200 Subject: [PATCH 037/196] Secrets map: Heimdall batch 6 and Soul pass 6 (hold; invalid index, collation takeover, predicate normaliser) Co-Authored-By: Claude Opus 5.5 --- docs/secret-files-cut.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/secret-files-cut.md b/docs/secret-files-cut.md index 5bdffab..38dd679 100644 --- a/docs/secret-files-cut.md +++ b/docs/secret-files-cut.md @@ -29,6 +29,20 @@ happen again". - **S5-4 to S5-7.** Test gaps: canary position, the `errorIdentity` filters, pool close, spread-env reads. - A flaky live-Postgres test. - Noted, not in scope: **S5-8**, forced RLS with no policy passes the check. +- **Batch 6** (`b5d7b6b..6c0c876`): S5-1 through S5-7 fixed, and the flaky live test fixed (template DB clone, `fixturePool`). + Result: 241 passed, 24/24 live, 40 of 41 mutants killed. The survivor, X13, is equivalent: pg-pool destroys a + failed client. +- **Soul pass 6: hold.** Batch 7 is in Hands. + - **F1.** An invalid or not-ready unique index passes the check. That breaks sign-in (42P10) and R21.1, because a + consume takes 2 rows. + - **F2.** A key index under a nondeterministic collation passes, and the upsert then moves one account's identity + row to another account. + - **F3.** The predicate normaliser is unsound: it strips parentheses and lowercases literals. + - **F4.** Some databases the store could serve are refused: indexes are required by name, and column-level grants + are not accepted. + - **F5.** The recipe exploration's all-at-once run stops early; mutants Z4, Z5 and Z6 survive. + - **F6-F8.** An inspected `cause` is not covered, a narrowed predicate has no fixture, and an added NOT NULL on a + nullable column is not caught. CHECK constraints and triggers are out of scope, as S5-8 is. **Rulings and defaults, 2026-09-30 (Self).** - **Q3 A is the operator's ruling:** "I'll rotate before prod". Cuts 1-4 land with the current values; rotation From 889e23629c8afe0da1267166d30d75c5da9faf9d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 21:40:21 +0200 Subject: [PATCH 038/196] Retire the legacy bifrost-persona-feedback deploy route The operator ruled "delete the Bifrost lane". This route was that lane's live trigger: /srv/odin/deploy-manifests/bifrost-persona-feedback runs `docker compose up persona-feedback`, and the provider actually running on Yggdrasil is the native bifrost-persona-feedback.service. Firing the route would start a container next to it. Following deploy/legacy's own order, the grant goes first, then the dispatch: - sudoers: drop `deploy bifrost-persona-feedback *`. - idunn-yggdrasil: drop the target's policy case and its dispatch token, plus the Bifrost release-authority check that only it used (requires_bifrost_authority and the `idunn validate-release-authority` call). A deploy request that claims a Bifrost authority is still refused. The manifest environment keeps IDUNN_REQUIRES_BIFROST_AUTHORITY=false, because the installed Epiphany manifest checks for it. - README: record the retirement and the counts. Also record that on 2026-09-30 the installed actuator matched this file, but the sudoers grant was not installed: the idunn user cannot sudo. Rebinding persona-feedback to a current Idunn binding is a separate follow-up. Co-Authored-By: Claude Opus 5.5 --- deploy/legacy/README.md | 32 ++++++++++++----- deploy/legacy/idunn-yggdrasil | 50 ++++----------------------- deploy/legacy/idunn-yggdrasil.sudoers | 7 ++-- 3 files changed, 34 insertions(+), 55 deletions(-) diff --git a/deploy/legacy/README.md b/deploy/legacy/README.md index dd91a25..54bb919 100644 --- a/deploy/legacy/README.md +++ b/deploy/legacy/README.md @@ -4,11 +4,16 @@ kept because they are what is *running*, and a migration needs both sides. `idunn-yggdrasil` is a root shell actuator reached through the narrow sudo grant -in `idunn-yggdrasil.sudoers`. It hardcodes eleven targets, their repositories +in `idunn-yggdrasil.sudoers`. It hardcodes ten targets, their repositories and their admitted refs, then delegates to executable manifests under -`/srv/odin/deploy-manifests/`. It calls `idunn validate-release-authority` -and `idunn validate-minimum-source-revision` — subcommands the current binary -does not have. +`/srv/odin/deploy-manifests/`. It calls `idunn validate-minimum-source-revision`, +a subcommand the current binary does not have. + +Checked on yggdrasil 2026-09-30: `/usr/local/libexec/idunn-yggdrasil` is installed +and matches this file byte for byte (before the retirement below), but +`idunn-yggdrasil.sudoers` is not installed. `sudo -l -U idunn` says the `idunn` +user may not run sudo, so the daemon cannot reach this actuator; only an +operator running it as root can. ## What replaced it @@ -16,10 +21,10 @@ Everything this script does by convention, the current Idunn does by contract: | Legacy mechanism | Current owner | |---|---| -| `deployment_target_policy()` — a `case` over eleven target names | one operator binding per target, in `--bindings-dir` | +| `deployment_target_policy()` — a `case` over ten target names | one operator binding per target, in `--bindings-dir` | | `upstream_ref=refs/heads/...` in shell | `[repository] admitted_ref` | | `minimum_source_revision_required` | `[repository] minimum_revision` | -| `requires_bifrost_authority` + `--release-authority-store` | `[repository] selection = "signed-release"`, per target | +| `requires_bifrost_authority` + `--release-authority-store` (retired with its only target) | `[repository] selection = "signed-release"`, per target | | `/srv/odin/deploy-manifests/` shell scripts | recipe `[[steps]]` in the target's own repository | | `run_manifest_while_holding_authority_locks` | the deployment transaction and process write-lease | | `flock` on a brake store | `[brakes] deployment_store` and `lifecycle_store` | @@ -45,15 +50,26 @@ This is the real migration scope, read out of `deployment_target_policy()`: | `heimdall` | GameCult/Heimdall | `main` | restart is `docker compose restart` | | `epiphany` | GameCult/Epiphany | `codex/epiphany-shakedown-live` | | | `epiphany-capstone-17` | GameCult/Epiphany | `codex/epiphany-shakedown-live` | second target, same repo | -| `bifrost-persona-feedback` | GameCult/Bifrost | `main` | **only** target requiring Bifrost release authority | +| ~~`bifrost-persona-feedback`~~ | GameCult/Bifrost | `main` | **retired 2026-09-30**, see below | | `repixelizer` | GameCult/repixelizer | `main` | | | `streampixels` | GameCult/StreamPixels | `main` | two units, one target | | `gjallar` | GameCult/Gjallar | `codex/yggdrasil-aggregate-daemon` | | -Seven of eleven are pinned to a non-`main` branch. That is its own problem — the +Seven of the ten live targets are pinned to a non-`main` branch. That is its own problem — the swarm goal is everyone on `main` and current with each other — and migrating a target is the moment to fix its ref, not to copy it forward. +## Retired targets + +- `bifrost-persona-feedback`, 2026-09-30. Its manifest ran the Bifrost compose lane + (`docker compose up persona-feedback`), while the live provider is the native + `bifrost-persona-feedback.service`, so firing it would have started a container + beside the native unit. The operator ruled "delete the Bifrost lane". Its grant, + dispatch case and the Bifrost release-authority check it alone used are gone; a + deploy request that claims a Bifrost authority is refused. It was the only + caller of `idunn validate-release-authority`. Rebinding persona-feedback to a + current Idunn binding is a separate follow-up. + ## When these files die When every target above has a binding and a recipe, and `idunn serve` owns diff --git a/deploy/legacy/idunn-yggdrasil b/deploy/legacy/idunn-yggdrasil index 1d83b52..26970c0 100644 --- a/deploy/legacy/idunn-yggdrasil +++ b/deploy/legacy/idunn-yggdrasil @@ -96,14 +96,12 @@ deployment_target_policy() { daemon_id=yggdrasil-odin repository_full_name=GameCult/Odin upstream_ref=refs/heads/codex/ygg-idunn-independent-bootstrap - requires_bifrost_authority=false manifest_takes_action=true ;; codex-connector) daemon_id=yggdrasil-codex-connector repository_full_name=GameCult/CodexConnector upstream_ref=refs/heads/codex/ghostlight-release-binding - requires_bifrost_authority=false minimum_source_revision_required=true manifest_takes_action=true ;; @@ -111,7 +109,6 @@ deployment_target_policy() { daemon_id=yggdrasil-ghostlight repository_full_name=GameCult/Ghostlight upstream_ref=refs/heads/codex/ghostlight-dungeon-mvp - requires_bifrost_authority=false minimum_source_revision_required=true manifest_takes_action=true ;; @@ -119,52 +116,39 @@ deployment_target_policy() { daemon_id=yggdrasil-voidbot repository_full_name=GameCult/VoidBot upstream_ref=refs/heads/main - requires_bifrost_authority=false manifest_takes_action=true ;; heimdall) daemon_id=yggdrasil-heimdall repository_full_name=GameCult/Heimdall upstream_ref=refs/heads/main - requires_bifrost_authority=false ;; epiphany-capstone-17) daemon_id=yggdrasil-epiphany-capstone-17 repository_full_name=GameCult/Epiphany upstream_ref=refs/heads/codex/epiphany-shakedown-live - requires_bifrost_authority=false manifest_takes_action=true ;; epiphany) daemon_id=yggdrasil-epiphany repository_full_name=GameCult/Epiphany upstream_ref=refs/heads/codex/epiphany-shakedown-live - requires_bifrost_authority=false manifest_takes_action=true ;; - bifrost-persona-feedback) - daemon_id=yggdrasil-bifrost-persona-feedback - repository_full_name=GameCult/Bifrost - upstream_ref=refs/heads/main - requires_bifrost_authority=true - ;; repixelizer) daemon_id=yggdrasil-repixelizer repository_full_name=GameCult/repixelizer upstream_ref=refs/heads/main - requires_bifrost_authority=false ;; streampixels) daemon_id=yggdrasil-streampixels repository_full_name=GameCult/StreamPixels upstream_ref=refs/heads/main - requires_bifrost_authority=false ;; gjallar) daemon_id=yggdrasil-gjallar repository_full_name=GameCult/Gjallar upstream_ref=refs/heads/codex/yggdrasil-aggregate-daemon - requires_bifrost_authority=false ;; *) echo "deployment target has no root actuator policy: $target" >&2; exit 1 ;; esac @@ -190,19 +174,10 @@ require_signed_deployment_grant() { *) echo "signed deployment request is not for root target $target" >&2; exit 1 ;; esac - case "$requires_bifrost_authority" in - true) - [ -n "$release_authority_id" ] || { echo "missing Bifrost release authority id" >&2; exit 1; } - case "$release_authority_sha256" in - *[!0-9a-f]*|'') echo "invalid or missing Bifrost release authority digest" >&2; exit 1 ;; - esac - [ "${#release_authority_sha256}" -eq 64 ] || { echo "Bifrost release authority digest must be 64 lowercase hex characters" >&2; exit 1; } - ;; - false) - [ -z "$release_authority_id" ] || { echo "source-observed release cannot claim Bifrost authority id" >&2; exit 1; } - [ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; } - ;; - esac + # No target carries Bifrost release authority: its only one, bifrost-persona-feedback, + # was retired on 2026-09-30. A request that claims one is refused. + [ -z "$release_authority_id" ] || { echo "source-observed release cannot claim Bifrost authority id" >&2; exit 1; } + [ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; } require_root_owned_resolved_executable "$idunn_provision" require_root_owned_regular_path "$brake_store" @@ -234,18 +209,6 @@ require_signed_deployment_grant() { --daemon "$daemon_id" \ --source-revision "$source_commit" fi - - if [ "$requires_bifrost_authority" = true ]; then - require_root_owned_resolved_executable /usr/local/bin/idunn - /usr/local/bin/idunn validate-release-authority \ - --store /var/lib/gamecult/idunn/yggdrasil.keepalive.cc \ - --release-authority-store /srv/bifrost/state/repository-release-authority.cc \ - --repository-full-name "$repository_full_name" \ - --upstream-ref "$upstream_ref" \ - --source-revision "$source_commit" \ - --release-authority-id "$release_authority_id" \ - --release-authority-sha256 "$release_authority_sha256" - fi } manifest_group_exists() { @@ -355,7 +318,7 @@ case "$action:$target" in run_manifest_while_holding_authority_locks "$manifest" restart exit $? ;; - deploy:voidbot|deploy:epiphany|deploy:epiphany-capstone-17|deploy:odin|deploy:codex-connector|deploy:ghostlight|deploy:heimdall|deploy:repixelizer|deploy:streampixels|deploy:bifrost-persona-feedback|deploy:gjallar) + deploy:voidbot|deploy:epiphany|deploy:epiphany-capstone-17|deploy:odin|deploy:codex-connector|deploy:ghostlight|deploy:heimdall|deploy:repixelizer|deploy:streampixels|deploy:gjallar) manifest="/srv/odin/deploy-manifests/$target" require_root_owned_regular_path "$manifest" [ -x "$manifest" ] || { echo "root actuator manifest is not executable: $manifest" >&2; exit 1; } @@ -368,7 +331,8 @@ case "$action:$target" in export BIFROST_RELEASE_AUTHORITY_ID="$release_authority_id" export BIFROST_RELEASE_AUTHORITY_SHA256="$release_authority_sha256" export IDUNN_DEPLOYMENT_REQUEST_ID="$deployment_request_id" - export IDUNN_REQUIRES_BIFROST_AUTHORITY="$requires_bifrost_authority" + # Installed manifests (Epiphany's) still require this to read false. + export IDUNN_REQUIRES_BIFROST_AUTHORITY=false if [ "$manifest_takes_action" = true ]; then run_manifest_while_holding_authority_locks "$manifest" "$action" exit $? diff --git a/deploy/legacy/idunn-yggdrasil.sudoers b/deploy/legacy/idunn-yggdrasil.sudoers index 8872c25..a855527 100644 --- a/deploy/legacy/idunn-yggdrasil.sudoers +++ b/deploy/legacy/idunn-yggdrasil.sudoers @@ -1,7 +1,7 @@ # Narrow root actuator grant for the Idunn Yggdrasil deployment/continuity path. -# One line per live target. codex-connector and gjallar were retired 2026-09-06; -# their grants are removed here first, because the privileged half of a retired -# target must not outlive its consumer. +# One line per live target. codex-connector and gjallar were retired 2026-09-06, +# and bifrost-persona-feedback on 2026-09-30; their grants are removed here first, +# because the privileged half of a retired target must not outlive its consumer. Cmnd_Alias IDUNN_YGGDRASIL_DEPLOY = \ /usr/local/libexec/idunn-yggdrasil deploy odin *, \ /usr/local/libexec/idunn-yggdrasil deploy ghostlight *, \ @@ -9,7 +9,6 @@ Cmnd_Alias IDUNN_YGGDRASIL_DEPLOY = \ /usr/local/libexec/idunn-yggdrasil deploy heimdall *, \ /usr/local/libexec/idunn-yggdrasil deploy epiphany-capstone-17 *, \ /usr/local/libexec/idunn-yggdrasil deploy epiphany *, \ - /usr/local/libexec/idunn-yggdrasil deploy bifrost-persona-feedback *, \ /usr/local/libexec/idunn-yggdrasil deploy repixelizer *, \ /usr/local/libexec/idunn-yggdrasil deploy streampixels * Cmnd_Alias IDUNN_YGGDRASIL_RESTART = \ From 1cfbeff083756c4281be6e8c627be71d9a8010a2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 22:30:18 +0200 Subject: [PATCH 039/196] B5 pins: firewall kept for a live incumbent, whole-id request match, backoff growth Tests only, from Soul's last B5 check: - F2: a rollback or withdrawal back to an incumbent route deletes no firewall rule (S-R2 survived). - F3: expiry requests whose ids are a strict prefix of the transaction's command id, or extend it, leave it untouched (S-ME1, S-ME2 survived). - F4: the second and third failures wait two and four polls, as back_off writes them (S-MJ1, S-MJ2 and a clamp survived). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 61 +++++++++++++++++++++++++++++++------------- src/drivers.rs | 3 +++ 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 00a7775..2c5fb3d 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14272,9 +14272,26 @@ mod tests { world.engine.run_scheduler_tick()?; } assert_eq!(attempts(), 1, "the wedged step was retried inside its backoff"); - set_due(1); - world.engine.run_scheduler_tick()?; - assert_eq!(attempts(), 2, "the step was not retried once its backoff was due"); + // Each further failure doubles the wait back_off writes. + for (attempt, polls) in [(2, 2), (3, 4)] { + set_due(1); + let before = now_millis()?; + world.engine.run_scheduler_tick()?; + let after = now_millis()?; + assert_eq!(attempts(), attempt, "the step was not retried once its backoff was due"); + let written = world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&id) + .map(|backoff| backoff.not_before_unix_millis) + .expect("the failure set no backoff"); + assert!( + (before + polls * poll..=after + polls * poll).contains(&written), + "failure {attempt} waits {written}, not {polls} polls ({poll} ms each) after it" + ); + } Ok(()) } @@ -18673,31 +18690,39 @@ mod tests { requested_by: "operator".into(), requested_at_unix_millis: now, }; + // And commands whose ids are a strict prefix of this one's, and + // extend it: a match is the whole id, in neither direction a part. + let prefix = ExpiryRequest { + command_id: "continuity-servic".into(), + ..request.clone() + }; + let extension = ExpiryRequest { + command_id: "continuity-service-2".into(), + ..request.clone() + }; let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); + let absent = |r#type: &str, key: &str| CultCacheExpectedEnvelope { + r#type: r#type.into(), + key: key.into(), + current: None, + }; assert!(store.compare_exchange( &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: other.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: sealing.transaction_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: ExpiryRequest::TYPE.into(), - key: request.command_id.clone(), - current: None, - }, + absent(DeploymentCommand::TYPE, &other.command_id), + absent(DeploymentTransaction::TYPE, &sealing.transaction_id), + absent(ExpiryRequest::TYPE, &request.command_id), + absent(ExpiryRequest::TYPE, &prefix.command_id), + absent(ExpiryRequest::TYPE, &extension.command_id), ], &[ command_envelope(&other, now)?, transaction_envelope(&sealing, now)?, expiry_request_envelope(&request, now)?, + expiry_request_envelope(&prefix, now)?, + expiry_request_envelope(&extension, now)?, ], )?); + assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.len(), 3); let snapshot = ControlSnapshot::read(&routed.world.state_store)?; assert_eq!(snapshot.transactions.iter().filter(|t| !t.value.is_terminal()).count(), 2); let mine = snapshot diff --git a/src/drivers.rs b/src/drivers.rs index b8375e1..072d49d 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -11952,10 +11952,13 @@ Content-Le".to_vec()), |_| {}), driver.rollback(&candidate, &digest('a'), &receipt, &Unmetered)?; assert_eq!(logged(&calls, "systemctl reload"), 2, "the rollback did not reload"); assert_eq!(fs::read(&driver.binding.config_path)?, incumbent); + // The incumbent still routes: its endpoint's firewall allow stays. + assert_eq!(logged(&calls, "ufw delete"), 0, "the rollback withdrew a live endpoint"); driver.install(&candidate, &digest('a'), &receipt, true, &Unmetered)?; driver.withdraw_candidate_membership(&receipt, &Unmetered)?; assert_eq!(logged(&calls, "systemctl reload"), 4, "the withdrawal did not reload"); assert_eq!(fs::read(&driver.binding.config_path)?, incumbent); + assert_eq!(logged(&calls, "ufw delete"), 0, "the withdrawal withdrew a live endpoint"); Ok(()) } From f1b45666c69fe5bc0dc02111f772f2965ca12559 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 22:36:02 +0200 Subject: [PATCH 040/196] Route map: B5 merged (fcb8ba1) and installed on Yggdrasil Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index baabb18..a278165 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -254,6 +254,27 @@ The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map - **H.** AwaitingReady and Routing keep the F16 shape. It is unreachable at today's heartbeats. - **I.** An `idunn expire` request written while Idunn is stopped lingers, and there is no verb to withdraw it, which is a downgrade hazard. +- **B5 MERGED and INSTALLED (2026-09-30).** Merged to main at `fcb8ba1` (Soul passes 1-3). Built on Yggdrasil: + `/srv/build/idunn-fcb8ba1`, 360 passed. `idunn` sha256 `e96bcefa...`, `idunn-provision` `127902ef...`, unit unchanged. + Installed at 22:35 CEST. + - **Backup**, taken with Idunn stopped and byte-compared: + `/root/idunn-store-backup-20260930T203515Z-pre-route-b5/`. It holds `control.cc` `fab5b66b...`, `history.cc` + `a2627ea4...` and `idunn.running` `848c08e5...` (a96ad9d). + - **Preflight** found one live transaction: the raven-muninn continuity `990dfc25` in AwaitingReady for about 40 + min. Raven had rebooted, and its actuator timed out at 21:36 and 21:49 CEST. The operator ruled "Feel free to + install". + - **After install:** + - No managed target's PID changed. Raven reattached after 15 s, and nginx made 0 reloads. + - B5 aborted the stale continuity at its passed deadline, and supervision opened `continuity-26737b91`. + - A single log line, "could not resume transaction tx-bcfbb693...: transaction disappeared while checking + scheduler progress", appeared at 20:35:38 UTC. It looks like a same-tick ordering quirk between the + resolver and resume. It goes to the next Soul pass. + - **Rollback rule:** stop Idunn and check `control.cc` for any `idunn.expiry_request` envelope. If one exists, + restore the backup instead of swapping the binary. + - **Named tradeoff (Soul batch-3 F1):** a persistent `ufw delete` failure while aborting a first routed deploy now + leaves the aborted candidate routed until ufw recovers. Before B5 the cost was reload churn. Q3-ii's abort + deadline bounds it. + - Test pins F2-F4 are in Hands (`hands/b5-pins`). - **B5 batch 2 (`c06ff2b..aef0da3`) and Soul pass 2 (2026-09-30): merge and install after batch 3.** - A, B, E, F and J hold. Verify: 356 passed. cargo-mutants caught 14 of 14 viable mutants, and Soul's hand mutants killed 16 of 20. From b69a8e81bafb493b86625e8bb43c78a79e6b0d20 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 22:49:57 +0200 Subject: [PATCH 041/196] Secrets map: Heimdall secrets merged at ca2366c after Soul pass 7 Co-Authored-By: Claude Opus 5.5 --- docs/secret-files-cut.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/secret-files-cut.md b/docs/secret-files-cut.md index 38dd679..71d3a60 100644 --- a/docs/secret-files-cut.md +++ b/docs/secret-files-cut.md @@ -43,6 +43,25 @@ happen again". - **F5.** The recipe exploration's all-at-once run stops early; mutants Z4, Z5 and Z6 survive. - **F6-F8.** An inspected `cause` is not covered, a narrowed predicate has no fixture, and an added NOT NULL on a nullable column is not caught. CHECK constraints and triggers are out of scope, as S5-8 is. +- **Batch 7** (`6c0c876..ca2366c`): F1-F8 fixed. + - Keys must be `indisvalid` with deterministic collations. + - Predicates are printed by Postgres, using a TEMP table in a rolled-back transaction, and compared as sets of + AND conditions. + - Index names are not required. Column privileges are checked with `has_column_privilege`. A nullable column must + stay nullable. + - Results: 241 passed, 39/39 live, 60/61 mutants killed. +- **Soul pass 7: merge.** Nothing blocks. + - 1400 predicate pairs were checked, and no two different predicates compared equal. + - The `indisvalid` argument held under interrupted concurrent DDL. + - TEMP is never refused for the real deployment, because the service role owns the database. +- **MERGED: Heimdall main fast-forwarded to `ca2366c` (2026-09-30).** +- Follow-up branch in Hands: + - S7-1: the recipe exploration never runs the production start shape. + - S7-2: extra unique indexes and similar documented as out of scope. + - S7-3: the column collation must be deterministic. + - S7-4: one more fixture each for Y3, Y4, Y5 and Y8. +- **Recorded:** a least-privilege column-subset grant is refused (F4(b)). That is theoretically servable, but it is + not how Heimdall is deployed. **Rulings and defaults, 2026-09-30 (Self).** - **Q3 A is the operator's ruling:** "I'll rotate before prod". Cuts 1-4 land with the current values; rotation From bca3d8539403c48f5149371d1a466841b6e3baef Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 23:09:34 +0200 Subject: [PATCH 042/196] Watchdog target: no tick waits on a lock someone else can hold Operator ruled the topology-lock stall into this campaign (target r3). Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-target.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index 962f118..ac9ea82 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -66,6 +66,13 @@ the machine has a detector with no record, and a deliverer that nothing feeds. clears, one closure notice follows, and only if the opening notice was sent. Repeated delivery attempts for either notice are idempotent at the recipient's end. +- **No tick waits on a lock someone else can hold.** Soul found that any + local uid could freeze Idunn's scheduler tick by holding a published + `.lock` file. This was true for incidents during the campaign, and for + `topology.cc` before it. Idunn's writes in its projection directory are + non-blocking exchanges. The locks behind them are private to Idunn, + whatever the directory's default ACL says. The operator ruled on + 2026-09-30 that the topology fix belongs to this campaign. - **Nothing sensitive leaves.** A notice names the target, the condition and the time. It never carries a binding value, a path to a secret, a token, or a line of configuration. From 4ac96e959d69f3c0b68dfae2f85fc23e69fa8ead Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 23:13:52 +0200 Subject: [PATCH 043/196] Route connect test: the listener never accepts, so the test cannot hang route_connect_retries_connection_refused_until_listener_is_ready blocked in accept() with no deadline. If the client connected to another listener on the reused port, accept() waited forever and the whole verify job held its slot for 39 minutes. The kernel completes a handshake from a listener's backlog, so the connect succeeds without an accept: the listener now only outlives the connect, waiting on a channel with a 30 s ceiling. A lost port race fails the test instead of hanging it. Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 9bf4372..bc4873f 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -9327,18 +9327,25 @@ Content-Le".to_vec()), |_| {}), let address = reservation.local_addr()?; drop(reservation); + // The listener never accepts: the kernel completes a handshake from + // its backlog, so the client connects as soon as it is bound. Nothing + // here can block on a connection that never arrives; the listener only + // has to outlive the connect, and gives up waiting after a bound. + let (connected, connect_done) = std::sync::mpsc::channel::<()>(); let server = thread::spawn(move || -> std::io::Result<()> { thread::sleep(Duration::from_millis(75)); let listener = std::net::TcpListener::bind(address)?; - let (_stream, _) = listener.accept()?; + let _ = connect_done.recv_timeout(Duration::from_secs(30)); + drop(listener); Ok(()) }); - let stream = connect_route_socket(address)?; - drop(stream); + let stream = connect_route_socket(address); + let _ = connected.send(()); server .join() .map_err(|_| anyhow!("route readiness test server panicked"))??; + drop(stream?); Ok(()) } From 9549d00fb9731620e625c57871ccd260c0528bd2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 23:13:52 +0200 Subject: [PATCH 044/196] Incident lock created 0600 before CultCache opens it; fault keys per decision site; no wall-clock bounds in lock tests lock-acl-world-openable: CultCache creates a lock with mode 0666, so under Yggdrasil's default ACL (u::rw g::r o::r) the incident lock landed 0644 and any uid could hold it and keep every incident from being recorded. Every incident write (incidents.cc and incident-history.cc) now goes through exchange(), which first creates the lock with O_CREAT mode 0600 (the create mode is intersected with the ACL, so 0600 holds and no wider file ever exists) and fchmods an earlier, wider lock to 0600 on the descriptor it just opened. The test sets the live default ACL on its directory, asserts a control file made the way CultCache does reads 0644 there, asserts both locks are 0600 after open, close and retire, and, as root with setpriv, that uid 65534 cannot open them. close-fault-repeats: record_incident keys its fault by operation and subject (open/close per target, * for the reconcile and retire passes), so one site's Ok no longer clears another site's standing fault. mutation-count-poisoned: a_held_incident_lock_changes_nothing_continuity_decides failed under parallel load because run_while_held asserted the work took under 2 s of wall clock, and the work is a whole continuity sequence. The holder now lets go when the work returns, or after a 30 s stuck ceiling; the test fails only if the work was freed by the ceiling. eventually() and the control-plane retry use the same ceiling. Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 47 +++++++--- src/incident.rs | 206 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 213 insertions(+), 40 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 1c22d7b..4029362 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -4660,12 +4660,14 @@ impl Engine { } /// Settle the outcome of one incident operation. A failure never leaves - /// this function: it is traced once, under its operation, and continuity - /// decides exactly as it would have with a healthy store. It names the - /// operation and the root cause and never a path or a value. Incident - /// writes are never progress. - fn record_incident(&self, operation: &str, outcome: Result) { - let key = format!("incident-store:{operation}"); + /// this function: it is traced once per episode, under the operation and + /// the subject it was decided for (`*` for a pass over the whole store), + /// and continuity decides exactly as it would have with a healthy store. + /// Each decision site has its own key, so one site's success never ends + /// another site's episode. The message names the operation and the root + /// cause and never a path or a value. Incident writes are never progress. + fn record_incident(&self, operation: &str, subject: &str, outcome: Result) { + let key = format!("incident-store:{operation}:{subject}"); match outcome { Ok(_) => self.clear_fault(&key), Err(error) => self.report_once( @@ -4699,8 +4701,8 @@ impl Engine { } Ok(()) }; - self.record_incident("close", close_unadmitted()); - self.record_incident("retire", now_millis().and_then(|now| store.retire_closed(now))); + self.record_incident("close", "*", close_unadmitted()); + self.record_incident("retire", "*", now_millis().and_then(|now| store.retire_closed(now))); } fn host_access(&self) -> Result> { @@ -5738,6 +5740,7 @@ impl Engine { // The window sliding alone is not health and never closes one. self.record_incident( "close", + ¤t.value.target, now_millis().and_then(|now| { // Running is not enough while the window still holds // the restarts that exhausted it: a release that @@ -5837,6 +5840,7 @@ impl Engine { // this record's shadow, never its substitute. self.record_incident( "open", + ¤t.value.target, self.incidents().open( IncidentCondition::ContinuityExhausted, ¤t.value.target, @@ -14929,6 +14933,29 @@ mod tests { assert_eq!(incident_store, PathBuf::from("/tmp/j.cc")); } + /// A standing fault is traced once. A key is cleared only by an operation + /// that succeeded, so another site's short-circuit must not end it: a + /// target that is running but still exhausted closes nothing, and that + /// used to clear the reconcile pass's fault and print it again every pass. + #[test] + fn a_failing_reconcile_is_traced_once_while_a_running_target_stays_exhausted() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + std::fs::write(incident_path(&world), b"not an incident store")?; + spend_the_window(&world)?; + + for pass in 0..6 { + world.engine.supervise_one_admitted_generation()?; + let reports = world.engine.fault_reports.lock().unwrap(); + assert!( + reports.contains_key("incident-store:close:*"), + "pass {pass}: the standing reconcile fault was cleared, so it prints again" + ); + } + Ok(()) + } + #[test] fn a_release_that_runs_one_pass_after_each_restart_keeps_one_incident() -> Result<()> { let workload = SwitchWorkload::new(); @@ -15002,7 +15029,7 @@ mod tests { #[cfg(unix)] #[test] fn a_held_incident_lock_changes_nothing_continuity_decides() -> Result<()> { - use crate::incident::tests::{hold_lock_of, run_while_held}; + use crate::incident::tests::{STUCK_AFTER, hold_lock_of, run_while_held}; let healthy_workload = SwitchWorkload::new(); let healthy = EngineFixture::with_workload(healthy_workload.clone())?; @@ -15034,7 +15061,7 @@ mod tests { // The lock is free again: the next pass retries and the incident lands. // (A child another test forked while it was held can keep it a moment.) - let deadline = std::time::Instant::now() + Duration::from_secs(2); + let deadline = std::time::Instant::now() + STUCK_AFTER; while !incidents_of(&world)?.iter().any(|record| record.is_open()) { assert!(std::time::Instant::now() < deadline, "the incident never landed"); world.engine.supervise_one_admitted_generation()?; diff --git a/src/incident.rs b/src/incident.rs index 95030c1..4aacc10 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -177,8 +177,8 @@ impl Snapshot { /// scheduler tick, so no write waits: each is one nonblocking compare-exchange /// of the whole file against the snapshot it was decided on. A lost race or a /// held lock is an error and a no-op, retried on a later pass. The lock beside -/// the file is not published: the readers of `incidents.cc` take no lock, so -/// only Idunn can ever hold it. +/// the file is not published and is created 0600 (`secure_lock`): the readers +/// of `incidents.cc` take no lock, so only Idunn can ever hold it. pub(crate) struct IncidentStore { path: PathBuf, } @@ -231,6 +231,7 @@ impl IncidentStore { expected: &[CultCacheEnvelope], replacements: &[CultCacheEnvelope], ) -> Result<()> { + secure_lock(path)?; match SingleFileMessagePackBackingStore::new(path) .try_compare_exchange_snapshot(expected, replacements)? { @@ -334,6 +335,46 @@ impl IncidentStore { } } +/// Make the lock beside `store` Idunn's alone, before CultCache opens it. +/// +/// CultCache creates a lock with mode 0666, so the published directory decides +/// who may open it: under Yggdrasil's default ACL (`u::rw g::r o::r`) that is +/// 0644, and any uid could hold the lock and keep every incident from being +/// recorded. Created here with 0600 the create mode is intersected with the +/// ACL and still holds, and no wider file ever exists. A lock left wider by an +/// earlier run is tightened on the descriptor just opened (a descriptor some +/// other uid already holds is not revoked). Readers of the store take no lock, +/// so nothing but Idunn needs to open this file. +fn secure_lock(store: &Path) -> Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + + let mut name = store + .file_name() + .context("the incident store has no file name")? + .to_owned(); + name.push(".lock"); + let lock = store.with_file_name(name); + if let Some(parent) = lock.parent() { + std::fs::create_dir_all(parent).context("creating the incident store directory")?; + } + std::fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .mode(0o600) + .open(&lock) + .context("opening the incident lock")? + .set_permissions(std::fs::Permissions::from_mode(0o600)) + .context("making the incident lock private")?; + } + #[cfg(not(unix))] + let _ = store; + Ok(()) +} + /// The lines `idunn status` prints for one target's incidents: each open one, /// and each closed one still inside retention. pub(crate) fn render_incidents(records: &[IncidentRecord], target: &str, now: u64) -> Vec { @@ -600,10 +641,9 @@ pub(crate) mod tests { ); } - /// Idunn runs with `UMask=027`, so a file it creates lands 0640 unless the - /// store publishes it. The umask is process-wide, so it is held only around - /// each write and only one test at a time may hold it. - #[cfg(unix)] + /// Idunn runs with `UMask=027`. The umask is process-wide, so it is held + /// only around each call and only one test at a time may hold it. + #[cfg(target_os = "linux")] fn under_service_umask(write: impl FnOnce() -> T) -> T { static UMASK: std::sync::Mutex<()> = std::sync::Mutex::new(()); let _held = UMASK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); @@ -614,36 +654,126 @@ pub(crate) mod tests { result } - #[cfg(unix)] + /// The body's condition: a directory carrying the default ACL + /// `u::rw g::r o::r` that Yggdrasil's idunn-projection directory has. Under + /// it the umask is ignored and a file created with mode 0666 lands 0644. + #[cfg(target_os = "linux")] + fn give_default_acl(dir: &Path) { + use std::os::unix::ffi::OsStrExt; + + // posix_acl_xattr: version 2, then (tag, perm, id) entries. The tags + // are ACL_USER_OBJ, ACL_GROUP_OBJ and ACL_OTHER; the id is undefined. + let mut value = 2u32.to_le_bytes().to_vec(); + for (tag, perm) in [(0x01u16, 6u16), (0x04, 4), (0x20, 4)] { + value.extend(tag.to_le_bytes()); + value.extend(perm.to_le_bytes()); + value.extend(u32::MAX.to_le_bytes()); + } + let dir = std::ffi::CString::new(dir.as_os_str().as_bytes()).unwrap(); + // SAFETY: both strings are NUL-terminated and the value is a live buffer. + let rc = unsafe { + libc::setxattr( + dir.as_ptr(), + c"system.posix_acl_default".as_ptr(), + value.as_ptr().cast(), + value.len(), + 0, + ) + }; + assert_eq!( + rc, + 0, + "the test filesystem must accept a default POSIX ACL: {}", + std::io::Error::last_os_error() + ); + } + + /// Whether uid 65534 can read `path`; `None` when the uid cannot be + /// switched (not root, or no setpriv), which leaves the question unproven. + #[cfg(target_os = "linux")] + fn readable_by_another_uid(path: &Path) -> Option { + std::process::Command::new("setpriv") + .args(["--reuid=65534", "--regid=65534", "--clear-groups", "cat"]) + .arg(path) + .output() + .ok() + .map(|output| output.status.success()) + } + + #[cfg(target_os = "linux")] #[test] - fn published_incident_store_is_world_readable_and_its_lock_is_not() { - use std::os::unix::fs::PermissionsExt; + fn under_a_default_acl_the_store_is_published_and_its_locks_are_private() { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; let temp = TempDir::new().unwrap(); - let path = temp.path().join("incidents.cc"); - let lock = temp.path().join("incidents.cc.lock"); + let dir = temp.path(); + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + give_default_acl(dir); + let path = dir.join("incidents.cc"); + let lock = dir.join("incidents.cc.lock"); + let history_lock = dir.join("incident-history.cc.lock"); let store = IncidentStore::new(&path); let condition = IncidentCondition::ContinuityExhausted; let mode = |path: &Path| std::fs::metadata(path).unwrap().permissions().mode() & 0o777; + // Control: the lock CultCache would have made here, open(2) mode 0666 + // under Idunn's umask. Without the ACL in force this reads 0640 and the + // test below proves nothing about the body. + let control = dir.join("control.lock"); + under_service_umask(|| { + std::fs::OpenOptions::new().create(true).write(true).mode(0o666).open(&control) + }) + .unwrap(); + assert_eq!(mode(&control), 0o644, "the default ACL is not in force"); + assert!(under_service_umask(|| store.open(condition, "service", OPENED)).unwrap()); assert_eq!(mode(&path), 0o644, "after open"); - assert_eq!(mode(&lock) & 0o007, 0, "the lock is open to others after open"); + assert_eq!(mode(&lock), 0o600, "the lock after open"); - // Each write replaces the file, and the replacement is published too. assert!( under_service_umask(|| store.close(condition, "service", CloseReason::Recovered, CLOSED)) .unwrap() ); assert_eq!(mode(&path), 0o644, "after close"); - assert_eq!(mode(&lock) & 0o007, 0, "the lock is open to others after close"); + assert_eq!(mode(&lock), 0o600, "the lock after close"); assert!( under_service_umask(|| store.retire_closed(CLOSED + INCIDENT_RETENTION_MILLIS + 1)) .unwrap() ); assert_eq!(mode(&path), 0o644, "after retire"); - assert_eq!(mode(&lock) & 0o007, 0, "the lock is open to others after retire"); + assert_eq!(mode(&lock), 0o600, "the lock after retire"); + assert_eq!(mode(&history_lock), 0o600, "the history lock after retire"); + + // The consequence: another uid cannot open the lock, so it cannot hold + // it. Provable only as root with setpriv, and only if the same uid can + // read the control file, which shows the switch itself works. + if readable_by_another_uid(&control) == Some(true) { + assert_eq!(readable_by_another_uid(&lock), Some(false)); + assert_eq!(readable_by_another_uid(&history_lock), Some(false)); + } else { + eprintln!("UNPROVEN: no uid switch here; the lock modes above stand alone"); + } + } + + #[cfg(unix)] + #[test] + fn a_lock_left_wider_by_an_earlier_run_is_tightened_before_it_is_used() { + use std::os::unix::fs::PermissionsExt; + + let temp = TempDir::new().unwrap(); + let path = temp.path().join("incidents.cc"); + let lock = temp.path().join("incidents.cc.lock"); + std::fs::write(&lock, b"").unwrap(); + std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o666)).unwrap(); + + assert!( + IncidentStore::new(&path) + .open(IncidentCondition::ContinuityExhausted, "service", OPENED) + .unwrap() + ); + let mode = std::fs::metadata(&lock).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); } /// An exclusive lock on a store's lock file, held through another file @@ -671,30 +801,46 @@ pub(crate) mod tests { HeldLock(file) } - /// Run `work` while `held` is held, and fail if it waited on it. The - /// holder lets go after three seconds so that work which does wait ends, - /// late, and the test fails instead of hanging. + /// How long a test lets a held lock, or a released lock's last straggler, + /// stand in its way before it calls the store stuck. Nothing correct ever + /// spends it: the code under test finishes in milliseconds, and the limit + /// is only what stops a blocked write from hanging the suite. No + /// assertion compares elapsed time to a bound the machine's load can move. + #[cfg(unix)] + pub(crate) const STUCK_AFTER: Duration = Duration::from_secs(30); + + /// Run `work` while `held` is held, and fail if it waited on it. The holder + /// lets go only when `work` has returned, or after `STUCK_AFTER`: work that + /// waits on the lock is freed by that second release and is then seen to + /// have waited, so the test fails instead of hanging. #[cfg(unix)] pub(crate) fn run_while_held(held: HeldLock, work: impl FnOnce() -> T) -> T { - let (release, released) = std::sync::mpsc::channel::<()>(); - let holder = std::thread::spawn(move || { - let _ = released.recv_timeout(Duration::from_secs(3)); - drop(held); - }); - let started = Instant::now(); + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, mpsc}; + + let (finished, work_done) = mpsc::channel::<()>(); + let gave_up = Arc::new(AtomicBool::new(false)); + let holder = { + let gave_up = Arc::clone(&gave_up); + std::thread::spawn(move || { + if work_done.recv_timeout(STUCK_AFTER) == Err(mpsc::RecvTimeoutError::Timeout) { + gave_up.store(true, Ordering::SeqCst); + } + drop(held); + }) + }; let result = work(); - let waited = started.elapsed(); - let _ = release.send(()); + let _ = finished.send(()); holder.join().unwrap(); - assert!(waited < Duration::from_secs(2), "the work waited {waited:?} on a held lock"); + assert!(!gave_up.load(Ordering::SeqCst), "the work waited on a held lock"); result } - /// Retry `attempt` for up to two seconds. A released lock can stay held for - /// a moment by a child another test forked while it was held. + /// Retry `attempt` until it lands. A released lock can stay held for a + /// moment by a child another test forked while it was held. #[cfg(unix)] fn eventually(mut attempt: impl FnMut() -> Result) -> T { - let deadline = Instant::now() + Duration::from_secs(2); + let deadline = Instant::now() + STUCK_AFTER; loop { match attempt() { Ok(value) => return value, From 8d0786dcaf6b2eb31791027664f2ebdcaa4943e6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 23:30:44 +0200 Subject: [PATCH 045/196] Watchdog map: topology lock probes C29-C40 Only the daemon writes topology.cc; Odin and Idunn read it lock-free; the root backup's shared lock survives a private 0600 lock. A chmod does not revoke an existing holder, so the shared primitive renames a fresh 0600 lock over a non-private one. Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-map.md | 64 +++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index 70bcd93..5d98c4d 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -118,6 +118,32 @@ file ran alone with the TAP reporter, then the whole glob ran once. | C27 | Suite with the fix | scratch commit `a6770c5` on `42728a67`: the scoped name at the seven live C21 sites, a skip guard on both Rust tests, and the `CARGO_TARGET_DIR` override removed. Same command. | rust-smoke 1/0/0/1 (`# SKIP`); delivery 9/9/0/0; permit 2/2/0/0; rudp 1/0/0/1 (`# SKIP`); feedback-cli 8/8/0/0; idunn-health 3/3/0/0. Glob total 24 tests, 22 pass, 0 fail, 2 skipped, exit 0. `agent-transport.mjs` and `governance-threads.mjs` were checked only by name resolution (C22), because no test loads them. | | C28 | C3 at the reader's install pin | `git show 36ea08d3:packages/cultcache-ts/src/single-file-messagepack-backing-store.ts` | `pullAll` is at `:43`, and the file has no lock. C3 holds at `36ea08d3`, which is the CultLib that cut `ops-notice-deploy` installs. | +### Topology-lock cut probes (2026-09-30, 21:00-21:25 UTC) + +Pinned heads: Idunn origin/main `bca3d85`; `hands/watchdog-idunn-incident` +local `9549d00` (h3, not yet pushed at probe time; origin at `3ed5f26`); +`hands/idunn-projection-mode` `15f7e18` (unmerged); Odin `665dc08`; +CodexConnector `6519289`; Ghostlight `eeb3a68`; gamecult-ops `fdb4356`; +cultcache-rs at Idunn's pin, CultLib `61549aa`. Line numbers marked *trial* +are from a scratch merge of `9549d00` into `bca3d85` (clean, no conflicts), +never in `F:\Projects\Idunn`. Yggdrasil probes were read-only over `ssh ygg`; +the semantics probe (C36) ran in a container through `ygg-verify.sh`. + +| # | What | Where / command | Result | +|---|---|---|---| +| C29 | Every writer of `topology.cc` | trial `git grep` for `CultCacheTopologyDriver {`, `Engine::open(`, `mutate(` | One writer: the `idunn serve` daemon. `CultCacheTopologyDriver::mutate` (trial `src/drivers.rs:4393-4411`) is the only write path; its seven callers are `demote_to_expected_only` `:4455`, `withdraw_stale_incarnation` `:4567`, and the `TopologyPort` methods `:4778, 4846, 4897, 4940, 4992`. `Engine::open` has one production caller, `boot` (`src/control_plane.rs:5330`); the CLI verbs `up`, `status`, `cancel`, `expire` and `validate` write only `control.cc`. `idunn-host` and `idunn-provision` never touch the projection. | +| C30 | Every reader of `topology.cc`, across repos | `git grep -e 'topology\.cc' -e idunn-projection HEAD` in every repo under `F:\Projects` | Readers: Odin (`crates/odin-daemon/src/lib.rs` `CultCacheIdunnProjectionSource::entries`, `pull_all_read_only_snapshot`, lock-free since its first commit `eedce4f`, 2026-09-04), Idunn itself (`snapshot`, trial `drivers.rs:4383-4388`, lock-free; every `control_plane.rs` read is `pull_all_read_only_snapshot`), and the nightly backup (C33). No managed target reads the file: CodexConnector, Ghostlight, Muninn and StreamPixels name only `odin.verse-topology.v1`, and they receive their Expected record in the runtime bundle (`GAMECULT_IDUNN_RUNTIME_BUNDLE`). The unit comment (`deploy/idunn-yggdrasil.service:77-80`) and `docs/migration.md:505-509` say that "every managed target must read it", which is stale. | +| C31 | Shared-lock readers that rely on blocking a writer | cultcache-rs `61549aa` `lib.rs:337-395` (`with_read_only_shared_snapshot`), `:905-910` (`pull_all`, which takes `lock_shared`); cross-repo grep for `with_read_only_shared_snapshot` | That API holds the sibling lock shared so that no writer can replace the snapshot during `action`. Its production users are CodexConnector (`src/idunn_health.rs:441, 562`, `src/daemon.rs:205`) and Ghostlight (`crates/ghostlight-dungeon/src/idunn_health.rs:901`). All of them read the runtime authority record, the process-write-lease record, or their own configuration. None reads `topology.cc`. The write-lease lock is a different file under `idunn-authority` (`CultCacheWriteLeaseDriver`, trial `drivers.rs:4173, 4218`, already `try_compare_exchange_snapshot`). No reader of `topology.cc` takes a lock to get a consistent view. | +| C32 | How a reader stays consistent without the lock | cultcache-rs `61549aa` `lib.rs:666-688` (`write_all_unlocked`) | Every write stages `.tmp`, fsyncs it, `replace_file_atomically` renames it over the store, and fsyncs the directory. A reader that opens the path sees one whole inode. Consistency comes from the rename, not the lock. | +| C33 | The nightly backup's lock use | live `/usr/local/sbin/gamecult-state-backup` sha256 `9d8ebd84...`, identical to gamecult-ops `scripts/gamecult-state-backup-yggdrasil.sh`; lines 98-126; `systemctl cat gamecult-state-backup.service` | Every file with a `.lock` sibling is re-copied under `flock -s -w 20 -E 75 cp -a`. Past the wait it falls back to a double-read. The unit has no `User=`, so it runs as root: a `0600 root` lock still opens (`flock(1)` opens the path), and the backup keeps its shared lock. It skips a store whose lock is absent (`[[ -f lock ]] || continue`) and never creates one. The copy is milliseconds of `cp` once a night. Under a try-lock, a publish in that window is contended and retried on a later pass (C35). The backup does not need the lock for a whole copy (C32). | +| C34 | Live state of the projection dir | `ls -la`, `getfacl -p /var/lib/gamecult/idunn-projection`; `/proc/locks` against the lock's inode; `find /proc/*/fd -lname` | `drwxr-xr-x+ root`, `default:user::rw- default:group::r-- default:other::r--`. `topology.cc` `0644`, 13436 bytes, written 21:07. `topology.cc.lock` is `0644 root`, inode 8227809, created 2026-09-06. No flock is held on it and no process has either file open. `incidents.cc` does not exist yet, because the incident cut is not installed. The default ACL **is** documented, in Idunn `docs/migration.md:523` (it closes the chmod window), though gamecult-ops does not record it. `idunn-yggdrasil.service`: `User=root`, `UMask=0027`, `WatchdogUSec=0`. Odin runs as DynamicUser uid 62025. | +| C35 | What a failed topology publish does inside the tick | trial `src/control_plane.rs` at each call site | Every tick-path write returns `Err` through `?` **before** any evidence of it is persisted, and the step is retried. `advance_starting` records `expected_publication_sha256` or `activation_publication_sha256` only after publish returns `Ok` (`:6605`, `:6664`). `advance_leasing` does the same for the write-lease projection digest (`:7004`, `:7073`). `advance_fencing` withdraws the lease projection before recording (`:6894`, `:6926`). The post-fencing abort and pre-fencing cleanup write `CleanupEvidence::Complete` only after `reconcile_failed_candidate_projection` (`:8956`, `:9046`, `:9180-9204`). A failed continuity demotion (`:6099`) mints no restart and defers continuity. The admitted Expected repair (`:5870`) is logged and re-checked every pass. `retire_stale_incarnations` (`:7914`) returns `Err`. **The one path that is not retried is boot** (`reconcile_failed_continuity_projections`, `:5228-5275`, called once from `boot` `:5332`). It logs a failure and does not retry, and it is outside the tick. | +| C36 | Lock semantics under the live ACL | `ygg-verify.sh Idunn bca3d85 python:3.12-slim` running scratch `probe_lock.py`, as root | (1) A file created with mode 0666 under this default ACL and umask 027 lands `0644`, CultCache's shape. (2) A separate process holding `LOCK_EX` on it makes a nonblocking try from the parent contend. (3) **`fchmod 0600` on the same inode does not revoke that holder**: the try still contends. (4) A 0600 `O_CREAT|O_EXCL` staged file renamed over the lock lands `0600`, and the try on the new inode **succeeds** while the old holder still holds the orphaned inode. (5) uid 65534 cannot open the replaced lock (`Permission denied`). | +| C37 | The incident cut's private-lock mechanism | `git show 9549d00:src/incident.rs` `:229-245` (`exchange`), `:336-375` (`secure_lock`) | `exchange` calls `secure_lock(path)`, then `try_compare_exchange_snapshot`, and turns `Mismatch` and `LockContended` into errors. `secure_lock` opens `.lock` with `create` and mode `0600`, then runs `set_permissions(0600)` on that descriptor. Its doc says that a descriptor another uid already holds is not revoked. So a lock left `0644` (C34's live `topology.cc.lock`) stays holdable by whoever opened it before (C36 step 3). It builds the lock name by hand, duplicating `authority_lock_path` (trial `drivers.rs:8194`). | +| C38 | `hands/b5` | `git merge-base --is-ancestor` | `hands/b5` (`f85e743`) and `hands/b5-pins` (`1cfbeff`) are both ancestors of origin/main (merged at `fcb8ba1` and `ddc031a`). The topology cut has no B5 conflict. | +| C39 | Unmerged branches that touch the topology driver | for each branch not in origin/main: `git diff origin/main... -- src/drivers.rs` | Two. `hands/watchdog-idunn-incident` (the dependency). `hands/idunn-projection-mode` (`15f7e18`, 4 commits, 2026-09-30 01:15-02:40 UTC, no campaign doc) deletes `publish_projection_mode`, has `mutate` set a default ACL on the directory by `setxattr` (`publish_directory_default_mode`), and replaces the mode test with `the_published_projection_is_never_visible_in_another_mode`. That test asserts the **lock** is always `0644`, which contradicts `tick-lock-private`. It conflicts textually with this cut at `mutate` and at `publish_projection_mode`. | +| C40 | The private root's locks | `ls -ld /var/lib/gamecult/idunn`; `sudo ls -la` inside it; `getent passwd idunn`; `ps` | `/var/lib/gamecult/idunn` is `drwxr-x--- idunn:idunn` (uid 986, group 978, which has no members; no process runs as `idunn`). `control.cc.lock` is `0644 root`. The tick's control-store writes use blocking `compare_exchange` (e.g. trial `control_plane.rs:5471`), so root (the backup's `flock -s`) or uid `idunn` could hold one. Nothing runs as `idunn` today. | + ## Model page A row per persistent kind this campaign creates or depends on. "New" means the @@ -127,6 +153,7 @@ cut spec fixes them. | Kind | Identity | Lifecycle | Authority | |---|---|---|---| | **Incident record** (new, `idunn.operator_incident.v1`, its own file `/var/lib/gamecult/idunn-projection/incidents.cc`, published 0644 beside `topology.cc`) | Key `::`. Namespace: `incidents.cc`, one type; the file refuses any other. It is never written to `control.cc`, which bails on unknown types, so a rollback would stop continuity (C1). `condition` is a closed enum (`continuity-exhausted` first; `odin-store-contested`, `odin-store-unpersisted` and `operator-required` join when their sources exist). `subject` is the target for target conditions and the `transaction_id` for `operator-required`. `opened_at` is written once by Idunn's clock. Injective because of one rule: **at most one open incident per `(condition, subject)`**, checked by a compare-exchange create. | **Created** at the site that decides the condition, after that decision and before its trace line. **Closed** by writing `closed_at` and `close_reason` on the same record, never by deleting it. Continuity-exhausted closes when the pass sees the workload running (`recovered`) or the target is no longer admitted, **not** when the window slides (I2). Odin conditions close when the catalog stops reporting them. `operator-required` has **no closing path yet** (I6; follow_up `b5-operator-required-no-exit`). **Reopened**: never; a recurrence is a new record with a new `opened_at`. **Replayed after an Idunn restart**: the durable open record makes the site write nothing. **Pruned**: closed more than 7 days ago, it retires to `history.cc`. **Store failure**: reported once as a fault and never returned into the tick (`survival-independent`). | Idunn's daemon, at the decision site, is the only writer. The same file is the record and what Bifrost reads, so no separate projection exists to drift. Forbidden: the CLI (`status` reads only), Bifrost and every other reader, Odin, `ReportOnce`, `control.cc`, a repair loop that opens incidents from log text. | +| **Projection lock files** (exist: `topology.cc.lock`; new with the incident cut: `incidents.cc.lock`, `incident-history.cc.lock`) | `.lock` beside its store, named by `authority_lock_path`. One per store file. | **Created** by Idunn at mode `0600` before its first exchange on that store. **Replaced** by a staged `0600` file renamed over it whenever Idunn finds it wider than `0600` or not owned by Idunn's euid; a chmod cannot revoke an existing holder (C36). **Held** only for one nonblocking exchange. **Never published**: the store is `0644`, and its lock is not. | Idunn's daemon alone opens it for writing. The root backup may take it shared for its copy (C33), and a contended exchange is retried. Forbidden: any reader opening it (none does, C30-C31), `publish_file_mode` on it, a default ACL deciding its mode. | | **Stderr trace** (exists) | `report_once` key, in memory | Printed once per process lifetime per fault; lost on restart (I1). | **Demoted**: `ReportOnce` is no longer an owner of "has the operator been told". It de-dups stderr only. | | **Delivery request / journal entry** (Bifrost store) | Keyed by the Idunn incident key plus the notice kind (`opened`, and `closed` if Q `closure-notice` says so). Injective because the incident key is. Not derived from payload text or time (BF1's `raisedAt` hash is the counter-example). | **Created** by Bifrost when its reader first sees a notice-worthy incident state in the projection. **Revised**: `pending` -> `sent` or `failed`, with an attempt count and last error. **Retried** until sent, with backoff; a create for an existing key writes nothing (V1's unconditional `put` is the counter-example). **Replayed after a Bifrost restart**: journal is durable; a `pending` entry is retried. **Pruned** only after the incident key has left Idunn's projection. | Bifrost's reader only. Forbidden: Idunn (it never writes Bifrost's store, even though Y2 shows it physically could), VoidBot, the operator by hand. Which Bifrost process: see Rationale. | | **Delivery receipt** (Bifrost, `discord_post_receipt.v1` or the journal's `sent` state) | Same key as the request. | Written once on success with the Discord message id. | Bifrost. **Idunn does nothing with it**: it neither reads it in the daemon nor waits on it (`survival-independent`). `idunn status` does not show delivery state; Bifrost's journal is where a failed delivery is visible. | @@ -217,10 +244,13 @@ and moves on. An unreadable store means no incident opens (the stderr trace still prints); it never changes what continuity decides. The Idunn cut's survival test pins this with an unwritable store path. -**Bifrost reads without the lock.** Idunn's writes block on an exclusive lock -(C2). A reader that took the lock could stall a tick. cultcache-ts `pullAll` -takes none (C3, C4), and Idunn's rename is atomic, so the reader only ever sees -a whole snapshot. The reader never opens Idunn's file through a CultMesh node, +**Bifrost reads without the lock.** Idunn's writes once blocked on an +exclusive lock (C2). The incident and topology cuts make every write in the +projection directory a nonblocking exchange behind a private lock. A reader +that took the lock is therefore unnecessary. It would also be refused, because +it cannot open a `0600` lock. cultcache-ts `pullAll` takes no lock (C3, C4), +and Idunn's rename is atomic (C32), so the reader only ever sees a whole +snapshot. The reader never opens Idunn's file through a CultMesh node, which could flush. **Delivery at most once, with the nonce as the second guard.** This follows @@ -293,3 +323,29 @@ goes, because it is wrong on every host except one workstation. two branches merge in either order without conflict. The reader depends on both. It is written against a tree with one Discord command vocabulary, and it is verified by a suite that loads. + +**The topology write stops waiting, and the lock stops being published.** The +lock was published `0644` because a comment assumed that readers open it +(`publish_projection_mode`, from `fbb9626`). None does (C30, C31), and +consistency comes from the rename (C32). So the lock is Idunn's alone, and a +contended publish is an error, not a wait. That is safe because every tick path +treats a failed publish as not done: no publication digest, phase transition, +cleanup evidence or continuity restart is persisted before the write returns +`Ok` (C35). A contended publish is retried on a later pass or after backoff, +and Idunn never believes it published what readers do not see. Boot +reconciliation is the exception. It runs once, outside the tick, and it already +logs failures and moves on. Only root processes can contend once the lock is +private: the backup's copy, a few milliseconds a night. No retry machinery is +added for it. + +**One lock rule for the whole projection directory.** The topology cut moves +the incident cut's `secure_lock` into one crate-level exchange primitive that +both stores call, rather than a second copy. It also strengthens the rule: +a lock wider than `0600`, or owned by someone else, is replaced by a rename, +not chmodded. A chmod does not revoke a holder (C36), and the live +`topology.cc.lock` has been `0644` since 2026-09-06 (C34), so tightening it in +place would leave any existing holder able to block every topology publish +forever. The replacement runs only while the lock is not yet private. Once it +is, no other uid can open it, so a later replacement never splits exclusion +between two inodes. A root backup holding the old inode during the first +replacement copies one whole inode anyway (C32). From 7e4163645d493ac24c359b778112d999a2a86db7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 23:54:55 +0200 Subject: [PATCH 046/196] Watchdog target: tick-lock-private is uid-scoped Operator ruled tick-lock-scope = uid-scoped (target r4). Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-target.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index ac9ea82..381565d 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -66,7 +66,9 @@ the machine has a detector with no record, and a deliverer that nothing feeds. clears, one closure notice follows, and only if the opening notice was sent. Repeated delivery attempts for either notice are idempotent at the recipient's end. -- **No tick waits on a lock someone else can hold.** Soul found that any +- **No tick waits on a lock another uid can hold.** Root holders, such as + the nightly backup's millisecond shared lock, are trusted and bounded. The + operator ruled this scope on 2026-09-30, choosing uid-scoped. Soul found that any local uid could freeze Idunn's scheduler tick by holding a published `.lock` file. This was true for incidents during the campaign, and for `topology.cc` before it. Idunn's writes in its projection directory are From 102c1298911bceee51ba7906400d135b055999b8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 01:28:05 +0200 Subject: [PATCH 047/196] Topology publishes never wait: one private-lock exchange primitive exchange_behind_private_lock (drivers.rs) is the single write primitive for the projection directory: one nonblocking compare-exchange behind a lock only Idunn can hold. An absent lock is created O_EXCL 0600; a lock that is not a regular file, ours, mode 0600 is replaced by a fresh 0600 file renamed over it (a chmod does not revoke a descriptor another uid already holds); a private lock is opened as is. The topology driver's mutate and the incident store both go through it; the blocking compare_exchange_snapshot call, publish_projection_mode (which published the lock 0644 on a false premise that readers open it) and incident.rs secure_lock are deleted. mutate now retries only a lost race; a held lock is Err "the topology projection is locked by another holder" and a no-op, so no caller records a digest, evidence or phase for a publish that did not land. Test helpers: the lock holder is a separate flock process, and run_while_held counts opens of the lock (inotify), so ANY wait on a held lock fails, not only one past the 30 s ceiling. Tests reproduce the live default ACL and use setpriv for the other-uid half. This commit has not been built; the verify run follows. Co-Authored-By: Claude Sonnet 5.5 --- deploy/idunn-yggdrasil.service | 5 +- docs/migration.md | 2 +- src/control_plane.rs | 90 ++++++- src/drivers.rs | 421 ++++++++++++++++++++++++++++++--- src/incident.rs | 268 ++++++++++++++------- 5 files changed, 651 insertions(+), 135 deletions(-) diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index 9758127..da82bd2 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -77,7 +77,10 @@ ReadWritePaths=/etc/ufw # idunn-projection is deliberately outside Idunn's own 0750 directory: the # topology store is Idunn's *published* surface, and every managed target must # read it to verify its own Expected incarnation against the Idunn anchor. It -# was inside the private root, where no workload could traverse to it. +# was inside the private root, where no workload could traverse to it. The +# published topology store is read lock-free (Odin) and incidents.cc is read by +# Bifrost's reader; managed targets get Expected from their runtime bundle. Only +# the stores are readable: each .lock beside them is Idunn's alone, 0600. ReadWritePaths=/var/lib/gamecult/idunn /var/lib/gamecult/idunn-authority /var/lib/gamecult/idunn-projection ReadWritePaths=/srv/build # Each managed target gets only its bound roots. A target Idunn cannot write is diff --git a/docs/migration.md b/docs/migration.md index 0c810fc..a78b4a0 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -520,7 +520,7 @@ started, ran, and then died on a file it could not open: | `runtime_root` and every directory above it | traversable by the workload: `o+x`, or group-owned by the state group with `g+x` | the workload is a `DynamicUser` and owns nothing, and `ReadOnlyPaths=` binds the bundle's leaf into the namespace without granting traversal above it. Idunn now checks the whole chain and names the closed directory | | `runtime_root` | setgid, group-owned by the state group (`2750 root:`) | `harden_root_authority_file` requires the write-lease record's group to equal its parent's, and only setgid makes a newly created record inherit it. Without it the record lands `root:root` and the workload cannot read its own lease | | the write-lease record | `root:`, mode `0640` | follows from the setgid parent; the workload reads it through the state group. Idunn writes it, so fixing an existing wrong-group record means deleting it, and that is safe only when no transaction is live | -| `/var/lib/gamecult/idunn-projection` | default ACL `u::rw-,g::r--,o::r--` | Idunn runs with `UMask=027`, so each publish creates the store and its `.lock` at `0640`. The code re-opens them after every write, but that leaves a window; a default ACL sets the mode at creation and closes it | +| `/var/lib/gamecult/idunn-projection` | default ACL `u::rw-,g::r--,o::r--` | Idunn runs with `UMask=027`, so a store is created `0640` and each publish re-opens it `0644`; the default ACL sets the mode at creation and closes that window for stores. The `.lock` is not published and no reader opens it: Idunn creates it `0600` (replacing one found wider, so a descriptor another uid already holds is orphaned), and the ACL never decides its mode | | `/etc/nginx/idunn-stream-routes` | `root:root`, not group-writable | "route authority parent is not canonical root-owned and nonwritable". It was `root:idunn 0775` from the generation where Idunn ran unprivileged and needed to write it as a group member | Two shapes are worth stating plainly because they are easy to get backwards. diff --git a/src/control_plane.rs b/src/control_plane.rs index d3edbf8..75af38f 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15496,7 +15496,7 @@ mod tests { Ok(()) } - #[cfg(unix)] + #[cfg(target_os = "linux")] #[test] fn a_held_incident_lock_changes_nothing_continuity_decides() -> Result<()> { use crate::incident::tests::{STUCK_AFTER, hold_lock_of, run_while_held}; @@ -15510,7 +15510,8 @@ mod tests { let world = EngineFixture::with_workload(workload.clone())?; admit_incumbent(&world)?; let path = incident_path(&world); - let actual = run_while_held(hold_lock_of(&path), || { + // Three ticks reach the open of an exhausted target, one exchange each. + let actual = run_while_held(hold_lock_of(&path), 3, || { continuity_decisions(&world, &world.engine, &workload) })?; assert_eq!(actual, expected, "a held incident lock changed a continuity decision"); @@ -15540,6 +15541,91 @@ mod tests { assert_control_store_holds_no_incident(&world) } + /// A publish that could not land is not done, at the tick: the demotion + /// continuity needs before a restart is one exchange on the topology lock, + /// and another holder of it defers the restart instead of waiting for it. + #[cfg(target_os = "linux")] + #[test] + fn a_held_topology_lock_defers_continuity_and_mints_nothing() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let store = world.engine.options.topology_store.clone(); + let projected = projected_under(&world, &incumbent.expected)?; + assert!(projected.contains(&IdunnRuntimeActivationRecord::TYPE.to_owned())); + let before = std::fs::read(&store)?; + workload.kill(); + + // One exchange: the demotion. The window is not spent, so continuity + // would restart if the projection could be demoted. + let moved = run_while_held(hold_lock_of(&store), 1, || { + world.engine.supervise_one_admitted_generation() + })?; + assert!(moved, "the deferral is a write the tick made"); + assert_eq!(std::fs::read(&store)?, before, "a contended publish changed the projection"); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!(snapshot.transactions.is_empty(), "continuity minted over a projection it could not demote"); + let recorded = snapshot.supervision_for("service").context("no deferral recorded")?.value.clone(); + assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); + let reason = recorded.continuity_deferral_reason.clone().context("no deferral reason")?; + assert!(reason.contains("locked by another holder"), "{reason}"); + assert!(!reason.contains(&world.root.display().to_string()), "{reason}"); + + // The holder is gone and the deferral has run out: the demotion lands + // and the restart proceeds. + let mut ran_out = recorded; + ran_out.continuity_deferred_until = Some(1); + set_meters(&world, &ran_out)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + /// A deployment phase records a publication digest only after the publish + /// returned: the digest is the claim that it landed. + #[cfg(target_os = "linux")] + #[test] + fn a_contended_expected_publish_is_not_recorded_as_published() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Warming)?; + let stored = resident(&world)?; + let mut starting = seeded.clone(); + starting.phase = DeploymentPhase::Starting; + starting.expected_publication_sha256 = None; + starting.activation = None; + starting.workload = None; + starting.activation_publication_sha256 = None; + starting.warming = None; + replace_transaction(&world.state_store, &stored, &starting)?; + let store = world.engine.options.topology_store.clone(); + let stored = resident(&world)?; + + let error = run_while_held(hold_lock_of(&store), 1, || { + world.engine.advance_transaction(&stored) + }) + .expect_err("a contended publish must not advance the phase"); + assert!(format!("{error:#}").contains("locked by another holder"), "{error:#}"); + assert!(!store.exists(), "a contended publish wrote the projection"); + let after = resident(&world)?.value; + assert_eq!(after.phase, DeploymentPhase::Starting); + assert_eq!(after.expected_publication_sha256, None); + + // Free again: the same step publishes and records what it published. + world.engine.advance_transaction(&resident(&world)?)?; + let expected = seeded.expected.context("seeded transaction has no Expected")?; + assert_eq!( + resident(&world)?.value.expected_publication_sha256, + Some(expected.canonical_sha256()?) + ); + Ok(()) + } + #[test] fn status_prints_an_open_incident_under_its_target() -> Result<()> { let workload = SwitchWorkload::new(); diff --git a/src/drivers.rs b/src/drivers.rs index 8fd3192..5dea8ad 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -4247,6 +4247,10 @@ impl WriteLeasePort for CultCacheWriteLeaseDriver { /// facts here. Service presence is absent by construction; only Odin may /// correlate these records with signed runtime observation into Present/Ready. /// +/// The store is published 0644 and read lock-free by Odin and Idunn; its lock +/// is Idunn's alone (`exchange_behind_private_lock`), so no other uid can hold +/// it and stall a publish. +/// /// Every record is keyed by the incarnation it describes, never by the target /// alone. A target being replaced has two incarnations at once -- the admitted /// incumbent and the sealed candidate -- and both are projected side by side. @@ -4390,22 +4394,28 @@ impl CultCacheTopologyDriver { /// Apply one whole-snapshot mutation with compare-and-swap. `mutate` /// returns `None` when the snapshot already has the shape it wants, and /// the replacement set otherwise; every writer below is one of these. + /// + /// Never waits: a publish is done exactly when this returns `Ok`. Only a + /// lost race (the store moved on) is retried at once; a held lock is an + /// error and a no-op, retried by the caller's own retry. fn mutate(&self, mutate: F) -> Result<()> where F: Fn(&[CultCacheEnvelope]) -> Result>>, { - if let Some(parent) = self.projection_store.parent() { - fs::create_dir_all(parent)?; - } - let store = SingleFileMessagePackBackingStore::new(&self.projection_store); for _ in 0..8 { let entries = self.snapshot()?; let Some(replacement) = mutate(&entries)? else { return Ok(()); }; - if store.compare_exchange_snapshot(&entries, &replacement)? { - publish_projection_mode(&self.projection_store)?; - return Ok(()); + match exchange_behind_private_lock(&self.projection_store, &entries, &replacement)? { + TryCompareExchangeSnapshotOutcome::Exchanged => { + publish_file_mode(&self.projection_store)?; + return Ok(()); + } + TryCompareExchangeSnapshotOutcome::Mismatch => {} + TryCompareExchangeSnapshotOutcome::LockContended => { + bail!("the topology projection is locked by another holder") + } } } bail!("CultCache projection changed repeatedly during publication") @@ -7710,19 +7720,95 @@ pub(crate) fn publish_file_mode(path: &Path) -> Result<()> { Ok(()) } -/// The topology store is Idunn's *published* surface: every managed target -/// reads it to verify its own Expected incarnation against the Idunn anchor. +/// The one write primitive for a store in the projection directory: one +/// nonblocking compare-exchange of the whole store against the snapshot it was +/// decided on, behind a lock only Idunn can hold. /// -/// Idunn runs with `UMask=027`, which is right for its private state and wrong -/// for this one file -- it lands `0640 root:root`, and a `DynamicUser` workload -/// gets EACCES. Integrity here comes from the signatures over the records, not -/// from the mode, so the published copy is readable. The lock sibling too: -/// CultCache opens it alongside the store, so a 0640 lock denies the read just -/// as surely as a 0640 store, and it is created fresh under Idunn's umask on -/// every publish. -fn publish_projection_mode(path: &Path) -> Result<()> { - publish_file_mode(path)?; - publish_file_mode(&authority_lock_path(path)) +/// Readers of a published store take no lock, so nobody but Idunn needs to +/// open its lock, and a lock another uid can open is a lock another uid can +/// hold to stall every publish. CultCache creates a lock with mode 0666, so +/// the directory's default ACL would decide who may (Yggdrasil's lands it +/// 0644). This makes the lock private first, by this rule: an absent lock is +/// created `O_EXCL` 0600; a lock that is not (a regular file, ours, 0600) is +/// replaced by a fresh 0600 file renamed over it, because a chmod does not +/// revoke a descriptor another uid already holds, while the rename leaves that +/// holder only an orphaned inode. A private lock is opened as it is. No +/// existing lock is ever chmodded. Nothing here names a path in an error. +pub(crate) fn exchange_behind_private_lock( + store: &Path, + expected: &[CultCacheEnvelope], + replacements: &[CultCacheEnvelope], +) -> Result { + make_lock_private(store)?; + SingleFileMessagePackBackingStore::new(store) + .try_compare_exchange_snapshot(expected, replacements) +} + +#[cfg(unix)] +fn make_lock_private(store: &Path) -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let lock = authority_lock_path(store); + if let Some(parent) = lock.parent() { + fs::create_dir_all(parent).context("creating the store directory")?; + } + // SAFETY: geteuid has no preconditions. + let euid = unsafe { libc::geteuid() }; + // The daemon is the only writer, so a lost create race is defensive only. + for _ in 0..3 { + match fs::symlink_metadata(&lock) { + Ok(metadata) + if metadata.is_file() + && metadata.uid() == euid + && metadata.permissions().mode() & 0o777 == 0o600 => + { + return Ok(()); + } + Ok(_) => return replace_with_private_lock(&lock), + Err(error) if error.kind() == ErrorKind::NotFound => { + match create_private_file(&lock) { + Ok(()) => return Ok(()), + Err(error) if error.kind() == ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("creating a private store lock"), + } + } + Err(error) => return Err(error).context("inspecting a store lock"), + } + } + bail!("a store lock kept changing while it was made private") +} + +#[cfg(not(unix))] +fn make_lock_private(store: &Path) -> Result<()> { + let _ = store; + Ok(()) +} + +/// A new file nothing else has open, mode 0600 (the create mode is intersected +/// with the umask and the directory's default ACL, which only narrow it). +#[cfg(unix)] +fn create_private_file(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::OpenOptionsExt; + + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .map(drop) +} + +/// Stage a fresh private file beside `lock` and rename it over the lock. +#[cfg(unix)] +fn replace_with_private_lock(lock: &Path) -> Result<()> { + let mut staged = lock.as_os_str().to_os_string(); + staged.push(format!(".{}", Uuid::new_v4().simple())); + let staged = PathBuf::from(staged); + create_private_file(&staged).context("staging a private store lock")?; + fs::rename(&staged, lock).map_err(|error| { + let _ = fs::remove_file(&staged); + anyhow::Error::new(error).context("replacing a store lock with a private one") + }) } fn build_machine_id(workspace: &Path) -> Result { @@ -9670,27 +9756,284 @@ Content-Le".to_vec()), |_| {}), )); } - #[cfg(unix)] - #[test] - fn the_published_projection_is_readable_by_a_workload() { - use std::os::unix::fs::PermissionsExt; + /// A native process that is the Expected's: what `publish_observed_activation` + /// checks the activation against. + #[cfg(target_os = "linux")] + fn observation_of( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> WorkloadObservation { + let mut host = host_audit_observation(1, 1); + host.runtime_instance_id = activation.runtime_instance_id.clone(); + host.executable_sha256 = expected.artifact_sha256.clone(); + WorkloadObservation::Host(host) + } - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join("topology.cc"); - std::fs::write(&path, b"x").unwrap(); - // What Idunn's UMask=027 leaves behind. - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o640)).unwrap(); - let lock = temp.path().join("topology.cc.lock"); - std::fs::write(&lock, b"").unwrap(); - std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o640)).unwrap(); - publish_projection_mode(&path).unwrap(); - let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; - assert_eq!( - mode, 0o644, - "every target must be able to read the projection" - ); - let lock_mode = std::fs::metadata(&lock).unwrap().permissions().mode() & 0o777; - assert_eq!(lock_mode, 0o644, "the lock is opened alongside the store"); + #[cfg(target_os = "linux")] + fn mode_of(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path).unwrap().permissions().mode() & 0o777 + } + + #[cfg(target_os = "linux")] + fn inode_of(path: &Path) -> u64 { + use std::os::unix::fs::MetadataExt; + fs::metadata(path).unwrap().ino() + } + + /// Idunn's `UMask=027` alone lands a file 0640: the store still has to be + /// published, and its lock has to stay private. + #[cfg(target_os = "linux")] + #[test] + fn a_published_store_is_readable_by_a_workload_and_its_lock_is_not() -> Result<()> { + use crate::incident::tests::under_service_umask; + + let temp = tempfile::tempdir()?; + let (expected, _, _, anchor) = authenticated_warming(temp.path())?; + let driver = topology_driver(temp.path(), "topology"); + under_service_umask(|| driver.publish_expected(&expected, &anchor))?; + assert_eq!(mode_of(&driver.projection_store), 0o644); + assert_eq!(mode_of(&authority_lock_path(&driver.projection_store)), 0o600); + Ok(()) + } + + /// The body's condition: the projection directory carries the default ACL + /// `u::rw g::r o::r`, under which the umask is ignored and a lock CultCache + /// creates lands 0644. Every writer publishes the store 0644 and leaves + /// the lock 0600, and another uid cannot open the lock. + #[cfg(target_os = "linux")] + #[test] + fn under_a_default_acl_the_projection_is_published_and_its_lock_is_private() -> Result<()> { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + use crate::incident::tests::{give_default_acl, readable_by_another_uid, under_service_umask}; + + let temp = tempfile::tempdir()?; + let (expected, activation, _, anchor) = authenticated_warming(temp.path())?; + let observation = observation_of(&expected, &activation); + let dir = temp.path().join("projection"); + fs::create_dir(&dir)?; + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755))?; + give_default_acl(&dir); + let driver = topology_driver(&dir, "topology"); + let store = driver.projection_store.clone(); + let lock = authority_lock_path(&store); + + // Control: the lock CultCache would have made here. Without the ACL in + // force this reads 0640 and the assertions below prove nothing. + let control = dir.join("control.lock"); + under_service_umask(|| { + OpenOptions::new().create(true).write(true).mode(0o666).open(&control) + })?; + assert_eq!(mode_of(&control), 0o644, "the default ACL is not in force"); + + let published = |step: &str| { + assert_eq!(mode_of(&store), 0o644, "the store after {step}"); + assert_eq!(mode_of(&lock), 0o600, "the lock after {step}"); + }; + under_service_umask(|| driver.publish_expected(&expected, &anchor))?; + published("publish_expected"); + let private_lock = inode_of(&lock); + under_service_umask(|| { + driver.publish_observed_activation(&expected, &activation, &observation) + })?; + published("publish_observed_activation"); + under_service_umask(|| { + driver.demote_to_expected_only(&expected, &anchor, &activation, None) + })?; + published("demote_to_expected_only"); + under_service_umask(|| driver.withdraw_incarnation(&expected, &anchor, None, None))?; + published("withdraw_incarnation"); + assert_eq!(inode_of(&lock), private_lock, "a private lock was replaced"); + + // The consequence: another uid cannot open the lock, so it cannot hold + // it. Provable only as root with setpriv, and only if that uid can + // read the control file, which shows the switch itself works. + if readable_by_another_uid(&control) == Some(true) { + assert_eq!(readable_by_another_uid(&lock), Some(false)); + } else { + eprintln!("UNPROVEN: no uid switch here; the lock modes above stand alone"); + } + Ok(()) + } + + /// Another process holds the lock: the publish is an error and a no-op, and + /// it lands once the lock is free. Nothing waits (`run_while_held` counts + /// the opens of the lock), and the error names the store kind, not a path. + #[cfg(target_os = "linux")] + #[test] + fn a_topology_publish_never_waits_on_a_held_lock_and_lands_when_freed() -> Result<()> { + use crate::incident::tests::{eventually, hold_lock_of, run_while_held}; + + let temp = tempfile::tempdir()?; + let (expected, activation, _, anchor) = authenticated_warming(temp.path())?; + let observation = observation_of(&expected, &activation); + let driver = topology_driver(temp.path(), "topology"); + + let error = run_while_held(hold_lock_of(&driver.projection_store), 1, || { + driver.publish_expected(&expected, &anchor) + }) + .unwrap_err(); + assert_eq!(error.to_string(), "the topology projection is locked by another holder"); + assert!(!driver.projection_store.exists(), "a contended publish wrote the store"); + + eventually(|| driver.publish_expected(&expected, &anchor)); + let landed = fs::read(&driver.projection_store)?; + + // An existing store is unchanged by a contended write of any kind. + let error = run_while_held(hold_lock_of(&driver.projection_store), 1, || { + driver.publish_observed_activation(&expected, &activation, &observation) + }) + .unwrap_err(); + assert_eq!(error.to_string(), "the topology projection is locked by another holder"); + assert_eq!(fs::read(&driver.projection_store)?, landed); + eventually(|| driver.publish_observed_activation(&expected, &activation, &observation)); + assert_ne!(fs::read(&driver.projection_store)?, landed); + Ok(()) + } + + /// The load-bearing rule (a chmod does not revoke a holder): a wider lock + /// that another process already holds is replaced by a fresh private one, + /// so the holder keeps only an orphaned inode and cannot block a publish. + #[cfg(target_os = "linux")] + #[test] + fn a_holder_of_a_wider_lock_cannot_block_a_publish() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + use crate::incident::tests::hold_lock_of; + + let temp = tempfile::tempdir()?; + let (expected, _, _, anchor) = authenticated_warming(temp.path())?; + let driver = topology_driver(temp.path(), "topology"); + let lock = authority_lock_path(&driver.projection_store); + fs::write(&lock, b"")?; + fs::set_permissions(&lock, fs::Permissions::from_mode(0o644))?; + let wider = inode_of(&lock); + let holder = hold_lock_of(&driver.projection_store); + + driver.publish_expected(&expected, &anchor)?; + assert_eq!(mode_of(&lock), 0o600); + assert_ne!(inode_of(&lock), wider, "the wider lock was chmodded, not replaced"); + assert!(holder.is_held(), "the publish ended the holder instead of outliving it"); + assert_eq!(driver.snapshot()?.len(), 2); + Ok(()) + } + + /// Every lock that is not a regular file, ours, mode 0600 is replaced, and + /// a private one is left alone. + #[cfg(target_os = "linux")] + #[test] + fn every_lock_that_is_not_private_is_replaced_and_a_private_one_is_not() -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let temp = tempfile::tempdir()?; + let (expected, activation, _, anchor) = authenticated_warming(temp.path())?; + let observation = observation_of(&expected, &activation); + let driver = topology_driver(temp.path(), "topology"); + let lock = authority_lock_path(&driver.projection_store); + driver.publish_expected(&expected, &anchor)?; + // Each call changes the projection, so each one exchanges. + let toggle = || -> Result<()> { + if driver.projected_activation_is_present(&expected)? { + driver + .demote_to_expected_only(&expected, &anchor, &activation, None) + .map(drop) + } else { + driver + .publish_observed_activation(&expected, &activation, &observation) + .map(drop) + } + }; + // SAFETY: geteuid has no preconditions. + let euid = unsafe { libc::geteuid() }; + + for (shape, mode, owner) in [ + ("0640", 0o640, None), + ("0666", 0o666, None), + ("0700", 0o700, None), + ("0400", 0o400, None), + ("0604", 0o604, None), + ("0600 owned by another uid", 0o600, Some(65534)), + ] { + if owner.is_some() && euid != 0 { + eprintln!("UNPROVEN: not root, so a lock owned by another uid cannot be made"); + continue; + } + fs::set_permissions(&lock, fs::Permissions::from_mode(mode))?; + if let Some(uid) = owner { + std::os::unix::fs::chown(&lock, Some(uid), Some(uid))?; + } + let before = inode_of(&lock); + toggle()?; + assert_ne!(inode_of(&lock), before, "a {shape} lock was kept"); + assert_eq!(mode_of(&lock), 0o600, "after a {shape} lock"); + assert_eq!(fs::metadata(&lock)?.uid(), euid, "after a {shape} lock"); + } + + // A lock that is a symlink is replaced, never followed. + let target = temp.path().join("elsewhere"); + fs::write(&target, b"")?; + fs::set_permissions(&target, fs::Permissions::from_mode(0o600))?; + fs::remove_file(&lock)?; + std::os::unix::fs::symlink(&target, &lock)?; + toggle()?; + assert!(!fs::symlink_metadata(&lock)?.file_type().is_symlink()); + assert_eq!(mode_of(&lock), 0o600); + assert!(target.exists()); + + // A private lock is opened as it is: two more exchanges, one inode. + let kept = inode_of(&lock); + toggle()?; + toggle()?; + assert_eq!(inode_of(&lock), kept); + Ok(()) + } + + /// A lost race is retried at once, against the store as it now is; a race + /// lost every time is an error rather than a loop. + #[cfg(target_os = "linux")] + #[test] + fn a_lost_race_is_retried_against_the_new_store_and_never_forever() -> Result<()> { + use std::cell::Cell; + + let temp = tempfile::tempdir()?; + let driver = topology_driver(temp.path(), "topology"); + let rival = |key: &str| CultCacheEnvelope { + key: key.into(), + r#type: "test.rival".into(), + payload: vec![0x90], + stored_at: "2026-01-01T00:00:00+00:00".into(), + schema_id: Some("test.rival.v1".into()), + }; + + // Lost once: a rival writes between the snapshot and the exchange. + let calls = Cell::new(0); + driver.mutate(|entries| { + calls.set(calls.get() + 1); + if calls.get() == 1 { + upsert_record(&driver.projection_store, rival("first")).unwrap(); + } + let mut next = entries.to_vec(); + next.push(rival(&format!("mine-{}", calls.get()))); + Ok(Some(next)) + })?; + assert_eq!(calls.get(), 2); + let keys = driver.snapshot()?.into_iter().map(|e| e.key).collect::>(); + assert_eq!(keys, ["first", "mine-2"]); + + // Lost every time: bounded, and the store is the rival's. + let calls = Cell::new(0); + let error = driver + .mutate(|entries| { + calls.set(calls.get() + 1); + upsert_record(&driver.projection_store, rival(&format!("rival-{}", calls.get()))).unwrap(); + let mut next = entries.to_vec(); + next.push(rival("never")); + Ok(Some(next)) + }) + .unwrap_err(); + assert!(error.to_string().contains("changed repeatedly"), "{error}"); + assert_eq!(calls.get(), 8); + assert!(driver.snapshot()?.iter().all(|entry| entry.key != "never")); + Ok(()) } #[cfg(unix)] diff --git a/src/incident.rs b/src/incident.rs index 4aacc10..b1b413a 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -20,7 +20,7 @@ use cultcache_rs::{ use serde::{Deserialize, Serialize}; use crate::control_plane::{decode_record, require_id, typed_envelope}; -use crate::drivers::publish_file_mode; +use crate::drivers::{exchange_behind_private_lock, publish_file_mode}; const INCIDENT_SCHEMA: &str = "idunn.operator_incident.v1"; @@ -177,8 +177,9 @@ impl Snapshot { /// scheduler tick, so no write waits: each is one nonblocking compare-exchange /// of the whole file against the snapshot it was decided on. A lost race or a /// held lock is an error and a no-op, retried on a later pass. The lock beside -/// the file is not published and is created 0600 (`secure_lock`): the readers -/// of `incidents.cc` take no lock, so only Idunn can ever hold it. +/// the file is not published and is Idunn's alone +/// (`exchange_behind_private_lock`): the readers of `incidents.cc` take no +/// lock, so only Idunn can ever hold it. pub(crate) struct IncidentStore { path: PathBuf, } @@ -231,10 +232,7 @@ impl IncidentStore { expected: &[CultCacheEnvelope], replacements: &[CultCacheEnvelope], ) -> Result<()> { - secure_lock(path)?; - match SingleFileMessagePackBackingStore::new(path) - .try_compare_exchange_snapshot(expected, replacements)? - { + match exchange_behind_private_lock(path, expected, replacements)? { TryCompareExchangeSnapshotOutcome::Exchanged => Ok(()), TryCompareExchangeSnapshotOutcome::Mismatch => { bail!("the incident store changed while it was being written") @@ -335,46 +333,6 @@ impl IncidentStore { } } -/// Make the lock beside `store` Idunn's alone, before CultCache opens it. -/// -/// CultCache creates a lock with mode 0666, so the published directory decides -/// who may open it: under Yggdrasil's default ACL (`u::rw g::r o::r`) that is -/// 0644, and any uid could hold the lock and keep every incident from being -/// recorded. Created here with 0600 the create mode is intersected with the -/// ACL and still holds, and no wider file ever exists. A lock left wider by an -/// earlier run is tightened on the descriptor just opened (a descriptor some -/// other uid already holds is not revoked). Readers of the store take no lock, -/// so nothing but Idunn needs to open this file. -fn secure_lock(store: &Path) -> Result<()> { - #[cfg(unix)] - { - use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; - - let mut name = store - .file_name() - .context("the incident store has no file name")? - .to_owned(); - name.push(".lock"); - let lock = store.with_file_name(name); - if let Some(parent) = lock.parent() { - std::fs::create_dir_all(parent).context("creating the incident store directory")?; - } - std::fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .mode(0o600) - .open(&lock) - .context("opening the incident lock")? - .set_permissions(std::fs::Permissions::from_mode(0o600)) - .context("making the incident lock private")?; - } - #[cfg(not(unix))] - let _ = store; - Ok(()) -} - /// The lines `idunn status` prints for one target's incidents: each open one, /// and each closed one still inside retention. pub(crate) fn render_incidents(records: &[IncidentRecord], target: &str, now: u64) -> Vec { @@ -399,7 +357,7 @@ pub(crate) fn render_incidents(records: &[IncidentRecord], target: &str, now: u6 #[cfg(test)] pub(crate) mod tests { - #[cfg(unix)] + #[cfg(target_os = "linux")] use std::time::{Duration, Instant}; use tempfile::TempDir; @@ -644,7 +602,7 @@ pub(crate) mod tests { /// Idunn runs with `UMask=027`. The umask is process-wide, so it is held /// only around each call and only one test at a time may hold it. #[cfg(target_os = "linux")] - fn under_service_umask(write: impl FnOnce() -> T) -> T { + pub(crate) fn under_service_umask(write: impl FnOnce() -> T) -> T { static UMASK: std::sync::Mutex<()> = std::sync::Mutex::new(()); let _held = UMASK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); // SAFETY: umask only swaps the process file-creation mask. @@ -658,7 +616,7 @@ pub(crate) mod tests { /// `u::rw g::r o::r` that Yggdrasil's idunn-projection directory has. Under /// it the umask is ignored and a file created with mode 0666 lands 0644. #[cfg(target_os = "linux")] - fn give_default_acl(dir: &Path) { + pub(crate) fn give_default_acl(dir: &Path) { use std::os::unix::ffi::OsStrExt; // posix_acl_xattr: version 2, then (tag, perm, id) entries. The tags @@ -691,7 +649,7 @@ pub(crate) mod tests { /// Whether uid 65534 can read `path`; `None` when the uid cannot be /// switched (not root, or no setpriv), which leaves the question unproven. #[cfg(target_os = "linux")] - fn readable_by_another_uid(path: &Path) -> Option { + pub(crate) fn readable_by_another_uid(path: &Path) -> Option { std::process::Command::new("setpriv") .args(["--reuid=65534", "--regid=65534", "--clear-groups", "cat"]) .arg(path) @@ -758,47 +716,154 @@ pub(crate) mod tests { #[cfg(unix)] #[test] - fn a_lock_left_wider_by_an_earlier_run_is_tightened_before_it_is_used() { - use std::os::unix::fs::PermissionsExt; + fn a_lock_left_wider_by_an_earlier_run_is_replaced_before_it_is_used() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; let temp = TempDir::new().unwrap(); let path = temp.path().join("incidents.cc"); let lock = temp.path().join("incidents.cc.lock"); std::fs::write(&lock, b"").unwrap(); std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o666)).unwrap(); + let before = std::fs::metadata(&lock).unwrap().ino(); assert!( IncidentStore::new(&path) .open(IncidentCondition::ContinuityExhausted, "service", OPENED) .unwrap() ); - let mode = std::fs::metadata(&lock).unwrap().permissions().mode() & 0o777; - assert_eq!(mode, 0o600); + let after = std::fs::metadata(&lock).unwrap(); + assert_eq!(after.permissions().mode() & 0o777, 0o600); + assert_ne!(after.ino(), before, "the wider lock was chmodded, not replaced"); } - /// An exclusive lock on a store's lock file, held through another file - /// description. flock conflicts between descriptions, so the holder need - /// not be another process or another uid: the test does not depend on who - /// it runs as. - #[cfg(unix)] - pub(crate) struct HeldLock(#[allow(dead_code)] std::fs::File); - - #[cfg(unix)] - pub(crate) fn hold_lock_of(store: &Path) -> HeldLock { - use std::os::unix::io::AsRawFd; - + /// The lock beside a store, spelled independently of the code under test. + #[cfg(target_os = "linux")] + fn lock_beside(store: &Path) -> PathBuf { let mut name = store.file_name().unwrap().to_owned(); name.push(".lock"); - let file = std::fs::OpenOptions::new() + store.with_file_name(name) + } + + /// An exclusive flock on a store's lock, held by a SEPARATE PROCESS + /// (`flock -x sleep`), so the holder shares no descriptor, thread + /// or address space with the code under test. A watch on the lock counts + /// every open of it from the moment it is held. + #[cfg(target_os = "linux")] + pub(crate) struct HeldLock { + holder: std::sync::Arc>, + watch: std::os::fd::OwnedFd, + } + + /// Hold the lock of `store`. An absent lock is created as production keeps + /// it (0600, ours); a lock a test put there first is held as it is, never + /// chmodded. + #[cfg(target_os = "linux")] + pub(crate) fn hold_lock_of(store: &Path) -> HeldLock { + use std::os::fd::{FromRawFd, OwnedFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::OpenOptionsExt; + use std::os::unix::io::AsRawFd; + + let lock = lock_beside(store); + std::fs::OpenOptions::new() .create(true) - .read(true) .write(true) .truncate(false) - .open(store.with_file_name(name)) + .mode(0o600) + .open(&lock) .unwrap(); - // SAFETY: flock on a descriptor this function owns. - assert_eq!(unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) }, 0); - HeldLock(file) + let mut holder = std::process::Command::new("flock") + .args(["--no-fork", "--exclusive"]) + .arg(&lock) + .args(["sleep", "3600"]) + .spawn() + .expect("flock (util-linux) must be installed to hold a lock from another process"); + + // Held once a nonblocking exclusive flock from here is refused. + let deadline = Instant::now() + STUCK_AFTER; + loop { + assert!(holder.try_wait().unwrap().is_none(), "the lock holder exited"); + let probe = std::fs::File::open(&lock).unwrap(); + // SAFETY: flock on a descriptor this function owns. + let refused = unsafe { libc::flock(probe.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0; + if refused { + assert_eq!( + std::io::Error::last_os_error().kind(), + std::io::ErrorKind::WouldBlock + ); + break; + } + drop(probe); + assert!(Instant::now() < deadline, "the lock was never held"); + std::thread::sleep(Duration::from_millis(10)); + } + + // Open and close alternate, so two opens are never identical adjacent + // events for the kernel to coalesce into one. + // SAFETY: plain syscalls on descriptors and strings this function owns. + let watch = unsafe { + let fd = libc::inotify_init1(libc::IN_CLOEXEC | libc::IN_NONBLOCK); + assert!(fd >= 0, "{}", std::io::Error::last_os_error()); + let watch = OwnedFd::from_raw_fd(fd); + let path = std::ffi::CString::new(lock.as_os_str().as_bytes()).unwrap(); + let wd = libc::inotify_add_watch( + fd, + path.as_ptr(), + libc::IN_OPEN | libc::IN_CLOSE_WRITE | libc::IN_CLOSE_NOWRITE, + ); + assert!(wd >= 0, "{}", std::io::Error::last_os_error()); + watch + }; + HeldLock { + holder: std::sync::Arc::new(std::sync::Mutex::new(holder)), + watch, + } + } + + #[cfg(target_os = "linux")] + impl HeldLock { + /// Whether the holder process is still running and so still holds. + pub(crate) fn is_held(&self) -> bool { + self.holder.lock().unwrap().try_wait().unwrap().is_none() + } + + /// How many times the lock has been opened since it was held. + fn opens(&self) -> usize { + use std::os::unix::io::AsRawFd; + + let mut opens = 0; + let mut buffer = [0u8; 4096]; + loop { + // SAFETY: reads into a live buffer of the stated length. + let read = unsafe { + libc::read(self.watch.as_raw_fd(), buffer.as_mut_ptr().cast(), buffer.len()) + }; + if read <= 0 { + return opens; + } + let read = usize::try_from(read).unwrap(); + let mut at = 0; + // struct inotify_event: wd i32, mask u32, cookie u32, len u32, name[len]. + while at + 16 <= read { + let word = |offset: usize| { + u32::from_ne_bytes(buffer[at + offset..at + offset + 4].try_into().unwrap()) + }; + if word(4) & libc::IN_OPEN != 0 { + opens += 1; + } + at += 16 + usize::try_from(word(12)).unwrap(); + } + } + } + } + + #[cfg(target_os = "linux")] + impl Drop for HeldLock { + fn drop(&mut self) { + let mut holder = self.holder.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let _ = holder.kill(); + let _ = holder.wait(); + } } /// How long a test lets a held lock, or a released lock's last straggler, @@ -806,40 +871,57 @@ pub(crate) mod tests { /// spends it: the code under test finishes in milliseconds, and the limit /// is only what stops a blocked write from hanging the suite. No /// assertion compares elapsed time to a bound the machine's load can move. - #[cfg(unix)] + #[cfg(target_os = "linux")] pub(crate) const STUCK_AFTER: Duration = Duration::from_secs(30); - /// Run `work` while `held` is held, and fail if it waited on it. The holder - /// lets go only when `work` has returned, or after `STUCK_AFTER`: work that - /// waits on the lock is freed by that second release and is then seen to - /// have waited, so the test fails instead of hanging. - #[cfg(unix)] - pub(crate) fn run_while_held(held: HeldLock, work: impl FnOnce() -> T) -> T { + /// Run `work` while `held` is held, and fail if it waited on it. The rule + /// is observed where it is decided, at the lock, not on a clock: every + /// exchange opens the lock exactly once, so `exchanges` is the number of + /// exchanges `work` makes against this lock, and a wait of any length (a + /// sleep and retry, a poll, a second attempt) opens it more often. A wait + /// that never ends is freed by the holder's release after `STUCK_AFTER`, + /// and is then seen to have waited, so the test fails instead of hanging. + /// The holder is still holding when `work` returns, and is released on the + /// way out. + #[cfg(target_os = "linux")] + pub(crate) fn run_while_held( + held: HeldLock, + exchanges: usize, + work: impl FnOnce() -> T, + ) -> T { use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, mpsc}; let (finished, work_done) = mpsc::channel::<()>(); let gave_up = Arc::new(AtomicBool::new(false)); - let holder = { + let watchdog = { let gave_up = Arc::clone(&gave_up); + let holder = Arc::clone(&held.holder); std::thread::spawn(move || { if work_done.recv_timeout(STUCK_AFTER) == Err(mpsc::RecvTimeoutError::Timeout) { gave_up.store(true, Ordering::SeqCst); + let _ = holder.lock().unwrap().kill(); } - drop(held); }) }; let result = work(); + let still_held = held.is_held(); let _ = finished.send(()); - holder.join().unwrap(); + watchdog.join().unwrap(); assert!(!gave_up.load(Ordering::SeqCst), "the work waited on a held lock"); + assert!(still_held, "the holder let go before the work returned"); + assert_eq!( + held.opens(), + exchanges, + "the work opened the held lock other than once per exchange: it waited or retried" + ); result } - /// Retry `attempt` until it lands. A released lock can stay held for a - /// moment by a child another test forked while it was held. - #[cfg(unix)] - fn eventually(mut attempt: impl FnMut() -> Result) -> T { + /// Retry `attempt` until it lands. A lock just released can take a moment + /// to be seen free. + #[cfg(target_os = "linux")] + pub(crate) fn eventually(mut attempt: impl FnMut() -> Result) -> T { let deadline = Instant::now() + STUCK_AFTER; loop { match attempt() { @@ -850,7 +932,7 @@ pub(crate) mod tests { } } - #[cfg(unix)] + #[cfg(target_os = "linux")] #[test] fn no_incident_write_waits_on_a_held_lock_and_each_lands_when_it_is_freed() { let temp = TempDir::new().unwrap(); @@ -863,15 +945,17 @@ pub(crate) mod tests { let before = std::fs::read(&path).unwrap(); let retired = OPENED + 1 + INCIDENT_RETENTION_MILLIS + 1; - let outcomes = run_while_held(hold_lock_of(&path), || { + // One exchange each: open, close, and the retirement's store delete. + let outcomes = run_while_held(hold_lock_of(&path), 3, || { ( store.open(condition, "new", CLOSED), store.close(condition, "open", CloseReason::Recovered, CLOSED), store.retire_closed(retired), ) }); - for outcome in [outcomes.0, outcomes.1, outcomes.2] { - assert!(outcome.is_err()); + for outcome in [outcomes.0.map(drop), outcomes.1.map(drop), outcomes.2.map(drop)] { + let error = outcome.unwrap_err().to_string(); + assert_eq!(error, "the incident store is locked by another holder"); } assert_eq!(std::fs::read(&path).unwrap(), before, "a contended write changed the file"); @@ -885,7 +969,7 @@ pub(crate) mod tests { assert_eq!(history.pull_all_read_only_snapshot().unwrap().len(), 1); } - #[cfg(unix)] + #[cfg(target_os = "linux")] #[test] fn a_held_history_lock_keeps_the_incident_where_it_is() { let temp = TempDir::new().unwrap(); @@ -898,7 +982,7 @@ pub(crate) mod tests { let before = std::fs::read(&path).unwrap(); let retired = OPENED + 1 + INCIDENT_RETENTION_MILLIS + 1; - let outcome = run_while_held(hold_lock_of(&history), || store.retire_closed(retired)); + let outcome = run_while_held(hold_lock_of(&history), 1, || store.retire_closed(retired)); assert!(outcome.is_err()); assert_eq!(std::fs::read(&path).unwrap(), before); assert!(eventually(|| store.retire_closed(retired))); From 827cfad0bb7d269f6f6faf90086f39478e6f0bea Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 02:05:05 +0200 Subject: [PATCH 048/196] Topology lock: test the lock-creation error classes and the create race Mutation of the first cut left make_lock_private's error classification and lost-create-race branch unkilled. Two behavioural tests at the primitive: an uninspectable lock (name one byte past NAME_MAX) reports an inspection error, an uncreatable one (procfs) a creation error, and neither is retried as a lost race; eight publishers released together to create the same lock all find it private, so none fails for having lost. Tests only; builds. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 5dea8ad..ada688c 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -9987,6 +9987,57 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// A lock that cannot be inspected is an inspection error, and one that + /// cannot be created is a creation error: neither is mistaken for the + /// other, and neither is retried as if another creator had won. + #[cfg(target_os = "linux")] + #[test] + fn a_lock_that_cannot_be_inspected_or_created_says_which() { + let temp = tempfile::tempdir().unwrap(); + // The lock's name is one byte past NAME_MAX: inspecting it fails with + // an error that is not "not found". + let long = temp.path().join("a".repeat(251)); + let error = exchange_behind_private_lock(&long, &[], &[]).unwrap_err(); + assert!(format!("{error:#}").contains("inspecting a store lock"), "{error:#}"); + + // Absent, but nothing can be created beside it: procfs refuses. + let error = exchange_behind_private_lock(Path::new("/proc/self/idunn-store.cc"), &[], &[]) + .unwrap_err(); + assert!(format!("{error:#}").contains("creating a private store lock"), "{error:#}"); + } + + /// Many first publishers race to create the same lock. Whoever loses finds + /// a private lock and carries on: no call fails for having lost. + #[cfg(target_os = "linux")] + #[test] + fn publishers_racing_to_create_the_lock_all_find_it_private() { + use std::sync::Barrier; + + const PUBLISHERS: usize = 8; + for round in 0..100 { + let temp = tempfile::tempdir().unwrap(); + let store = temp.path().join("topology.cc"); + let start = Barrier::new(PUBLISHERS); + let outcomes = thread::scope(|scope| { + (0..PUBLISHERS) + .map(|_| { + scope.spawn(|| { + start.wait(); + exchange_behind_private_lock(&store, &[], &[]).map(drop) + }) + }) + .collect::>() + .into_iter() + .map(|publisher| publisher.join().unwrap()) + .collect::>() + }); + for outcome in outcomes { + outcome.unwrap_or_else(|error| panic!("round {round}: {error:#}")); + } + assert_eq!(mode_of(&authority_lock_path(&store)), 0o600); + } + } + /// A lost race is retried at once, against the store as it now is; a race /// lost every time is an error rather than a loop. #[cfg(target_os = "linux")] From 424300d67219b59f741fd53eb92f99b996925f77 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 03:31:48 +0200 Subject: [PATCH 049/196] A publish that loses to a held lock is retried before the fence, not aborted resume_candidate turned any failed pre-Fencing step into a pre-fencing abort, so a topology publish that met a held lock (the nightly backup's shared lock, an operator) ended the deploy or continuity restart on its first attempt. LockContended is now a typed error (drivers::LockContended, raised by mutate and IncidentStore::exchange with the same texts as before) and resume_candidate's one classification keeps such a transaction in its phase with last_error set; the backoff retries it and the phase deadline bounds the wait. Other publish errors still abort. Tests at the resume path: a contended Starting publish stays in Starting across ticks and lands after release with one continuity restart; a non-contention publish error still aborts. Both builds. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 91 +++++++++++++++++++++++++++++++++++++++++++- src/drivers.rs | 18 ++++++++- src/incident.rs | 4 +- 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 75af38f..dd2962f 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -40,7 +40,7 @@ use crate::deployment_plan::{ use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, - ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, + LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, @@ -5576,7 +5576,14 @@ impl Engine { // Before the fence an abort that cannot finish a step is // resumable, never a post-fence abort: that path refuses a // pre-fence phase, so choosing it only wedged the tick. - if latest.value.pre_fencing_abort.is_none() { + // + // A publish that lost to a held lock is not a failed step: the + // holder lets go, so the transaction stays in its phase with + // the error recorded and the backoff above retries it. The + // phase deadline, not an abort, bounds the wait. + if latest.value.pre_fencing_abort.is_none() + && error.downcast_ref::().is_none() + { self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -15626,6 +15633,86 @@ mod tests { Ok(()) } + /// A Continuity transaction parked in Starting with nothing published yet, + /// plus the stored record the tick will see. + #[cfg(target_os = "linux")] + fn parked_in_starting(world: &EngineFixture) -> Result { + let seeded = transaction_at(world, DeploymentPhase::Warming)?; + let stored = resident(world)?; + let mut starting = seeded.clone(); + starting.phase = DeploymentPhase::Starting; + starting.expected_publication_sha256 = None; + starting.activation = None; + starting.workload = None; + starting.activation_publication_sha256 = None; + starting.warming = None; + replace_transaction(&world.state_store, &stored, &starting)?; + Ok(seeded) + } + + /// A publish that lost to a held lock is transient: before the fence the + /// transaction stays in its phase with the error recorded, no abort is + /// begun, and the next tick after the release lands the publish. A + /// continuity restart is a Continuity transaction on this same path, so + /// its attempt is minted once and a contended publish spends no second one. + #[cfg(target_os = "linux")] + #[test] + fn a_contended_starting_publish_is_retried_on_a_later_tick_not_aborted() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = parked_in_starting(&world)?; + let mut meters = TargetSupervision::new("service"); + meters.continuity_restarts = vec![now_millis()?]; + set_meters(&world, &meters)?; + let store = world.engine.options.topology_store.clone(); + + for tick in 0..2 { + run_while_held(hold_lock_of(&store), 1, || world.engine.resume_one_transaction())?; + world.engine.resume_backoff.lock().unwrap().clear(); + let held = resident(&world)?.value; + assert_eq!(held.phase, DeploymentPhase::Starting, "tick {tick}"); + assert!(held.pre_fencing_abort.is_none(), "tick {tick}: a held lock aborted the transaction"); + assert!(!held.is_terminal(), "tick {tick}"); + assert_eq!(held.expected_publication_sha256, None, "tick {tick}"); + let error = held.last_error.context("a contended publish left no last_error")?; + assert!(error.contains("locked by another holder"), "{error}"); + assert!(!store.exists(), "a contended publish wrote the projection"); + } + + // Released: the next tick lands the publish and records its digest. + world.engine.resume_one_transaction()?; + let landed = resident(&world)?.value; + assert_eq!(landed.phase, DeploymentPhase::Starting); + assert!(landed.pre_fencing_abort.is_none()); + assert_eq!( + landed.expected_publication_sha256, + Some(seeded.expected.context("seeded transaction has no Expected")?.canonical_sha256()?) + ); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + /// Only contention is transient: a publish that fails for any other reason + /// still ends the transaction before the fence, as it always did. + #[cfg(target_os = "linux")] + #[test] + fn a_non_contention_publish_error_still_aborts_before_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + parked_in_starting(&world)?; + // A directory where the store should be: the snapshot cannot be read. + std::fs::create_dir_all(&world.engine.options.topology_store)?; + world.engine.resume_one_transaction()?; + let after = resident(&world)?.value; + assert!(after.pre_fencing_abort.is_some(), "a non-contention error did not abort"); + assert!( + !after.last_error.as_deref().unwrap_or_default().contains("locked by another holder"), + "{:?}", + after.last_error + ); + Ok(()) + } + #[test] fn status_prints_an_open_incident_under_its_target() -> Result<()> { let workload = SwitchWorkload::new(); diff --git a/src/drivers.rs b/src/drivers.rs index ada688c..e005ec5 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -4414,7 +4414,7 @@ impl CultCacheTopologyDriver { } TryCompareExchangeSnapshotOutcome::Mismatch => {} TryCompareExchangeSnapshotOutcome::LockContended => { - bail!("the topology projection is locked by another holder") + return Err(LockContended { store: "topology projection" }.into()); } } } @@ -5107,6 +5107,22 @@ pub enum RouteActuation { Survival, } +/// A publish that lost to a held projection-directory lock. Transient by +/// kind: the holder lets go, and the caller's own retry decides what a later +/// attempt does. It names the store kind, never its path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct LockContended { + pub(crate) store: &'static str, +} + +impl std::fmt::Display for LockContended { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "the {} is locked by another holder", self.store) + } +} + +impl std::error::Error for LockContended {} + /// A forward route actuation the ceiling refused. `reopens_at_unix_millis` is /// when the oldest counted actuation leaves the window. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/incident.rs b/src/incident.rs index b1b413a..1d7f095 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -20,7 +20,7 @@ use cultcache_rs::{ use serde::{Deserialize, Serialize}; use crate::control_plane::{decode_record, require_id, typed_envelope}; -use crate::drivers::{exchange_behind_private_lock, publish_file_mode}; +use crate::drivers::{LockContended, exchange_behind_private_lock, publish_file_mode}; const INCIDENT_SCHEMA: &str = "idunn.operator_incident.v1"; @@ -238,7 +238,7 @@ impl IncidentStore { bail!("the incident store changed while it was being written") } TryCompareExchangeSnapshotOutcome::LockContended => { - bail!("the incident store is locked by another holder") + Err(LockContended { store: "incident store" }.into()) } } } From 6667f0f9a66f7c71446fcae15e0713127978a394 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 10:25:36 +0200 Subject: [PATCH 050/196] Watchdog reporting postmortem First typed Eureka campaign (idunn-watchdog): 8 cuts, 17 Soul verdicts, 55 findings; landed about 2.5x its estimate. Idunn pages the operator on Discord through Bifrost; closure DM still owed. Co-Authored-By: Claude Opus 5.5 --- docs/watchdog-reporting-postmortem.md | 244 ++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 docs/watchdog-reporting-postmortem.md diff --git a/docs/watchdog-reporting-postmortem.md b/docs/watchdog-reporting-postmortem.md new file mode 100644 index 0000000..3249d2d --- /dev/null +++ b/docs/watchdog-reporting-postmortem.md @@ -0,0 +1,244 @@ +# Watchdog reporting postmortem + +Campaign `idunn-watchdog`, 2026-09-30 to 2026-10-01. This was the first Eureka +campaign whose state was typed in Huginn's mind (instance `eureka`) rather than +kept in a prose map. Document ids below are `idunn-watchdog::`. +Read them with `view`; this page does not restate them. + +## Summary + +Idunn is now a watchdog that reaches the operator on Discord. + +- **Idunn** opens a durable incident at the site where a continuity budget is + exhausted. The incident is written to `incidents.cc`, a world-readable file + in its projection directory with a private 0600 lock, and closes when the + target runs again and its restart window is no longer exhausted. +- **Bifrost's watchdog-notice reader** runs from a one-minute timer. It sends + one opening DM and at most one closure DM per incident, keeps its own + journal, and retries a failed send forever with capped backoff. +- **Scale:** 8 cuts across Idunn, Bifrost and gamecult-ops: 13 spec revisions, + 17 Hands reports, 17 Soul verdicts and 55 findings. Measured against the + in-force estimates: + +| Repo | Estimate (lines) | Landed (lines) | +|---|---|---| +| Idunn | +930/−101 | +2618/−354 | +| Bifrost | +246/−335 | +1182/−429 | +| gamecult-ops | +280/−35 | +505/−104 | + +- **Installed on Yggdrasil:** + - Idunn `a909f93` at 2026-09-30 23:21 UTC, then `cb60965` at 2026-10-01 + 02:02 UTC. + - The Bifrost reader at `160fac20` with CultLib `36ea08d3`, at 2026-10-01 + 08:04 UTC. +- **First live notice:** the incident's first real input was raven-muninn, + which exhausted its budget minutes after the incident cut was installed. Its + opening DM was delivered at 08:04:41 UTC, and the operator confirmed it. +- **Still owed:** the closure DM for that same incident + (`follow_up:closure-dm-observed`). It depends on raven-muninn recovering, and + raven-muninn's own fault lies outside this campaign. + +## Scope and invariants + +Target `r4`. Its invariants are: + +- `incident-owner` +- `record-before-delivery` +- `delivery-owner` +- `survival-independent` +- `one-per-incident` +- `no-sensitive-egress` +- `tick-lock-private` + +The operator directions are `ruling:operator-watchdog-direction` and +`ruling:operator-bifrost-bridge`. The second made Bifrost the bridge to +Discord, so Idunn holds no credential and speaks no Discord API. + +Excluded: + +- **Odin's store condition and B5 `OperatorRequired`.** Neither can page yet. + Odin's O2 cut is unbuilt, and B5 has no operator exit + (`follow_up:b5-operator-required-no-exit`). +- **VoidBot's `notify_owner`.** +- **The Persona Discord crossing.** +- **Eve lowering.** + +The earlier attempt was Bifrost `c063f27` (2026-09-04): an alarm was a command +written into Bifrost's store. It never ran, because no pump was deployed and +the deployed pump lacked the DM verb. It was also reversed by ruling: +`ruling:delivery-crossing` has Bifrost read Idunn's projection, so no write +crosses an owner boundary. + +## Timeline + +| Cut | Reports | Soul passes | Notable finding | +|---|---|---|---| +| ops-retire-alarm | h1 | 1 | none | +| bifrost-retire-alarm | h1–h2 | 2 | `silent-skip`: a stray `discord-dm` command sat pending forever | +| bifrost-suite | h1 | 1 | `suite-cultcache-scope` (pre-existing): 9 of 17 tests failed at base on a CultLib rename | +| bifrost-notice-reader | h1–h3 | 3 | `prune-keyed-on-presence`, `bridge-exit-is-failed`: duplicate DMs and blocked closures | +| bifrost-notice-retry | h1–h2 | 2 | `future-stamp-stalls-retry` | +| ops-notice-deploy | h1–h3 | 3 | `retry-window-five-minutes`: a 5-minute outage dropped a notice for good | +| idunn-incident | h1–h3 | 3 | `lock-stalls-tick` (Blocker); `lock-acl-world-openable` (High) | +| idunn-topology-lock | h1–h2 | 2 | `contended-publish-aborts`; the live stall it fixed was pre-existing | + +## Structural delta + +The landed code was about 2.5× its estimate, and the misses were not explained +cut by cut as the skill asks. + +- **idunn-incident:** +1813 against +700. Its fix batches added the lock + primitive, load-independent lock tests, and per-operation fault keys. +- **bifrost-notice-reader:** +891, against r3's estimate of +140. That r3 + figure described only the revision's own delta, not the cut, so r3 carried + the wrong baseline. + +Deleted: + +- Bifrost's `publish-idunn-alarm` and the pump's `discord-dm` verb. +- gamecult-ops' dead `--operator-alarm-command` scripts. +- Idunn's `publish_projection_mode` and its 0644-lock test. +- The reader's prune-on-absence logic and its `WATCHDOG_NOTICE_NOW_MS` + production clock seam. + +Added: + +- `idunn.operator_incident.v1` (`incidents.cc`) and + `bifrost.watchdog_notice_execution.v1` (the journal). +- `bifrost-watchdog-notice.{service,timer}`. + +Nothing was parked. + +## What Soul caught + +Green suites passed every defect below. + +- **Survival and locking.** + - Incident writes took a *blocking* flock inside Idunn's scheduler tick, on + a world-readable lock. Any local uid could freeze continuity for every + target, and `WatchdogUSec=0` meant nothing would restart Idunn + (`s1.lock-stalls-tick`). + - After the fix, the live directory's default ACL still made the lock 0644, + so any uid could stop incidents opening. The test ran in a temp directory + with no ACL (`s2.lock-acl-world-openable`). + - The same probe exposed a pre-existing live hazard: `topology.cc.lock` 0644 + with a blocking write in the tick, measured at 5.07 s at base + (`s2.topology-lock-stalls-tick`). The operator ruled it into the campaign; + after the topology cut the tick takes 0.09 s. +- **Split authority.** + - The reader pruned its "already sent" journal whenever a key was absent + from Idunn's store. Restoring a backup would have re-sent old DMs. + - The reader decided subject validity with its own grammar, narrower than + Idunn's. + - Retention was decided by two predicates in Idunn. +- **Lifecycle.** + - One running observation closed an exhaustion incident. A release that + crashes after each restart would have sent about 12 DMs an hour + (`s1.flap-reopens`). + - A bridge that died after Discord accepted a message was retried up to 5 + times, and the closure was then blocked forever. + - Five attempts on a one-minute timer dropped a notice after a 5-minute + Discord outage. +- **Decorative checks.** + - A compare-exchange expected `None` on a key that embeds `now`, so it + guarded nothing (`s1.cas-decorative`). + - A 2-second wall-clock bound on a lock test flaked under load and counted + equivalent mutants as caught. +- **The verifier itself.** Soul's s3 found that the stopgap forced one + `CARGO_TARGET_DIR` on every job, so cargo-mutants' parallel workers tested + each other's binaries. Two runs of the same two mutants disagreed + (`s3.mutants-share-target-dir`). Every parallel mutation count reported + through the stopgap before 2026-10-01 is unproven, in both sessions' work. + +## Operator corrections + +- **"Bifrost is indeed intended to be the bridge to external services like + Discord."** The agents had found two delivery paths: VoidBot's + `notify_owner` and Bifrost's command store. The ownership boundary was + missing from both repos' summaries. +- **"Stewardship is currently universal…"** Self had filed a gap treating + follow-up repos as needing stewardship. It was withdrawn; the rule was not + written in the skill. +- **"…we seem to be saving some tokens… but we're very very slow."** Self + blamed cold builds, measured, and was wrong. Cold builds took about 51 s and + warm builds 29 s. The cost was slot queueing, repeated verify jobs, a test + that hung a slot for 39 minutes, and a 4-hour job from the other session + holding a fifth of capacity. +- **"Yep, fixing Mind issues is highest priority, immediately once this + campaign closes"** (`ruling:mind-fixes-next`). + +## Incidents + +- **Network outages (ENOTFOUND).** Two outages killed three agents + mid-pass. Each was resumed by `SendMessage` with context intact, and nothing + was lost, because each agent's state was in commits and the mind. +- **A shared stopgap target directory** made mutation evidence noise. It is + fixed in Eureka `a755a60`. +- **Self's own brief inflated runtime.** It told Hands to mutate serially to + dodge a flake, and was corrected to skipping the flaky test by name. +- **A hung test held a verify slot for 39 minutes** + (`s2.route-connect-test-hangs`). It is fixed. +- **A Hands test helper overwrote the container's `/usr/bin/node`** through a + symlink. It was caught by the agent's own baseline and then confined + (`s2.test-clobbers-usr-bin-node`). +- **A worktree directory stays locked** by a finished process + (`F:\Projects\gamecult-ops-watchdog-deploy`, empty). Removing it is left to + the operator. + +## What worked + +- **Soul attacks the layer where the rule fails, not the test.** Every + High/Blocker finding came from a probe the shipped tests could not see: a + separate-process lock holder, the live directory's ACL, or a restored store. +- **Point Soul at the load-bearing invariant in the brief.** Naming "can the + lock hang the tick" produced the Blocker on the first pass. +- **Read before touching live state.** Self installed only after checking the + installed revision, the in-flight transactions and the unit diff. The repo + unit carried a BOM the live one did not. +- **One verify job per pass.** Once briefs required it, a reader fix batch + took 11 minutes, where the first reader Hands took 38. + +## What to change in the pipeline + +Already changed in the skill: + +- The substrate gaps moved from a table to the mind. +- Imagination owns the model page. +- Soul pays for each rerun once. +- The rust image uses sccache. +- The stopgap no longer forces `CARGO_TARGET_DIR`. + +Still to change: + +- **Explain every ledger miss in the next report's deviations.** This campaign + ran at about 2.5× its estimate and nobody explained it per cut. +- **Make a revision's `estimate` the whole cut.** r3's estimate covered only + its own delta. +- **Verify the merged tree of a long-lived branch before pushing.** Self did + this once, for the incident cut over B5, and it should be the rule. +- **Briefs name the standard sibling build command.** The stopgap cannot + provision siblings (`follow_up:gap-verify-no-sibling-repos`), and one + hand-rolled build produced four false failures. + +## Open follow-ups + +Fifty follow-ups are in force. Twenty are substrate gaps (`gap-*`): the work +the operator ruled next. Query them with the substrate-gaps recipe in the +skill. Seven findings remain unresolved: + +- **`cut-idunn-incident.s3.mutants-share-target-dir`.** It is fixed in the + stopgap but awaits a Soul mutation run under the fixed script. +- **Six findings whose resolution keys exceed 64 bytes** + (`follow_up:gap-unresolvable-finding-key`). Their substance is recorded in + `follow_up:notice-retry-hardening`, `follow_up:ops-refusal-test-hardening`, + `follow_up:bifrost-pump-input-hygiene` and + `follow_up:contended-publish-bounds`. + +Operational follow-ups that matter most: + +- `closure-dm-observed` +- `notice-unknown-settle` +- `contended-publish-bounds` +- `idunn-load-flaky-tests` +- `cultcache-rs-lock-mode` +- `cultcache-ts-readonly-pull` From c2477d43d42d394dfdc1f4069936e1d265cef465 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 22:51:31 +0200 Subject: [PATCH 051/196] Reincarnation target: a reboot is not a fault, one owner per target at every instant Co-Authored-By: Claude Opus 5.5 --- docs/reincarnation-target.md | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/reincarnation-target.md diff --git a/docs/reincarnation-target.md b/docs/reincarnation-target.md new file mode 100644 index 0000000..a5de465 --- /dev/null +++ b/docs/reincarnation-target.md @@ -0,0 +1,92 @@ +# Reincarnation after reboot: target + +Status: target, Self, 2026-10-18 (campaign `idunn-reincarnation` in Eureka's +mind). It states the ends. The body facts, model page and cut sequence will +live in `reincarnation-map.md`. + +## Why + +Yggdrasil rebooted at 11:46 UTC on 2026-10-18. Admitted bodies run as +transient systemd units under `/run/systemd/transient`, and the reboot wiped +them all. nginx failed at boot, because it bound `10.77.0.1:17871` before +`wg-gamecult` carried that address. Every stable route, Idunn's own +`127.0.0.1:8831-8833` included, stayed down until a manual +`systemctl restart nginx` at about 19:46. + +What Idunn did: + +- From 11:50 onward continuity spent streampixels-service's whole restart + budget (6 per hour) and logged "The target is free for a deployment to + replace it: systemd unit … is absent". The log repeated in hourly bursts as + the window slid. +- When nginx returned, ghostlight and streampixels-web were re-incarnated, and + streampixels-service was not. +- An operator-approved `idunn up streampixels-service` (tx-e69f5402) sat in + Sealing for about 15 minutes. It then failed with "admitted incumbent failed + before candidate fencing; deployment yielded to continuity". Continuity had + freed the target for a deployment. The deployment handed it back to + continuity. Neither actuated. +- At 20:46:43 continuity re-incarnated the service under generation + tx-0804185a, once the window had slid to 4/6. So the outage was not ended by + the deployment the operator approved. The restart window bounded it, a full + hour after its cause had cleared. +- Throughout, several times a second: "preserved admitted … after rejecting + topology observation: topology correlation does not bind the current + activation", and "rejected admitted streampixels-service route continuity: + admitted process write lease is no longer exact". + +## Invariants + +1. **A reboot is not a fault of the body.** If an admitted body's unit is + absent because the host lost its transient units, continuity + re-materializes it from the installed body. It needs no deployment, no + operator, and no part of the restart budget for that absence. +2. **The restart budget measures the body.** A failure caused by a shared host + dependency, such as nginx, the stable route's listener or Odin, never + charges a target's restart budget. A dependency outage can never exhaust + the budget of every target behind it. +3. **Exactly one authority owns a target's actuation at every instant, and + the handover is explicit.** When continuity stops, it records that it has + stopped, and from then on deployment owns the target. A deployment never + yields to a continuity owner that has stopped. A deployment whose incumbent + is absent replaces it, with nothing to fence. There is no reachable state + in which continuity waits for a deployment while that deployment waits for + continuity. A negative test proves it. +4. **Continuity is daemon survival; deployment is change** (`F:\Projects\CLAUDE.md`). + Restarting the admitted installed body never needs a deployment and is + never gated by a deployment brake. A deployment brake gates only artifact, + revision, configuration, schema, unit or authority changes. +5. **Rejections of stale observations are not a log stream.** An observation + that cannot bind the current activation is either resolved by the owner + that can refresh it, or reported once as an incident. It is never + re-rejected several times a second. +6. **Existing invariants hold.** `route-continuity-target.md` 1 to 7 (in + particular no actuation from observation, bounded backoff, every + post-fencing phase ends) and `authority-map.md` (Idunn's own recovery + depends on no managed target). + +## Scope + +In: + +- Idunn's continuity of admitted bodies across a reboot: absent units, the + restart budget and what charges it. +- The handover between continuity and deployment, in both directions. +- Classifying the topology-correlation and write-lease rejections: the same + fault, or a separate one with its own cut. +- Re-admitting streampixels-service on Yggdrasil. It is running again under + tx-0804185a, and the closing proof checks that it survives the fix's + install. + +Out, and not consumers of this change: + +- **nginx's boot order on Yggdrasil.** It is host configuration, owned by + `gamecult-ops`. It gets its own operator-approved change and record, not an + Idunn cut. +- **Ghostlight's degraded route** (53 consecutive failures, incident open) is + in scope only if the map shows the same fault. Otherwise it is a follow-up. +- **Heimdall's onboarding failures and its environment.** +- **Retention of the ~130 dead transient units** on Yggdrasil. It is an ops + follow-up already recorded under route continuity. +- **Luna's 2026-09-27 weekend patches** are not foundation to build on. Where + this map meets them, it cuts them. From f54c776417919069db0e6825bbc2167fb70bd65f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:17:41 +0200 Subject: [PATCH 052/196] Reincarnation map: body facts from the 2026-10-18 reboot, model page, cut sequence Co-Authored-By: Claude Opus 5.5 --- docs/reincarnation-map.md | 283 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 docs/reincarnation-map.md diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md new file mode 100644 index 0000000..82f8dc7 --- /dev/null +++ b/docs/reincarnation-map.md @@ -0,0 +1,283 @@ +# Reincarnation after reboot: map + +Status: map, Imagination, 2026-10-18 (campaign `idunn-reincarnation`, session +`idunn-reincarnation-2026-10-18`). The ends are in +[`reincarnation-target.md`](reincarnation-target.md). Questions, rulings, the +first cut spec and follow-ups live in Eureka's mind; this page holds what has +no kind: body facts, the model page, the authority map's rationale and the +cut sequence. Line numbers are at `c2477d4`. + +In force from the mind: `idunn-reincarnation:cut_spec:cut-c1-durable-stop.r1`; +questions `idunn-reincarnation:question:handover-pre-fencing` and +`idunn-reincarnation:question:budget-charge`; defaulted rulings +`stop-is-control-state`, `stop-rearms-on-window`, +`rematerialize-once-per-admission`; follow-ups +`ghostlight-abort-lease-before-stop`, `host-actuator-reboot`, +`odin-route-record-stale`. Cited from `idunn-watchdog`: +`exhaustion-close-rule-op` (an exhaustion incident closes when the target runs +and the window no longer holds it) and `paging-conditions` (page on +exhaustion). + +## 1. Body facts + +Evidence: the `idunn-yggdrasil` journal from 11:40 UTC (94,670 lines), the +per-unit journals of the streampixels-service candidates and of Odin +(`idunn-odin-7cfe5cd1…`), `idunn status` and `idunn status --command ` +on Yggdrasil, `journalctl -u nginx`, and the source at `c2477d4`. All times +UTC, 2026-10-18. + +**B1. The reboot did not stop continuity from re-materializing anything.** +Every streampixels-service continuity attempt minted a fresh transient unit +from the installed release `sha256-df663f0a…`: 54 continuity commands from +`continuity-85887ef2` (11:46) to `continuity-52bbbd66` (19:57), most listed by `idunn status` as +`continuity-… failed systemd unit idunn-streampixels-service-.service +is not running`. A continuity transaction is a full `DeploymentTransaction` +built by `from_continuity` (`control_plane.rs:1678`) from the admitted plan, +sealed release, installed release and Expected. It runs every phase: +Starting, Warming, Fencing, Leasing, AwaitingReady, Routing, Committing. The +absent admitted unit `461c6443…` was never the problem. Each new candidate +was. + +**B2. Every candidate died of a dependency, not of itself.** The +streampixels-service process exits about 30 s after start when Odin does not +admit its initial presence. Unit `2bc02905…` at 11:47:21 logged "StreamPixels +service failed to enter its Idunn incarnation: Error: Odin did not admit the +initial Idunn runtime presence ... Timed out waiting for CultNet RUDP response +after 5000 ms." The same is logged by `42ecf363…` at 13:47:13, `03acf44d…` +at 19:47:14, `45276008…` at 19:49:54 and `599b705b…` at 19:51:18. Odin's +RUDP endpoint is `rudp://10.77.0.1:17871`, which nginx proxies. nginx failed +at 11:46:26 ("bind() to 10.77.0.1:17871 failed (99: Cannot assign requested +address)") and started at 19:46:24. Between 11:46 and 19:44 Odin's journal +shows no presence message from streampixels-service at all: none arrived. +After nginx returned, the first one arrived at 19:49:25 and was rejected: +"runtime presence has no exact current activation and provider anchor". The +attempts minted at about 19:53 and 19:55 got past Fencing and then failed +"the Leasing phase passed its deadline" (FailedAfterFencing, RestoreIncumbent). +Other early failures: "observed write lease is not the current Idunn grant" +(11:46:47, the first attempt after boot), and three "reading process stat +/proc/N/stat: No such file or directory" races. + +**B3. Every attempt was charged at mint.** `record_restart(now)` runs inside +the mint CAS (`control_plane.rs:6156-6157`). Nothing reads a continuity's +outcome back into the log. The only exemption anywhere is `HostUnobservable` +(`:5944`), and it applies to an observation, not to a candidate. So the 6-per- +hour window (`CONTINUITY_RESTART_ATTEMPTS` = 6, window 3,600,000 ms, `:71-72`) +filled at 11:50:52 and refilled every hour as entries slid out. The bursts are +in the journal at :47, :48, :49, :51, :52 and :53 of every hour, 39 times +"Idunn stopped restarting admitted streampixels-service". This answers Q2: +the nginx outage counted against the body's budget, and the place that decides +it is the unconditional charge at `:6156`. + +**B4. The deployment was killed by a window slide, not by a live +continuity.** `up-3ad6b05b` (release `sha256-67e227b2…`, a new release) was +recorded at 20:24:51. It was still before Fencing at 20:46:41, when it was +completed `FailedBeforeFencing` with "admitted incumbent failed before +candidate fencing; deployment yielded to continuity: systemd unit +idunn-streampixels-service-461c6443….service is absent". Two seconds later +continuity minted `tx-0804185a` with the OLD release `df663f0a…`, and it was +admitted. The restart that left the window was minted at about 19:46:40 (its +unit `03acf44d…` started at 19:46:42), exactly one window before the abort. +The mechanism, from the source: + +- `:5805-5821`: a live transaction for the target is the `blocker`. From + Fencing on (or Starting for stop-then-start) supervision skips the target + entirely. +- `:6033-6061`: with a blocker present, continuity never restarts and never + reports. It may abort the blocker, and the predicate for that is + `Deploy && !supervision.restarts_exhausted(now) && phase < Fencing && no + abort && not held` (`:6045-6050`). +- `:6062-6084`: only with no blocker does an exhausted continuity open the + incident and say "The target is free for a deployment to replace it". + +So from 20:24:51 the deployment owned the target (`owns_target_authority`, +phase != Complete, `:1772`). Continuity deferred to it through the blocker +`continue` at `:6060`. The deployment deferred to continuity whenever the +window had a free slot, through `:6046`. "Stopped" existed only as a durable +incident (incidents.cc, a projection) and a report. The ownership decision +read the window. When the window gained a slot, continuity reclaimed the +target with no record that it had resumed. This answers Q3: the two owners are +the live `DeploymentTransaction` (it owns while not Complete) and +continuity-as-window (it owns whenever the window is not full). Continuity +defers to the deployment while any transaction is live. The deployment defers +to continuity while the window has room. The "free for a deployment" message +names the first ownership, and the code acts on the second. + +**B5. Fencing an absent incumbent already works.** `advance_fencing` +(`:6869-6952`) stops the exact incumbent and revokes its lease. The systemd +driver's `stop` returns `Ok(())` for an absent unit (`drivers.rs:4003-4007`). +A deployment that is not aborted therefore replaces an absent incumbent with +nothing to fence, which is target invariant 3's last clause, already true. +`exact_incumbent` (`:7556-7569`) fails "target incumbent changed after +transaction creation" if a continuity commit replaced the generation during +the deployment. That is why a phase-split handover needs a rebase +(`handover-pre-fencing`). + +**B6. The two rejection streams are consequences of a dead admitted +generation, not separate faults.** "preserved admitted streampixels-service +after rejecting topology observation: topology correlation does not bind the +current activation" was logged 46,271 times, from 11:46:49 to exactly +20:46:48, about 5,300 an hour. Ghostlight's 42,482 identical lines ran to +19:56:29. Since 20:47, raven-muninn, newly exhausted, shows the same line +621 times. `refresh_admitted_topology` (`:6395-6457`) authenticates Odin's +correlation against the admitted generation's activation. When that +incarnation is dead and Odin's latest record names another one (a +continuity candidate's, or a stale one), authentication fails with an `Err` +rather than `Ok(false)`, and `:5834-5840` prints it on every pass. The pass +order is what does it: route supervision (`:5822`) and topology refresh +(`:5834`) run before the workload is observed (`:5891`). So a dead +generation is "supervised" every tick. "rejected admitted +streampixels-service route continuity: admitted process write lease is no +longer exact" (3,615, from 20:02:23 to 20:46:48) followed the two +post-fencing aborts at 19:54 and 19:57. Fencing revoked the dead incumbent's +lease, "restore incumbent" cannot re-grant a lease to a process that does not +exist, and `supervise_admitted_route` (`:6250-6255`) then refuses the +admitted generation on every challenge. Both streams stopped the moment +`tx-0804185a` was admitted. This answers Q4: both streams follow from one +model gap, that a dead admitted generation is treated as a live incumbent. +That gap is cut 4. They are not the deadlock. + +**B7. Ghostlight is a separate fault.** Its continuity `tx-039b91e9` is +wedged in a post-fencing abort. It was "lease-not-adopted" at its AwaitingReady +deadline, and the abort withdraws the candidate's lease (`:8895-8928`) before +it stops the candidate (`:8929-8946`). The candidate unit `6c51e35e…` has +been running since 19:55:15 and holds the lease's shared lock, so every +attempt fails "process write lease is held by a shared-lock consumer" +(`drivers.rs:4179/4224`). The transaction owns the target, so continuity is +locked out and the route degraded (53 consecutive failures). This is follow-up +`ghostlight-abort-lease-before-stop` (route-continuity invariant 5), not the +stopped-owner deadlock. + +**B8. Unexplained, recorded.** Idunn's record of Odin's stable route reads +"degraded-since 11:46:31, consecutive-failures 1, next-challenge-at 11:47:01" +about nine hours later. It has not been re-challenged since. This is follow-up +`odin-route-record-stale`, and it blocks the recommended answer to +`budget-charge`, which reads dependency health from this record. + +**B9. Luna's weekend commits on this path (Q5).** None of them caused the +deadlock. The yield predicate dates from the route-continuity and watchdog +campaigns, not from the weekend. Where they touch: + +| Commit | On the path? | Status | +|---|---|---| +| `9f00e7a` replay provider receipts at admission time | Yes: the deployment's Sealing compile (`advance_sealing` → `current_ready_provider_tokens`) logged "excluded non-current provider ghostlight / raven-muninn: admitted provider's latest receipt is not Ready at its admission time" at 20:24:53 (`:8588`, `:8609`) | Still live: route-continuity B4 has not landed. It did not block this deployment | +| `29946bb` wait for the stable route listener | Yes: the route proofs during the outage were "Connection refused" on 127.0.0.1:8831-8833 | Kept by route-continuity (honest scope) | +| `8ae00a1` cancel live stateless candidates | No: no cancel ran | Still present (`:2888`); route-continuity S2 owns its deletion | +| `ae8ac45`, `8651906`, `e976d32`, `5e94eb8` identity and launch evidence | Only in `observe` and `stop`; nothing here drifted | Kept | +| `a148802` | No | Reverted (`87fa57d`) | + +The target forbids building on them. Cut 1 touches none of their lines. + +## 2. Model page + +One row per persistent kind the campaign touches. + +| Kind | What names it | Lifecycle | Who decides | +|---|---|---|---| +| Continuity budget: `TargetSupervision.continuity_restarts` (control.cc, `idunn.target_supervision.v1`) | key = target | Ascending times, at most 6, sliding 1 h window. It outlives every generation. Charged at mint today, and after `budget-charge` at outcome | Continuity supervision, in the mint CAS | +| Continuity stop (new, cut 1: keys 7-8) | target, plus the admitted `generation_id` it names | Written when the window is spent. Cleared by the next continuity mint (an explicit resume, `stop-rearms-on-window`). Inert once another generation is admitted | Continuity supervision only (`stop-is-control-state`) | +| Re-materialization bound (new, cut 1: key 9) | target | Set by the one uncharged re-materialization. Spent while it is not older than the admitted generation's `admitted_at` | Continuity supervision, from the workload driver's evidence | +| Admitted generation (`idunn.admitted_generation`) | key = target, `generation_id` = `generation-` | Replaced atomically by every committed deployment or continuity. A dead incarnation keeps its record until replaced | The committing transaction | +| Incident `ContinuityExhausted` (incidents.cc) | condition + subject (target) | Opened on exhaustion. Closed when the target runs and the window no longer holds it (`exhaustion-close-rule-op`). Retired after retention | Idunn's incident writer, a notification projection: never read by ownership | +| Transaction and pre-fencing abort (`idunn.deployment_transaction.v4`, history.cc once terminal) | `tx-` and its `command_id` (`up-…` or `continuity-…`) | Sealing … Complete. A pre-fencing abort completes `FailedBeforeFencing`, a post-fencing abort `FailedAfterFencing` with a named recovery. Retired to history | The scheduler advances it. Continuity supervision may begin a pre-fencing abort of a Deploy only while continuity is live | +| Unit materialization (systemd transient unit `idunn--`) | the activation's hash | Created by Starting. Lost on a host reboot (`/run/systemd/transient`). A failed unit stays loaded; a cleanly stopped one is collected | The workload driver actuates. Whether a loss was a reboot is the driver's evidence (`WorkloadPort::lost_to_host_reboot`, cut 1) | + +No cell is open. The two forks are questions with recommendations, and cut 1 +does not depend on either of them. + +## 3. Authority map: continuity and deployment + +The authority map for cut 1 is in its spec. This section is the campaign-level +map that the spec is one step of. + +- **Owner.** Exactly one authority actuates a target at each instant. A live + `DeploymentTransaction` owns the target from Fencing on. Before Fencing, + ownership is the open fork `handover-pre-fencing`; the recommendation is that + the deployment owns only its candidate and continuity owns the incumbent + slot. Continuity owns the admitted generation's survival, and whether it is + live or stopped is its own durable record. +- **Inputs.** For continuity: the admitted generation, the workload driver's + observation and reboot evidence, its restart log, and the live + transaction's kind and phase. For a deployment: its own plan, its brake, and + the admitted generation at Fencing. +- **Outputs.** Continuity: mints, charged or uncharged, a stop record, and an + abort of a pre-fencing deployment only while it is live (until + `handover-pre-fencing` removes even that). Deployment: a new admitted + generation, or an abort that names its recovery. +- **Derived state.** `restarts_exhausted(now)` is derived and is no longer an + ownership input. The ContinuityExhausted incident and the "free for a + deployment" report are notifications. `idunn status` is display. +- **Forbidden writers.** The sliding window may not decide a yield. The + incident store may not decide ownership. No deployment path writes + continuity's stop. Continuity never aborts a deployment that it has stopped + in front of. A window slide never resumes continuity while any transaction + owns the target. +- **Shared paths.** Crash restart, reboot re-materialization and resume all + mint through the one mint CAS (`:6158-6183`). Deployments, whether operator + or scheduled, go through `advance_transaction`. +- **Deletion line.** The window term in the yield (`:6046`) dies first. + Under `handover-pre-fencing` = phase-split, the whole yield + (`:6033-6061`) and the blocker gate on continuity die in cut 2. + +Demotions, named: `restarts_exhausted` is no longer an owner of the handover. +It is derived from the log and feeds the stop write, the incident and status. +The ContinuityExhausted incident is no longer read as "stopped". It is derived +from exhaustion. + +## 4. Rationale + +The fault is a split brain, not a missing rule. Continuity's "I have stopped" +was written in two places, a durable incident and a log message. Neither was +the place the ownership decision read; the yield re-derived "stopped" from a +clock. Each answer was right at the moment it was computed, and they disagreed +as soon as the clock moved. The tempting patch, not yielding if an exhaustion +incident is open, would make a projection store decide ownership and add a +second reader of a record whose writes "never wait" and may fail. The coherent +cut moves "stopped" into the record the decision reads (control.cc, beside the +log it summarises), deletes the clock from the decision, and binds the stop to +the generation, so a new admission ends it without another writer. + +The reboot invariant needs the evidence of what happened, not a guess. +"Unit absent" alone is not a reboot: a cleanly stopped transient unit is +collected too. "Host booted after admission" alone is not a loss: the unit +might have been started this boot. The systemd driver owns both facts (a +`systemctl show` of nothing, and `/proc/stat` btime), so it answers one +question, `lost_to_host_reboot`, and supervision keeps the budget. The bound, +once per admission, keeps a reboot loop finite without inventing a boot +ledger. It compares two wall clocks (Idunn's admission time and the kernel's +boot time). Admission happens at least one full transaction, tens of seconds, +after boot, so skew of a few seconds cannot misclassify it. + +What cut 1 deliberately does not fix: dependency failures still charge +(`budget-charge`), a live continuity still kills a pre-fencing deployment +(`handover-pre-fencing`), and a dead generation is still supervised every tick +(cut 4). Replayed against 2026-10-18 with only cut 1: the reboot +re-materialization at 11:46 is free, the next five attempts are charged and +fail on the nginx outage, and the stop is recorded at about 11:53. The +operator's `idunn up` at 20:24 is then not aborted at 20:46. It proceeds over +the absent incumbent and admits the new release. Without that deployment, +continuity resumes explicitly each hour, as it did, under +`stop-rearms-on-window`. + +## 5. Cut sequence + +1. **c1-durable-stop**: spec + `idunn-reincarnation:cut_spec:cut-c1-durable-stop.r1`. Depends on + nothing. +2. **Handover before Fencing**: waits on `handover-pre-fencing`. With + phase-split it deletes the yield and the blocker gate on continuity, admits + one continuity and one pre-fencing deployment per target, and makes the + deployment rebase its incumbent at Fencing (`:7556`). +3. **Budget measures the body**: waits on `budget-charge` and on follow-up + `odin-route-record-stale`. It moves the charge from the mint to the + terminal outcome, classified against Idunn's own dependency observation. +4. **A dead generation is not supervised as alive** (target invariant 5): + observe the workload before route supervision and topology refresh, so that + a dead admitted generation gets no topology authentication and no + lease-exactness challenge. Its non-binding correlation is reported once, + not every tick. It waits on cut 1, because it rewrites the same function. +5. **Closing proof on Yggdrasil**: an operator-approved install of the merged + cuts. streampixels-service survives the install, and an `idunn status` + check is made before and after. It waits on 1-4. + +Ops, out of scope: nginx boot order (`gamecult-ops`), and retention of the +roughly 130 dead transient units. From 74a700817fed8c0724bb139c43531fe19889797e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:22:33 +0200 Subject: [PATCH 053/196] Research: restart budgets and rollout/restart handover prior art (idunn-reincarnation forks) Co-Authored-By: Claude Opus 5.5 --- docs/research/restart-budget-prior-art.md | 124 ++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 docs/research/restart-budget-prior-art.md diff --git a/docs/research/restart-budget-prior-art.md b/docs/research/restart-budget-prior-art.md new file mode 100644 index 0000000..a320c42 --- /dev/null +++ b/docs/research/restart-budget-prior-art.md @@ -0,0 +1,124 @@ +# Restart-budget and handover prior art (Eyes facts pass) + +Serves two operator questions in Eureka campaign `idunn-reincarnation` (session idunn-reincarnation-2026-10-18): + +- `idunn-reincarnation:question:budget-charge` (what charges a target's continuity restart budget) +- `idunn-reincarnation:question:handover-pre-fencing` (who acts when the incumbent dies while a deployment is before Fencing) + +Facts and evidence pointers only. No recommendation. Gathered 2026-10-18 from official docs and upstream source (main branches at fetch time). Quotes are under 15 words. "Not found" means the primary source read does not state it, not that it is false. + +## 1. Kubernetes + +### 1.1 CrashLoopBackOff schedule, cap, reset (kubelet) + +- Docs, pod lifecycle, "Container restarts": backoff is "exponential backoff delay (10s, 20s, 40s, ...)", "capped at 300 seconds (5 minutes)". Reset: after the container has run "for 10 minutes without any problems". https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ +- Source pkg/kubelet/kubelet.go (https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet.go): `initialCrashLoopBackOff = time.Second * 10`; max is `MaxCrashLoopBackOff = v1beta1.MaxContainerBackOff` (value lives in the kubelet config API package, not re-read; docs say 300 s). Under alpha gate `ReduceDefaultCrashLoopBackOffDecay`: initial 1 s, max 60 s. Gate `KubeletCrashLoopBackOffMax` lets the config set the max (`CrashLoopBackOff.MaxContainerRestartPeriod`). +- Reset rule in code: `klet.crashLoopBackOff.HasExpiredFunc` returns `eventTime.Sub(lastUpdate) > 600*time.Second` (kubelet.go ~line 1102). I.e. the entry resets when more than 600 s have passed since the entry was last updated (last restart event). The client-go default is 2*maxDuration; the kubelet overrides it. +- Growth: client-go `Backoff.Next` doubles (`delay := entry.backoff * 2`) and caps with `min(delay, p.maxDuration)` (https://github.com/kubernetes/client-go/blob/master/util/flowcontrol/backoff.go). +- Window anchor: the exited container's `FinishedAt`; keyed per pod+container; event text "Back-off restarting failed container" (pkg/kubelet/kuberuntime/kuberuntime_manager.go, `doBackOff`). +- The backoff table is an in-memory object created in `NewMainKubelet` (`flowcontrol.NewBackOff(base, boMax)`); no persistence found in the code read. Not found: any doc statement that a node reboot does or does not reset or count toward it. Docs only say restart policy "only applies to restarts that make replacement containers inside the same Pod and running on the same node." Docs on kubelet restarts: pods "continue to run even during the restart" (kubelet restart, not node reboot). + +### 1.2 Probes + +Docs (pod lifecycle "Container probes"; https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/): + +- Liveness: past the failure threshold "the kubelet restarts that container" (per pod restartPolicy). +- Readiness: the EndpointSlice controller "removes the Pod's IP address from the EndpointSlices"; the container is not restarted. +- Startup: until it succeeds "Kubernetes does not execute liveness or readiness probes"; on failure "the kubelet kills the container", subject to restart policy. +- Backoff keys on a container having exited (`doBackOff` reads the last exited container), so crash, liveness kill and startup-probe kill all reach it; readiness failure leaves the container running and does not. The docs do not describe backoff per probe type. + +### 1.3 Dependency down + +Docs, same page, "Dependencies and init containers": init containers "run only once (if successful), during Pod startup"; failed init containers are restarted by the kubelet when pod restartPolicy is OnFailure or Always. Readiness gates: pod is ready only when all `readinessGates` conditions are True. No general documented rule for a main container whose external dependency is down: an app that exits is restarted under backoff; an app that stays up and fails readiness is only removed from endpoints. + +### 1.4 RollingUpdate and the old ReplicaSet + +- Deployment docs (https://kubernetes.io/docs/concepts/workloads/controllers/deployment/): "Once new Pods are ready, old ReplicaSet can be scaled down further"; at the end "all old ReplicaSets is scaled to 0". maxUnavailable and maxSurge default 25%. progressDeadlineSeconds default 600; when exceeded, "Kubernetes takes no action on a stalled Deployment" beyond the Progressing=False condition. +- ReplicaSet docs: it fulfills its purpose by "creating and deleting Pods as needed to reach the desired number." https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +- Deployment controller source, pkg/controller/deployment/rolling.go (https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/deployment/rolling.go): `reconcileOldReplicaSets` computes `minAvailable = replicas - maxUnavailable`, runs `cleanupUnhealthyReplicas` first (comment: "Clean up unhealthy replicas first, otherwise unhealthy replicas will block deployment"), then `scaleDownOldReplicaSetsForRollingUpdate`. The Deployment controller decides old-RS replica counts during rollout, scaling unavailable ones down first. +- Layering: restarts of a dying container are done by the kubelet (restartPolicy with backoff), not the ReplicaSet. The ReplicaSet recreates a pod only when the pod object is gone or terminal. Not found: a statement in those words that the old ReplicaSet keeps recreating its dying pods until scaled down. + +## 2. Nomad + +### 2.1 restart block (https://developer.hashicorp.com/nomad/docs/job-specification/restart) + +- `attempts` restarts allowed within `interval`; interval "starts when the first task starts". Defaults: service/system attempts 2, interval 30m, delay 15s, mode fail; batch attempts 3, interval 24h, delay 15s, mode fail. Delay has up to 25% jitter. +- `mode = "fail"` (default): after attempts are used the allocation is marked failed and goes to `reschedule`. `mode = "delay"`: "The task waits another full interval before restarting"; the allocation is never failed by restarts. +- Source client/allocrunner/taskrunner/restarts/restarts.go (https://github.com/hashicorp/nomad/blob/main/client/allocrunner/taskrunner/restarts/restarts.go): in-memory `RestartTracker` with `count` and `startTime` set at construction (`time.Now()`); a new interval resets count; `count > Attempts` returns TaskNotRestarting in fail mode, or TaskRestarting with delay to interval end in delay mode. Constructed in `NewTaskRunner` (task_runner.go ~line 468) from the alloc's job type and the task group's policy. `SetKilled` returns TaskKilled without counting. A restart triggered with failure=false (operator/API) returns TaskRestarting without consulting the policy. Unrecoverable start errors return TaskNotRestarting. +- Client restart: restarts.go keeps no persisted counter. `TaskRunner.Restore` (task_runner.go ~line 1244) restores local state and the driver handle (`RecoverTask`); if the handle cannot be restored for a non-system job the task "will not run until server is contacted" (`waitOnServers`). The tracker is rebuilt at task-runner construction. Persisted `TaskState` carries restart counts separately; `GetState` does not read them. + +### 2.2 reschedule block (https://developer.hashicorp.com/nomad/docs/job-specification/reschedule) + +- Applies when an allocation fails to set up or its tasks fail more than `restart` allows. Drains trigger migrations and lost nodes trigger replacements, so reschedule is ignored in both cases. Not applied to system/sysbatch. +- Service defaults: `delay_function = exponential`, delay starting 30s, `max_delay` 1h, `unlimited = true`. Batch defaults: 1 attempt per 24h, constant 5s, unlimited false. `delay_function`: constant, exponential (doubling), fibonacci. `max_delay` ignored for constant. Minimum delay 1 s; docs warn very low delay with `unlimited = true` can cause runaway rescheduling. +- With `unlimited = true`: "The `progress_deadline` in the `update` block still applies". + +### 2.3 Deployments and restarts of old allocations + +- Update block (https://developer.hashicorp.com/nomad/docs/job-specification/update): `max_parallel` 1, `min_healthy_time` 10s, `healthy_deadline` 5m, `progress_deadline` 10m ("the deployment is marked as failed"), `auto_revert` false, `canary` 0. The page says nothing on restarts of old allocations. Not found: docs on deployment/restart interaction. +- Restart is client-local ("Restarts happen on the client that is running the task"); deployment is server-side. Reconciler source (scheduler/reconciler/reconcile_cluster.go, `placeAllocs`): when the deployment is not place-ready (paused/failed/canarying), rescheduling of failed allocs still proceeds, with comment about avoiding "odd user experiences", except where the failed alloc is part of the failed deployment (`partOfFailedDeployment` skips it). + +## 3. systemd + +### 3.1 Start limit (https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html) + +- A unit started more than `StartLimitBurst=` times within `StartLimitIntervalSec=` may not start again until the interval passes. Applies to all starts, including manual, not only `Restart=`; it is checked after condition checks, so failing conditions do not count. Units with `Restart=` that hit it are no longer restarted automatically; `systemctl reset-failed` flushes the counter. `StartLimitAction=` default none. Counters are cleared when the unit is unloaded. +- Source: `unit_test_start_limit` in src/core/unit.c (`ratelimit_below(&u->start_ratelimit)`; on hit logs "Start request repeated too quickly", returns -ECANCELED); `unit_reset_failed` calls `ratelimit_reset`. + +### 3.2 Job result "dependency" and the start limit + +- src/core/unit.c `unit_start` order: conditions (-ECOMM), asserts (-EPROTO), `unit_verify_deps` (-ENOLINK, "The necessary dependencies are not fulfilled"), then `UNIT_VTABLE(u)->test_startable`. For services, `service_test_startable` (src/core/service.c ~6381) calls `unit_test_start_limit`. A start failing on unfulfilled dependencies returns before the start-limit counter is touched. +- src/core/job.c: `-ENOLINK` maps to `JOB_DEPENDENCY` (`job_run_and_invalidate`); `job_fail_dependencies` also finishes dependent START/VERIFY_ACTIVE jobs with `JOB_DEPENDENCY` ("Dependency failed for %s."). `n_failed_jobs` increments only for JOB_FAILED, JOB_INVALID, JOB_FROZEN, JOB_CONCURRENCY. JOB_DEPENDENCY triggers `OnFailure=` handling. +- Requires=: if the required unit fails to activate and the dependent has After= on it, the dependent is not started; without After= it may still start. Wants=: failure "has no impact on the validity of the transaction as a whole." Dependent is stopped when the required unit is explicitly stopped or restarted; not stopped if the required unit stops on its own. + +### 3.3 Restart= and dependency failure + +- systemd.service(5) (https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html): the restart table is by exit cause of the service's own process (exit code, signal, timeout, watchdog, OOM); no row for dependency failure. Page: "service restart is subject to unit start rate limiting." Not found: a doc statement on whether Restart= acts on a dependency failure. From the code above, a dependency-failed start job finishes with JOB_DEPENDENCY and never passes through the service's own failure state (inference, not an explicit doc statement). +- Backoff: `RestartSec=` default 100ms; `RestartSteps=` with `RestartMaxDelaySec=` give exponential growth to a cap (doc example 10s, 20s, 40s, 80s, 160s, 160s...). `RestartMaxDelaySec` default infinity; both must be set. `RestartRandomizedDelaySec=` (v262) adds jitter. + +### 3.4 Transient units across reboot + +- systemd.unit(5) path table (man/systemd.unit.xml ~lines 403, 465): "Dynamic configuration for transient units" lives in `/run/systemd/transient` (system) and `$XDG_RUNTIME_DIR/systemd/transient` (user). That `/run` is volatile is not stated in this file. +- systemd-run docs: failed transient units are "kept in memory until the user explicitly resets their failure state"; `--collect` is `CollectMode=inactive-or-failed`. systemd-run and https://systemd.io/TRANSIENT-SETTINGS/ do not mention reboot. Not found: an explicit sentence on survival across reboot. + +## 4. Other supervisors + +### 4.1 Erlang/OTP supervisor (https://www.erlang.org/doc/apps/stdlib/supervisor.html, https://www.erlang.org/doc/system/sup_princ.html) + +- `intensity` (MaxR) and `period` (MaxT), defaults 1 and 5. If more than MaxR restarts occur within the last MaxT seconds, the supervisor "terminates all the child processes and then itself" (exit reason `shutdown`). The parent supervisor then restarts it or itself terminates; failure escalates up the tree. Stated purpose: prevent a process from repeatedly crashing for the same reason and being restarted indefinitely. +- Restart types: `permanent` (always), `transient` (only abnormal exit), `temporary` (never). Every restart the strategy performs counts toward intensity; the docs describe no per-cause charging and no backoff delay. Multi-level: total restarts before the top gives up is roughly the product of intensities above the failing child. Strategy `rest_for_one` restarts the failed child and those started after it. + +### 4.2 runit and s6 + +- runit `runsv` (https://smarden.org/runit/runsv.8.html): if ./run exits immediately runsv "waits a second" before restarting; no rate limit or retry cap described; keeps restarting unless told otherwise (`o`, `d`, `x`). +- s6-supervise (https://skarnet.org/software/s6/s6-supervise.html): "There is a minimum 1-second delay between two ./run spawns." A ./finish script exiting 125 makes supervise stop restarting; otherwise it restarts indefinitely. The give-up decision is delegated to a user script, not a built-in counter. + +### 4.3 Borg (Verma et al., EuroSys 2015; https://research.google/pubs/large-scale-cluster-management-at-google-with-borg/, PDF text read) + +- Borglet "restarts them if they fail"; Borg "monitors the health-check URL and restarts tasks that do not respond promptly" (sections 2.6, 3.3). +- Section 4 (Availability): Borg "automatically reschedules evicted tasks"; "limits the allowed rate of task disruptions and the number of tasks from a job that can be simultaneously down"; rate-limits finding new places for tasks from unreachable machines because it cannot tell machine failure from network partition; "avoids repeating task::machine pairings that cause task or machine crashes". +- Section 2.3 (updates): "Updates are generally done in a rolling fashion", with a limit on task disruptions an update causes; changes that would exceed it "are skipped". +- Not found in the Borg paper: a crash-loop backoff schedule, a restart budget, or any distinction between charging task faults and dependency faults. Omega paper not fetched. + +## 5. Facts grouped by the brief's sub-topics + +- (a) Charging only body faults: Kubernetes backoff keys on container exit regardless of cause (readiness failure does not exit and does not advance it). systemd runs dependency verification (-ENOLINK, JOB_DEPENDENCY) before the start-limit test, and JOB_DEPENDENCY is not in `n_failed_jobs`. Nomad's tracker counts any recoverable task failure or start error, with no dependency classification. OTP counts every restart. Borg avoids repeating task::machine pairings that cause crashes (placement, not budget). +- (b) Give up vs capped forever: OTP gives up and escalates. Kubernetes caps at 300 s with no give-up. Nomad restart fail-mode gives up locally, then reschedule continues (service default unlimited, exponential to 1 h); delay-mode never fails. systemd start limit stops auto-restart until the interval passes or reset-failed; `RestartMaxDelaySec` default infinity. runit restarts forever at 1 s; s6 restarts forever unless finish exits 125. +- (c) Rollout vs restart supervisor: Kubernetes splits by layer (kubelet restarts containers; ReplicaSet maintains pod count; Deployment controller scales old RS during rollout). Nomad splits by node vs server (client restart tracker vs server reconciler; reconciler skips rescheduling allocs of a failed deployment). Borg bounds update-induced disruptions with a rate limit. + +## 6. Memory claims vs sources + +budget-charge: + +1. "Kubernetes separates readiness from liveness; only liveness feeds CrashLoopBackOff": CORRECTED. Readiness never restarts and never advances backoff (confirmed). Backoff keys on any container exit (crash, liveness kill, startup-probe kill), so it is not fed only by liveness. +2. "CrashLoopBackOff caps at 5 min": CONFIRMED (docs 300 s; initial 10 s, doubling). Reset: docs say 10 min healthy; code says 600 s since the entry's last update. 60 s cap only under alpha gate. +3. "systemd does not run a unit whose required dependency failed (job result 'dependency')": CONFIRMED with qualifier: Requires= plus After= blocks the start; Requires= without After= may still start; Wants= does not block. +4. "Nomad reschedules with capped exponential delay": CONFIRMED for service jobs (exponential from 30 s, max_delay 1 h, unlimited by default); batch default is constant 5 s, 1 attempt per 24 h. +5. Dependency failure vs start limit (not in the question's claims, material): systemd checks dependencies before counting a start (confirmed by code order). Whether Restart= acts on dependency failure: UNSUPPORTED by docs. +6. Whether restarts after a node reboot count toward CrashLoopBackOff: UNSUPPORTED by docs; code read shows the kubelet backoff map in memory. Nomad's restart tracker is likewise in memory and rebuilt on client restart. + +handover-pre-fencing: + +1. "In a Kubernetes rolling update the old ReplicaSet keeps recreating its dying pods until the rollout scales it down": PARTLY CONFIRMED, PARTLY CORRECTED. ReplicaSets do create pods to reach their desired count, and the Deployment controller scales old RS down, unhealthy first. Container restarts are the kubelet's, not the ReplicaSet's; the RS recreates only pods that are gone. +2. "Nomad restarts old allocations by their own restart policy, independent of the deployment": PARTLY SUPPORTED. Restart is client-local using the alloc's task-group policy (code), so independent of the server-side deployment; no doc statement on the interaction. Server-side, failed allocs are rescheduled even when the deployment is not place-ready, except allocs of a failed deployment. +3. Any source saying an aborted or stalled rollout automatically hands the slot back to the old generation: UNSUPPORTED (Kubernetes takes no action on a stalled Deployment; Nomad `auto_revert` defaults false). From f079389be0266eb15ec659bb721d040b70427fd5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:44:14 +0200 Subject: [PATCH 054/196] Reincarnation: continuity never stops (target r2), cut 1 never-stop mapped Co-Authored-By: Claude Opus 5.5 --- docs/reincarnation-map.md | 284 +++++++++++++++++++++++------------ docs/reincarnation-target.md | 15 +- 2 files changed, 196 insertions(+), 103 deletions(-) diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md index 82f8dc7..84fdaac 100644 --- a/docs/reincarnation-map.md +++ b/docs/reincarnation-map.md @@ -3,20 +3,17 @@ Status: map, Imagination, 2026-10-18 (campaign `idunn-reincarnation`, session `idunn-reincarnation-2026-10-18`). The ends are in [`reincarnation-target.md`](reincarnation-target.md). Questions, rulings, the -first cut spec and follow-ups live in Eureka's mind; this page holds what has +cut spec and follow-ups live in Eureka's mind; this page holds what has no kind: body facts, the model page, the authority map's rationale and the -cut sequence. Line numbers are at `c2477d4`. +cut sequence. Line numbers are at `74a7008`; `src/` is byte-identical to +`c2477d4`, so the earlier anchors still hold. -In force from the mind: `idunn-reincarnation:cut_spec:cut-c1-durable-stop.r1`; -questions `idunn-reincarnation:question:handover-pre-fencing` and -`idunn-reincarnation:question:budget-charge`; defaulted rulings -`stop-is-control-state`, `stop-rearms-on-window`, -`rematerialize-once-per-admission`; follow-ups -`ghostlight-abort-lease-before-stop`, `host-actuator-reboot`, -`odin-route-record-stale`. Cited from `idunn-watchdog`: -`exhaustion-close-rule-op` (an exhaustion incident closes when the target runs -and the window no longer holds it) and `paging-conditions` (page on -exhaustion). +Read the in-force set with the rulings-in-force and open-follow-ups recipes +for root `idunn-reincarnation`; the cut in flight is the specs-with-no-report +recipe. The first spec (`cut-c1-durable-stop.r1`) was withdrawn when the +operator ruled that continuity never stops; its replacement is cut +`c1-never-stop`. Cited from `idunn-watchdog`: `exhaustion-close-rule-op` and +`paging-conditions`. ## 1. Body facts @@ -150,8 +147,10 @@ stopped-owner deadlock. **B8. Unexplained, recorded.** Idunn's record of Odin's stable route reads "degraded-since 11:46:31, consecutive-failures 1, next-challenge-at 11:47:01" about nine hours later. It has not been re-challenged since. This is follow-up -`odin-route-record-stale`, and it blocks the recommended answer to -`budget-charge`, which reads dependency health from this record. +`odin-route-record-stale`. It blocks cut 3. Under `budget-never-stop`, a +failure attributed to a declared dependency must not grow the backoff, and the +only dependency-health record Idunn holds for Odin's stable route is this +one. **B9. Luna's weekend commits on this path (Q5).** None of them caused the deadlock. The yield predicate dates from the route-continuity and watchdog @@ -167,117 +166,208 @@ campaigns, not from the weekend. Where they touch: The target forbids building on them. Cut 1 touches none of their lines. +**B10. The stop was the only door a deployment had into a crash loop.** +Source read at `74a7008`. A queued `idunn up` becomes a transaction only in +`freeze_command`, which refuses while any transaction for any of its targets +`blocks_new_target_mutation` (`:5741-5747`); the command just stays queued. +The scheduler tick runs `retire_one_terminal_transaction` (returns early when +it retires), then `resume_one_transaction`, then +`supervise_one_admitted_generation`, then `freeze_one_queued_command` +(`:5414-5423`). Today's doubling wait is anchored at the previous attempt's +**mint** (`next_restart_at`, `:1163-1170`). So once an attempt lasts longer than +its wait, the tick after its transaction is retired mints the next attempt +before the freeze runs, and the queued deployment never finds the target +free. At `c2477d4` that ended when the window filled and continuity stopped. +Delete the stop without opening another door and a crash-looping release can +never be replaced. The door the cut uses is kubelet's own anchor +(`docs/research/restart-budget-prior-art.md` §1.1, the exited container's +`FinishedAt`): the wait runs from the first supervision pass that finds the +target dead **and free**. That leaves at least one wait (5 s or more) with no +live transaction, and the freeze runs every tick. This is reasoning over the +source, not a probe. The cut pins it with a tick-level test. + +**B11. The yield cannot outlive the stop.** With the window term gone from +`:6046`, the yield aborts every pre-fencing Deploy whenever the incumbent is +dead (`:6045-6059`). For a crash-looping release the incumbent is always +dead, so the deployment that would fix it is aborted at the first +supervision pass after it is frozen. That is the 2026-10-18 failure: on that +day the operator's `idunn up` existed to replace a release that would not +start. Ruling `handover-phase-split` already says nothing yields before +Fencing. So cut 1 deletes the whole yield. Continuity keeps deferring to any +live transaction for its target (`:6033`, the `continue`). Cut 2 keeps +the incumbent alive alongside the build. Between them, a dead incumbent stays +down while a pre-fencing deployment over it builds. That is bounded by the +deployment's own deadlines, and the deployment then fences it as nothing +(B5). + +**B12. The doubling function already exists once.** `backoff_wait` +(`:4000-4007`) doubles from a base up to `RESUME_BACKOFF_CEILING_MILLIS`, and +has one caller (`back_off`, `:5621`). Continuity's wait is a second spelling +of the same rule (`5_000 << (live.len() - 1)`, `:1169`). The cut gives +`backoff_wait` a ceiling parameter and deletes the second spelling. +`#[cultcache(key = N, default)]` is how an absent key decodes as its default +(precedent `DeploymentTransaction` key 34, `:1552-1557`). New +`TargetSupervision` keys use it, because every record on Yggdrasil predates +them. + ## 2. Model page One row per persistent kind the campaign touches. | Kind | What names it | Lifecycle | Who decides | |---|---|---|---| -| Continuity budget: `TargetSupervision.continuity_restarts` (control.cc, `idunn.target_supervision.v1`) | key = target | Ascending times, at most 6, sliding 1 h window. It outlives every generation. Charged at mint today, and after `budget-charge` at outcome | Continuity supervision, in the mint CAS | -| Continuity stop (new, cut 1: keys 7-8) | target, plus the admitted `generation_id` it names | Written when the window is spent. Cleared by the next continuity mint (an explicit resume, `stop-rearms-on-window`). Inert once another generation is admitted | Continuity supervision only (`stop-is-control-state`) | -| Re-materialization bound (new, cut 1: key 9) | target | Set by the one uncharged re-materialization. Spent while it is not older than the admitted generation's `admitted_at` | Continuity supervision, from the workload driver's evidence | +| Paging log: `TargetSupervision.continuity_restarts` (control.cc, `idunn.target_supervision.v1`, key 3) | key = target | The newest 6 charged attempt times, ascending, read through a sliding 1 h window. Outlives every generation. After cut 1 it is the paging threshold only (`restarts_page`): it opens and holds the ContinuityExhausted incident and is shown in status. It gates no mint, wait or yield | Continuity supervision, pushed in the mint CAS | +| Attempt streak (new, cut 1: key 7, `default`) | target | Charged attempts since the admitted body last ran a sustained run. +1 in each charged mint CAS. Reset to 0 when the dead generation was admitted after the last charged attempt and ran ≥ 600 s before it was found dead. Saturating, and the wait is capped, so it can be unbounded | Continuity supervision only | +| Backoff anchor (new, cut 1: key 8, `default`) | target | Written on the first pass that finds the admitted generation dead and the target free after an attempt. The next attempt is due at anchor + `backoff_wait(5 s, streak - 1, 300 s)`. Cleared by every mint. Stale, and ignored, once older than the admitted generation's `admitted_at` | Continuity supervision only | +| Re-materialization bound (new, cut 1: key 9, `default`) | target | Set by the one attempt that is not a backoff step, when the driver reports a reboot loss. Spent while it is not older than the admitted generation's `admitted_at` | Continuity supervision, from the workload driver's evidence | +| Projection deferral (keys 4-5) | target | Unchanged: a failed demotion defers the next mint 30 s. It spaces a re-materialization too | Continuity supervision | | Admitted generation (`idunn.admitted_generation`) | key = target, `generation_id` = `generation-` | Replaced atomically by every committed deployment or continuity. A dead incarnation keeps its record until replaced | The committing transaction | -| Incident `ContinuityExhausted` (incidents.cc) | condition + subject (target) | Opened on exhaustion. Closed when the target runs and the window no longer holds it (`exhaustion-close-rule-op`). Retired after retention | Idunn's incident writer, a notification projection: never read by ownership | -| Transaction and pre-fencing abort (`idunn.deployment_transaction.v4`, history.cc once terminal) | `tx-` and its `command_id` (`up-…` or `continuity-…`) | Sealing … Complete. A pre-fencing abort completes `FailedBeforeFencing`, a post-fencing abort `FailedAfterFencing` with a named recovery. Retired to history | The scheduler advances it. Continuity supervision may begin a pre-fencing abort of a Deploy only while continuity is live | +| Incident `ContinuityExhausted` (incidents.cc) | condition + subject (target) | Opened at 6 charged attempts in the hour while the target is dead and free. Closed when the target runs and the window no longer holds 6 (`exhaustion-close-rule-op`). Retired after retention. The name is the wire name and stays, though nothing is exhausted any more | Idunn's incident writer, a notification projection: never read by ownership | +| Transaction and pre-fencing abort (`idunn.deployment_transaction.v4`, history.cc once terminal) | `tx-` and its `command_id` (`up-…` or `continuity-…`) | Sealing … Complete. A pre-fencing abort completes `FailedBeforeFencing`, a post-fencing abort `FailedAfterFencing` with a named recovery. Retired to history | The scheduler advances it. After cut 1, continuity supervision never aborts a transaction; the transaction's own steps do | | Unit materialization (systemd transient unit `idunn--`) | the activation's hash | Created by Starting. Lost on a host reboot (`/run/systemd/transient`). A failed unit stays loaded; a cleanly stopped one is collected | The workload driver actuates. Whether a loss was a reboot is the driver's evidence (`WorkloadPort::lost_to_host_reboot`, cut 1) | -No cell is open. The two forks are questions with recommendations, and cut 1 -does not depend on either of them. +No cell is open. Both forks are ruled (`budget-never-stop`, +`handover-phase-split`). The numbers that `budget-never-stop` left open are +the defaulted ruling `continuity-backoff-defaults`. ## 3. Authority map: continuity and deployment -The authority map for cut 1 is in its spec. This section is the campaign-level -map that the spec is one step of. +The authority map for cut 1 is in its spec. This section is the +campaign-level map, and cut 1 is one step of it. - **Owner.** Exactly one authority actuates a target at each instant. A live - `DeploymentTransaction` owns the target from Fencing on. Before Fencing, - ownership is the open fork `handover-pre-fencing`; the recommendation is that - the deployment owns only its candidate and continuity owns the incumbent - slot. Continuity owns the admitted generation's survival, and whether it is - live or stopped is its own durable record. -- **Inputs.** For continuity: the admitted generation, the workload driver's - observation and reboot evidence, its restart log, and the live - transaction's kind and phase. For a deployment: its own plan, its brake, and - the admitted generation at Fencing. -- **Outputs.** Continuity: mints, charged or uncharged, a stop record, and an - abort of a pre-fencing deployment only while it is live (until - `handover-pre-fencing` removes even that). Deployment: a new admitted + `DeploymentTransaction` owns the target from Fencing on, and a stop-first + rollout owns it from Starting. Before Fencing (`handover-phase-split`), a + deployment owns only its private candidate, and continuity owns the + incumbent slot. Continuity owns the admitted generation's survival and + never stops. When it next acts is a function of its own record: the + streak, the backoff anchor and the re-materialization bound. +- **Inputs.** For continuity: the admitted generation (id, `admitted_at`, + workload, plan), the workload observation and the driver's reboot + evidence, its own record, and whether a live transaction blocks the + target. After cut 3 it also reads Idunn's dependency-health observation. + For a deployment: its own plan, its brake, and at Fencing whatever + generation is admitted at that moment (cut 2). +- **Outputs.** Continuity: mints, either charged or the one re-materialization; + backoff anchors; deferrals; and, as notifications only, the + ContinuityExhausted incident and its once-report. Deployment: a new admitted generation, or an abort that names its recovery. -- **Derived state.** `restarts_exhausted(now)` is derived and is no longer an - ownership input. The ContinuityExhausted incident and the "free for a - deployment" report are notifications. `idunn status` is display. -- **Forbidden writers.** The sliding window may not decide a yield. The - incident store may not decide ownership. No deployment path writes - continuity's stop. Continuity never aborts a deployment that it has stopped - in front of. A window slide never resumes continuity while any transaction - owns the target. -- **Shared paths.** Crash restart, reboot re-materialization and resume all - mint through the one mint CAS (`:6158-6183`). Deployments, whether operator - or scheduled, go through `advance_transaction`. -- **Deletion line.** The window term in the yield (`:6046`) dies first. - Under `handover-pre-fencing` = phase-split, the whole yield - (`:6033-6061`) and the blocker gate on continuity die in cut 2. +- **Derived state.** `restarts_page(now)` (6 charged attempts in the hour) is + derived from the paging log and is notification-only: incident open and + close, and status. The next attempt's due time is derived from the + anchor and the streak. The incident and `idunn status` are projections. +- **Forbidden writers.** The paging window gates no mint, no wait and no + yield. Continuity supervision never aborts a deployment: + `begin_pre_fencing_abort` has no caller in it. No deployment path writes + continuity's fields. The incident store decides nothing. No stopped state + exists: no field, and no branch that ends a pass because a count is + reached. +- **Shared paths.** A crash restart, a restart after a sustained run, and a + reboot re-materialization all mint through the one CAS (`:6158-6183`). The + dead-and-free pass is the only writer of the backoff anchor. Every + deployment, operator or scheduled, enters through `freeze_command` and + advances through `advance_transaction`. +- **Deletion line.** Cut 1 deletes the yield (`:6034-6059`), the + exhaustion `continue` and its "free for a deployment" text (`:6062-6085`), + and the mint-anchored wait (`:1160-1184`). Cut 2 deletes the blocker + `continue` for a pre-fencing deployment (`:6033`), so continuity restarts + alongside the build, and the "incumbent changed after transaction + creation" refusal at Fencing (`:7556-7569`). -Demotions, named: `restarts_exhausted` is no longer an owner of the handover. -It is derived from the log and feeds the stop write, the incident and status. -The ContinuityExhausted incident is no longer read as "stopped". It is derived -from exhaustion. +Demotions, named: `restarts_exhausted` is no longer an owner of anything. It +is renamed `restarts_page`, and it opens and holds the incident. The +ContinuityExhausted incident was never read as "stopped", and there is now no +stopped state for it to stand for. The "free for a deployment" report is +deleted: every target is free for a deployment between attempts (B10). ## 4. Rationale The fault is a split brain, not a missing rule. Continuity's "I have stopped" -was written in two places, a durable incident and a log message. Neither was -the place the ownership decision read; the yield re-derived "stopped" from a -clock. Each answer was right at the moment it was computed, and they disagreed -as soon as the clock moved. The tempting patch, not yielding if an exhaustion -incident is open, would make a projection store decide ownership and add a -second reader of a record whose writes "never wait" and may fail. The coherent -cut moves "stopped" into the record the decision reads (control.cc, beside the -log it summarises), deletes the clock from the decision, and binds the stop to -the generation, so a new admission ends it without another writer. +was written in two places, a durable incident and a log message, and neither +was what the ownership decision read: the yield re-derived "stopped" from a +clock. The withdrawn first spec moved "stopped" into control state. The +operator's ruling is better, and it is also smaller: there is no stopped +state. The window it was derived from becomes a pager. Every case the stop +existed for is then handled by an owner that already exists. A broken release +is replaced by a deployment, which gets in through the gap after each +attempt (B10) and is never yielded (B11). A host-dependency outage costs one +attempt every five minutes or so, and pages. -The reboot invariant needs the evidence of what happened, not a guess. -"Unit absent" alone is not a reboot: a cleanly stopped transient unit is -collected too. "Host booted after admission" alone is not a loss: the unit -might have been started this boot. The systemd driver owns both facts (a -`systemctl show` of nothing, and `/proc/stat` btime), so it answers one -question, `lost_to_host_reboot`, and supervision keeps the budget. The bound, -once per admission, keeps a reboot loop finite without inventing a boot -ledger. It compares two wall clocks (Idunn's admission time and the kernel's -boot time). Admission happens at least one full transaction, tens of seconds, -after boot, so skew of a few seconds cannot misclassify it. +Why the wait is anchored at "found dead and free" rather than at the mint: +the anchor is the door. A mint-anchored wait can be swallowed by a long +attempt, and then no tick ever sees the target free (B10). Kubelet anchors +CrashLoopBackOff on `FinishedAt` for the same reason, so this is the +established spelling, not a new one. A streak is stored, not recomputed from +the paging log. The log holds only six entries and slides by the hour, while +the streak must outlive both. Reading the backoff from the window count was +today's spelling, and it is exactly the mutant the cut's tests must kill. -What cut 1 deliberately does not fix: dependency failures still charge -(`budget-charge`), a live continuity still kills a pre-fencing deployment -(`handover-pre-fencing`), and a dead generation is still supervised every tick -(cut 4). Replayed against 2026-10-18 with only cut 1: the reboot -re-materialization at 11:46 is free, the next five attempts are charged and -fail on the nginx outage, and the stop is recorded at about 11:53. The -operator's `idunn up` at 20:24 is then not aborted at 20:46. It proceeds over -the absent incumbent and admits the new release. Without that deployment, -continuity resumes explicitly each hour, as it did, under -`stop-rearms-on-window`. +The reset, "admitted after the last charged attempt and ran at least 600 s +before it was found dead", needs no new writer. Supervision observes the +admitted generation on every pass, so the first pass that finds it dead is +within one poll of the death. A death hidden behind a live transaction or an +unobservable host overstates the run, and so resets early. That is the safe +side, because it shortens one wait. + +The reboot invariant needs evidence of what happened, not a guess. "Unit +absent" alone is not a reboot, because a cleanly stopped transient unit is +collected too. "Host booted after admission" alone is not a loss, because +the unit might have been started this boot. The systemd driver owns both +facts (a `systemctl show` that returns nothing, and the `/proc/stat` btime). +So it answers one question, `lost_to_host_reboot`, and supervision keeps the +budget. The once-per-admission bound is still needed under never-stop. A +failing re-materialization leaves the same admitted generation, unit absent, +booted after admission. Without the bound it would repeat with no step, no +wait and no page, every tick. The clock comparison is safe: admission +happens at least one full transaction, tens of seconds, after boot. + +What cut 1 deliberately does not fix. A dependency failure still grows the +streak (cut 3). A dead incumbent stays down while a pre-fencing deployment +builds over it (cut 2). A dead generation is still route-supervised and +topology-refreshed every tick (cut 4). Paging residual: the cap gives 3600 / +(300 + D) attempts an hour, where D is the attempt's duration, so a crash +loop whose attempts last longer than 300 s settles under 6 an hour. Its +incident opens in the first hour's burst and stays open while the release +never runs, but a release that runs for one pass after each restart can see +it close and not reopen. That is the operator's threshold, applied as ruled. +It is recorded here, not changed. + +Replayed against 2026-10-18 with only cut 1. The reboot loss at 11:46 is +re-materialized at once, with no step. It fails on the nginx outage. The +next attempt resets the streak, because the generation had run for hours, +and then waits 5, 10, 20, 40, 80, 160 s, and 300 s after that. The incident +opens within the first hour and stays open (the release never runs), so the +operator is paged during the outage, not at 20:24. The `idunn up` at 20:24 +freezes in the next gap. Continuity defers to it, nothing aborts it, and it +fences the absent incumbent as nothing and admits the new release. ## 5. Cut sequence -1. **c1-durable-stop**: spec - `idunn-reincarnation:cut_spec:cut-c1-durable-stop.r1`. Depends on - nothing. -2. **Handover before Fencing**: waits on `handover-pre-fencing`. With - phase-split it deletes the yield and the blocker gate on continuity, admits - one continuity and one pre-fencing deployment per target, and makes the - deployment rebase its incumbent at Fencing (`:7556`). -3. **Budget measures the body**: waits on `budget-charge` and on follow-up - `odin-route-record-stale`. It moves the charge from the mint to the - terminal outcome, classified against Idunn's own dependency observation. +1. **c1-never-stop**: spec in the mind (specs-with-no-report recipe). Depends + on nothing. +2. **Handover before Fencing** (`handover-phase-split`): continuity + restarts the incumbent while a pre-fencing deployment builds. This deletes + the blocker `continue` for a pre-fencing Deploy, and lets one continuity and + one pre-fencing deployment coexist per target, which touches + `blocks_new_target_mutation`, `freeze_command` and candidate ports. The + deployment takes the target at Fencing over whatever is admitted + (`exact_incumbent`, `:7556`). Entering Fencing needs a hand-off with a + live continuity transaction, decided as data rather than by tick order. + Waits on cut 1, because it rewrites the same block. +3. **Dependency failures do not grow the backoff** + (`budget-never-stop`): an attempt that failed because of a declared + dependency (Odin's stable route, the target's own stable route + listener), as Idunn itself observed it, leaves the streak unchanged. + Waits on follow-up `odin-route-record-stale`, because the record it + would read is not trustworthy, and on cut 1. 4. **A dead generation is not supervised as alive** (target invariant 5): - observe the workload before route supervision and topology refresh, so that - a dead admitted generation gets no topology authentication and no + observe the workload before route supervision and topology refresh, so + that a dead admitted generation gets no topology authentication and no lease-exactness challenge. Its non-binding correlation is reported once, - not every tick. It waits on cut 1, because it rewrites the same function. + not every tick. Waits on cut 1, because it rewrites the same function. 5. **Closing proof on Yggdrasil**: an operator-approved install of the merged - cuts. streampixels-service survives the install, and an `idunn status` - check is made before and after. It waits on 1-4. + cuts. streampixels-service survives the install, and `idunn status` is + checked before and after. Waits on 1-4. Ops, out of scope: nginx boot order (`gamecult-ops`), and retention of the roughly 130 dead transient units. diff --git a/docs/reincarnation-target.md b/docs/reincarnation-target.md index a5de465..72e50f7 100644 --- a/docs/reincarnation-target.md +++ b/docs/reincarnation-target.md @@ -45,12 +45,15 @@ What Idunn did: dependency, such as nginx, the stable route's listener or Odin, never charges a target's restart budget. A dependency outage can never exhaust the budget of every target behind it. -3. **Exactly one authority owns a target's actuation at every instant, and - the handover is explicit.** When continuity stops, it records that it has - stopped, and from then on deployment owns the target. A deployment never - yields to a continuity owner that has stopped. A deployment whose incumbent - is absent replaces it, with nothing to fence. There is no reachable state - in which continuity waits for a deployment while that deployment waits for +3. **Exactly one authority owns each slot at every instant, and nothing + waits on an owner that has stopped.** Continuity never stops (operator, + 2026-10-18: capped backoff, and the 6 per hour count only pages). It never + aborts a deployment. Before Fencing a deployment owns only its candidate, + and continuity owns the incumbent slot. At Fencing the deployment takes the + target and fences whatever is admitted then; an absent incumbent fences as + nothing. Every continuity attempt leaves a free gap in which a queued + deployment can be admitted. There is no reachable state in which + continuity waits for a deployment while that deployment waits for continuity. A negative test proves it. 4. **Continuity is daemon survival; deployment is change** (`F:\Projects\CLAUDE.md`). Restarting the admitted installed body never needs a deployment and is From e36cad32f0995cd3a65edf4b74863b2982e29404 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:46:01 +0200 Subject: [PATCH 055/196] Negative test: a pre-fencing deployment over a dead paged incumbent is never yielded Written first against base API; expected to fail at this commit (deployment aborted 'yielded to continuity' once the window slides). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index dd2962f..9fff4b9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15901,6 +15901,54 @@ mod tests { Ok(()) } + /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment + /// waiting before Fencing. Continuity never stops and never yields a + /// deployment (budget-never-stop, handover-phase-split), so the window + /// sliding must not turn the deployment into a casualty. + #[test] + fn a_deployment_before_fencing_over_a_dead_incumbent_that_paged_is_never_yielded_and_lands() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + set_meters( + &world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + world.engine.supervise_one_admitted_generation()?; + // The oldest leaves the window: base would now yield the deployment. + set_meters( + &world, + &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + for _ in 0..3 { + let _ = world.engine.supervise_one_admitted_generation()?; + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + assert!( + after.pre_fencing_abort.is_none(), + "the deployment was yielded: {:?}", + after.pre_fencing_abort + ); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| transaction.command_kind != CommandKind::Continuity), + "continuity minted beside a pre-fencing deployment" + ); + } + workload.revive(); + odin_reports_ready(&world, &redeploy, seeded_sequence(Some(&incumbent)) + 1)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + let landed = latest(&world)?; + assert_eq!(landed.transaction_id, redeploy.transaction_id); + assert_eq!(landed.phase, DeploymentPhase::Complete); + assert!(landed.pre_fencing_abort.is_none()); + Ok(()) + } + /// A source that resolves one fixed recipe and never freezes anything. struct FixedSource { recipe: String, From 6eb7931a8ce2ba3a642482eb5b7af9e47567530d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:47:04 +0200 Subject: [PATCH 056/196] Continuity never stops: delete the yield, the exhaustion stop and the window-derived wait The paging window now only pages: restarts_page opens the incident and reports; it ends no pass and aborts no transaction. Deletes next_restart_at, continuity_is_waiting and the five tests that pinned the stop, the yield and the mint-anchored spacing. The streak backoff follows in the next commit. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 283 +++++++------------------------------------ 1 file changed, 45 insertions(+), 238 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 9fff4b9..fda70fb 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -62,15 +62,14 @@ const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v1"; const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1"; -/// How many times continuity will restart one target inside one window before -/// it concludes the target itself is the problem. More than one because a -/// start can fail for a passing reason -- a port still held, a peer not yet up -/// -- and bounded because each attempt holds the target against any -/// deployment. Counted per target, so a restart that succeeds and dies again -/// does not start the count over. -const CONTINUITY_RESTART_ATTEMPTS: usize = 6; +/// The number of charged continuity attempts inside +/// `CONTINUITY_RESTART_WINDOW_MILLIS` at which the ContinuityExhausted incident +/// opens and is held. It gates no actuation: continuity never stops +/// (ruling budget-never-stop). +const CONTINUITY_PAGING_RESTARTS: usize = 6; const CONTINUITY_RESTART_WINDOW_MILLIS: u64 = 3_600_000; -/// The wait after the first restart in a window; it doubles with each further one. +/// The base of the streak backoff: the wait after the first charged attempt; +/// it doubles with each further one in the streak. const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; /// Forward route actuations (fragment write, firewall, `nginx -t`, reload, @@ -1047,7 +1046,8 @@ struct TargetSupervision { /// ascending: the only entries the ceiling counts. #[cultcache(key = 2)] route_actuations: Vec, - /// The newest `CONTINUITY_RESTART_ATTEMPTS` restart times, ascending. + /// The newest `CONTINUITY_PAGING_RESTARTS` restart times, ascending: the + /// paging threshold's log. #[cultcache(key = 3)] continuity_restarts: Vec, /// A restart whose projection could not be prepared waits until this time. @@ -1154,38 +1154,21 @@ impl TargetSupervision { live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len() } - fn restarts_exhausted(&self, now: u64) -> bool { - self.restarts_used(now) >= CONTINUITY_RESTART_ATTEMPTS + /// The paging threshold of the ContinuityExhausted incident; it gates no actuation. + fn restarts_page(&self, now: u64) -> bool { + self.restarts_used(now) >= CONTINUITY_PAGING_RESTARTS } - /// The next restart is due one doubling wait after the last, from the - /// number of restarts still inside the window. - fn next_restart_at(&self, now: u64) -> Option { - let live = live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS); - if live.is_empty() { - return None; - } - let last = *self.continuity_restarts.last()?; - Some(last.saturating_add(CONTINUITY_RESTART_BACKOFF_MILLIS << (live.len() - 1))) - } - - /// Whether continuity must still wait: the doubling wait after the last - /// restart, or the deferral after a projection that would not demote. - fn continuity_is_waiting(&self, now: u64) -> bool { - self.next_restart_at(now).is_some_and(|due| { - is_waiting( - now, - due, - CONTINUITY_RESTART_BACKOFF_MILLIS << (CONTINUITY_RESTART_ATTEMPTS - 1), - ) - }) || self - .continuity_deferred_until + /// Whether continuity must still wait out the deferral after a projection + /// that would not demote. + fn continuity_deferred(&self, now: u64) -> bool { + self.continuity_deferred_until .is_some_and(|due| is_waiting(now, due, CONTINUITY_DEFERRAL_MILLIS)) } /// One restart scheduled. It ends any deferral: the projection was prepared. fn record_restart(&mut self, now: u64) { - record_entry(&mut self.continuity_restarts, now, CONTINUITY_RESTART_ATTEMPTS); + record_entry(&mut self.continuity_restarts, now, CONTINUITY_PAGING_RESTARTS); self.continuity_deferred_until = None; self.continuity_deferral_reason = None; } @@ -1203,7 +1186,7 @@ impl TargetSupervision { require_id(&self.target, "supervised target")?; for (log, limit, what) in [ (&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"), - (&self.continuity_restarts, CONTINUITY_RESTART_ATTEMPTS, "continuity restart"), + (&self.continuity_restarts, CONTINUITY_PAGING_RESTARTS, "continuity restart"), (&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"), ] { ensure!(log.len() <= limit, "{what} log exceeds its bound"); @@ -4486,10 +4469,9 @@ fn render_supervision( lines.push(format!(" held: {reason}")); } lines.push(format!( - " continuity restarts {}/{} next-restart-at {}", + " continuity attempts-in-hour {} pages-at {}", meters.restarts_used(now), - CONTINUITY_RESTART_ATTEMPTS, - at(meters.next_restart_at(now)) + CONTINUITY_PAGING_RESTARTS, )); if let (Some(until), Some(reason)) = ( meters.continuity_deferred_until, @@ -5972,7 +5954,7 @@ impl Engine { // same incident. if snapshot .supervision_or_new(¤t.value.target) - .restarts_exhausted(now) + .restarts_page(now) { return Ok(false); } @@ -5993,8 +5975,7 @@ impl Engine { // continuity minted over it copies the undeclared Expected, is // held at once, and owns the target forever, so the declaring // redeploy that would clear the hold could never freeze. The dead - // held target stays down and free. Nothing yields to it either: - // there is no continuity to yield to. + // held target stays down and free. if current.value.readiness().is_err() { self.note_fault( "holds a dead admitted generation down; declare readiness in its recipe and redeploy", @@ -6003,16 +5984,12 @@ impl Engine { ); continue; } - // Continuity restarts a release; it cannot repair one. When the - // admitted release will not start, rescheduling it forever keeps - // the target permanently occupied -- and a target with a live - // transaction accepts no deployment, so the one action that could - // fix it is exactly the one that is locked out. Restarts are - // counted in the target's own log, in a sliding window, and spaced - // by a doubling wait. The log outlives every generation, so a - // release that starts, dies and starts again is bounded the same as - // one that never starts. Nothing here reads history: an unreadable - // history file must not stop crash recovery. + // Continuity restarts a release; it cannot repair one, and it + // never stops (budget-never-stop). Every charged attempt is spaced + // by a doubling, capped wait, so a queued deployment always gets + // a free gap (map B10). The log outlives every generation and only + // pages. Nothing here reads history: an unreadable history file + // must not stop crash recovery. let now = now_millis()?; let supervision = snapshot.supervision_or_new(¤t.value.target); // A stepped-back clock leaves entries in the future. Settle them @@ -6030,36 +6007,12 @@ impl Engine { } let continuity_key = format!("continuity:{}", current.value.target); - if let Some(blocker) = blocker { - // Yielding a *deployment* to continuity is right: the incumbent - // died, so changing it can wait. Doing the same to a continuity - // transaction is self-defeating -- that transaction exists to - // restart the very workload whose absence triggers the yield, - // so aborting it schedules another, which is aborted in turn. - // Odin sat in that loop, down, while Idunn cancelled its own - // recovery every few seconds. - // Only yield to a continuity that is still trying. Once it has - // given up on this release there is nothing to yield to, and - // the deployment is the only thing left that can fix the - // target -- aborting it would close the last door. - if blocker.value.command_kind == CommandKind::Deploy - && !supervision.restarts_exhausted(now) - && blocker.value.phase < DeploymentPhase::Fencing - && blocker.value.pre_fencing_abort.is_none() - // A held record is reported and never aborted by Idunn. - && blocker.value.held_disagreement().is_none() - { - self.begin_pre_fencing_abort( - blocker, - anyhow!( - "admitted incumbent failed before candidate fencing; deployment yielded to continuity: {workload_error:#}" - ), - )?; - progressed = true; - } + // Continuity defers to any live transaction for its target and + // never aborts one (handover-phase-split). + if blocker.is_some() { continue; } - if supervision.restarts_exhausted(now) { + if supervision.restarts_page(now) { // The record comes first and the trace after it: the trace is // this record's shadow, never its substitute. self.record_incident( @@ -6074,18 +6027,15 @@ impl Engine { self.report_once( &continuity_key, format!( - "Idunn stopped restarting admitted {}: {} restarts inside the window. The target is free for a deployment to replace it{}: {workload_error:#}", + "Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff: {workload_error:#}", current.value.target, supervision.restarts_used(now), - supervision.route_reopens_at(now).map_or_else(String::new, |at| format!( - ", but its route ceiling refuses the route change until it reopens at unix ms {at}" - )), ), ); - continue; + } else { + self.clear_fault(&continuity_key); } - self.clear_fault(&continuity_key); - if supervision.continuity_is_waiting(now) { + if supervision.continuity_deferred(now) { continue; } @@ -15775,132 +15725,6 @@ mod tests { ); } - #[test] - fn continuity_stops_when_the_targets_own_log_has_spent_the_window() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - // Six restarts inside the window, the last long enough ago that its - // doubling wait (160 s) is over: only the ceiling holds. - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - assert!(!world.engine.supervise_one_admitted_generation()?); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - assert!( - world - .engine - .fault_reports - .lock() - .unwrap() - .contains_key("continuity:service"), - "exhaustion was not reported" - ); - - // The oldest leaves the window: five remain, so one more is allowed. - set_meters( - &world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); - Ok(()) - } - - #[test] - fn an_exhausted_target_says_when_its_route_ceiling_reopens_only_if_it_is_full() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - let said = || -> String { - world - .engine - .fault_reports - .lock() - .unwrap() - .get("continuity:service") - .and_then(|report| report.last.lock().unwrap().clone()) - .unwrap_or_default() - }; - let mut meters = - restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?; - set_meters(&world, &meters)?; - assert!(!world.engine.supervise_one_admitted_generation()?); - let open = said(); - assert!(open.contains("free for a deployment"), "{open}"); - assert!(!open.contains("route ceiling"), "{open}"); - - world.engine.clear_fault("continuity:service"); - let now = now_millis()?; - meters.route_actuations = (0..ROUTE_ACTUATION_CEILING as u64).map(|n| now - 1_000 + n).collect(); - set_meters(&world, &meters)?; - let reopens_at = meters.route_reopens_at(now).unwrap(); - assert!(!world.engine.supervise_one_admitted_generation()?); - let full = said(); - assert!(full.contains(&format!("reopens at unix ms {reopens_at}")), "{full}"); - Ok(()) - } - - #[test] - fn continuity_restarts_are_spaced_by_a_doubling_wait_and_counted_in_the_mint() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - let restarts = |world: &EngineFixture| -> Result { - let progressed = world.engine.supervise_one_admitted_generation()?; - let minted = !ControlSnapshot::read(&world.state_store)?.transactions.is_empty(); - assert_eq!(progressed, minted); - clear_live_transactions(world)?; - Ok(minted) - }; - - // The first restart is minted, and the same write counts it. - assert!(restarts(&world)?); - assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); - // At once again: the 5 s wait has not passed. - assert!(!restarts(&world)?); - // The second wait is 10 s and the third 20 s: each is the last one doubled. - set_meters(&world, &restarted_ago(&[100_000, 6_000])?)?; - assert!(!restarts(&world)?); - set_meters(&world, &restarted_ago(&[100_000, 11_000])?)?; - assert!(restarts(&world)?); - assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 3); - set_meters(&world, &restarted_ago(&[200_000, 100_000, 15_000])?)?; - assert!(!restarts(&world)?); - set_meters(&world, &restarted_ago(&[200_000, 100_000, 21_000])?)?; - assert!(restarts(&world)?); - Ok(()) - } - - #[test] - fn a_deployment_yields_to_a_restart_only_while_the_target_has_restarts_left() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - workload.kill(); - // The target's restarts are spent: nothing is left to yield to, and - // aborting the deployment would close the last door. - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!(after.pre_fencing_abort.is_none(), "yielded with no restarts left"); - - set_meters(&world, &restarted_ago(&[200_000])?)?; - world.engine.supervise_one_admitted_generation()?; - assert!(resident(&world)?.value.pre_fencing_abort.is_some()); - Ok(()) - } - /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment /// waiting before Fencing. Continuity never stops and never yields a /// deployment (budget-never-stop, handover-phase-split), so the window @@ -16232,23 +16056,6 @@ mod tests { /// The control for the test above: the same redeploy over a dead incumbent /// that is not held is yielded to continuity. - #[test] - fn a_declaring_redeploy_still_yields_to_continuity_when_the_incumbent_is_not_held() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - assert!(incumbent.readiness().is_ok()); - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - workload.kill(); - world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - let abort = after.pre_fencing_abort.context("the redeploy was not yielded")?; - assert!(abort.error.contains("yielded to continuity"), "{}", abort.error); - Ok(()) - } - /// `seeded` with its own Expected undeclared, so Idunn holds it. fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { let mut bare = seeded.clone(); @@ -16417,24 +16224,24 @@ mod tests { let now = t - 86_400_000; let mut meters = TargetSupervision::new("service"); meters.route_actuations = vec![t; ROUTE_ACTUATION_CEILING]; - meters.continuity_restarts = vec![t; CONTINUITY_RESTART_ATTEMPTS]; + meters.continuity_restarts = vec![t; CONTINUITY_PAGING_RESTARTS]; meters.continuity_deferred_until = Some(t); meters.continuity_deferral_reason = Some("demotion failed".into()); meters.validate()?; // Entries from the future count as now: the meters hold for one window. let read_only = meters.clone(); - assert!(read_only.restarts_exhausted(now)); + assert!(read_only.restarts_page(now)); let refused = meters.charge_route(now, RouteActuation::Forward).unwrap_err(); assert!(refused.reopens_at_unix_millis <= now + ROUTE_ACTUATION_WINDOW_MILLIS); // The clamp is written, so the window runs from the step and no longer. assert_eq!(meters.route_actuations, vec![now; ROUTE_ACTUATION_CEILING]); - assert_eq!(meters.continuity_restarts, vec![now; CONTINUITY_RESTART_ATTEMPTS]); - assert!(!meters.restarts_exhausted(now + CONTINUITY_RESTART_WINDOW_MILLIS)); + assert_eq!(meters.continuity_restarts, vec![now; CONTINUITY_PAGING_RESTARTS]); + assert!(!meters.restarts_page(now + CONTINUITY_RESTART_WINDOW_MILLIS)); meters.charge_route(now + ROUTE_ACTUATION_WINDOW_MILLIS, RouteActuation::Forward)?; // Waits set for the future are due, not stalled for the length of the step. - assert!(!read_only.continuity_is_waiting(now)); + assert!(!read_only.continuity_deferred(now)); let route = RouteSupervisionState { next_challenge_at_unix_millis: Some(t), ..RouteSupervisionState::default() @@ -19035,7 +18842,7 @@ mod tests { // it first: the write ends the pass, and nothing is minted from // entries that only look recent because the clock stepped back. let mut restarts = TargetSupervision::new("service"); - restarts.continuity_restarts = vec![ahead; CONTINUITY_RESTART_ATTEMPTS]; + restarts.continuity_restarts = vec![ahead; CONTINUITY_PAGING_RESTARTS]; set_meters(&routed.world, &restarts)?; routed.workload.kill(); assert!(routed.tick()?); @@ -19210,12 +19017,12 @@ mod tests { &routed.world, &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, )?; - for _ in 0..2 * CONTINUITY_RESTART_ATTEMPTS { + for _ in 0..2 * CONTINUITY_PAGING_RESTARTS { routed.world.engine.charge_route_actuation("service", RouteActuation::Survival)?; } let spent = meters_of(&routed.world, "service")?; - assert!(spent.restarts_exhausted(now_millis()?)); - assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_RESTART_ATTEMPTS); + assert!(spent.restarts_page(now_millis()?)); + assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_PAGING_RESTARTS); // Warming's preflight and Routing's install are both admitted. routed.step()?; From 9a6b4a128c03123a338a521a92fd962a1ebfed71 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:52:13 +0200 Subject: [PATCH 057/196] Continuity backoff: capped streak anchored when the target is found free; one uncharged re-materialization TargetSupervision gains the attempt streak, the backoff anchor and the re-materialization bound (keys 7-9, same schema; pre-key records decode with them defaulted). Every charged attempt waits backoff_wait(base, streak - 1, 300 s) from the first dead-and-free pass; a sustained run resets the streak. WorkloadPort::lost_to_host_reboot lets a reboot-lost unit be re-materialized once per generation with no step. The streak backoff, reboot and negative tests land with it. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 544 +++++++++++++++++++++++++++++++++++++++++-- src/drivers.rs | 103 ++++++++ src/host_actuator.rs | 6 + 3 files changed, 633 insertions(+), 20 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index fda70fb..05db41d 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -71,6 +71,12 @@ const CONTINUITY_RESTART_WINDOW_MILLIS: u64 = 3_600_000; /// The base of the streak backoff: the wait after the first charged attempt; /// it doubles with each further one in the streak. const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; +/// The longest streak backoff between charged continuity attempts (ruling +/// continuity-backoff-defaults). +const CONTINUITY_BACKOFF_CEILING_MILLIS: u64 = 300_000; +/// How long an admitted generation must have run, with no attempt failing on +/// it, before the attempt streak resets (ruling continuity-backoff-defaults). +const CONTINUITY_SUSTAINED_RUN_MILLIS: u64 = 600_000; /// Forward route actuations (fragment write, firewall, `nginx -t`, reload, /// private-mount validation) one target may perform per sliding window. A @@ -1059,6 +1065,33 @@ struct TargetSupervision { /// Visibility only: never counted, never refused. #[cultcache(key = 6)] route_survivals: Vec, + // Keys 7-9 are absent from every record written before them; `default` + // decodes that as no streak, no anchor and no re-materialization. + /// Charged attempts since the last sustained run. Owned by continuity + /// supervision; it decides the next attempt's backoff step. + #[cultcache(key = 7, default)] + continuity_attempt_streak: u32, + /// When a dead-and-free pass began waiting out the next attempt. Owned by + /// continuity supervision; it decides when the wait is over. Older than + /// the admitted generation, it decides nothing. + #[cultcache(key = 8, default)] + continuity_backoff_from_unix_millis: Option, + /// When the admitted generation's one uncharged re-materialization was + /// minted. Owned by continuity supervision; it can only withhold a free + /// re-materialization, never grant one. + #[cultcache(key = 9, default)] + continuity_rematerialized_at_unix_millis: Option, +} + +/// What continuity supervision does about the next attempt for a dead target. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ContinuityBackoff { + /// Mint now, carrying this streak into the attempt. + Due { streak: u32 }, + /// Start the wait: write the anchor and mint nothing this pass. + Begin, + /// The wait has not run out. + Waiting, } /// The entries of a sliding log still inside `window` at `now`. An entry from @@ -1101,9 +1134,33 @@ impl TargetSupervision { continuity_deferred_until: None, continuity_deferral_reason: None, route_survivals: Vec::new(), + continuity_attempt_streak: 0, + continuity_backoff_from_unix_millis: None, + continuity_rematerialized_at_unix_millis: None, } } + /// Decode a stored record. Records written before keys 7-9 are the first + /// seven elements of the tuple and decode with those keys defaulted; the + /// next write stores all ten. Anything else must re-encode byte for byte. + fn decode(envelope: &CultCacheEnvelope) -> Result { + let value: Self = rmp_serde::from_slice(&envelope.payload)?; + let before_backoff_keys = rmp_serde::to_vec(&( + &value.schema_version, + &value.target, + &value.route_actuations, + &value.continuity_restarts, + &value.continuity_deferred_until, + &value.continuity_deferral_reason, + &value.route_survivals, + ))?; + ensure!( + rmp_serde::to_vec(&value)? == envelope.payload || before_backoff_keys == envelope.payload, + "Idunn control store contains a noncanonical record" + ); + Ok(value) + } + /// Clamp every log to `now`. Callers that decide from a log write the /// result back when it changed. fn settle(&mut self, now: u64) { @@ -1166,13 +1223,81 @@ impl TargetSupervision { .is_some_and(|due| is_waiting(now, due, CONTINUITY_DEFERRAL_MILLIS)) } - /// One restart scheduled. It ends any deferral: the projection was prepared. - fn record_restart(&mut self, now: u64) { + /// When the wait for the next charged attempt ends, from the anchor and + /// the streak; the one spelling `continuity_backoff` and `status` share. + /// An anchor older than the admitted generation decides nothing. + fn backoff_due(&self, admitted_at: u64) -> Option { + let from = self + .continuity_backoff_from_unix_millis + .filter(|&from| from >= admitted_at)?; + let wait = self.continuity_attempt_streak.checked_sub(1).map_or(0, |steps| { + backoff_wait( + CONTINUITY_RESTART_BACKOFF_MILLIS, + steps, + CONTINUITY_BACKOFF_CEILING_MILLIS, + ) + }); + Some(from.saturating_add(wait)) + } + + /// What to do about the next attempt for the dead admitted generation + /// admitted at `admitted_at`. The streak resets only after a sustained + /// run: the generation has lived the sustained time and no attempt has + /// been minted since it was admitted. + fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { + let streak = self.continuity_attempt_streak; + if let Some(due) = self.backoff_due(admitted_at) { + return if streak != 0 + && is_waiting(now, due, CONTINUITY_BACKOFF_CEILING_MILLIS) + { + ContinuityBackoff::Waiting + } else { + ContinuityBackoff::Due { streak } + }; + } + let sustained = self + .continuity_restarts + .last() + .is_none_or(|&last| admitted_at >= last) + && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; + let streak = if sustained { 0 } else { streak }; + if streak == 0 { + ContinuityBackoff::Due { streak: 0 } + } else { + ContinuityBackoff::Begin + } + } + + /// One charged attempt scheduled at `streak`. It ends the wait and any + /// deferral: the projection was prepared. + fn record_restart(&mut self, now: u64, streak: u32) { record_entry(&mut self.continuity_restarts, now, CONTINUITY_PAGING_RESTARTS); + self.continuity_attempt_streak = streak.saturating_add(1); + self.continuity_backoff_from_unix_millis = None; self.continuity_deferred_until = None; self.continuity_deferral_reason = None; } + /// The target was found dead and free: the wait for its next attempt runs from now. + fn begin_backoff(&mut self, now: u64) { + self.continuity_backoff_from_unix_millis = Some(now); + } + + /// The one uncharged re-materialization scheduled. The log and the streak + /// are left alone. + fn record_rematerialization(&mut self, now: u64) { + self.continuity_rematerialized_at_unix_millis = Some(now); + self.continuity_backoff_from_unix_millis = None; + self.continuity_deferred_until = None; + self.continuity_deferral_reason = None; + } + + /// Whether the generation admitted at `admitted_at` has had its free re-materialization. + fn rematerialization_spent(&self, admitted_at: u64) -> bool { + self.continuity_rematerialized_at_unix_millis + .is_some_and(|at| at >= admitted_at) + } + fn defer_continuity(&mut self, now: u64, reason: &str) { self.continuity_deferred_until = Some(now.saturating_add(CONTINUITY_DEFERRAL_MILLIS)); self.continuity_deferral_reason = Some(reason.to_owned()); @@ -1195,6 +1320,11 @@ impl TargetSupervision { "{what} log is not an ascending list of times" ); } + ensure!( + self.continuity_backoff_from_unix_millis != Some(0) + && self.continuity_rematerialized_at_unix_millis != Some(0), + "continuity backoff anchor or re-materialization has no time" + ); ensure!( self.continuity_deferred_until.is_some() == self.continuity_deferral_reason.is_some(), "continuity deferral has a time or a reason but not both" @@ -3034,7 +3164,7 @@ impl ControlSnapshot { envelope.schema_id.as_deref() == Some(TARGET_SUPERVISION_SCHEMA), "Idunn control store contains an unsupported target supervision record" ); - let value: TargetSupervision = decode_record(&envelope)?; + let value = TargetSupervision::decode(&envelope)?; value.validate()?; ensure!( envelope.key == value.target, @@ -3983,10 +4113,10 @@ fn read_history_transactions(state_store: &Path) -> Vec { /// How long to wait after `failures` consecutive failures: one poll interval /// after the first, doubling each time, never past the ceiling. -fn backoff_wait(poll_millis: u64, failures: u32) -> u64 { - poll_millis +fn backoff_wait(base_millis: u64, failures: u32, ceiling_millis: u64) -> u64 { + base_millis .saturating_mul(1u64 << failures.min(20)) - .min(RESUME_BACKOFF_CEILING_MILLIS) + .min(ceiling_millis) } /// Whether an attempt due at `not_before` must still wait at `now`. A due time @@ -4469,10 +4599,15 @@ fn render_supervision( lines.push(format!(" held: {reason}")); } lines.push(format!( - " continuity attempts-in-hour {} pages-at {}", + " continuity attempts-in-hour {} pages-at {} streak {} next-attempt-at {}", meters.restarts_used(now), CONTINUITY_PAGING_RESTARTS, + meters.continuity_attempt_streak, + at(generation.and_then(|value| meters.backoff_due(value.admitted_at_unix_millis))) )); + if let Some(rematerialized) = meters.continuity_rematerialized_at_unix_millis { + lines.push(format!(" continuity rematerialized-at {rematerialized}")); + } if let (Some(until), Some(reason)) = ( meters.continuity_deferred_until, &meters.continuity_deferral_reason, @@ -5600,7 +5735,7 @@ impl Engine { let now = now_millis()?; let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); - let wait = backoff_wait(self.options.poll_millis, failures); + let wait = backoff_wait(self.options.poll_millis, failures, RESUME_BACKOFF_CEILING_MILLIS); all.insert( transaction_id.to_string(), ResumeBackoff { @@ -6027,9 +6162,10 @@ impl Engine { self.report_once( &continuity_key, format!( - "Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff: {workload_error:#}", + "Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff (streak {}): {workload_error:#}", current.value.target, supervision.restarts_used(now), + supervision.continuity_attempt_streak, ), ); } else { @@ -6038,6 +6174,43 @@ impl Engine { if supervision.continuity_deferred(now) { continue; } + // A unit the host lost to a reboot is re-materialized once per + // generation, uncharged and with no wait; the driver owns the + // evidence and supervision owns the budget. An unanswerable + // driver is not evidence of a reboot. + let admitted_at = current.value.admitted_at_unix_millis; + let rematerialize = !supervision.rematerialization_spent(admitted_at) + && match self.workload_for(¤t.value.plan).and_then(|workload| { + workload.lost_to_host_reboot(¤t.value.workload, admitted_at) + }) { + Ok(lost) => lost, + Err(error) => { + self.note_fault( + "could not tell whether the host rebooted under", + &format!("reboot:{}", current.value.target), + &error, + ); + false + } + }; + let mut streak = 0; + if !rematerialize { + match supervision.continuity_backoff(now, admitted_at) { + ContinuityBackoff::Waiting => continue, + ContinuityBackoff::Begin => { + let mut began = supervision.clone(); + began.begin_backoff(now); + self.write_target_supervision( + snapshot.supervision_for(¤t.value.target), + &began, + now, + )?; + progressed = true; + continue; + } + ContinuityBackoff::Due { streak: due } => streak = due, + } + } // The projected activation names an incarnation that is gone. Every // restart is issued a fresh one, so leaving the old record standing @@ -6102,10 +6275,15 @@ impl Engine { command.validate()?; let transaction = DeploymentTransaction::from_continuity(&command, ¤t.value, now)?; - // The restart is counted in the same CAS that mints it, so a crash - // cannot schedule one the ceiling never saw. + // The attempt is charged, or recorded as the one re-materialization, + // in the same CAS that mints it, so a crash cannot schedule one the + // meters never saw. let mut counted = supervision; - counted.record_restart(now); + if rematerialize { + counted.record_rematerialization(now); + } else { + counted.record_restart(now, streak); + } ensure!( SingleFileMessagePackBackingStore::new(&self.options.state_store) .compare_exchange( @@ -11999,6 +12177,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(false) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } /// A Continuity transaction for an unrouted, stateless target with no @@ -13270,18 +13451,29 @@ mod tests { /// `StillWorkload`'s. struct SwitchWorkload { alive: std::sync::atomic::AtomicBool, + rebooted: std::sync::atomic::AtomicBool, } impl SwitchWorkload { fn new() -> Arc { Arc::new(Self { alive: std::sync::atomic::AtomicBool::new(true), + rebooted: std::sync::atomic::AtomicBool::new(false), }) } fn revive(&self) { self.alive .store(true, std::sync::atomic::Ordering::SeqCst); + self.rebooted + .store(false, std::sync::atomic::Ordering::SeqCst); + } + + /// The host rebooted under the unit: it is gone and the driver can tell. + fn reboot(&self) { + self.rebooted + .store(true, std::sync::atomic::Ordering::SeqCst); + self.kill(); } fn kill(&self) { @@ -13342,6 +13534,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(false) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) + } } /// Admit a generation for "service" through the real phase machine and @@ -13449,6 +13644,8 @@ mod tests { let mut meters = TargetSupervision::new("service"); meters.continuity_restarts = ages.iter().map(|age| now - age).collect(); meters.continuity_restarts.sort_unstable(); + // Every logged attempt belongs to one streak. + meters.continuity_attempt_streak = ages.len() as u32; Ok(meters) } @@ -14140,6 +14337,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(true) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } #[test] @@ -14265,6 +14465,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(true) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } fn queue_deploy_for_nowhere(world: &EngineFixture) -> Result { @@ -14450,6 +14653,9 @@ mod tests { fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { Ok(true) } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } #[test] @@ -14521,12 +14727,17 @@ mod tests { #[test] fn backoff_doubles_caps_and_waits_exactly() { - assert_eq!(backoff_wait(500, 0), 500); - assert_eq!(backoff_wait(500, 1), 1000); - assert_eq!(backoff_wait(500, 2), 2000); - assert_eq!(backoff_wait(500, 3), 4000); - assert_eq!(backoff_wait(500, 30), RESUME_BACKOFF_CEILING_MILLIS); - assert_eq!(backoff_wait(u64::MAX, 5), RESUME_BACKOFF_CEILING_MILLIS); + let ceiling = RESUME_BACKOFF_CEILING_MILLIS; + assert_eq!(backoff_wait(500, 0, ceiling), 500); + assert_eq!(backoff_wait(500, 1, ceiling), 1000); + assert_eq!(backoff_wait(500, 2, ceiling), 2000); + assert_eq!(backoff_wait(500, 3, ceiling), 4000); + assert_eq!(backoff_wait(500, 30, ceiling), ceiling); + assert_eq!(backoff_wait(u64::MAX, 5, ceiling), ceiling); + // The ceiling is the caller's: continuity's is lower than the resume path's. + assert_eq!(backoff_wait(5_000, 5, 300_000), 160_000); + assert_eq!(backoff_wait(5_000, 6, 300_000), 300_000); + assert_eq!(backoff_wait(5_000, 20, 300_000), 300_000); assert!(is_waiting(9, 10, 5)); assert!(!is_waiting(10, 10, 5)); assert!(!is_waiting(11, 10, 5)); @@ -15121,7 +15332,8 @@ mod tests { spend_the_window(&world)?; assert!(!incident_path(&world).exists()); - assert!(!world.engine.supervise_one_admitted_generation()?); + // The first pass over the dead target starts its wait: progress, no mint. + assert!(world.engine.supervise_one_admitted_generation()?); let opened = incidents_of(&world)?; assert_eq!(opened.len(), 1); assert_eq!(opened[0].subject, "service"); @@ -15158,7 +15370,7 @@ mod tests { // The window slides while the workload is still dead: exhaustion is // over, the target is not healthy, and the incident stays open. slide_the_window(&world)?; - assert!(!world.engine.supervise_one_admitted_generation()?); + let _ = world.engine.supervise_one_admitted_generation()?; assert_eq!(incidents_of(&world)?, first); // Seen running, it closes as recovered. @@ -15773,6 +15985,291 @@ mod tests { Ok(()) } + /// The incumbent was admitted this long ago, so an anchor written "in the + /// past" is not older than its generation. + fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { + edit_incumbent(world, |generation| generation.admitted_at_unix_millis -= millis)?; + Ok(()) + } + + /// Put the target's backoff anchor `ago` milliseconds back. + fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { + let mut meters = meters_of(world, "service")?; + meters.continuity_backoff_from_unix_millis = Some(now_millis()? - ago); + set_meters(world, &meters) + } + + fn continuity_mints(world: &EngineFixture) -> Result { + Ok(every_transaction(world)? + .iter() + .filter(|transaction| transaction.command_kind == CommandKind::Continuity) + .count()) + } + + #[test] + fn continuity_keeps_restarting_past_the_paging_threshold() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + // Six charged attempts in the hour: the incident pages, nothing stops. + set_meters( + &world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + let opened = incidents_of(&world)?; + assert_eq!(opened.len(), 1); + assert!(opened[0].is_open()); + + // The streak-6 wait is 160 s. + anchor_ago(&world, 159_000)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + anchor_ago(&world, 160_001)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let after = meters_of(&world, "service")?; + assert_eq!(after.continuity_restarts.len(), 6); + assert!(after.continuity_restarts.last().is_some_and(|&last| now_millis().unwrap() - last < 60_000)); + assert_eq!(after.continuity_attempt_streak, 7); + assert_eq!(after.continuity_backoff_from_unix_millis, None); + assert_eq!(incidents_of(&world)?, opened); + Ok(()) + } + + #[test] + fn continuity_backoff_is_a_capped_doubling_of_the_attempt_streak() { + let from = 10_000_000_u64; + for (streak, seconds) in [ + (1_u32, 5_u64), + (2, 10), + (3, 20), + (4, 40), + (5, 80), + (6, 160), + (7, 300), + (8, 300), + (9, 300), + ] { + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = streak; + meters.continuity_backoff_from_unix_millis = Some(from); + // The paging log disagrees with the streak on purpose: the wait + // is a function of the streak, not of the window's count. + meters.continuity_restarts = vec![from - 10; if streak == 1 { 6 } else { 2 }]; + let due = from + seconds * 1_000; + assert_eq!(meters.backoff_due(0), Some(due), "streak {streak}"); + assert_eq!( + meters.continuity_backoff(due - 1, 0), + ContinuityBackoff::Waiting, + "streak {streak}" + ); + assert_eq!( + meters.continuity_backoff(due, 0), + ContinuityBackoff::Due { streak }, + "streak {streak}" + ); + } + // No streak, no wait. + let mut fresh = TargetSupervision::new("service"); + fresh.continuity_backoff_from_unix_millis = Some(from); + assert_eq!(fresh.continuity_backoff(from, 0), ContinuityBackoff::Due { streak: 0 }); + } + + #[test] + fn a_sustained_run_resets_the_streak_and_only_a_sustained_run() { + let now = 100_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 7; + meters.continuity_restarts = vec![1_000]; + // Lived the sustained time, and no attempt since admission. + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), + ContinuityBackoff::Due { streak: 0 } + ); + // A moment short of it. + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + 10_000), + ContinuityBackoff::Begin + ); + // Attempts after admission failed on this generation: it has not run well. + meters.continuity_restarts = vec![now - 100_000]; + assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); + // Never attempted over this log, long lived: reset. + meters.continuity_restarts = Vec::new(); + assert_eq!( + meters.continuity_backoff(now, now - 700_000), + ContinuityBackoff::Due { streak: 0 } + ); + assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); + // An anchor older than its generation decides nothing. + meters.continuity_backoff_from_unix_millis = Some(now - 800_000); + assert_eq!(meters.backoff_due(now - 700_000), None); + assert_eq!( + meters.continuity_backoff(now, now - 700_000), + ContinuityBackoff::Due { streak: 0 } + ); + assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); + // Streak zero with no history is simply due. + let fresh = TargetSupervision::new("service"); + assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); + } + + #[test] + fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + + // The last attempt was long ago, but the wait starts now: a wait + // anchored on the mint would already be over. + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // Streak 3 waits 20 s. + anchor_ago(&world, 19_000)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + anchor_ago(&world, 20_001)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 4); + Ok(()) + } + + #[test] + fn a_queued_deployment_freezes_in_the_gap_after_a_continuity_attempt() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[3_000_000])?; + meters.continuity_attempt_streak = 2; + set_meters(&world, &meters)?; + queue_declaring_redeploy(&world, "up-service-queued")?; + + for _ in 0..4 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-queued"), + "the queued deployment never froze" + ); + assert_eq!(continuity_mints(&world)?, 0); + for _ in 0..3 { + let _ = world.engine.run_scheduler_tick(); + assert_eq!(continuity_mints(&world)?, 0); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| transaction.pre_fencing_abort.is_none()), + "the deployment was yielded" + ); + } + Ok(()) + } + + #[test] + fn a_unit_lost_to_a_host_reboot_is_rematerialized_once_with_no_backoff_step() -> Result<()> { + let fresh_world = |workload: &Arc| -> Result { + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + meters.continuity_backoff_from_unix_millis = Some(now_millis()?); + set_meters(&world, &meters)?; + Ok(world) + }; + let workload = SwitchWorkload::new(); + let world = fresh_world(&workload)?; + let before = meters_of(&world, "service")?; + + // Control: dead but not rebooted, the ordinary 20 s wait holds. + workload.kill(); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // The host rebooted under the unit: one mint, charged to nothing. + workload.reboot(); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let after = meters_of(&world, "service")?; + assert_eq!(after.continuity_restarts, before.continuity_restarts); + assert_eq!(after.continuity_attempt_streak, 3); + assert!(after.continuity_rematerialized_at_unix_millis.is_some()); + assert_eq!(after.continuity_backoff_from_unix_millis, None); + + // The bound is spent: the next death of this generation waits like any other. + clear_live_transactions(&world)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // Control: a deferred projection spaces a re-materialization too. + let workload = SwitchWorkload::new(); + let world = fresh_world(&workload)?; + let mut deferred = meters_of(&world, "service")?; + deferred.defer_continuity(now_millis()?, "demotion failed"); + set_meters(&world, &deferred)?; + workload.reboot(); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + Ok(()) + } + + #[test] + fn target_supervision_decodes_without_keys_7_to_9() -> Result<()> { + let world = EngineFixture::new()?; + let meters = restarted_ago(&[200_000])?; + // A record as every one on a running host was written: seven keys. + let payload = rmp_serde::to_vec(&( + &meters.schema_version, + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + let envelope = CultCacheEnvelope { + key: "service".into(), + r#type: TargetSupervision::TYPE.into(), + payload, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(TARGET_SUPERVISION_SCHEMA.into()), + }; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent(envelope)?; + let read = meters_of(&world, "service")?; + assert_eq!(read.continuity_restarts, meters.continuity_restarts); + assert_eq!(read.continuity_attempt_streak, 0); + assert_eq!(read.continuity_backoff_from_unix_millis, None); + assert_eq!(read.continuity_rematerialized_at_unix_millis, None); + + let mut anchored = TargetSupervision::new("service"); + anchored.continuity_backoff_from_unix_millis = Some(0); + assert!(anchored.validate().is_err()); + anchored.continuity_backoff_from_unix_millis = None; + anchored.continuity_rematerialized_at_unix_millis = Some(0); + assert!(anchored.validate().is_err()); + Ok(()) + } + /// A source that resolves one fixed recipe and never freezes anything. struct FixedSource { recipe: String, @@ -16242,6 +16739,13 @@ mod tests { // Waits set for the future are due, not stalled for the length of the step. assert!(!read_only.continuity_deferred(now)); + let mut anchored = TargetSupervision::new("service"); + anchored.continuity_attempt_streak = 7; + anchored.continuity_backoff_from_unix_millis = Some(t); + assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Due { streak: 7 }); + anchored.continuity_attempt_streak = 1; + anchored.continuity_backoff_from_unix_millis = Some(now + 10_000); + assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Waiting); let route = RouteSupervisionState { next_challenge_at_unix_millis: Some(t), ..RouteSupervisionState::default() diff --git a/src/drivers.rs b/src/drivers.rs index e005ec5..fdf925c 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -587,6 +587,17 @@ pub trait WorkloadPort { /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; + + /// True only when the host lost the workload's unit to a reboot after the + /// generation was admitted at `admitted_at_unix_millis`. The driver owns + /// the evidence; continuity supervision owns the budget it buys (one + /// uncharged re-materialization per generation). Every implementation + /// answers explicitly. + fn lost_to_host_reboot( + &self, + observation: &WorkloadObservation, + admitted_at_unix_millis: u64, + ) -> Result; } pub trait TopologyPort { @@ -4000,6 +4011,29 @@ impl WorkloadPort for SystemdTransientWorkloadDriver { Ok(unit.properties.get("ActiveState").map(String::as_str) == Some("failed")) } + fn lost_to_host_reboot( + &self, + observation: &WorkloadObservation, + admitted_at_unix_millis: u64, + ) -> Result { + let observation = observation.systemd()?; + if self.show_unit(&observation.unit)?.is_some() { + return Ok(false); + } + // A transient unit does not survive a reboot, so an absent unit is a + // reboot's loss only when the kernel booted after admission. + let stat = std::fs::read_to_string(self.proc_root.join("stat")) + .context("could not read the kernel boot time")?; + let btime = stat + .lines() + .find_map(|line| line.strip_prefix("btime ")) + .context("the kernel boot time is missing from stat")? + .trim() + .parse::() + .context("the kernel boot time is not a number")?; + Ok(btime.saturating_mul(1000) > admitted_at_unix_millis) + } + fn stop(&self, observation: &WorkloadObservation) -> Result<()> { let observation = observation.systemd()?; let Some(unit_observation) = self.show_unit(&observation.unit)? else { @@ -9576,6 +9610,75 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// Continuity's one free re-materialization rests on this: an absent unit + /// is a reboot's loss only when the kernel booted after admission. + #[cfg(unix)] + #[test] + fn lost_to_host_reboot_needs_an_absent_unit_and_a_boot_after_admission() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let systemctl = temp.path().join("systemctl"); + let state = temp.path().join("systemctl.state"); + std::fs::write(&systemctl, "#!/bin/sh +cat \"$0.state\" +exit 0 +")?; + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; + let proc_root = temp.path().join("proc"); + std::fs::create_dir(&proc_root)?; + let driver = SystemdTransientWorkloadDriver { + systemctl_program: systemctl, + proc_root: proc_root.clone(), + ..Default::default() + }; + let observation = audit_workload_observation("no"); + let absent = "LoadState=not-found +ActiveState=inactive +SubState=dead +"; + let failed = "LoadState=loaded +ActiveState=failed +SubState=failed +InvocationID=inv +Description=Idunn test +"; + let boot = |btime: &str| -> Result<()> { + Ok(std::fs::write( + proc_root.join("stat"), + format!("cpu 1 2 3 +btime {btime} +processes 9 +"), + )?) + }; + let admitted_at = 1_700_000_000_000_u64; + + std::fs::write(&state, absent)?; + boot("1700000001")?; + assert!(driver.lost_to_host_reboot(&observation, admitted_at)?); + boot("1700000000")?; + assert!(!driver.lost_to_host_reboot(&observation, admitted_at)?, "booted at admission"); + boot("1699999999")?; + assert!(!driver.lost_to_host_reboot(&observation, admitted_at)?, "booted before admission"); + + // A unit systemd still knows died on its own, not to a reboot. + std::fs::write(&state, failed)?; + boot("1700000001")?; + assert!(!driver.lost_to_host_reboot(&observation, admitted_at)?); + + // No boot time is not evidence either way. + std::fs::write(&state, absent)?; + std::fs::write(proc_root.join("stat"), "cpu 1 2 3 +")?; + assert!(driver.lost_to_host_reboot(&observation, admitted_at).is_err()); + boot("soon")?; + assert!(driver.lost_to_host_reboot(&observation, admitted_at).is_err()); + std::fs::remove_file(proc_root.join("stat"))?; + assert!(driver.lost_to_host_reboot(&observation, admitted_at).is_err()); + Ok(()) + } + fn host_audit_observation(pid: u32, created: u64) -> HostWorkloadObservation { HostWorkloadObservation { host: "raven".into(), diff --git a/src/host_actuator.rs b/src/host_actuator.rs index 90592be..92d01e3 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -963,6 +963,12 @@ impl WorkloadPort for HostActuatorWorkloadDriver<'_> { }; Ok(!running) } + + /// A managed host's reboot is not yet observable here: follow-up + /// host-actuator-reboot. Until then continuity charges its restarts. + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } } #[cfg(test)] From 61fc89bbd75a1e566cc18231bf4e8484e0c4b4df Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:53:40 +0200 Subject: [PATCH 058/196] Tests: count only continuity minted over an incumbent; status shows streak and next attempt Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 05db41d..465e721 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15968,10 +15968,9 @@ mod tests { "the deployment was yielded: {:?}", after.pre_fencing_abort ); - assert!( - every_transaction(&world)? - .iter() - .all(|transaction| transaction.command_kind != CommandKind::Continuity), + assert_eq!( + continuity_mints(&world)?, + 0, "continuity minted beside a pre-fencing deployment" ); } @@ -15999,10 +15998,15 @@ mod tests { set_meters(world, &meters) } + /// Continuity transactions minted over an incumbent (the one that admitted + /// it was seeded over nothing). fn continuity_mints(world: &EngineFixture) -> Result { Ok(every_transaction(world)? .iter() - .filter(|transaction| transaction.command_kind == CommandKind::Continuity) + .filter(|transaction| { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.is_some() + }) .count()) } @@ -16929,16 +16933,30 @@ mod tests { meters.continuity_restarts = vec![now - 9_000, now - 4_000]; meters.continuity_deferred_until = Some(now + 500); meters.continuity_deferral_reason = Some("projection would not demote".into()); + meters.continuity_attempt_streak = 2; + meters.continuity_backoff_from_unix_millis = Some(now - 4_000); let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; generation.last_error = Some("held".into()); + generation.admitted_at_unix_millis = 1; let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); assert!(lines.contains(&format!("target service {}", generation.generation_id)), "{lines}"); assert!(lines.contains("held: held"), "{lines}"); assert!(lines.contains(&format!( - "continuity restarts 2/6 next-restart-at {}", + "continuity attempts-in-hour 2 pages-at 6 streak 2 next-attempt-at {}", now - 4_000 + 10_000 )), "{lines}"); + assert!(!lines.contains("rematerialized-at"), "{lines}"); + meters.continuity_rematerialized_at_unix_millis = Some(now - 1_000); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" +"); + assert!(lines.contains(&format!("continuity rematerialized-at {}", now - 1_000)), "{lines}"); + // An anchor older than the generation is not shown as a pending wait. + generation.admitted_at_unix_millis = now; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" +"); + assert!(lines.contains("streak 2 next-attempt-at none"), "{lines}"); + generation.admitted_at_unix_millis = 1; assert!(lines.contains("continuity deferred until 10000500: projection would not demote"), "{lines}"); assert!(lines.contains(&format!( "route actuations 12/12 in window, reopens-at {}", From 69464b11ef6ed9ff000083edc881586a34ef963c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:54:47 +0200 Subject: [PATCH 059/196] Test: name the abort when the queued deployment does not survive the gap Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 465e721..d2b2d44 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16176,12 +16176,11 @@ mod tests { for _ in 0..3 { let _ = world.engine.run_scheduler_tick(); assert_eq!(continuity_mints(&world)?, 0); - assert!( - every_transaction(&world)? - .iter() - .all(|transaction| transaction.pre_fencing_abort.is_none()), - "the deployment was yielded" - ); + let aborts = every_transaction(&world)? + .into_iter() + .filter_map(|transaction| transaction.pre_fencing_abort) + .collect::>(); + assert!(aborts.is_empty(), "the deployment was aborted: {aborts:?}"); } Ok(()) } From 2fae3af133f3f8339a3fe2236712f42cd8e091c7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:55:26 +0200 Subject: [PATCH 060/196] Test: a queued deployment may fail on its own account, never to continuity Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d2b2d44..617a86b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16180,7 +16180,12 @@ mod tests { .into_iter() .filter_map(|transaction| transaction.pre_fencing_abort) .collect::>(); - assert!(aborts.is_empty(), "the deployment was aborted: {aborts:?}"); + // The deployment may fail on its own account (a root test runner + // cannot resolve its source); continuity must never be the reason. + assert!( + aborts.iter().all(|abort| !abort.error.contains("continuity")), + "the deployment was yielded: {aborts:?}" + ); } Ok(()) } From b1dfe829a1fe9c66c48f06b693766451a6ab515d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 18 Oct 2026 23:55:47 +0200 Subject: [PATCH 061/196] Remove an orphaned doc line left by the deleted yield control test Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 617a86b..8da3a36 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16559,8 +16559,6 @@ mod tests { Ok(()) } - /// The control for the test above: the same redeploy over a dead incumbent - /// that is not held is yielded to continuity. /// `seeded` with its own Expected undeclared, so Idunn holds it. fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { let mut bare = seeded.clone(); From 32ff87979071c02e42f0defb832b069f9ae80883 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 00:24:48 +0200 Subject: [PATCH 062/196] Test: a managed-host workload is never claimed lost to a reboot (kills the surviving Ok(true) mutant) Co-Authored-By: Claude Opus 5.5 --- src/host_actuator.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/host_actuator.rs b/src/host_actuator.rs index 92d01e3..f7bf4c7 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1050,6 +1050,43 @@ mod tests { Ok(()) } + /// A managed host's reboot is not observable yet (follow-up + /// host-actuator-reboot): the driver never claims a reboot, so continuity + /// never grants a free re-materialization it cannot justify. + #[test] + fn a_managed_host_workload_is_never_claimed_lost_to_a_reboot() -> Result<()> { + let (_temp, idunn, _host) = identities()?; + let hub = HostActuatorHub::bind("127.0.0.1:0".parse()?)?; + let driver = HostActuatorWorkloadDriver { + access: HostActuatorAccess { + hub: Arc::new(Mutex::new(hub)), + anchors: BTreeMap::new(), + signer: &idunn, + }, + }; + let observation = WorkloadObservation::Host(HostWorkloadObservation { + host: "raven".into(), + actuator_identity_id: "actuator".into(), + process_id: 1, + process_creation_time: 1, + session_id: 1, + user_sid: "S-1-5-21-1-2-3-1001".into(), + executable: "C:/GameCult/idunn/releases/muninn/x/muninn.exe".into(), + executable_sha256: format!("sha256-{}", "1".repeat(64)), + command_line_sha256: format!("sha256-{}", "2".repeat(64)), + environment_names: Vec::new(), + environment_contract_sha256: format!("sha256-{}", "3".repeat(64)), + runtime_bundle: "C:/GameCult/idunn/runtime/muninn/x".into(), + runtime_instance_id: format!("sha256-{}", "a".repeat(64)), + activation_signer_identity_id: "activation".into(), + activation_signer_public_key: vec![1; 32], + exit_code: Some(1), + }); + assert!(!driver.lost_to_host_reboot(&observation, 1)?); + assert!(!driver.lost_to_host_reboot(&observation, u64::MAX)?); + Ok(()) + } + /// A real hub and a real client socket on loopback: Hello attaches under /// the bound anchor, a wrong host is refused, a request is answered. #[test] From 07a1d4169239d88a58bdbb594b31028cf6f52862 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 01:22:34 +0200 Subject: [PATCH 063/196] Continuity: reset counts the re-materialization, a running pass ends the wait; supervision v2 Fixes Soul pass 1 of cut c1-never-stop: - the sustained-run reset reads the newest of the paging log and the re-materialization, so a failed re-materialization is followed by a gap - an observed-running pass clears the backoff anchor, so a standing anchor cannot suppress the reset on a later death - TargetSupervision with keys 7-9 is schema v2; the dual-shape decoder is deleted and v1 records are lifted and migrated once at store open (deletion line named at TARGET_SUPERVISION_SCHEMA_V1) - tests: continuity mints at every streak 1..=12 with a full or short log, the mint stores the reset streak, the schema ids name one shape each - ROUTE_ACTUATION_CEILING doc no longer says continuity spends restarts Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 392 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 338 insertions(+), 54 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8da3a36..ee06aae 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -60,7 +60,13 @@ const ADMITTED_GENERATION_SCHEMA_V3: &str = "idunn.admitted_generation.v3"; const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; /// The capability whose declaration makes a target Odin-correlated. pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; -const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v1"; +const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v2"; +/// The seven-key layout written before continuity's streak, anchor and +/// re-materialization bound. Read and migrated by `read_supervision_record` +/// and `migrate_control_store_to_current_schema`; delete it, `LegacyTargetSupervisionV1` +/// and those two arms once every store has opened under a build that +/// migrates (the key-34 precedent's deletion line). +const TARGET_SUPERVISION_SCHEMA_V1: &str = "idunn.target_supervision.v1"; const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1"; /// The number of charged continuity attempts inside /// `CONTINUITY_RESTART_WINDOW_MILLIS` at which the ContinuityExhausted incident @@ -83,12 +89,14 @@ const CONTINUITY_SUSTAINED_RUN_MILLIS: u64 = 600_000; /// healthy target performs none; a legitimate deployment performs one or two. /// Only deployments count: a Survival actuation is recorded beside this /// ceiling, never counted against it and never refused by it, so a target -/// whose continuity spent its restarts can still be rescued by a deploy. -/// Survival is bounded elsewhere: a continuity's actuations by the restart log -/// times what one transaction can do before its phase deadline ends it (none, -/// over its admitted route), and an admitted-route repair by the challenge -/// backoff. A withdrawal that keeps failing retries, reloading each time, -/// under the resume backoff alone: the abort has no deadline. Nothing bounds +/// that is being restarted can still be rescued by a deploy. +/// Survival is bounded elsewhere: a continuity's actuations by the streak +/// backoff (at most one attempt per `CONTINUITY_BACKOFF_CEILING_MILLIS` once +/// the streak is long) times what one transaction can do before its phase +/// deadline ends it (none, over its admitted route), and an admitted-route +/// repair by the challenge backoff. A withdrawal that keeps failing retries, +/// reloading each time, under the resume backoff alone: the abort has no +/// deadline. Nothing bounds /// reloads across the host. const ROUTE_ACTUATION_CEILING: usize = 12; const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; @@ -1041,7 +1049,7 @@ impl RouteSupervisionState { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.target_supervision", - schema = "idunn.target_supervision.v1" + schema = "idunn.target_supervision.v2" )] struct TargetSupervision { #[cultcache(key = 0)] @@ -1065,21 +1073,22 @@ struct TargetSupervision { /// Visibility only: never counted, never refused. #[cultcache(key = 6)] route_survivals: Vec, - // Keys 7-9 are absent from every record written before them; `default` - // decodes that as no streak, no anchor and no re-materialization. + // Keys 7-9 are why this is v2: a v1 record is lifted by + // `LegacyTargetSupervisionV1` with no streak, no anchor and no + // re-materialization. /// Charged attempts since the last sustained run. Owned by continuity /// supervision; it decides the next attempt's backoff step. - #[cultcache(key = 7, default)] + #[cultcache(key = 7)] continuity_attempt_streak: u32, /// When a dead-and-free pass began waiting out the next attempt. Owned by /// continuity supervision; it decides when the wait is over. Older than /// the admitted generation, it decides nothing. - #[cultcache(key = 8, default)] + #[cultcache(key = 8)] continuity_backoff_from_unix_millis: Option, /// When the admitted generation's one uncharged re-materialization was /// minted. Owned by continuity supervision; it can only withhold a free /// re-materialization, never grant one. - #[cultcache(key = 9, default)] + #[cultcache(key = 9)] continuity_rematerialized_at_unix_millis: Option, } @@ -1140,27 +1149,6 @@ impl TargetSupervision { } } - /// Decode a stored record. Records written before keys 7-9 are the first - /// seven elements of the tuple and decode with those keys defaulted; the - /// next write stores all ten. Anything else must re-encode byte for byte. - fn decode(envelope: &CultCacheEnvelope) -> Result { - let value: Self = rmp_serde::from_slice(&envelope.payload)?; - let before_backoff_keys = rmp_serde::to_vec(&( - &value.schema_version, - &value.target, - &value.route_actuations, - &value.continuity_restarts, - &value.continuity_deferred_until, - &value.continuity_deferral_reason, - &value.route_survivals, - ))?; - ensure!( - rmp_serde::to_vec(&value)? == envelope.payload || before_backoff_keys == envelope.payload, - "Idunn control store contains a noncanonical record" - ); - Ok(value) - } - /// Clamp every log to `now`. Callers that decide from a log write the /// result back when it changed. fn settle(&mut self, now: u64) { @@ -1242,8 +1230,10 @@ impl TargetSupervision { /// What to do about the next attempt for the dead admitted generation /// admitted at `admitted_at`. The streak resets only after a sustained - /// run: the generation has lived the sustained time and no attempt has - /// been minted since it was admitted. + /// run: the generation has lived the sustained time and no attempt, charged + /// or the one re-materialization, has been minted since it was admitted. + /// A standing anchor belongs to a death the supervision pass has seen + /// through; an observed-running pass clears it, so it never hides a reset. fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { let streak = self.continuity_attempt_streak; if let Some(due) = self.backoff_due(admitted_at) { @@ -1258,7 +1248,9 @@ impl TargetSupervision { let sustained = self .continuity_restarts .last() - .is_none_or(|&last| admitted_at >= last) + .into_iter() + .chain(&self.continuity_rematerialized_at_unix_millis) + .all(|&last| admitted_at >= last) && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; let streak = if sustained { 0 } else { streak }; if streak == 0 { @@ -1283,6 +1275,12 @@ impl TargetSupervision { self.continuity_backoff_from_unix_millis = Some(now); } + /// The admitted workload was seen running: the death the anchor waited + /// out is over, and the next death waits from its own discovery. + fn end_backoff(&mut self) { + self.continuity_backoff_from_unix_millis = None; + } + /// The one uncharged re-materialization scheduled. The log and the streak /// are left alone. fn record_rematerialization(&mut self, now: u64) { @@ -3160,12 +3158,7 @@ impl ControlSnapshot { snapshot.admitted.push(Stored { envelope, value }); } TargetSupervision::TYPE => { - ensure!( - envelope.schema_id.as_deref() == Some(TARGET_SUPERVISION_SCHEMA), - "Idunn control store contains an unsupported target supervision record" - ); - let value = TargetSupervision::decode(&envelope)?; - value.validate()?; + let value = read_supervision_record(&envelope)?; ensure!( envelope.key == value.target, "target supervision key is not its target" @@ -3879,6 +3872,74 @@ fn read_generation_record(envelope: &CultCacheEnvelope) -> Result, + #[cultcache(key = 3)] + continuity_restarts: Vec, + #[cultcache(key = 4)] + continuity_deferred_until: Option, + #[cultcache(key = 5)] + continuity_deferral_reason: Option, + #[cultcache(key = 6)] + route_survivals: Vec, +} + +impl LegacyTargetSupervisionV1 { + /// No streak, no anchor, no re-materialization: the same defaults a fresh + /// record starts with. + fn into_current(self) -> TargetSupervision { + TargetSupervision { + schema_version: TARGET_SUPERVISION_SCHEMA.into(), + target: self.target, + route_actuations: self.route_actuations, + continuity_restarts: self.continuity_restarts, + continuity_deferred_until: self.continuity_deferred_until, + continuity_deferral_reason: self.continuity_deferral_reason, + route_survivals: self.route_survivals, + continuity_attempt_streak: 0, + continuity_backoff_from_unix_millis: None, + continuity_rematerialized_at_unix_millis: None, + } + } +} + +/// Read one stored target supervision record, lifting a v1 record to the +/// current shape by the same rule as `read_generation_record`. One schema id +/// names one shape: a v1 record that carries ten keys, or a v2 record that +/// carries seven, is refused. +fn read_supervision_record(envelope: &CultCacheEnvelope) -> Result { + let value = match envelope.schema_id.as_deref() { + Some(TARGET_SUPERVISION_SCHEMA) => decode_record::(envelope)?, + Some(TARGET_SUPERVISION_SCHEMA_V1) => { + let legacy: LegacyTargetSupervisionV1 = decode_record(envelope) + .context("decoding a v1 target supervision record")?; + ensure!( + legacy.schema_version == TARGET_SUPERVISION_SCHEMA_V1, + "stored supervision schema differs from its envelope" + ); + legacy.into_current() + } + _ => bail!("Idunn control store contains an unsupported target supervision record"), + }; + value.validate()?; + ensure!( + envelope.key == value.target, + "target supervision key is not its target" + ); + Ok(value) +} + /// Rewrite every older record as current, once, before the engine runs. /// /// The read path lifts older records on its own, so this is convergence rather @@ -3907,6 +3968,8 @@ fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { schema, Some(ADMITTED_GENERATION_SCHEMA_V2 | ADMITTED_GENERATION_SCHEMA_V3) )) + || (envelope.r#type == TargetSupervision::TYPE + && schema == Some(TARGET_SUPERVISION_SCHEMA_V1)) }) .collect::>(); let mut migrated = 0; @@ -3914,6 +3977,10 @@ fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { let (key, next) = cleanup_evidence::migrate_transaction_record(&envelope)?; (DeploymentTransaction::TYPE, key, next) + } else if envelope.r#type == TargetSupervision::TYPE { + let value = read_supervision_record(&envelope)?; + let next = target_supervision_envelope(&value, now_millis()?)?; + (TargetSupervision::TYPE, value.target, next) } else { let value = read_generation_record(&envelope)?; let next = admitted_envelope(&value, value.admitted_at_unix_millis)?; @@ -6076,6 +6143,14 @@ impl Engine { } }; if observation.is_some() { + let seen = snapshot.supervision_for(¤t.value.target); + if let Some(stored) = seen.filter(|stored| { + stored.value.continuity_backoff_from_unix_millis.is_some() + }) { + let mut next = stored.value.clone(); + next.end_backoff(); + self.write_target_supervision(Some(stored), &next, now_millis()?)?; + } // The admitted workload was seen running: the only evidence // that ends an exhaustion incident, once the window no longer holds it. // The window sliding alone is not health and never closes one. @@ -16241,13 +16316,23 @@ mod tests { Ok(()) } + fn supervision_envelope_raw(payload: Vec, schema: &str) -> Result { + Ok(CultCacheEnvelope { + key: "service".into(), + r#type: TargetSupervision::TYPE.into(), + payload, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(schema.into()), + }) + } + #[test] - fn target_supervision_decodes_without_keys_7_to_9() -> Result<()> { + fn a_v1_target_supervision_record_is_lifted_and_migrated_once() -> Result<()> { let world = EngineFixture::new()?; let meters = restarted_ago(&[200_000])?; - // A record as every one on a running host was written: seven keys. + // A record as every one on a running host was written: seven keys, v1. let payload = rmp_serde::to_vec(&( - &meters.schema_version, + TARGET_SUPERVISION_SCHEMA_V1, &meters.target, &meters.route_actuations, &meters.continuity_restarts, @@ -16255,20 +16340,22 @@ mod tests { &meters.continuity_deferral_reason, &meters.route_survivals, ))?; - let envelope = CultCacheEnvelope { - key: "service".into(), - r#type: TargetSupervision::TYPE.into(), - payload, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(TARGET_SUPERVISION_SCHEMA.into()), - }; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent(envelope)?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + supervision_envelope_raw(payload, TARGET_SUPERVISION_SCHEMA_V1)?, + )?; let read = meters_of(&world, "service")?; assert_eq!(read.continuity_restarts, meters.continuity_restarts); assert_eq!(read.continuity_attempt_streak, 0); assert_eq!(read.continuity_backoff_from_unix_millis, None); assert_eq!(read.continuity_rematerialized_at_unix_millis, None); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 0); + let stored = ControlSnapshot::read(&world.state_store)?; + let envelope = &stored.supervision_for("service").unwrap().envelope; + assert_eq!(envelope.schema_id.as_deref(), Some(TARGET_SUPERVISION_SCHEMA)); + assert_eq!(meters_of(&world, "service")?.continuity_restarts, meters.continuity_restarts); + let mut anchored = TargetSupervision::new("service"); anchored.continuity_backoff_from_unix_millis = Some(0); assert!(anchored.validate().is_err()); @@ -16278,6 +16365,203 @@ mod tests { Ok(()) } + #[test] + fn one_target_supervision_schema_id_names_one_shape() -> Result<()> { + let meters = restarted_ago(&[200_000])?; + let ten = rmp_serde::to_vec(&meters)?; + let seven = rmp_serde::to_vec(&( + TARGET_SUPERVISION_SCHEMA, + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + // The current id refuses the old shape and the old id refuses the new one. + assert!( + read_supervision_record(&supervision_envelope_raw(seven, TARGET_SUPERVISION_SCHEMA)?) + .is_err() + ); + assert!( + read_supervision_record(&supervision_envelope_raw( + ten.clone(), + TARGET_SUPERVISION_SCHEMA_V1 + )?) + .is_err() + ); + assert_eq!( + read_supervision_record(&supervision_envelope_raw(ten, TARGET_SUPERVISION_SCHEMA)?)?, + meters + ); + Ok(()) + } + + #[test] + fn continuity_keeps_minting_at_every_streak_and_log_length() -> Result<()> { + // No stop at any streak, however long, with the paging log full or not. + let full = [3_000_000_u64, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000]; + for streak in 1..=12_u32 { + for log in [&[1_000_000_u64][..], &full[..]] { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(log)?; + meters.continuity_attempt_streak = streak; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the wait begins"); + anchor_ago(&world, 300_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1, "streak {streak} log {} stopped", log.len()); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, streak + 1); + } + } + Ok(()) + } + + #[test] + fn a_mint_after_a_sustained_run_stores_the_reset_streak() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + workload.kill(); + let mut meters = restarted_ago(&[3_000_000])?; + meters.continuity_attempt_streak = 7; + set_meters(&world, &meters)?; + // Ran 700 s since the last attempt: no wait, and the streak restarts at 1. + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + + #[test] + fn a_re_materialization_counts_as_the_last_attempt_for_the_sustained_reset() { + let admitted_at = 10_000_000_u64; + let now = admitted_at + 700_000; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 5; + meters.continuity_restarts = vec![admitted_at - 1_000]; + // Before admission: a sustained run, reset. + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at - 1); + assert_eq!( + meters.continuity_backoff(now, admitted_at), + ContinuityBackoff::Due { streak: 0 } + ); + // After admission: that attempt failed on this generation. + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at + 1); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); + } + + mod after_a_reboot { + use super::*; + + fn resident_continuity(world: &EngineFixture) -> Result { + Ok(ControlSnapshot::read(&world.state_store)? + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .count()) + } + + /// The attempt failed and was retired: drop continuity transactions + /// and their commands only. + fn drop_continuity(world: &EngineFixture) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let ids = snapshot + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .map(|s| s.value.command_id.clone()) + .collect::>(); + let envelopes = snapshot + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .map(|s| s.envelope.clone()) + .chain( + snapshot + .commands + .iter() + .filter(|s| ids.contains(&s.value.command_id)) + .map(|s| s.envelope.clone()), + ) + .collect::>(); + if !envelopes.is_empty() { + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&envelopes)? + ); + } + Ok(()) + } + + /// (attempts minted on tick 1, on tick 2, whether the queued deploy froze by then). + fn gap_run(reboot: bool) -> Result<(usize, usize, bool)> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + if reboot { + workload.reboot(); + } else { + workload.kill(); + } + queue_declaring_redeploy(&world, "up-gap")?; + // One tick, in the tick's order: supervise, then freeze. + world.engine.supervise_one_admitted_generation()?; + let first = resident_continuity(&world)?; + world.engine.freeze_one_queued_command()?; + // That attempt failed and was retired. + drop_continuity(&world)?; + world.engine.supervise_one_admitted_generation()?; + let second = resident_continuity(&world)?; + world.engine.freeze_one_queued_command()?; + let frozen = every_transaction(&world)?.iter().any(|t| t.command_id == "up-gap"); + Ok((first, second, frozen)) + } + + #[test] + fn a_failed_re_materialization_is_followed_by_a_gap_a_queued_deployment_can_take( + ) -> Result<()> { + // Control: an ordinary attempt after a sustained run leaves a gap. + assert_eq!(gap_run(false)?, (1, 0, true)); + // The re-materialization is an attempt too: the next pass waits. + assert_eq!(gap_run(true)?, (1, 0, true)); + Ok(()) + } + + #[test] + fn a_running_pass_ends_the_wait_so_a_later_death_after_a_sustained_run_resets( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis, None); + + // It runs on for days, then dies: no wait, and the streak restarts. + age_incumbent(&world, 200_000)?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + } + /// A source that resolves one fixed recipe and never freezes anything. struct FixedSource { recipe: String, From a75e31330aeccaa04b6652248b5e6290402eb681 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 01:24:09 +0200 Subject: [PATCH 064/196] Continuity: the attempt after a re-materialization waits the base step The re-materialization charges no streak, so a streak of 0 gave Due{0} and a failed re-materialization was followed at once by a charged attempt. It is an attempt that failed on the generation: the next one waits the base step, so a queued deployment gets its gap. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 40932 +++++++++++++++++++++-------------------- 1 file changed, 20478 insertions(+), 20454 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index ee06aae..3037ab8 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1,20454 +1,20478 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::env; -use std::fs::{self, File, OpenOptions}; -use std::io::Write; -use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; -use std::thread; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcache_rs::{ - CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, -}; -use cultnet_rs::{ - AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, - IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, - IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, - IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, - IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, - IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, - OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, - ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, - authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, - correlate_runtime_presence_claim, derive_service_identity_id, - evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, - verify_idunn_deployment_brake_authorization, verify_runtime_authority, -}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use uuid::Uuid; - -use crate::deployment::{ - DependencyKind, OperatorBinding, RolloutStrategy, RouteBinding, WorkloadBinding, - capability_compatible, -}; -use crate::deployment_plan::{ - CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, -}; -use crate::drivers::{ - ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, - GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, - LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, - SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, - WriteLeasePort, -}; -use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; -use crate::host_actuator::{ - HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, - HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, -}; - -const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2"; -const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v4"; -const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3"; -const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2"; -const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v4"; -const ADMITTED_GENERATION_SCHEMA_V3: &str = "idunn.admitted_generation.v3"; -const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; -/// The capability whose declaration makes a target Odin-correlated. -pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; -const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v2"; -/// The seven-key layout written before continuity's streak, anchor and -/// re-materialization bound. Read and migrated by `read_supervision_record` -/// and `migrate_control_store_to_current_schema`; delete it, `LegacyTargetSupervisionV1` -/// and those two arms once every store has opened under a build that -/// migrates (the key-34 precedent's deletion line). -const TARGET_SUPERVISION_SCHEMA_V1: &str = "idunn.target_supervision.v1"; -const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1"; -/// The number of charged continuity attempts inside -/// `CONTINUITY_RESTART_WINDOW_MILLIS` at which the ContinuityExhausted incident -/// opens and is held. It gates no actuation: continuity never stops -/// (ruling budget-never-stop). -const CONTINUITY_PAGING_RESTARTS: usize = 6; -const CONTINUITY_RESTART_WINDOW_MILLIS: u64 = 3_600_000; -/// The base of the streak backoff: the wait after the first charged attempt; -/// it doubles with each further one in the streak. -const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; -/// The longest streak backoff between charged continuity attempts (ruling -/// continuity-backoff-defaults). -const CONTINUITY_BACKOFF_CEILING_MILLIS: u64 = 300_000; -/// How long an admitted generation must have run, with no attempt failing on -/// it, before the attempt streak resets (ruling continuity-backoff-defaults). -const CONTINUITY_SUSTAINED_RUN_MILLIS: u64 = 600_000; - -/// Forward route actuations (fragment write, firewall, `nginx -t`, reload, -/// private-mount validation) one target may perform per sliding window. A -/// healthy target performs none; a legitimate deployment performs one or two. -/// Only deployments count: a Survival actuation is recorded beside this -/// ceiling, never counted against it and never refused by it, so a target -/// that is being restarted can still be rescued by a deploy. -/// Survival is bounded elsewhere: a continuity's actuations by the streak -/// backoff (at most one attempt per `CONTINUITY_BACKOFF_CEILING_MILLIS` once -/// the streak is long) times what one transaction can do before its phase -/// deadline ends it (none, over its admitted route), and an admitted-route -/// repair by the challenge backoff. A withdrawal that keeps failing retries, -/// reloading each time, under the resume backoff alone: the abort has no -/// deadline. Nothing bounds -/// reloads across the host. -const ROUTE_ACTUATION_CEILING: usize = 12; -const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; -/// How many Survival route actuation times a target keeps, for `status` -/// only: they are a record, not a meter, and no decision reads them. -const ROUTE_SURVIVAL_RECORD: usize = 12; -/// The longest wait between route challenges while proofs keep failing. The -/// shortest is the observation max age. -const ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS: u64 = 600_000; - -/// How long continuity waits after it could not prepare a restart (the -/// projection would not demote) before trying again. -const CONTINUITY_DEFERRAL_MILLIS: u64 = 30_000; - -const DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS: u64 = 30_000; -const DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS: u64 = 2_000; - -/// An immutable request. Execution state deliberately does not fit in this -/// record; status is derived from the transactions created for it. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.deployment_command", - schema = "idunn.deployment_command.v2" -)] -struct DeploymentCommand { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - command_id: String, - #[cultcache(key = 2)] - kind: CommandKind, - #[cultcache(key = 3)] - selector: String, - #[cultcache(key = 4)] - requested_by: String, - #[cultcache(key = 5)] - requested_at_unix_millis: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -enum CommandKind { - Deploy, - Continuity, -} - -impl DeploymentCommand { - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == DEPLOYMENT_COMMAND_SCHEMA, - "deployment command schema is unsupported" - ); - require_id(&self.command_id, "deployment command id")?; - require_selector(&self.selector)?; - require_value(&self.requested_by, "deployment requester")?; - ensure!( - self.requested_at_unix_millis > 0, - "deployment command has no request time" - ); - match self.kind { - CommandKind::Deploy => ensure!( - !self.command_id.starts_with("continuity-"), - "operator deployment uses the continuity command namespace" - ), - CommandKind::Continuity => ensure!( - self.command_id.starts_with("continuity-") - && !self.selector.starts_with("profile:"), - "continuity command identity or selector is invalid" - ), - } - Ok(()) - } -} - -/// An operator's request that a live transaction's post-fencing phase end now -/// (`idunn expire`). The CLI only writes this record. The daemon's deadline -/// resolver reads it as a deadline that has passed, and the record is deleted -/// once its transaction no longer has a phase to end. No CLI writes a -/// transaction field. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache(type = "idunn.expiry_request", schema = "idunn.expiry_request.v1")] -struct ExpiryRequest { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - command_id: String, - #[cultcache(key = 2)] - requested_by: String, - #[cultcache(key = 3)] - requested_at_unix_millis: u64, -} - -impl ExpiryRequest { - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == EXPIRY_REQUEST_SCHEMA, - "expiry request schema is unsupported" - ); - require_id(&self.command_id, "expiry request command id")?; - require_value(&self.requested_by, "expiry requester")?; - ensure!( - self.requested_at_unix_millis > 0, - "expiry request has no request time" - ); - Ok(()) - } -} - -fn expiry_request_envelope(value: &ExpiryRequest, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.command_id, - ExpiryRequest::TYPE, - EXPIRY_REQUEST_SCHEMA, - value, - now, - ) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -#[repr(u8)] -enum DeploymentPhase { - Sealing, - Starting, - Warming, - Fencing, - Leasing, - AwaitingReady, - Routing, - Committing, - Complete, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct DeploymentAuthorization { - authorization_id: String, - brake_sha256: String, - canonical_brake_bytes: Vec, - authorized_at_unix_millis: u64, -} - -impl DeploymentAuthorization { - fn validate_shape(&self) -> Result<()> { - require_id(&self.authorization_id, "deployment authorization id")?; - ensure!( - !self.canonical_brake_bytes.is_empty() - && self.brake_sha256 == sha256_id(&self.canonical_brake_bytes) - && self.authorized_at_unix_millis > 0, - "deployment authorization receipt is incomplete" - ); - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&self.canonical_brake_bytes)?; - record.validate()?; - ensure!( - rmp_serde::to_vec(&record)? == self.canonical_brake_bytes - && record.authorization_id.as_deref() == Some(&self.authorization_id), - "deployment authorization receipt is noncanonical or mismatched" - ); - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct TopologyEvidence { - canonical_bytes: Vec, - canonical_sha256: String, - signer_identity_id: String, - publisher_sequence: u64, - admitted_at_unix_millis: u64, -} - -impl TopologyEvidence { - fn from_authenticated( - topology: &AuthenticatedOdinRuntimeTopologyCorrelation, - admitted_at_unix_millis: u64, - ) -> Result { - let record = topology.record(); - let evidence = Self { - canonical_bytes: topology.canonical_bytes().to_vec(), - canonical_sha256: sha256_id(topology.canonical_bytes()), - signer_identity_id: record.signer_identity_id.clone(), - publisher_sequence: record.publisher_sequence, - admitted_at_unix_millis, - }; - evidence.validate_shape()?; - Ok(evidence) - } - - fn validate_shape(&self) -> Result<()> { - ensure!( - !self.canonical_bytes.is_empty() - && self.canonical_sha256 == sha256_id(&self.canonical_bytes), - "topology evidence bytes or digest are invalid" - ); - require_id(&self.signer_identity_id, "Odin topology signer")?; - ensure!( - self.publisher_sequence > 0 && self.admitted_at_unix_millis > 0, - "topology evidence sequence or admission time is invalid" - ); - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct RuntimePresenceEvidence { - canonical_bytes: Vec, - canonical_sha256: String, - message_id: String, - challenged_at_unix_millis: u64, - admitted_at_unix_millis: u64, -} - -impl RuntimePresenceEvidence { - fn from_present( - present: &cultnet_rs::VerifiedRuntimePresence, - message_id: String, - challenged_at_unix_millis: u64, - admitted_at_unix_millis: u64, - ) -> Result { - let evidence = Self { - canonical_bytes: present.canonical_bytes().to_vec(), - canonical_sha256: present.signed_presence_sha256().to_owned(), - message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - }; - evidence.validate_shape()?; - Ok(evidence) - } - - fn validate_shape(&self) -> Result<()> { - ensure!( - !self.canonical_bytes.is_empty() - && self.canonical_sha256 == sha256_id(&self.canonical_bytes), - "runtime presence evidence bytes or digest are invalid" - ); - require_id(&self.message_id, "runtime presence challenge")?; - ensure!( - self.challenged_at_unix_millis > 0 - && self.admitted_at_unix_millis >= self.challenged_at_unix_millis, - "runtime presence evidence timeline is invalid" - ); - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "source", rename_all = "kebab-case", deny_unknown_fields)] -enum WarmingEvidence { - OdinTopology { evidence: TopologyEvidence }, - FirstOdinDirect { evidence: RuntimePresenceEvidence }, - /// A route-proof target's candidate answered Idunn's challenge itself. - RouteProofDirect { evidence: RuntimePresenceEvidence }, -} - -impl WarmingEvidence { - fn validate_shape(&self) -> Result<()> { - match self { - Self::OdinTopology { evidence } => evidence.validate_shape(), - Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { - evidence.validate_shape() - } - } - } - - fn voucher(&self) -> Voucher { - match self { - Self::OdinTopology { .. } | Self::FirstOdinDirect { .. } => Voucher::Odin, - Self::RouteProofDirect { .. } => Voucher::Candidate, - } - } - - /// The presence Idunn challenged for itself, when no Odin carried it. - fn direct(&self) -> Option<&RuntimePresenceEvidence> { - match self { - Self::OdinTopology { .. } => None, - Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { - Some(evidence) - } - } - } -} - -/// Opaque capability minted only after the transaction CAS stores exact -/// Warming evidence. Adjacent drivers receive only the candidate identity and -/// signed-presence digest needed to bind the write lease; they cannot inspect -/// or construct semantic admission. -#[derive(Clone, Debug)] -pub struct SequenceAdmittedWarming { - transaction_id: String, - signed_presence_sha256: String, - runtime_instance_id: String, -} - -impl SequenceAdmittedWarming { - fn from_topology( - transaction_id: String, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - ) -> Result { - let record = authenticated.record(); - let signed_presence_sha256 = record - .signed_presence_sha256 - .clone() - .context("Warming topology has no signed presence")?; - let runtime_instance_id = record - .runtime_instance_id - .clone() - .context("Warming topology has no runtime instance")?; - Ok(Self { - transaction_id, - signed_presence_sha256, - runtime_instance_id, - }) - } - - fn from_direct_presence( - transaction_id: String, - warming: WarmingEvidence, - present: cultnet_rs::VerifiedRuntimePresence, - ) -> Result { - let evidence = warming - .direct() - .context("direct Warming evidence is an Odin topology receipt")?; - ensure!( - present.canonical_bytes() == evidence.canonical_bytes - && present.signed_presence_sha256() == evidence.canonical_sha256, - "direct Warming evidence differs from its authenticated presence" - ); - let runtime_instance_id = present.record().runtime_instance_id.clone(); - Ok(Self { - transaction_id, - signed_presence_sha256: evidence.canonical_sha256.clone(), - runtime_instance_id, - }) - } - - #[cfg(test)] - pub(crate) fn for_test( - transaction_id: impl Into, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - ) -> Result { - let record = authenticated.record(); - ensure!( - record.present - && !record.ready - && record.observed_presence_state.as_deref() == Some("warming") - && record.observed_write_lease_sha256.is_none() - && record.disagreements.is_empty(), - "test receipt is not exact Warming evidence" - ); - let transaction_id = transaction_id.into(); - require_id(&transaction_id, "test transaction id")?; - let signed_presence_sha256 = record - .signed_presence_sha256 - .clone() - .context("test Warming receipt has no signed presence")?; - let runtime_instance_id = record - .runtime_instance_id - .clone() - .context("test Warming receipt has no runtime instance")?; - let token = Self::from_topology(transaction_id, authenticated)?; - ensure!( - token.signed_presence_sha256 == signed_presence_sha256 - && token.runtime_instance_id == runtime_instance_id, - "test Warming token extraction changed" - ); - Ok(token) - } - - pub(crate) fn transaction_id(&self) -> &str { - &self.transaction_id - } - - pub(crate) fn signed_presence_sha256(&self) -> &str { - &self.signed_presence_sha256 - } - - pub(crate) fn runtime_instance_id(&self) -> &str { - &self.runtime_instance_id - } -} - -/// Ready proof with the same construction boundary. Dependency planning and -/// promotion receive this token, never caller-supplied digest/sequence tuples. -#[derive(Clone, Debug)] -pub(crate) struct SequenceAdmittedReady { - transaction_id: String, - evidence: TopologyEvidence, - expected: IdunnExpectedIncarnationRecord, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, -} - -impl SequenceAdmittedReady { - #[cfg(test)] - pub(crate) fn for_test( - expected: &IdunnExpectedIncarnationRecord, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - admitted_at_unix_millis: u64, - ) -> Result { - expected.validate()?; - let record = authenticated.record(); - ensure!( - record.expected_projection_sha256 == expected.canonical_sha256()? - && record.target == expected.target - && record.runtime_id == expected.runtime_id - && record.present - && record.ready - && record.observed_presence_state.as_deref() == Some("active") - && record.disagreements.is_empty(), - "test receipt is not exact Ready evidence" - ); - let evidence = - TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; - Ok(Self { - transaction_id: "test-sequence-admission".into(), - evidence, - expected: expected.clone(), - authenticated, - }) - } - - pub(crate) fn authenticated(&self) -> &AuthenticatedOdinRuntimeTopologyCorrelation { - &self.authenticated - } - - pub(crate) fn expected(&self) -> &IdunnExpectedIncarnationRecord { - &self.expected - } - - pub(crate) fn transaction_id(&self) -> &str { - &self.transaction_id - } - - pub(crate) fn evidence_sha256(&self) -> &str { - &self.evidence.canonical_sha256 - } - - pub(crate) fn publisher_sequence(&self) -> u64 { - self.evidence.publisher_sequence - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] -enum FencingEvidence { - SkippedStateless, - Revoked { - incumbent_lease_sha256: Option, - candidate_lease_path_verified_empty: bool, - }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] -enum LeasingEvidence { - SkippedStateless, - Prepared { - lease: IdunnProcessWriteLeaseRecord, - lease_sha256: String, - }, - Granted { - lease: IdunnProcessWriteLeaseRecord, - lease_sha256: String, - }, -} - -impl LeasingEvidence { - fn lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { - match self { - Self::SkippedStateless | Self::Prepared { .. } => None, - Self::Granted { lease, .. } => Some(lease), - } - } - - /// The lease Idunn issued, granted or only prepared. A prepared lease may - /// already be on disk: the physical grant runs before `Granted` is durable. - fn issued_lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { - match self { - Self::SkippedStateless => None, - Self::Prepared { lease, .. } | Self::Granted { lease, .. } => Some(lease), - } - } - - fn prepared_lease(&self) -> Option<(&IdunnProcessWriteLeaseRecord, &str)> { - match self { - Self::Prepared { - lease, - lease_sha256, - } => Some((lease, lease_sha256)), - Self::SkippedStateless | Self::Granted { .. } => None, - } - } - - fn lease_sha256(&self) -> Option<&str> { - match self { - Self::SkippedStateless | Self::Prepared { .. } => None, - Self::Granted { lease_sha256, .. } => Some(lease_sha256), - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] -enum RoutingEvidence { - SkippedUnrouted, - Promoted { - observation: RouteObservation, - promoted_at_unix_millis: u64, - }, -} - -impl RoutingEvidence { - fn observation(&self) -> Option<&RouteObservation> { - match self { - Self::SkippedUnrouted => None, - Self::Promoted { observation, .. } => Some(observation), - } - } - - fn promoted_at_unix_millis(&self) -> Option { - match self { - Self::SkippedUnrouted => None, - Self::Promoted { - promoted_at_unix_millis, - .. - } => Some(*promoted_at_unix_millis), - } - } -} - -/// The states a route-proof candidate may answer its Warming challenge in. A -/// stateful one is warming until it holds a lease it cannot hold yet, and the -/// lease binds that warming presence. A stateless one may have finished -/// warming before the first challenge landed. -fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'static [&'static str] { - if expected.write_lease_required { - &["warming"] - } else { - &["warming", "active"] - } -} - -/// A challenged runtime answered as something that disagrees with current -/// authority. Each disagreement is named, so a capacity below the Expected -/// minimum reads as that and not as a generic refusal. It stays the error's -/// typed value; callers read `disagreements`, and the text is only its -/// rendering. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct PresenceDisagrees { - pub(crate) disagreements: Vec, -} - -impl std::fmt::Display for PresenceDisagrees { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let named = self - .disagreements - .iter() - .map(|disagreement| { - format!( - "{} (expected {}, observed {})", - disagreement.code, - disagreement.expected.as_deref().unwrap_or("none"), - disagreement.observed.as_deref().unwrap_or("none"), - ) - }) - .collect::>() - .join("; "); - write!( - formatter, - "route proof answered with a runtime that disagrees with current authority: {named}" - ) - } -} - -impl std::error::Error for PresenceDisagrees {} - -/// What a direct challenge to a candidate endpoint produced. -enum CandidateAnswer { - /// No answer: the reason it is still being waited on. - Silent(String), - Answered { - evidence: RuntimePresenceEvidence, - present: cultnet_rs::VerifiedRuntimePresence, - }, -} - -//// Which kind of proof admits a target as Ready, derived from what the target's -/// recipe declares and from nothing else. Idunn infers no way to prove -/// readiness: a target that provides `odin.verse-rendezvous` is Odin itself, one -/// that declares a `shared-infrastructure odin.verse-rendezvous` dependency -/// advertises into the Verse and is Odin-correlated, and a routed one that -/// declares neither answers Idunn's own challenge and is route-proof. A target -/// with no route and no Odin dependency declares no proof at all, and is -/// refused. Odin collects the same receipts an Odin-correlated target does, and -/// its admitted generation is what names the Odin authority, so it is never -/// route-proof. -/// -/// `of` is the only place a class is decided, and Expected is digest-bound, so -/// the class cannot drift from the incarnation it describes. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum ReadinessClass { - OdinSelf, - OdinCorrelated, - RouteProof, -} - -/// A recipe that declares neither a route nor an Odin dependency, and is not -/// Odin: it has said nothing about how it proves readiness. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct UndeclaredReadiness { - pub(crate) target: String, -} - -impl std::fmt::Display for UndeclaredReadiness { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - formatter, - "target {} declares no way to prove readiness: its recipe must declare a route \ - (Idunn challenges it) or a shared-infrastructure {ODIN_RENDEZVOUS_CAPABILITY} \ - dependency (Odin correlates it)", - self.target - ) - } -} - -impl std::error::Error for UndeclaredReadiness {} - -impl ReadinessClass { - /// The class of a recipe's declarations. Plan compilation and `of` both - /// call it, so admission refuses exactly what `of` would refuse. - pub(crate) fn declared( - target: &str, - provides_odin: bool, - declares_odin: bool, - routed: bool, - ) -> Result { - if provides_odin { - Ok(Self::OdinSelf) - } else if declares_odin { - Ok(Self::OdinCorrelated) - } else if routed { - Ok(Self::RouteProof) - } else { - Err(UndeclaredReadiness { - target: target.to_owned(), - }) - } - } - - pub(crate) fn of( - expected: &IdunnExpectedIncarnationRecord, - ) -> Result { - Self::declared( - &expected.target, - expected - .capabilities - .iter() - .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY), - expected.dependencies.iter().any(|dependency| { - dependency.kind == "shared-infrastructure" - && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY - }), - expected.route.is_some(), - ) - } - - /// The class of `expected`, provided every receipt collected under it was - /// vouched for by that class's voucher. The one way a stored record is - /// asked what class it is. - fn confirmed( - expected: &IdunnExpectedIncarnationRecord, - collected: impl IntoIterator, - ) -> Result { - let required = Self::of(expected).map_err(ReadinessDisagreement::Undeclared)?; - match collected - .into_iter() - .find(|voucher| *voucher != required.voucher()) - { - None => Ok(required), - Some(collected) => Err(ReadinessDisagreement::WrongVoucher { - target: expected.target.clone(), - required, - collected, - }), - } - } - - /// Who vouches for a target of this class: Odin's correlation, or the - /// candidate answering Idunn. - fn voucher(self) -> Voucher { - match self { - Self::OdinSelf | Self::OdinCorrelated => Voucher::Odin, - Self::RouteProof => Voucher::Candidate, - } - } -} - -/// Who vouched for a piece of stored readiness evidence. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Voucher { - Odin, - Candidate, -} - -/// A stored record whose readiness evidence is not what its own Expected -/// requires. A record admitted before the target's recipe declared how it -/// proves readiness looks like this. It is reported and held, never repaired: -/// re-deriving the evidence around it would make the evidence tag a second -/// authority for the class. -#[derive(Clone, Debug, PartialEq, Eq)] -enum ReadinessDisagreement { - Undeclared(UndeclaredReadiness), - WrongVoucher { - target: String, - required: ReadinessClass, - collected: Voucher, - }, -} - -impl std::fmt::Display for ReadinessDisagreement { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Undeclared(undeclared) => write!(formatter, "{undeclared}"), - Self::WrongVoucher { - target, - required, - collected, - } => write!( - formatter, - "target {target} is {required:?} but its stored readiness evidence was vouched \ - for by {collected:?}: it was collected under a different class, and is held \ - until the target is redeployed" - ), - } - } -} - -impl std::error::Error for ReadinessDisagreement {} - -/// The receipt that admitted a candidate as Ready, tagged by who vouched for -/// it. The tag records what was collected; what a target must collect is -/// `ReadinessClass::of(expected)`, and a tag that differs is a -/// `ReadinessDisagreement`, never a second opinion about the class. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)] -enum ReadinessEvidence { - OdinCorrelated { evidence: TopologyEvidence }, - RouteProof { evidence: RuntimePresenceEvidence }, -} - -impl ReadinessEvidence { - fn voucher(&self) -> Voucher { - match self { - Self::OdinCorrelated { .. } => Voucher::Odin, - Self::RouteProof { .. } => Voucher::Candidate, - } - } - - fn validate_shape(&self) -> Result<()> { - match self { - Self::OdinCorrelated { evidence } => evidence.validate_shape(), - Self::RouteProof { evidence } => evidence.validate_shape(), - } - } - - fn odin(&self) -> Option<&TopologyEvidence> { - match self { - Self::OdinCorrelated { evidence } => Some(evidence), - Self::RouteProof { .. } => None, - } - } -} - -// Signed evidence that the candidate process picked up the lease Idunn -/// granted: its presence carries `write_lease_sha256` equal to the grant. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LeaseAdoptionEvidence { - write_lease_sha256: String, - signed_presence_sha256: String, - source: AdoptionSource, - observed_at_unix_millis: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -enum AdoptionSource { - Direct, - OdinTopology, -} - -impl LeaseAdoptionEvidence { - /// The adoption a stateful candidate's Ready proves: its signed presence - /// held exactly the granted lease. `None` for a stateless candidate. - fn with_ready( - granted_lease_sha256: Option<&str>, - signed_presence_sha256: &str, - source: AdoptionSource, - observed_at_unix_millis: u64, - ) -> Option { - granted_lease_sha256.map(|lease| Self { - write_lease_sha256: lease.to_owned(), - signed_presence_sha256: signed_presence_sha256.to_owned(), - source, - observed_at_unix_millis, - }) - } - - /// Whether this evidence is about exactly the lease that was granted. - fn names(&self, leasing: &LeasingEvidence) -> bool { - leasing.lease_sha256() == Some(self.write_lease_sha256.as_str()) - } - - fn validate_shape(&self) -> Result<()> { - require_id(&self.write_lease_sha256, "adopted lease digest")?; - require_id(&self.signed_presence_sha256, "lease adoption presence digest")?; - ensure!( - self.observed_at_unix_millis > 0, - "lease adoption has no observation time" - ); - Ok(()) - } -} - -/// When the transaction entered its current post-fencing phase and when that -/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PhaseDeadline { - phase: DeploymentPhase, - entered_at_unix_millis: u64, - deadline_at_unix_millis: u64, -} - -impl PhaseDeadline { - /// `None` for a phase that has no deadline (pre-fencing and Complete). - fn entering( - phase: DeploymentPhase, - plan: &CompiledDeploymentPlan, - now: u64, - ) -> Option { - let deadlines = plan.phase_deadlines(); - let seconds = match phase { - DeploymentPhase::Fencing => deadlines.fencing_seconds, - DeploymentPhase::Leasing => deadlines.leasing_seconds, - DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, - DeploymentPhase::Routing => deadlines.routing_seconds, - DeploymentPhase::Committing => deadlines.committing_seconds, - _ => return None, - }; - Some(Self { - phase, - entered_at_unix_millis: now, - deadline_at_unix_millis: now.saturating_add(u64::from(seconds) * 1000), - }) - } -} - -/// How the deadline resolver ends a post-fencing phase. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum PhaseEnd { - /// Abort through the post-fencing path. `lease_not_adopted` names a - /// granted lease with no adoption evidence. - Abort { lease_not_adopted: bool }, - /// A stateful deployment whose candidate holds an issued lease: the phase - /// is reported and not resolved. - OperatorRequired, -} - -/// How a post-fencing failure recovers. `RestoreIncumbent` is what a -/// post-fencing abort has always done and is the default for records that -/// predate the field. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "action", rename_all = "kebab-case", deny_unknown_fields)] -enum TerminalRecovery { - RestartAdmitted, - #[default] - RestoreIncumbent, - OperatorRequired { - reason: String, - }, -} - -/// Route supervision's durable memory for one admitted routed incarnation: -/// challenge pacing and the degradation mark. It belongs to the incarnation, so -/// a new generation starts from `default()`. What a target may actuate outlives -/// every generation and lives in `TargetSupervision`. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct RouteSupervisionState { - last_challenge_at_unix_millis: Option, - consecutive_failures: u32, - next_challenge_at_unix_millis: Option, - /// Set while route proof keeps failing. Marks the route degraded so - /// dependents stop selecting it; it never authorizes a restart. - degraded_since_unix_millis: Option, -} - -impl RouteSupervisionState { - /// The next challenge is not due yet. - fn is_waiting(&self, now: u64, maximum_age_millis: u64) -> bool { - self.next_challenge_at_unix_millis.is_some_and(|next| { - is_waiting( - now, - next, - ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(maximum_age_millis), - ) - }) - } - - /// A failed proof, or a failed repair of the fragment it needs. Observation - /// state only: the route is marked degraded and the next challenge waits, - /// twice as long each time, from the observation max age up to a cap. - fn record_failed_challenge(&mut self, now: u64, maximum_age_millis: u64) { - self.consecutive_failures = self.consecutive_failures.saturating_add(1); - let doublings = self.consecutive_failures.saturating_sub(1).min(16); - let wait = maximum_age_millis - .saturating_mul(1 << doublings) - .min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS) - .max(maximum_age_millis); - self.last_challenge_at_unix_millis = Some(now); - self.next_challenge_at_unix_millis = Some(now.saturating_add(wait)); - self.degraded_since_unix_millis.get_or_insert(now); - } - - fn record_proved_challenge(&mut self, now: u64) { - self.consecutive_failures = 0; - self.last_challenge_at_unix_millis = Some(now); - self.next_challenge_at_unix_millis = None; - self.degraded_since_unix_millis = None; - } - - fn validate(&self) -> Result<()> { - if let Some(since) = self.degraded_since_unix_millis { - ensure!(since > 0, "route degradation has no start time"); - } - Ok(()) - } -} - -/// What one target has been allowed to do, kept across every generation and -/// present before the first: Forward route actuations and continuity restarts, -/// each a sliding log of the times they happened, and a record of Survival -/// route actuations. Owned by the meter primitives below; -/// the route driver only asks, and no generation write carries it. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.target_supervision", - schema = "idunn.target_supervision.v2" -)] -struct TargetSupervision { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - /// The newest `ROUTE_ACTUATION_CEILING` Forward actuation times, - /// ascending: the only entries the ceiling counts. - #[cultcache(key = 2)] - route_actuations: Vec, - /// The newest `CONTINUITY_PAGING_RESTARTS` restart times, ascending: the - /// paging threshold's log. - #[cultcache(key = 3)] - continuity_restarts: Vec, - /// A restart whose projection could not be prepared waits until this time. - #[cultcache(key = 4)] - continuity_deferred_until: Option, - #[cultcache(key = 5)] - continuity_deferral_reason: Option, - /// The newest `ROUTE_SURVIVAL_RECORD` Survival actuation times, ascending. - /// Visibility only: never counted, never refused. - #[cultcache(key = 6)] - route_survivals: Vec, - // Keys 7-9 are why this is v2: a v1 record is lifted by - // `LegacyTargetSupervisionV1` with no streak, no anchor and no - // re-materialization. - /// Charged attempts since the last sustained run. Owned by continuity - /// supervision; it decides the next attempt's backoff step. - #[cultcache(key = 7)] - continuity_attempt_streak: u32, - /// When a dead-and-free pass began waiting out the next attempt. Owned by - /// continuity supervision; it decides when the wait is over. Older than - /// the admitted generation, it decides nothing. - #[cultcache(key = 8)] - continuity_backoff_from_unix_millis: Option, - /// When the admitted generation's one uncharged re-materialization was - /// minted. Owned by continuity supervision; it can only withhold a free - /// re-materialization, never grant one. - #[cultcache(key = 9)] - continuity_rematerialized_at_unix_millis: Option, -} - -/// What continuity supervision does about the next attempt for a dead target. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ContinuityBackoff { - /// Mint now, carrying this streak into the attempt. - Due { streak: u32 }, - /// Start the wait: write the anchor and mint nothing this pass. - Begin, - /// The wait has not run out. - Waiting, -} - -/// The entries of a sliding log still inside `window` at `now`. An entry from -/// the future means the clock stepped back; it counts as `now`, so a backwards -/// step holds a meter for at most one window however far the clock moved. -fn live_entries(log: &[u64], now: u64, window: u64) -> Vec { - log.iter() - .map(|&at| at.min(now)) - .filter(|&at| now - at < window) - .collect() -} - -/// Record `now`, keeping the newest `limit` entries: all a meter needs to -/// decide "at most `limit` in any window". -fn record_entry(log: &mut Vec, now: u64, limit: usize) { - clamp_to(log, now); - log.push(now); - if log.len() > limit { - log.drain(..log.len() - limit); - } -} - -/// Make a stepped-back clock's future entries the present. `live_entries` -/// only reads them that way, so a log that is never written would keep -/// looking recent until the clock caught up; writing the clamp is what makes -/// a backwards step cost one window and no more. -fn clamp_to(log: &mut [u64], now: u64) { - for at in log.iter_mut() { - *at = (*at).min(now); - } -} - -impl TargetSupervision { - fn new(target: &str) -> Self { - Self { - schema_version: TARGET_SUPERVISION_SCHEMA.into(), - target: target.into(), - route_actuations: Vec::new(), - continuity_restarts: Vec::new(), - continuity_deferred_until: None, - continuity_deferral_reason: None, - route_survivals: Vec::new(), - continuity_attempt_streak: 0, - continuity_backoff_from_unix_millis: None, - continuity_rematerialized_at_unix_millis: None, - } - } - - /// Clamp every log to `now`. Callers that decide from a log write the - /// result back when it changed. - fn settle(&mut self, now: u64) { - clamp_to(&mut self.route_actuations, now); - clamp_to(&mut self.continuity_restarts, now); - clamp_to(&mut self.route_survivals, now); - } - - fn route_used(&self, now: u64) -> usize { - live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() - } - - /// When the oldest counted actuation leaves the window, if the ceiling is reached. - fn route_reopens_at(&self, now: u64) -> Option { - let live = live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS); - (live.len() >= ROUTE_ACTUATION_CEILING) - .then(|| live[live.len() - ROUTE_ACTUATION_CEILING] + ROUTE_ACTUATION_WINDOW_MILLIS) - } - - /// Charge one route actuation. A `Forward` change is counted against the - /// ceiling and refused at it; a `Survival` is only recorded, never - /// counted and never refused. - fn charge_route(&mut self, now: u64, kind: RouteActuation) -> Result<(), RouteActuationRefused> { - self.settle(now); - match kind { - RouteActuation::Forward => { - if let Some(reopens_at) = self.route_reopens_at(now) { - return Err(RouteActuationRefused { - target: self.target.clone(), - used: self.route_used(now), - reopens_at_unix_millis: reopens_at, - }); - } - record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING); - } - RouteActuation::Survival => { - record_entry(&mut self.route_survivals, now, ROUTE_SURVIVAL_RECORD); - } - } - Ok(()) - } - - fn survivals_in_window(&self, now: u64) -> usize { - live_entries(&self.route_survivals, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() - } - - fn restarts_used(&self, now: u64) -> usize { - live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len() - } - - /// The paging threshold of the ContinuityExhausted incident; it gates no actuation. - fn restarts_page(&self, now: u64) -> bool { - self.restarts_used(now) >= CONTINUITY_PAGING_RESTARTS - } - - /// Whether continuity must still wait out the deferral after a projection - /// that would not demote. - fn continuity_deferred(&self, now: u64) -> bool { - self.continuity_deferred_until - .is_some_and(|due| is_waiting(now, due, CONTINUITY_DEFERRAL_MILLIS)) - } - - /// When the wait for the next charged attempt ends, from the anchor and - /// the streak; the one spelling `continuity_backoff` and `status` share. - /// An anchor older than the admitted generation decides nothing. - fn backoff_due(&self, admitted_at: u64) -> Option { - let from = self - .continuity_backoff_from_unix_millis - .filter(|&from| from >= admitted_at)?; - let wait = self.continuity_attempt_streak.checked_sub(1).map_or(0, |steps| { - backoff_wait( - CONTINUITY_RESTART_BACKOFF_MILLIS, - steps, - CONTINUITY_BACKOFF_CEILING_MILLIS, - ) - }); - Some(from.saturating_add(wait)) - } - - /// What to do about the next attempt for the dead admitted generation - /// admitted at `admitted_at`. The streak resets only after a sustained - /// run: the generation has lived the sustained time and no attempt, charged - /// or the one re-materialization, has been minted since it was admitted. - /// A standing anchor belongs to a death the supervision pass has seen - /// through; an observed-running pass clears it, so it never hides a reset. - fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { - let streak = self.continuity_attempt_streak; - if let Some(due) = self.backoff_due(admitted_at) { - return if streak != 0 - && is_waiting(now, due, CONTINUITY_BACKOFF_CEILING_MILLIS) - { - ContinuityBackoff::Waiting - } else { - ContinuityBackoff::Due { streak } - }; - } - let sustained = self - .continuity_restarts - .last() - .into_iter() - .chain(&self.continuity_rematerialized_at_unix_millis) - .all(|&last| admitted_at >= last) - && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; - let streak = if sustained { 0 } else { streak }; - if streak == 0 { - ContinuityBackoff::Due { streak: 0 } - } else { - ContinuityBackoff::Begin - } - } - - /// One charged attempt scheduled at `streak`. It ends the wait and any - /// deferral: the projection was prepared. - fn record_restart(&mut self, now: u64, streak: u32) { - record_entry(&mut self.continuity_restarts, now, CONTINUITY_PAGING_RESTARTS); - self.continuity_attempt_streak = streak.saturating_add(1); - self.continuity_backoff_from_unix_millis = None; - self.continuity_deferred_until = None; - self.continuity_deferral_reason = None; - } - - /// The target was found dead and free: the wait for its next attempt runs from now. - fn begin_backoff(&mut self, now: u64) { - self.continuity_backoff_from_unix_millis = Some(now); - } - - /// The admitted workload was seen running: the death the anchor waited - /// out is over, and the next death waits from its own discovery. - fn end_backoff(&mut self) { - self.continuity_backoff_from_unix_millis = None; - } - - /// The one uncharged re-materialization scheduled. The log and the streak - /// are left alone. - fn record_rematerialization(&mut self, now: u64) { - self.continuity_rematerialized_at_unix_millis = Some(now); - self.continuity_backoff_from_unix_millis = None; - self.continuity_deferred_until = None; - self.continuity_deferral_reason = None; - } - - /// Whether the generation admitted at `admitted_at` has had its free re-materialization. - fn rematerialization_spent(&self, admitted_at: u64) -> bool { - self.continuity_rematerialized_at_unix_millis - .is_some_and(|at| at >= admitted_at) - } - - fn defer_continuity(&mut self, now: u64, reason: &str) { - self.continuity_deferred_until = Some(now.saturating_add(CONTINUITY_DEFERRAL_MILLIS)); - self.continuity_deferral_reason = Some(reason.to_owned()); - } - - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == TARGET_SUPERVISION_SCHEMA, - "target supervision schema is unsupported" - ); - require_id(&self.target, "supervised target")?; - for (log, limit, what) in [ - (&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"), - (&self.continuity_restarts, CONTINUITY_PAGING_RESTARTS, "continuity restart"), - (&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"), - ] { - ensure!(log.len() <= limit, "{what} log exceeds its bound"); - ensure!( - log.iter().all(|&at| at > 0) && log.windows(2).all(|pair| pair[0] <= pair[1]), - "{what} log is not an ascending list of times" - ); - } - ensure!( - self.continuity_backoff_from_unix_millis != Some(0) - && self.continuity_rematerialized_at_unix_millis != Some(0), - "continuity backoff anchor or re-materialization has no time" - ); - ensure!( - self.continuity_deferred_until.is_some() == self.continuity_deferral_reason.is_some(), - "continuity deferral has a time or a reason but not both" - ); - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "outcome", rename_all = "kebab-case", deny_unknown_fields)] -enum TransactionCompletion { - Admitted { generation_id: String }, - FailedBeforeFencing { error: String }, - FailedAfterFencing { - error: String, - #[serde(default)] - recovery: TerminalRecovery, - }, -} - -use cleanup_evidence::CleanupEvidence; - -/// Where the legacy marker lives, so that only the lift can make one. -/// -/// `LegacyLift` has a private field: outside this module no expression can -/// construct it, so no other code can write `CleanupEvidence::Legacy(..)`. -/// Decoding a stored record is the one other way in, and it stays here too. -/// `transaction_envelope` refuses a record carrying the marker; the schema -/// migration, which persists what the lift produced, writes through -/// `migrate_transaction_record`, and a fault note on an existing record through -/// `last_error_envelope`; both are defined here beside the only constructor, -/// and the encoder they share is private to this module. -mod cleanup_evidence { - use super::*; - - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub(super) struct LegacyLift(()); - - const LEGACY_MARKER: &str = "legacy-skipped-before-b1"; - - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub(super) enum CleanupEvidence { - Pending, - Skipped, - Complete, - /// Set only by the legacy lift, on a terminal continuity abort written - /// before the single resolution rule: it issued an activation and - /// recorded `Skipped`. Boot reconciliation, not the record, owes that - /// residue's demotion. The marker is what lets validation accept the - /// old shape without reopening a finished transaction. - Legacy(LegacyLift), - } - - impl CleanupEvidence { - pub(super) fn is_complete(self) -> bool { - !matches!(self, Self::Pending) - } - - pub(super) fn is_legacy_marker(self) -> bool { - matches!(self, Self::Legacy(_)) - } - } - - impl Serialize for CleanupEvidence { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_str(match self { - Self::Pending => "pending", - Self::Skipped => "skipped", - Self::Complete => "complete", - Self::Legacy(_) => LEGACY_MARKER, - }) - } - } - - impl<'de> Deserialize<'de> for CleanupEvidence { - fn deserialize>(deserializer: D) -> Result { - let text = String::deserialize(deserializer)?; - match text.as_str() { - "pending" => Ok(Self::Pending), - "skipped" => Ok(Self::Skipped), - "complete" => Ok(Self::Complete), - LEGACY_MARKER => Ok(Self::Legacy(LegacyLift(()))), - other => Err(serde::de::Error::unknown_variant( - other, - &["pending", "skipped", "complete", LEGACY_MARKER], - )), - } - } - } - - impl DeploymentTransaction { - pub(super) fn carries_legacy_marker(&self) -> bool { - self.pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - .into_iter() - .chain( - self.post_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation), - ) - .any(CleanupEvidence::is_legacy_marker) - } - } - - /// The schema migration's one entry point: read a stored v2, v3 or v4 - /// transaction (lifting older ones) and return its key and current-shape - /// envelope. The record may carry the marker the lift just set. - pub(super) fn migrate_transaction_record( - envelope: &CultCacheEnvelope, - ) -> Result<(String, CultCacheEnvelope)> { - let value = read_transaction_record(envelope)?; - let next = encode_transaction(&value, value.updated_at_unix_millis)?; - Ok((value.transaction_id, next)) - } - - /// The envelope for an existing record with only its `last_error` set. The - /// record may already carry the marker; this cannot add one. - pub(super) fn last_error_envelope( - value: &DeploymentTransaction, - detail: &str, - ) -> Result { - let mut next = value.clone(); - next.last_error = Some(detail.to_string()); - encode_transaction(&next, next.updated_at_unix_millis) - } - -/// A continuity abort written before the single resolution rule recorded no -/// projection cleanup although its transaction had issued an activation, and -/// so left that activation standing. -/// -/// Nothing is reopened. A record already terminal keeps its terminal phase and -/// completion and is marked the legacy marker, the one shape validation -/// accepts for a terminal abort that owes nothing yet issued an activation; -/// boot reconciliation, the single owner of that residue, demotes it by exact -/// activation. A record still in flight lifts to `Pending`: it already holds -/// its target's authority (a live record claims it before and after the lift), -/// so the lift adds no claimant, and its own abort demotes the activation. -/// Only the control store's read applies this: history describes what happened -/// and is lifted unchanged. -pub(super) fn owe_legacy_continuity_projection(transaction: &mut DeploymentTransaction) { - if transaction.command_kind != CommandKind::Continuity || transaction.activation.is_none() { - return; - } - let terminal = transaction.completion.is_some(); - if let Some(abort) = transaction.pre_fencing_abort.as_mut() - && abort.topology_reconciliation == CleanupEvidence::Skipped - { - abort.topology_reconciliation = if terminal { - CleanupEvidence::Legacy(LegacyLift(())) - } else { - CleanupEvidence::Pending - }; - } - if let Some(abort) = transaction.post_fencing_abort.as_mut() - && abort.topology_reconciliation == CleanupEvidence::Skipped - && !terminal - { - abort.topology_reconciliation = CleanupEvidence::Pending; - } -} - -} - -/// Rollback of a transaction that has already fenced the incumbent. -/// -/// Fencing stops the admitted incarnation and revokes its write lease, so a -/// candidate that then dies for good leaves the target with nothing serving it. -/// Retrying forever cannot fix that -- a transient unit with `Restart=no` will -/// not come back -- and it holds the target, leaving every later command queued -/// behind a transaction that can never finish. -/// -/// So the candidate's own artifacts are withdrawn in the order that never -/// leaves two writers: route first, then the write lease, then the process, -/// then the projection. Reconciling the projection restores the incumbent's -/// admitted Expected where there is one, which is what lets continuity bring -/// the incumbent back; where there is none, the failed Expected is withdrawn. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PostFencingAbort { - error: String, - route_restoration: CleanupEvidence, - lease_withdrawal: CleanupEvidence, - candidate_cleanup: CleanupEvidence, - topology_reconciliation: CleanupEvidence, - source_cleanup: CleanupEvidence, -} - -impl PostFencingAbort { - fn is_complete(&self) -> bool { - self.route_restoration.is_complete() - && self.lease_withdrawal.is_complete() - && self.candidate_cleanup.is_complete() - && self.topology_reconciliation.is_complete() - && self.source_cleanup.is_complete() - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PreFencingAbort { - error: String, - candidate_cleanup: CleanupEvidence, - topology_reconciliation: CleanupEvidence, - source_cleanup: CleanupEvidence, -} - -impl PreFencingAbort { - fn is_complete(&self) -> bool { - self.candidate_cleanup.is_complete() - && self.topology_reconciliation.is_complete() - && self.source_cleanup.is_complete() - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] -enum IncumbentCleanupEvidence { - SkippedNoIncumbent, - Pending { - generation_id: String, - workload: WorkloadObservation, - }, - Complete { - generation_id: String, - }, -} - -impl IncumbentCleanupEvidence { - fn is_complete(&self) -> bool { - matches!(self, Self::SkippedNoIncumbent | Self::Complete { .. }) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -enum SourceCleanupEvidence { - SkippedContinuity, - Pending, - Complete, -} - -impl SourceCleanupEvidence { - fn is_complete(self) -> bool { - !matches!(self, Self::Pending) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PostCommitCleanup { - incumbent: IncumbentCleanupEvidence, - source: SourceCleanupEvidence, -} - -impl PostCommitCleanup { - fn is_complete(&self) -> bool { - self.incumbent.is_complete() && self.source.is_complete() - } -} - -/// The sole owner of an in-flight deployment decision. Every actuator result -/// is durable here before a later phase is entered. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v4" -)] -struct DeploymentTransaction { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - transaction_id: String, - #[cultcache(key = 2)] - command_id: String, - #[cultcache(key = 3)] - command_kind: CommandKind, - #[cultcache(key = 4)] - target: String, - #[cultcache(key = 5)] - ordinal: u32, - #[cultcache(key = 6)] - phase: DeploymentPhase, - #[cultcache(key = 7)] - created_at_unix_millis: u64, - #[cultcache(key = 8)] - updated_at_unix_millis: u64, - #[cultcache(key = 9)] - incumbent_generation_id: Option, - #[cultcache(key = 10)] - plan: Option, - #[cultcache(key = 11)] - frozen_source: Option, - #[cultcache(key = 12)] - sealed_release: Option, - #[cultcache(key = 13)] - installed_release: Option, - #[cultcache(key = 14)] - expected: Option, - #[cultcache(key = 15)] - expected_publication_sha256: Option, - #[cultcache(key = 16)] - deployment_authorization: Option, - #[cultcache(key = 17)] - lifecycle_authorized_at_unix_millis: Option, - #[cultcache(key = 18)] - activation: Option, - #[cultcache(key = 19)] - workload: Option, - #[cultcache(key = 20)] - activation_publication_sha256: Option, - #[cultcache(key = 21)] - latest_odin_observation: Option, - #[cultcache(key = 22)] - warming: Option, - #[cultcache(key = 23)] - route_preflight: Option, - #[cultcache(key = 24)] - isolation: Option, - #[cultcache(key = 25)] - fencing: Option, - #[cultcache(key = 26)] - leasing: Option, - #[cultcache(key = 27)] - ready: Option, - #[cultcache(key = 28)] - routing: Option, - #[cultcache(key = 29)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 30)] - last_error: Option, - #[cultcache(key = 31)] - completion: Option, - #[cultcache(key = 32)] - pre_fencing_abort: Option, - #[cultcache(key = 33)] - post_commit_cleanup: Option, - // because every transaction already durable predates this field: - // an absent key 34 decodes as None, which is exactly "this transaction has - // no post-fencing abort". Without it the daemon refuses to read its own - // control store and crashloops. - #[cultcache(key = 34, default)] - post_fencing_abort: Option, - /// Set by `transition` on entry to each post-fencing phase. - #[cultcache(key = 35)] - phase_deadline: Option, - #[cultcache(key = 36)] - lease_adoption: Option, -} - -impl DeploymentTransaction { - /// The one writer of `phase`. It stamps the update time and the phase - /// deadline together, so a record outside Fencing..=Committing carries no - /// deadline by construction. No other code assigns `phase`. - fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { - self.phase = phase; - self.updated_at_unix_millis = now; - self.phase_deadline = self - .plan - .as_ref() - .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); - } - - /// A live transaction past the fence that is not already being aborted: - /// the only kind whose phase a deadline, or an operator's expiry, ends. - fn resolves_by_deadline(&self) -> bool { - self.completion.is_none() - && self.pre_fencing_abort.is_none() - && self.post_fencing_abort.is_none() - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) - } - - /// What an ended post-fencing phase resolves to (Q3). A candidate that was - /// never issued a write lease cannot have written, so the transaction - /// aborts to the incumbent through the post-fencing path, stateless or not. - /// A continuity always aborts: its candidate is the admitted release, and - /// supervision restarts that release under the continuity backoff. A - /// deployment whose candidate was issued a lease may have written under a - /// new state contract, and restoring the incumbent over that is not - /// Idunn's call: it is left for the operator. - fn phase_end(&self) -> PhaseEnd { - let leasing = self.leasing.as_ref(); - let granted = leasing.and_then(LeasingEvidence::lease).is_some(); - let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); - match self.command_kind { - CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, - _ => PhaseEnd::Abort { - lease_not_adopted: granted && self.lease_adoption.is_none(), - }, - } - } - - /// The recovery a completed post-fencing abort names. A continuity whose - /// candidate adopted its lease restarts the admitted release; everything - /// else restores the incumbent. - fn terminal_recovery(&self) -> TerminalRecovery { - if self.command_kind == CommandKind::Continuity && self.lease_adoption.is_some() { - TerminalRecovery::RestartAdmitted - } else { - TerminalRecovery::RestoreIncumbent - } - } - - /// Whether this transaction's binding declares stop-then-start. A - /// transaction without a plan yet cannot, so it reads as false. - fn rollout_stops_incumbent_first(&self) -> bool { - self.plan - .as_ref() - .and_then(|plan| plan.parsed_inputs().ok()) - .is_some_and(|(_, binding)| binding.rollout.strategy == RolloutStrategy::StopThenStart) - } - - fn new( - command: &DeploymentCommand, - target: String, - ordinal: u32, - incumbent: Option<&AdmittedGeneration>, - now: u64, - ) -> Result { - let transaction = Self { - schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), - transaction_id: format!("tx-{}", Uuid::new_v4()), - command_id: command.command_id.clone(), - command_kind: command.kind, - target, - ordinal, - phase: DeploymentPhase::Sealing, - created_at_unix_millis: now, - updated_at_unix_millis: now, - incumbent_generation_id: incumbent.map(|value| value.generation_id.clone()), - plan: None, - frozen_source: None, - sealed_release: None, - installed_release: None, - expected: None, - expected_publication_sha256: None, - deployment_authorization: None, - lifecycle_authorized_at_unix_millis: None, - activation: None, - workload: None, - activation_publication_sha256: None, - latest_odin_observation: None, - warming: None, - route_preflight: None, - isolation: None, - fencing: None, - leasing: None, - ready: None, - routing: None, - odin_publisher_sequence_cursor: incumbent - .map_or(0, |value| value.odin_publisher_sequence_cursor), - last_error: None, - completion: None, - pre_fencing_abort: None, - post_fencing_abort: None, - post_commit_cleanup: None, - phase_deadline: None, - lease_adoption: None, - }; - transaction.validate()?; - Ok(transaction) - } - - fn from_continuity( - command: &DeploymentCommand, - incumbent: &AdmittedGeneration, - now: u64, - ) -> Result { - let mut transaction = - Self::new(command, incumbent.target.clone(), 0, Some(incumbent), now)?; - transaction.plan = Some(incumbent.plan.clone()); - transaction.sealed_release = Some(incumbent.sealed_release.clone()); - transaction.installed_release = Some(incumbent.installed_release.clone()); - transaction.expected = Some(incumbent.expected.clone()); - transaction.validate()?; - Ok(transaction) - } - - /// Whether an abort owes the projection a reconciliation. A deployment - /// published its own Expected key. A continuity shares the admitted key, - /// whose Expected is never withdrawn, so it owes only the demotion of the - /// activation it issued. - fn abort_topology_reconciliation(&self) -> CleanupEvidence { - let owes = match self.command_kind { - CommandKind::Deploy => self.expected_publication_sha256.is_some(), - CommandKind::Continuity => self.activation.is_some(), - }; - if owes { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - } - } - - /// Whether an abort owes the candidate's durable cleanup: it does once an - /// activation was issued or a workload was observed, either of which leaves - /// something behind. - fn candidate_cleanup_owed(&self) -> CleanupEvidence { - if self.activation.is_some() || self.workload.is_some() { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - } - } - - /// Whether an abort's recorded candidate cleanup is what - /// `candidate_cleanup_owed` says it must be: owed cleanup is pending or - /// done, unowed cleanup is skipped. - fn candidate_cleanup_recorded(&self, recorded: CleanupEvidence) -> bool { - matches!( - (self.candidate_cleanup_owed(), recorded), - ( - CleanupEvidence::Pending, - CleanupEvidence::Pending | CleanupEvidence::Complete - ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) - ) - } - - fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { - let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; - let detail = truncate(&format!("{error:#}"), 2048); - transaction.enter_phase(DeploymentPhase::Complete, now); - transaction.last_error = Some(detail.clone()); - transaction.pre_fencing_abort = Some(PreFencingAbort { - error: detail.clone(), - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Skipped, - }); - transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { error: detail }); - transaction.validate()?; - Ok(transaction) - } - - fn is_terminal(&self) -> bool { - if self.phase != DeploymentPhase::Complete { - return false; - } - match &self.completion { - Some(TransactionCompletion::FailedBeforeFencing { .. }) => self - .pre_fencing_abort - .as_ref() - .is_some_and(PreFencingAbort::is_complete), - Some(TransactionCompletion::FailedAfterFencing { .. }) => self - .post_fencing_abort - .as_ref() - .is_some_and(PostFencingAbort::is_complete), - Some(TransactionCompletion::Admitted { .. }) => self - .post_commit_cleanup - .as_ref() - .is_some_and(PostCommitCleanup::is_complete), - None => false, - } - } - - /// Complete transactions retain retryable cleanup work, but admission has - /// already transferred current-incarnation authority to AdmittedGeneration. - fn owns_target_authority(&self) -> bool { - self.phase != DeploymentPhase::Complete - } - - /// Admission has moved to the new generation at Complete, but an exact - /// draining incumbent still reserves its process and candidate endpoint. - /// No later mutation for the same target may overlap that cleanup. - fn blocks_new_target_mutation(&self) -> bool { - self.owns_target_authority() - || self - .post_commit_cleanup - .as_ref() - .is_some_and(|cleanup| !cleanup.is_complete()) - } - - /// Where what this transaction has collected disagrees with what its own - /// Expected requires. There is nothing to disagree with before Expected. - fn readiness_disagreement(&self) -> Option { - let expected = self.expected.as_ref()?; - ReadinessClass::confirmed( - expected, - self.warming - .iter() - .map(WarmingEvidence::voucher) - .chain(self.ready.iter().map(ReadinessEvidence::voucher)), - ) - .err() - } - - /// The disagreement that holds this transaction: evidence collected under - /// another class is held and reported, not advanced and not repaired, since - /// no step can make it the right kind and running one anyway only fails it - /// every tick. What is finished, or being finished (a completion, or an - /// abort under way), never reads the class, so it is left to run. - fn held_disagreement(&self) -> Option { - let finishing = self.completion.is_some() - || self.pre_fencing_abort.is_some() - || self.post_fencing_abort.is_some(); - self.readiness_disagreement().filter(|_| !finishing) - } - - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, - "deployment transaction schema is unsupported" - ); - require_id(&self.transaction_id, "deployment transaction id")?; - require_id(&self.command_id, "transaction command id")?; - require_id(&self.target, "transaction target")?; - ensure!( - self.created_at_unix_millis > 0 && self.updated_at_unix_millis > 0, - "deployment transaction timestamps are invalid" - ); - if let Some(generation) = &self.incumbent_generation_id { - require_id(generation, "incumbent generation id")?; - } - if let Some(plan) = &self.plan { - plan.validate()?; - ensure!( - plan.parsed_inputs()?.1.target == self.target, - "transaction plan belongs to another target" - ); - } - if let (Some(plan), Some(frozen)) = (&self.plan, &self.frozen_source) { - frozen.validate_against(plan)?; - } - if let (Some(plan), Some(release)) = (&self.plan, &self.sealed_release) { - release.validate_against(plan)?; - } - if let Some(expected) = &self.expected { - expected.validate()?; - ensure!(expected.target == self.target, "Expected target differs"); - if let Some(plan) = &self.plan { - ensure!(expected.plan_id == plan.plan_id, "Expected plan differs"); - } - if let Some(release) = &self.sealed_release { - ensure!( - expected.sealed_release_id == release.sealed_release_id, - "Expected release differs" - ); - } - if let Some(digest) = &self.expected_publication_sha256 { - ensure!( - digest == &expected.canonical_sha256()?, - "Expected publication receipt differs" - ); - } - } - if let (Some(expected), Some(activation)) = (&self.expected, &self.activation) { - activation.validate()?; - ensure!( - activation.expected_projection_sha256 == expected.canonical_sha256()?, - "activation belongs to another Expected projection" - ); - if let Some(workload) = &self.workload { - ensure!( - workload.runtime_instance_id() == activation.runtime_instance_id, - "workload belongs to another activation" - ); - } - if let Some(digest) = &self.activation_publication_sha256 { - ensure!( - digest == &activation.canonical_sha256()?, - "activation publication receipt differs" - ); - } - } - ensure!( - self.workload.is_none() || self.activation.is_some(), - "workload observation exists without its prepared activation" - ); - let ready_odin = self.ready.as_ref().and_then(ReadinessEvidence::odin); - for evidence in [self.latest_odin_observation.as_ref(), ready_odin] - .into_iter() - .flatten() - { - evidence.validate_shape()?; - ensure!( - evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, - "topology evidence exceeds the transaction replay cursor" - ); - } - if let Some(ready) = &self.ready { - ready.validate_shape()?; - // A candidate-vouched receipt is only ever written by this code, so - // one on a target that is not route-proof is forged. The other - // direction -- an Odin receipt on a route-proof target -- is what a - // pre-B3 record looks like, and is a ReadinessDisagreement to hold, - // not a record to refuse: refusing it would refuse to boot. - if let (Voucher::Candidate, Some(expected)) = (ready.voucher(), &self.expected) { - ensure!( - ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof), - "route-proof Ready evidence for a target that is not route-proof" - ); - } - } - if let Some(warming) = &self.warming { - warming.validate_shape()?; - match warming { - WarmingEvidence::OdinTopology { evidence } => ensure!( - evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, - "Warming topology evidence exceeds the transaction replay cursor" - ), - WarmingEvidence::FirstOdinDirect { .. } => ensure!( - self.expected.as_ref().is_some_and(|expected| { - ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::OdinSelf) - }), - "direct Warming evidence is reserved for Odin observing itself" - ), - WarmingEvidence::RouteProofDirect { .. } => ensure!( - self.expected.as_ref().is_none_or(|expected| { - ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof) - }), - "direct route-proof Warming evidence for a target that is not route-proof" - ), - } - } - if let Some(leasing) = &self.leasing { - if let Some(expected) = &self.expected { - ensure!( - matches!( - (expected.write_lease_required, leasing), - (true, LeasingEvidence::Prepared { .. }) - | (true, LeasingEvidence::Granted { .. }) - | (false, LeasingEvidence::SkippedStateless) - ), - "lease evidence differs from the Expected state contract" - ); - } - let lease_and_sha256 = match leasing { - LeasingEvidence::SkippedStateless => None, - LeasingEvidence::Prepared { - lease, - lease_sha256, - } - | LeasingEvidence::Granted { - lease, - lease_sha256, - } => Some((lease, lease_sha256)), - }; - if let Some((lease, lease_sha256)) = lease_and_sha256 { - lease.validate()?; - ensure!( - lease.canonical_sha256()? == *lease_sha256, - "lease evidence digest differs" - ); - } - } - if let (Some(expected), Some(fencing)) = (&self.expected, &self.fencing) { - match fencing { - FencingEvidence::SkippedStateless => ensure!( - !expected.write_lease_required, - "stateful candidate claims stateless fencing" - ), - FencingEvidence::Revoked { - candidate_lease_path_verified_empty, - .. - } => ensure!( - *candidate_lease_path_verified_empty == expected.write_lease_required, - "candidate lease-path fencing differs from Expected" - ), - } - } - if let Some(error) = &self.last_error { - require_detail(error, "transaction error")?; - } - if let Some(deadline) = &self.phase_deadline { - ensure!( - deadline.phase == self.phase - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&deadline.phase) - && deadline.entered_at_unix_millis > 0 - && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, - "phase deadline does not describe the current post-fencing phase" - ); - } - if let Some(adoption) = &self.lease_adoption { - adoption.validate_shape()?; - ensure!( - self.leasing - .as_ref() - .is_some_and(|leasing| adoption.names(leasing)), - "lease adoption does not name the granted lease" - ); - } - if let Some(authorization) = &self.deployment_authorization { - authorization.validate_shape()?; - } - if let Some(abort) = &self.pre_fencing_abort { - require_detail(&abort.error, "pre-fencing abort error")?; - ensure!( - self.post_commit_cleanup.is_none(), - "aborted transaction also carries post-commit cleanup" - ); - ensure!( - self.candidate_cleanup_recorded(abort.candidate_cleanup), - "abort candidate cleanup differs from its prepared activation" - ); - ensure!( - matches!( - ( - self.abort_topology_reconciliation(), - abort.topology_reconciliation - ), - ( - CleanupEvidence::Pending, - CleanupEvidence::Pending | CleanupEvidence::Complete - ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) - ) || (self.command_kind == CommandKind::Continuity - && self.phase == DeploymentPhase::Complete - && self.completion.is_some() - && abort.topology_reconciliation.is_legacy_marker()), - "abort topology cleanup differs from what the transaction projected" - ); - ensure!( - matches!( - (self.command_kind, abort.source_cleanup), - ( - CommandKind::Deploy, - CleanupEvidence::Pending | CleanupEvidence::Complete - ) | (CommandKind::Continuity, CleanupEvidence::Skipped) - ) || (self.command_kind == CommandKind::Deploy - && abort.source_cleanup == CleanupEvidence::Skipped - && self.plan.is_none() - && self.frozen_source.is_none() - && self.expected.is_none() - && self.phase == DeploymentPhase::Complete), - "abort source cleanup differs from command work" - ); - ensure!( - self.phase < DeploymentPhase::Fencing || self.phase == DeploymentPhase::Complete, - "pre-fencing abort crossed the fencing boundary" - ); - } - - if let Some(abort) = &self.post_fencing_abort { - require_detail(&abort.error, "post-fencing abort error")?; - ensure!( - self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_none(), - "post-fencing abort collides with another terminal path" - ); - ensure!( - self.phase >= DeploymentPhase::Fencing, - "post-fencing abort exists before the fence" - ); - ensure!( - self.candidate_cleanup_recorded(abort.candidate_cleanup), - "abort candidate cleanup differs from its prepared activation" - ); - } - - let failed_after_fencing = matches!( - self.completion, - Some(TransactionCompletion::FailedAfterFencing { .. }) - ); - if failed_after_fencing { - ensure!( - self.phase == DeploymentPhase::Complete, - "terminal failure is not Complete" - ); - let abort = required( - &self.post_fencing_abort, - "terminal post-fence abort evidence", - )?; - let TransactionCompletion::FailedAfterFencing { error, recovery } = - self.completion.as_ref().unwrap() - else { - unreachable!() - }; - if let TerminalRecovery::OperatorRequired { reason } = recovery { - require_detail(reason, "operator-required recovery reason")?; - } - ensure!( - abort.is_complete() && abort.error == *error, - "terminal failure lacks complete matching abort evidence" - ); - return Ok(()); - } - - let failed = matches!( - self.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - ); - if failed { - ensure!( - self.phase == DeploymentPhase::Complete, - "terminal failure is not Complete" - ); - let abort = required(&self.pre_fencing_abort, "terminal abort evidence")?; - let TransactionCompletion::FailedBeforeFencing { error } = - self.completion.as_ref().unwrap() - else { - unreachable!() - }; - ensure!( - abort.is_complete() && abort.error == *error, - "terminal failure lacks complete matching abort evidence" - ); - } else { - ensure!( - !matches!( - self.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - ), - "non-failure branch carries failure completion" - ); - if self.phase >= DeploymentPhase::Starting { - ensure!( - self.plan.is_some() - && self.sealed_release.is_some() - && self.installed_release.is_some() - && self.expected.is_some(), - "Starting transaction lacks sealed release evidence" - ); - // Expected is published as Starting's first step, after the - // brake; from the activation onward it must be there. - ensure!( - self.expected_publication_sha256.is_some() - || (self.activation.is_none() && self.phase == DeploymentPhase::Starting), - "started transaction has not published its Expected" - ); - match self.command_kind { - CommandKind::Deploy => ensure!( - self.frozen_source.is_some() && self.deployment_authorization.is_some(), - "deployment entered Starting without frozen source and brake consumption" - ), - CommandKind::Continuity => ensure!( - self.lifecycle_authorized_at_unix_millis.is_some() - && self.deployment_authorization.is_none(), - "continuity entered Starting without its lifecycle gate" - ), - } - } - if self.phase >= DeploymentPhase::Warming { - ensure!( - self.activation.is_some() - && self.workload.is_some() - && self.activation_publication_sha256.is_some(), - "Warming transaction lacks observed activation" - ); - } - if self.phase >= DeploymentPhase::Fencing { - ensure!( - self.warming.is_some() && self.isolation.is_some(), - "Fencing transaction lacks warming or isolation evidence" - ); - ensure!( - self.expected - .as_ref() - .and_then(|expected| expected.route.as_ref()) - .is_some() - == self.route_preflight.is_some(), - "route preflight does not match routed Expected" - ); - if let Some(preflight) = &self.route_preflight { - preflight.validate()?; - } - } - if self.phase >= DeploymentPhase::Leasing { - ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence"); - } - if self.phase >= DeploymentPhase::AwaitingReady { - ensure!( - matches!( - self.leasing.as_ref(), - Some(LeasingEvidence::SkippedStateless) - | Some(LeasingEvidence::Granted { .. }) - ), - "AwaitingReady lacks finalized lease evidence" - ); - } - if self.phase >= DeploymentPhase::Routing { - ensure!(self.ready.is_some(), "Routing lacks Ready evidence"); - } - if self.phase >= DeploymentPhase::Committing { - ensure!(self.routing.is_some(), "Committing lacks routing evidence"); - } - if let Some(RoutingEvidence::Promoted { - observation, - promoted_at_unix_millis, - }) = &self.routing - { - observation.validate()?; - ensure!(*promoted_at_unix_millis > 0, "route promotion has no time"); - } - if self.phase == DeploymentPhase::Complete { - ensure!( - matches!( - self.completion, - Some(TransactionCompletion::Admitted { .. }) - ), - "successful Complete transaction lacks admission receipt" - ); - ensure!( - self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_some(), - "admitted transaction lacks exclusive post-commit cleanup evidence" - ); - let TransactionCompletion::Admitted { generation_id } = - self.completion.as_ref().unwrap() - else { - unreachable!() - }; - ensure!( - generation_id == &format!("generation-{}", self.transaction_id), - "admission completion names another transaction generation" - ); - } else { - ensure!( - self.completion.is_none() && self.post_commit_cleanup.is_none(), - "non-Complete transaction has completion or post-commit state" - ); - } - } - if let Some(cleanup) = &self.post_commit_cleanup { - ensure!( - self.phase == DeploymentPhase::Complete - && matches!( - self.completion, - Some(TransactionCompletion::Admitted { .. }) - ), - "post-commit cleanup exists without an admitted Complete transaction" - ); - match (&self.incumbent_generation_id, &cleanup.incumbent) { - (None, IncumbentCleanupEvidence::SkippedNoIncumbent) => {} - ( - Some(expected), - IncumbentCleanupEvidence::Pending { generation_id, .. } - | IncumbentCleanupEvidence::Complete { generation_id }, - ) if expected == generation_id => {} - _ => bail!("post-commit incumbent cleanup names another generation"), - } - ensure!( - matches!( - (self.command_kind, cleanup.source), - (CommandKind::Deploy, SourceCleanupEvidence::Pending) - | (CommandKind::Deploy, SourceCleanupEvidence::Complete) - | ( - CommandKind::Continuity, - SourceCleanupEvidence::SkippedContinuity - ) - ), - "post-commit source cleanup differs from command kind" - ); - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct AdmittedOdinAuthority { - signer_identity_id: String, - signer_public_key: Vec, -} - -impl AdmittedOdinAuthority { - fn from_anchor(anchor: &ServiceIdentityTrustAnchor) -> Result { - ensure!( - derive_service_identity_id::(&anchor.public_key)? - == anchor.identity_id, - "Odin topology anchor identity differs from its key" - ); - Ok(Self { - signer_identity_id: anchor.identity_id.clone(), - signer_public_key: anchor.public_key.clone(), - }) - } - - fn validate(&self) -> Result<()> { - require_id(&self.signer_identity_id, "admitted Odin signer")?; - ensure!( - derive_service_identity_id::(&self.signer_public_key)? - == self.signer_identity_id, - "admitted Odin signer identity differs from its key" - ); - Ok(()) - } -} - -/// The only current-incarnation owner. Unit state, route state, topology -/// projections, and CLI output are observations of this record, never peers. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v4" -)] -struct AdmittedGeneration { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - #[cultcache(key = 2)] - generation_id: String, - #[cultcache(key = 3)] - command_id: String, - #[cultcache(key = 4)] - transaction_id: String, - #[cultcache(key = 5)] - admitted_at_unix_millis: u64, - #[cultcache(key = 6)] - plan: CompiledDeploymentPlan, - #[cultcache(key = 7)] - sealed_release: SealedRelease, - #[cultcache(key = 8)] - installed_release: InstalledReleaseObservation, - #[cultcache(key = 9)] - expected: IdunnExpectedIncarnationRecord, - #[cultcache(key = 10)] - activation: IdunnRuntimeActivationRecord, - #[cultcache(key = 11)] - workload: WorkloadObservation, - #[cultcache(key = 12)] - leasing: LeasingEvidence, - /// The receipt that admitted this incarnation, tagged by proof class. - #[cultcache(key = 13)] - ready: ReadinessEvidence, - /// Present exactly when `ready` is Odin-correlated. - #[cultcache(key = 14)] - latest_odin_observation: Option, - #[cultcache(key = 15)] - routing: RoutingEvidence, - /// Present exactly when `ready` is Odin-correlated. - #[cultcache(key = 16)] - odin_authority: Option, - /// Meaningful only for an Odin-correlated generation. - #[cultcache(key = 17)] - odin_publisher_sequence_cursor: u64, - // Keys 18 (route repair start) and 20 (continuity backoff) are retired. - // The slots stay as gaps: the layout is positional, so a slot cannot be - // reused without a bump. - /// Present exactly when `routing` is Promoted. - #[cultcache(key = 19)] - route_supervision: Option, - /// Why this generation is held or down, for `status`. Cleared when the - /// generation is replaced. - #[cultcache(key = 21)] - last_error: Option, -} - -/// The Odin-side receipts of an Odin-correlated generation. Decisions that -/// consume Odin evidence read it through here, so a route-proof generation -/// cannot be mistaken for one. -struct OdinReceipts<'a> { - ready: &'a TopologyEvidence, - latest: &'a TopologyEvidence, -} - -impl AdmittedGeneration { - /// This generation's class, or where its evidence disagrees with it. The - /// class comes from its Expected, never from the tag on its Ready receipt. - fn readiness(&self) -> Result { - ReadinessClass::confirmed(&self.expected, [self.ready.voucher()]) - } - - fn odin_receipts(&self) -> Result> { - match ( - &self.ready, - &self.latest_odin_observation, - &self.odin_authority, - ) { - (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(_)) => { - Ok(OdinReceipts { - ready: evidence, - latest, - }) - } - _ => bail!("admitted generation of {} is not Odin-correlated", self.target), - } - } - - fn from_transaction( - transaction: &DeploymentTransaction, - odin_authority: Option, - now: u64, - ) -> Result { - ensure!( - transaction.phase == DeploymentPhase::Committing, - "only Committing can create an admitted generation" - ); - let ready = required(&transaction.ready, "Ready receipt")?.clone(); - let (latest_odin_observation, odin_authority) = match &ready { - ReadinessEvidence::OdinCorrelated { .. } => ( - Some( - required(&transaction.latest_odin_observation, "latest Odin receipt")? - .clone(), - ), - Some(required(&odin_authority, "Odin authority")?.clone()), - ), - ReadinessEvidence::RouteProof { .. } => (None, None), - }; - let routing = required(&transaction.routing, "route disposition")?.clone(); - let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. }) - .then(RouteSupervisionState::default); - let generation = Self { - schema_version: ADMITTED_GENERATION_SCHEMA.into(), - target: transaction.target.clone(), - generation_id: format!("generation-{}", transaction.transaction_id), - command_id: transaction.command_id.clone(), - transaction_id: transaction.transaction_id.clone(), - admitted_at_unix_millis: now, - plan: required(&transaction.plan, "transaction plan")?.clone(), - sealed_release: required(&transaction.sealed_release, "sealed release")?.clone(), - installed_release: required(&transaction.installed_release, "installed release")? - .clone(), - expected: required(&transaction.expected, "Expected projection")?.clone(), - activation: required(&transaction.activation, "activation")?.clone(), - workload: required(&transaction.workload, "workload")?.clone(), - leasing: required(&transaction.leasing, "lease disposition")?.clone(), - ready, - latest_odin_observation, - routing, - odin_authority, - odin_publisher_sequence_cursor: transaction.odin_publisher_sequence_cursor, - route_supervision, - last_error: None, - }; - generation.validate()?; - Ok(generation) - } - - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == ADMITTED_GENERATION_SCHEMA, - "admitted generation schema is unsupported" - ); - require_id(&self.target, "admitted target")?; - require_id(&self.generation_id, "admitted generation id")?; - require_id(&self.command_id, "admitted command id")?; - require_id(&self.transaction_id, "admitted transaction id")?; - ensure!(self.admitted_at_unix_millis > 0, "admission has no time"); - self.plan.validate()?; - self.sealed_release.validate_against(&self.plan)?; - self.expected.validate()?; - self.activation.validate()?; - self.ready.validate_shape()?; - match ( - &self.ready, - &self.latest_odin_observation, - &self.odin_authority, - ) { - (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(authority)) => { - latest.validate_shape()?; - authority.validate()?; - ensure!( - evidence.publisher_sequence <= self.odin_publisher_sequence_cursor - && latest.publisher_sequence == self.odin_publisher_sequence_cursor, - "admitted generation evidence does not describe one incarnation" - ); - } - (ReadinessEvidence::RouteProof { .. }, None, None) => ensure!( - ReadinessClass::of(&self.expected).is_ok_and(|class| class == ReadinessClass::RouteProof), - "route-proof Ready evidence for a target that is not route-proof" - ), - _ => bail!("admitted readiness evidence and Odin receipts disagree"), - } - ensure!( - self.target == self.expected.target - && self.expected.plan_id == self.plan.plan_id - && self.expected.sealed_release_id == self.sealed_release.sealed_release_id - && self.activation.expected_projection_sha256 - == self.expected.canonical_sha256()? - && self.activation.runtime_instance_id == self.workload.runtime_instance_id(), - "admitted generation evidence does not describe one incarnation" - ); - ensure!( - self.expected.write_lease_required == self.leasing.lease().is_some(), - "admitted write-lease disposition differs from Expected" - ); - ensure!( - self.expected.route.is_some() == self.routing.observation().is_some(), - "admitted route disposition differs from Expected" - ); - if let Some(route) = self.routing.observation() { - route.validate()?; - ensure!( - route.runtime_instance_id == self.activation.runtime_instance_id, - "admitted route observation belongs to another runtime instance" - ); - } - ensure!( - self.route_supervision.is_some() - == matches!(&self.routing, RoutingEvidence::Promoted { .. }), - "route supervision state exists exactly for a promoted route" - ); - if let Some(state) = &self.route_supervision { - state.validate()?; - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct RuntimeOptions { - state_store: PathBuf, - bindings_dir: PathBuf, - source_root: PathBuf, - staging_root: PathBuf, - topology_store: PathBuf, - /// Operator incidents, published world-readable beside the topology - /// projection. Not part of the control store: see `incident`. - incident_store: PathBuf, - odin_correlation_store: PathBuf, - odin_trust_anchor: PathBuf, - idunn_identity_store: PathBuf, - deployment_brake_operator_anchor: PathBuf, - source_identity: Option, - topology_maximum_age_millis: u64, - topology_maximum_future_skew_millis: u64, - poll_millis: u64, - /// Where host actuators dial in. `None` means no host-actuator workload - /// can be served; a binding that names one then fails at its first - /// driver call, not at startup, so an Idunn without hosts is unchanged. - host_actuator_bind: Option, - /// Where route drivers find nginx, systemctl, ufw and systemd-run, and - /// their scratch directory. Every route driver is built from this. - route_actuators: RouteActuators, -} - -impl Default for RuntimeOptions { - fn default() -> Self { - Self { - state_store: PathBuf::from("/var/lib/gamecult/idunn/control.cc"), - bindings_dir: PathBuf::from("/etc/gamecult/idunn/bindings"), - source_root: PathBuf::from("/var/lib/gamecult/idunn/sources"), - staging_root: PathBuf::from("/var/lib/gamecult/idunn/staging"), - topology_store: PathBuf::from("/var/lib/gamecult/idunn/topology.cc"), - incident_store: PathBuf::from("/var/lib/gamecult/idunn-projection/incidents.cc"), - odin_correlation_store: PathBuf::from( - "/var/lib/gamecult/odin/idunn-runtime-topology.cc", - ), - odin_trust_anchor: PathBuf::from("/etc/gamecult/idunn/odin-topology-anchor.cc"), - idunn_identity_store: PathBuf::from( - "/var/lib/gamecult/idunn/idunn-service-identity.cc", - ), - deployment_brake_operator_anchor: PathBuf::from( - "/etc/gamecult/idunn/deployment-brake-operator-anchor.cc", - ), - source_identity: None, - topology_maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, - topology_maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, - poll_millis: 500, - host_actuator_bind: None, - route_actuators: RouteActuators::default(), - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum Command { - Serve(RuntimeOptions), - Up { - selector: String, - requested_by: String, - state_store: PathBuf, - wait: bool, - timeout_seconds: u64, - }, - Status { - state_store: PathBuf, - incident_store: PathBuf, - command_id: Option, - }, - Cancel { - state_store: PathBuf, - command_id: String, - requested_by: String, - }, - Expire { - state_store: PathBuf, - command_id: String, - requested_by: String, - }, - Validate { - recipe: PathBuf, - binding: Option, - }, -} - -pub fn run(args: impl Iterator) -> Result<()> { - match parse(args)? { - Command::Serve(options) => serve(options), - Command::Up { - selector, - requested_by, - state_store, - wait, - timeout_seconds, - } => submit( - &state_store, - &selector, - &requested_by, - wait, - timeout_seconds, - ), - Command::Status { - state_store, - incident_store, - command_id, - } => status( - &mut std::io::stdout().lock(), - &state_store, - &incident_store, - command_id.as_deref(), - ), - Command::Cancel { - state_store, - command_id, - requested_by, - } => cancel(&state_store, &command_id, &requested_by), - Command::Expire { - state_store, - command_id, - requested_by, - } => expire(&state_store, &command_id, &requested_by), - Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), - } -} - -/// Offline admission check for authoring. Reads a recipe, optionally reads a -/// binding, and runs the same `admit` the daemon runs at load. It opens no -/// store, contacts no host, and actuates nothing, so it is safe to run against -/// a candidate binding before installing it. -fn validate(recipe_path: &Path, binding_path: Option<&Path>) -> Result<()> { - let recipe_text = fs::read_to_string(recipe_path) - .with_context(|| format!("reading recipe {}", recipe_path.display()))?; - let recipe = crate::deployment::TargetDeclaration::parse(&recipe_text) - .with_context(|| format!("validating recipe {}", recipe_path.display()))?; - println!( - "recipe ok: {} declares target {}", - recipe_path.display(), - recipe.target - ); - - let Some(binding_path) = binding_path else { - return Ok(()); - }; - let binding_text = fs::read_to_string(binding_path) - .with_context(|| format!("reading operator binding {}", binding_path.display()))?; - let binding = OperatorBinding::parse(&binding_text) - .with_context(|| format!("validating operator binding {}", binding_path.display()))?; - ensure!( - binding.target == recipe.target, - "operator binding targets {} but the recipe declares {}", - binding.target, - recipe.target - ); - binding - .admit(&recipe) - .with_context(|| format!("admitting {} against its recipe", binding.target))?; - println!( - "binding ok: {} admits target {}", - binding_path.display(), - binding.target - ); - Ok(()) -} - -fn parse(args: impl Iterator) -> Result { - let mut args = args.peekable(); - let command = args.next().ok_or_else(|| anyhow!(usage()))?; - match command.as_str() { - "serve" => parse_serve(args), - "up" => parse_up(args), - "status" => parse_status(args), - "cancel" => parse_cancel(args), - "expire" => parse_expire(args), - "validate" => parse_validate(args), - "--help" | "-h" | "help" => bail!(usage()), - _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), - } -} - -fn parse_serve(mut args: impl Iterator) -> Result { - let mut options = RuntimeOptions::default(); - let mut source_uid = None; - let mut source_gid = None; - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => options.state_store = path_value(&mut args, &argument)?, - "--bindings-dir" => options.bindings_dir = path_value(&mut args, &argument)?, - "--source-root" => options.source_root = path_value(&mut args, &argument)?, - "--staging-root" => options.staging_root = path_value(&mut args, &argument)?, - "--topology-store" => options.topology_store = path_value(&mut args, &argument)?, - "--incident-store" => options.incident_store = path_value(&mut args, &argument)?, - "--odin-correlation-store" => { - options.odin_correlation_store = path_value(&mut args, &argument)? - } - "--odin-trust-anchor" => options.odin_trust_anchor = path_value(&mut args, &argument)?, - "--idunn-identity-store" => { - options.idunn_identity_store = path_value(&mut args, &argument)? - } - "--deployment-brake-operator-anchor" => { - options.deployment_brake_operator_anchor = path_value(&mut args, &argument)? - } - "--topology-maximum-age-millis" => { - options.topology_maximum_age_millis = u64_value(&mut args, &argument)? - } - "--topology-maximum-future-skew-millis" => { - options.topology_maximum_future_skew_millis = u64_value(&mut args, &argument)? - } - "--source-uid" => source_uid = Some(u32_value(&mut args, &argument)?), - "--source-gid" => source_gid = Some(u32_value(&mut args, &argument)?), - "--poll-millis" => options.poll_millis = u64_value(&mut args, &argument)?, - "--host-actuator-bind" => { - let value = string_value(&mut args, &argument)?; - options.host_actuator_bind = Some( - value - .parse() - .with_context(|| format!("{argument} is not a socket address"))?, - ); - } - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn serve option {argument:?}"), - } - } - ensure!( - options.poll_millis > 0 && options.topology_maximum_age_millis > 0, - "poll and topology maximum age must be positive" - ); - options.source_identity = match (source_uid, source_gid) { - (Some(uid), Some(gid)) => { - ensure!( - uid > 0 && gid > 0, - "source UID and GID must be unprivileged" - ); - Some(ProcessIdentity { uid, gid }) - } - (None, None) => None, - _ => bail!("--source-uid and --source-gid must be supplied together"), - }; - Ok(Command::Serve(options)) -} - -fn parse_up(mut args: impl Iterator) -> Result { - let selector = args - .next() - .ok_or_else(|| anyhow!("idunn up requires a service or profile selector"))?; - require_selector(&selector)?; - let mut state_store = RuntimeOptions::default().state_store; - let mut requested_by = env::var("SUDO_USER") - .or_else(|_| env::var("USER")) - .or_else(|_| env::var("USERNAME")) - .unwrap_or_else(|_| "operator".into()); - let mut wait = true; - let mut timeout_seconds = 1800; - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => state_store = path_value(&mut args, &argument)?, - "--requested-by" => requested_by = string_value(&mut args, &argument)?, - "--no-wait" => wait = false, - "--timeout-seconds" => timeout_seconds = u64_value(&mut args, &argument)?, - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn up option {argument:?}"), - } - } - ensure!(timeout_seconds > 0, "--timeout-seconds must be positive"); - require_value(&requested_by, "deployment requester")?; - Ok(Command::Up { - selector, - requested_by, - state_store, - wait, - timeout_seconds, - }) -} - -fn parse_status(mut args: impl Iterator) -> Result { - let defaults = RuntimeOptions::default(); - let mut state_store = defaults.state_store; - let mut incident_store = defaults.incident_store; - let mut command_id = None; - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => state_store = path_value(&mut args, &argument)?, - "--incident-store" => incident_store = path_value(&mut args, &argument)?, - "--command" => command_id = Some(string_value(&mut args, &argument)?), - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn status option {argument:?}"), - } - } - Ok(Command::Status { - state_store, - incident_store, - command_id, - }) -} - -fn parse_expire(mut args: impl Iterator) -> Result { - let command_id = args - .next() - .ok_or_else(|| anyhow!("idunn expire requires a command id"))?; - require_id(&command_id, "command id")?; - let mut state_store = RuntimeOptions::default().state_store; - let mut requested_by = env::var("SUDO_USER") - .or_else(|_| env::var("USER")) - .or_else(|_| env::var("USERNAME")) - .unwrap_or_else(|_| "operator".into()); - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => state_store = path_value(&mut args, &argument)?, - "--requested-by" => requested_by = string_value(&mut args, &argument)?, - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn expire option {argument:?}"), - } - } - require_value(&requested_by, "expiry requester")?; - Ok(Command::Expire { - state_store, - command_id, - requested_by, - }) -} - -/// Ask the daemon to end a live transaction's post-fencing phase now. This -/// writes a request and nothing else: the daemon's deadline resolver ends the -/// phase exactly as it ends one whose deadline has passed. -fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { - let snapshot = ControlSnapshot::read(store_path)?; - let mut live = snapshot - .transactions - .iter() - .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); - let current = live - .next() - .context("command has no live transaction to expire")?; - ensure!( - live.next().is_none(), - "command has multiple live transactions" - ); - ensure!( - current.value.resolves_by_deadline(), - "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" - ); - let now = now_millis()?; - let request = ExpiryRequest { - schema_version: EXPIRY_REQUEST_SCHEMA.into(), - command_id: command_id.to_owned(), - requested_by: requested_by.to_owned(), - requested_at_unix_millis: now, - }; - ensure!( - SingleFileMessagePackBackingStore::new(store_path) - .insert_entry_if_absent(expiry_request_envelope(&request, now)?)?, - "an expiry request for this command is already waiting" - ); - println!("{command_id} expiry requested; Idunn ends its phase as an expired deadline"); - Ok(()) -} - -fn parse_cancel(mut args: impl Iterator) -> Result { - let command_id = args - .next() - .ok_or_else(|| anyhow!("idunn cancel requires a deployment command id"))?; - require_id(&command_id, "deployment command id")?; - let mut state_store = RuntimeOptions::default().state_store; - let mut requested_by = env::var("SUDO_USER") - .or_else(|_| env::var("USER")) - .or_else(|_| env::var("USERNAME")) - .unwrap_or_else(|_| "operator".into()); - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => state_store = path_value(&mut args, &argument)?, - "--requested-by" => requested_by = string_value(&mut args, &argument)?, - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn cancel option {argument:?}"), - } - } - require_value(&requested_by, "cancellation requester")?; - Ok(Command::Cancel { - state_store, - command_id, - requested_by, - }) -} - -/// Withdraw queued work, a held deployment that has not fenced, or ask Idunn to -/// clean up a live stateless candidate. Stateful candidates past fencing cannot be cancelled because their prior -/// writer may already be stopped and its state boundary needs explicit repair. -fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { - let snapshot = ControlSnapshot::read(store_path)?; - let command = snapshot - .commands - .iter() - .find(|stored| stored.value.command_id == command_id) - .context("deployment command is unknown or already retired")?; - ensure!( - command.value.kind == CommandKind::Deploy, - "only deployment commands can be cancelled; continuity is Idunn's own" - ); - let mut live = snapshot - .transactions - .iter() - .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); - if let Some(current) = live.next() { - ensure!( - live.next().is_none(), - "deployment command has multiple live transactions" - ); - ensure!( - cancel_is_safe_for_live_stateless(¤t.value) - || cancel_is_safe_for_held_prefence(¤t.value), - "live deployment can only be cancelled before fencing when Idunn holds it, or after fencing when its candidate is stateless" - ); - let error = format!("cancelled by {requested_by}"); - let mut next = current.value.clone(); - if cancel_is_safe_for_held_prefence(&next) { - next.pre_fencing_abort = Some(pre_fencing_abort_intent(&next, &error)); - } else { - next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); - } - next.last_error = Some(error); - next.updated_at_unix_millis = now_millis()?; - replace_transaction(store_path, current, &next)?; - println!("{command_id} cancellation requested; Idunn is performing exact candidate cleanup"); - return Ok(()); - } - let now = now_millis()?; - let refusal = DeploymentTransaction::rejected( - &command.value, - anyhow!("cancelled by {requested_by} before freezing"), - now, - )?; - let envelope = transaction_envelope(&refusal, now)?; - ensure!( - SingleFileMessagePackBackingStore::new(store_path).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command_id.to_owned(), - current: Some(command.envelope.clone()), - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: refusal.transaction_id.clone(), - current: None, - }, - ], - std::slice::from_ref(&envelope), - )?, - "deployment command changed before it could be cancelled" - ); - archive_terminal_transaction(store_path, &envelope)?; - println!("{command_id} cancelled"); - Ok(()) -} - -/// A deployment Idunn holds before it fenced anything has touched nothing of -/// the incumbent, so an operator may withdraw it. Idunn never does so itself. -fn cancel_is_safe_for_held_prefence(transaction: &DeploymentTransaction) -> bool { - transaction.command_kind == CommandKind::Deploy - && transaction.phase < DeploymentPhase::Fencing - && transaction.held_disagreement().is_some() -} - -fn cancel_is_safe_for_live_stateless(transaction: &DeploymentTransaction) -> bool { - transaction.command_kind == CommandKind::Deploy - && transaction.phase >= DeploymentPhase::Fencing - && matches!( - transaction.fencing.as_ref(), - Some(FencingEvidence::SkippedStateless) - ) - && transaction.completion.is_none() - && transaction.pre_fencing_abort.is_none() - && transaction.post_fencing_abort.is_none() - && transaction.post_commit_cleanup.is_none() -} - -fn parse_validate(mut args: impl Iterator) -> Result { - let mut recipe = None; - let mut binding = None; - while let Some(argument) = args.next() { - match argument.as_str() { - "--recipe" => recipe = Some(path_value(&mut args, &argument)?), - "--binding" => binding = Some(path_value(&mut args, &argument)?), - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn validate option {argument:?}"), - } - } - let recipe = recipe.ok_or_else(|| anyhow!("validate requires --recipe"))?; - Ok(Command::Validate { recipe, binding }) -} - -#[derive(Clone)] -struct Stored { - envelope: CultCacheEnvelope, - value: T, -} - -#[derive(Default)] -struct ControlSnapshot { - commands: Vec>, - transactions: Vec>, - admitted: Vec>, - targets: Vec>, - expiry_requests: Vec>, -} - -impl ControlSnapshot { - fn read(path: &Path) -> Result { - if !path.exists() { - return Ok(Self::default()); - } - let mut snapshot = Self::default(); - for envelope in SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot() - .context("reading Idunn control snapshot")? - { - match envelope.r#type.as_str() { - DeploymentCommand::TYPE => { - ensure!( - envelope.schema_id.as_deref() == Some(DEPLOYMENT_COMMAND_SCHEMA), - "Idunn control store contains an unsupported command" - ); - let value: DeploymentCommand = decode_record(&envelope)?; - value.validate()?; - ensure!( - envelope.key == value.command_id, - "deployment command key differs from its identity" - ); - snapshot.commands.push(Stored { envelope, value }); - } - DeploymentTransaction::TYPE => { - let value = read_transaction_record(&envelope)?; - ensure!( - envelope.key == value.transaction_id, - "deployment transaction key differs from its identity" - ); - snapshot.transactions.push(Stored { envelope, value }); - } - AdmittedGeneration::TYPE => { - let value = read_generation_record(&envelope)?; - ensure!( - envelope.key == value.target, - "admitted generation key is not its target" - ); - snapshot.admitted.push(Stored { envelope, value }); - } - TargetSupervision::TYPE => { - let value = read_supervision_record(&envelope)?; - ensure!( - envelope.key == value.target, - "target supervision key is not its target" - ); - snapshot.targets.push(Stored { envelope, value }); - } - ExpiryRequest::TYPE => { - ensure!( - envelope.schema_id.as_deref() == Some(EXPIRY_REQUEST_SCHEMA), - "Idunn control store contains an unsupported expiry request" - ); - let value: ExpiryRequest = decode_record(&envelope)?; - value.validate()?; - ensure!( - envelope.key == value.command_id, - "expiry request key is not its command" - ); - snapshot.expiry_requests.push(Stored { envelope, value }); - } - _ => bail!("Idunn control store contains a foreign document"), - } - } - snapshot.validate_relations()?; - Ok(snapshot) - } - - fn validate_relations(&self) -> Result<()> { - let command_ids = self - .commands - .iter() - .map(|stored| stored.value.command_id.as_str()) - .collect::>(); - ensure!( - command_ids.len() == self.commands.len(), - "Idunn control store contains duplicate commands" - ); - let transaction_ids = self - .transactions - .iter() - .map(|stored| stored.value.transaction_id.as_str()) - .collect::>(); - ensure!( - transaction_ids.len() == self.transactions.len(), - "Idunn control store contains duplicate transactions" - ); - let admitted_targets = self - .admitted - .iter() - .map(|stored| stored.value.target.as_str()) - .collect::>(); - ensure!( - admitted_targets.len() == self.admitted.len(), - "Idunn control store contains duplicate current generations" - ); - ensure!( - self.targets - .iter() - .map(|stored| stored.value.target.as_str()) - .collect::>() - .len() - == self.targets.len(), - "Idunn control store contains duplicate target supervision records" - ); - for transaction in &self.transactions { - let command = self - .commands - .iter() - .find(|candidate| candidate.value.command_id == transaction.value.command_id) - .context("deployment transaction has no immutable command")?; - ensure!( - command.value.kind == transaction.value.command_kind, - "transaction kind differs from its immutable command" - ); - } - let mut live_targets = BTreeSet::new(); - for transaction in self - .transactions - .iter() - .filter(|stored| stored.value.owns_target_authority()) - { - ensure!( - live_targets.insert(transaction.value.target.as_str()), - "multiple transactions claim current-incarnation authority for one target" - ); - if let Some(fencing) = transaction - .value - .fencing - .as_ref() - .filter(|_| transaction.value.phase < DeploymentPhase::Complete) - { - let incumbent = self.admitted_for(&transaction.value.target); - let incumbent_matches = - match (&transaction.value.incumbent_generation_id, incumbent) { - (None, None) => true, - (Some(expected), Some(current)) => current.value.generation_id == *expected, - _ => false, - }; - ensure!( - incumbent_matches, - "fenced transaction incumbent differs from current admission" - ); - let incumbent_lease_sha256 = incumbent - .and_then(|stored| stored.value.leasing.lease()) - .map(IdunnProcessWriteLeaseRecord::canonical_sha256) - .transpose()?; - match fencing { - FencingEvidence::SkippedStateless => ensure!( - incumbent_lease_sha256.is_none(), - "stateless fencing omitted the admitted incumbent lease" - ), - FencingEvidence::Revoked { - incumbent_lease_sha256: recorded, - .. - } => { - ensure!( - recorded == &incumbent_lease_sha256, - "fencing evidence names another incumbent lease" - ); - ensure!( - incumbent_lease_sha256.is_some() - || transaction - .value - .expected - .as_ref() - .is_some_and(|expected| expected.write_lease_required), - "stateless transaction did not use its explicit fencing skip" - ); - } - } - } - } - let mut authorizations = BTreeSet::new(); - for authorization in self - .transactions - .iter() - .filter_map(|stored| stored.value.deployment_authorization.as_ref()) - { - ensure!( - authorizations.insert(authorization.authorization_id.as_str()), - "deployment authorization was consumed more than once" - ); - } - Ok(()) - } - - /// Every stored record that concerns `target`, as read: the generation, the - /// meters and its transactions. Two reads with equal stamps saw no write - /// for the target between them. - fn target_stamp(&self, target: &str) -> Vec { - let mut stamp = self - .admitted - .iter() - .filter(|stored| stored.value.target == target) - .map(|stored| stored.envelope.clone()) - .chain( - self.targets - .iter() - .filter(|stored| stored.value.target == target) - .map(|stored| stored.envelope.clone()), - ) - .chain( - self.transactions - .iter() - .filter(|stored| stored.value.target == target) - .map(|stored| stored.envelope.clone()), - ) - .collect::>(); - stamp.sort_by(|a, b| (&a.r#type, &a.key).cmp(&(&b.r#type, &b.key))); - stamp - } - - fn admitted_for(&self, target: &str) -> Option<&Stored> { - self.admitted - .iter() - .find(|stored| stored.value.target == target) - } - - fn supervision_for(&self, target: &str) -> Option<&Stored> { - self.targets - .iter() - .find(|stored| stored.value.target == target) - } - - /// The target's meters, or empty ones: a target that was never metered has - /// used nothing. - fn supervision_or_new(&self, target: &str) -> TargetSupervision { - self.supervision_for(target) - .map_or_else(|| TargetSupervision::new(target), |stored| stored.value.clone()) - } - - /// The admitted generation of the target that provides - /// `odin.verse-rendezvous`: Odin is whichever target says so, not - /// whichever is named "odin". - fn admitted_odin(&self) -> Option<&Stored> { - self.admitted.iter().find(|stored| { - ReadinessClass::of(&stored.value.expected) - .is_ok_and(|class| class == ReadinessClass::OdinSelf) - }) - } - - fn transaction_for_command(&self, command_id: &str) -> Vec<&DeploymentTransaction> { - let mut values = self - .transactions - .iter() - .filter(|stored| stored.value.command_id == command_id) - .map(|stored| &stored.value) - .collect::>(); - values.sort_by_key(|value| value.ordinal); - values - } - - fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool { - self.transactions.iter().any(|stored| { - stored.value.owns_target_authority() - && stored.value.command_id == transaction.command_id - && stored.value.ordinal < transaction.ordinal - }) - } - - /// The highest Odin publisher sequence this target has ever *acted on*. - /// - /// A transaction that failed contributes nothing. Its observations were - /// never built upon -- the deployment was abandoned and its projection - /// withdrawn -- so treating them as admitted history only raises the bar - /// for the next attempt. Odin does not reuse a sequence, so nothing is - /// weakened by forgetting the readings of an attempt that came to nothing; - /// what would be weakened is refusing every retry after a failure, which is - /// how a target ends up unable to deploy at all. - fn max_odin_sequence(&self, target: &str, signer_identity_id: &str) -> u64 { - let transaction_max = self - .transactions - .iter() - .filter_map(|stored| { - stored - .value - .latest_odin_observation - .as_ref() - .filter(|evidence| { - stored.value.target == target - && evidence.signer_identity_id == signer_identity_id - && !matches!( - stored.value.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - | Some(TransactionCompletion::FailedAfterFencing { .. }) - ) - }) - .map(|evidence| evidence.publisher_sequence) - }) - .max() - .unwrap_or(0); - let admitted_max = self - .admitted - .iter() - .filter(|stored| { - stored.value.target == target - && stored - .value - .odin_authority - .as_ref() - .is_some_and(|authority| authority.signer_identity_id == signer_identity_id) - }) - .map(|stored| stored.value.odin_publisher_sequence_cursor) - .max() - .unwrap_or(0); - transaction_max.max(admitted_max) - } -} - -pub(crate) fn decode_record(envelope: &CultCacheEnvelope) -> Result -where - T: for<'de> Deserialize<'de> + Serialize, -{ - let value: T = rmp_serde::from_slice(&envelope.payload)?; - ensure!( - rmp_serde::to_vec(&value)? == envelope.payload, - "Idunn control store contains a noncanonical record" - ); - Ok(value) -} - -fn command_envelope(value: &DeploymentCommand, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.command_id, - DeploymentCommand::TYPE, - DEPLOYMENT_COMMAND_SCHEMA, - value, - now, - ) -} - -/// The transaction layout of v2 and v3 records: `ready` is a bare Odin receipt, -/// and there are no phase deadline or lease adoption slots. It exists only to -/// decode records written before v4. `migrate_control_store_to_current_schema` -/// rewrites every one in `control.cc` at boot, but `history.cc` is never -/// migrated or pruned, so this struct dies only when history is pruned or -/// migrated. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v3" -)] -struct LegacyDeploymentTransaction { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - transaction_id: String, - #[cultcache(key = 2)] - command_id: String, - #[cultcache(key = 3)] - command_kind: CommandKind, - #[cultcache(key = 4)] - target: String, - #[cultcache(key = 5)] - ordinal: u32, - #[cultcache(key = 6)] - phase: DeploymentPhase, - #[cultcache(key = 7)] - created_at_unix_millis: u64, - #[cultcache(key = 8)] - updated_at_unix_millis: u64, - #[cultcache(key = 9)] - incumbent_generation_id: Option, - #[cultcache(key = 10)] - plan: Option, - #[cultcache(key = 11)] - frozen_source: Option, - #[cultcache(key = 12)] - sealed_release: Option, - #[cultcache(key = 13)] - installed_release: Option, - #[cultcache(key = 14)] - expected: Option, - #[cultcache(key = 15)] - expected_publication_sha256: Option, - #[cultcache(key = 16)] - deployment_authorization: Option, - #[cultcache(key = 17)] - lifecycle_authorized_at_unix_millis: Option, - #[cultcache(key = 18)] - activation: Option, - #[cultcache(key = 19)] - workload: Option, - #[cultcache(key = 20)] - activation_publication_sha256: Option, - #[cultcache(key = 21)] - latest_odin_observation: Option, - #[cultcache(key = 22)] - warming: Option, - #[cultcache(key = 23)] - route_preflight: Option, - #[cultcache(key = 24)] - isolation: Option, - #[cultcache(key = 25)] - fencing: Option, - #[cultcache(key = 26)] - leasing: Option, - #[cultcache(key = 27)] - ready: Option, - #[cultcache(key = 28)] - routing: Option, - #[cultcache(key = 29)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 30)] - last_error: Option, - #[cultcache(key = 31)] - completion: Option, - #[cultcache(key = 32)] - pre_fencing_abort: Option, - #[cultcache(key = 33)] - post_commit_cleanup: Option, - // A v2 record has no key 34: it predates post-fencing aborts. - #[cultcache(key = 34, default)] - post_fencing_abort: Option, -} - -impl LegacyDeploymentTransaction { - /// Every legacy Ready receipt is an Odin receipt: no route-proof class - /// existed. The new slots start empty, which is what an older transaction - /// had. - fn into_current(self) -> DeploymentTransaction { - DeploymentTransaction { - schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), - transaction_id: self.transaction_id, - command_id: self.command_id, - command_kind: self.command_kind, - target: self.target, - ordinal: self.ordinal, - phase: self.phase, - created_at_unix_millis: self.created_at_unix_millis, - updated_at_unix_millis: self.updated_at_unix_millis, - incumbent_generation_id: self.incumbent_generation_id, - plan: self.plan, - frozen_source: self.frozen_source, - sealed_release: self.sealed_release, - installed_release: self.installed_release, - expected: self.expected, - expected_publication_sha256: self.expected_publication_sha256, - deployment_authorization: self.deployment_authorization, - lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, - activation: self.activation, - workload: self.workload, - activation_publication_sha256: self.activation_publication_sha256, - latest_odin_observation: self.latest_odin_observation, - warming: self.warming, - route_preflight: self.route_preflight, - isolation: self.isolation, - fencing: self.fencing, - leasing: self.leasing, - ready: self - .ready - .map(|evidence| ReadinessEvidence::OdinCorrelated { evidence }), - routing: self.routing, - odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, - last_error: self.last_error, - completion: self.completion, - pre_fencing_abort: self.pre_fencing_abort, - post_commit_cleanup: self.post_commit_cleanup, - post_fencing_abort: self.post_fencing_abort, - phase_deadline: None, - lease_adoption: None, - } - } -} - -/// The v2 admitted-generation layout. Same lifetime rule as -/// `LegacyDeploymentTransaction`. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v2" -)] -struct LegacyAdmittedGeneration { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - #[cultcache(key = 2)] - generation_id: String, - #[cultcache(key = 3)] - command_id: String, - #[cultcache(key = 4)] - transaction_id: String, - #[cultcache(key = 5)] - admitted_at_unix_millis: u64, - #[cultcache(key = 6)] - plan: CompiledDeploymentPlan, - #[cultcache(key = 7)] - sealed_release: SealedRelease, - #[cultcache(key = 8)] - installed_release: InstalledReleaseObservation, - #[cultcache(key = 9)] - expected: IdunnExpectedIncarnationRecord, - #[cultcache(key = 10)] - activation: IdunnRuntimeActivationRecord, - #[cultcache(key = 11)] - workload: WorkloadObservation, - #[cultcache(key = 12)] - leasing: LeasingEvidence, - #[cultcache(key = 13)] - ready: TopologyEvidence, - #[cultcache(key = 14)] - latest_odin_observation: TopologyEvidence, - #[cultcache(key = 15)] - routing: RoutingEvidence, - #[cultcache(key = 16)] - odin_authority: AdmittedOdinAuthority, - #[cultcache(key = 17)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 18)] - route_repair_started_at_unix_millis: Option, -} - -impl LegacyAdmittedGeneration { - /// Every v2 generation was admitted on Odin receipts, so it migrates as - /// Odin-correlated with those receipts unchanged. Route supervision state - /// starts empty for a promoted route. The v2 repair start decided nothing - /// after S1 and is not carried. - fn into_current(self) -> AdmittedGeneration { - let route_supervision = matches!(&self.routing, RoutingEvidence::Promoted { .. }) - .then(RouteSupervisionState::default); - AdmittedGeneration { - schema_version: ADMITTED_GENERATION_SCHEMA.into(), - target: self.target, - generation_id: self.generation_id, - command_id: self.command_id, - transaction_id: self.transaction_id, - admitted_at_unix_millis: self.admitted_at_unix_millis, - plan: self.plan, - sealed_release: self.sealed_release, - installed_release: self.installed_release, - expected: self.expected, - activation: self.activation, - workload: self.workload, - leasing: self.leasing, - ready: ReadinessEvidence::OdinCorrelated { - evidence: self.ready, - }, - latest_odin_observation: Some(self.latest_odin_observation), - routing: self.routing, - odin_authority: Some(self.odin_authority), - odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, - route_supervision, - last_error: None, - } - } -} - -/// The v3 admitted-generation layout. Its meters (`actuations`, the continuity -/// backoff, the repair start) are retired: no released Idunn ever wrote a -/// nonzero one, and a record that shows one is refused instead of lifted. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v3" -)] -struct LegacyAdmittedGenerationV3 { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - #[cultcache(key = 2)] - generation_id: String, - #[cultcache(key = 3)] - command_id: String, - #[cultcache(key = 4)] - transaction_id: String, - #[cultcache(key = 5)] - admitted_at_unix_millis: u64, - #[cultcache(key = 6)] - plan: CompiledDeploymentPlan, - #[cultcache(key = 7)] - sealed_release: SealedRelease, - #[cultcache(key = 8)] - installed_release: InstalledReleaseObservation, - #[cultcache(key = 9)] - expected: IdunnExpectedIncarnationRecord, - #[cultcache(key = 10)] - activation: IdunnRuntimeActivationRecord, - #[cultcache(key = 11)] - workload: WorkloadObservation, - #[cultcache(key = 12)] - leasing: LeasingEvidence, - #[cultcache(key = 13)] - ready: ReadinessEvidence, - #[cultcache(key = 14)] - latest_odin_observation: Option, - #[cultcache(key = 15)] - routing: RoutingEvidence, - #[cultcache(key = 16)] - odin_authority: Option, - #[cultcache(key = 17)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 18)] - route_repair_started_at_unix_millis: Option, - #[cultcache(key = 19)] - route_supervision: Option, - #[cultcache(key = 20)] - continuity_backoff: LegacyContinuityBackoffV3, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LegacyRouteSupervisionStateV3 { - last_challenge_at_unix_millis: Option, - consecutive_failures: u32, - next_challenge_at_unix_millis: Option, - degraded_since_unix_millis: Option, - actuations: LegacyActuationWindowV3, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LegacyActuationWindowV3 { - window_started_at_unix_millis: u64, - count: u32, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LegacyContinuityBackoffV3 { - window_started_at_unix_millis: Option, - attempts: u32, - next_restart_at_unix_millis: Option, -} - -impl LegacyAdmittedGenerationV3 { - /// Drops the retired meters. The lift creates no `TargetSupervision`: every - /// target starts unmetered, which is what no released binary ever changed. - fn into_current(self) -> Result { - ensure!( - self.continuity_backoff.attempts == 0 - && self - .route_supervision - .as_ref() - .is_none_or(|state| state.actuations.count == 0), - "admitted generation of {} records route actuations or continuity restarts that no released Idunn wrote; refusing to drop them", - self.target - ); - Ok(AdmittedGeneration { - schema_version: ADMITTED_GENERATION_SCHEMA.into(), - target: self.target, - generation_id: self.generation_id, - command_id: self.command_id, - transaction_id: self.transaction_id, - admitted_at_unix_millis: self.admitted_at_unix_millis, - plan: self.plan, - sealed_release: self.sealed_release, - installed_release: self.installed_release, - expected: self.expected, - activation: self.activation, - workload: self.workload, - leasing: self.leasing, - ready: self.ready, - latest_odin_observation: self.latest_odin_observation, - routing: self.routing, - odin_authority: self.odin_authority, - odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, - route_supervision: self.route_supervision.map(|state| RouteSupervisionState { - last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, - consecutive_failures: state.consecutive_failures, - next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, - degraded_since_unix_millis: state.degraded_since_unix_millis, - }), - last_error: None, - }) - } -} - -/// Decode a v2 or v3 transaction and lift it to the current shape, without -/// validating it. History reads stop here; the control store validates. -fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result { - let schema = match envelope.schema_id.as_deref() { - Some(schema @ (DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3)) => { - schema - } - _ => bail!("Idunn control store contains an unsupported transaction"), - }; - let value: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload) - .with_context(|| format!("decoding a {schema} deployment transaction"))?; - ensure!( - value.schema_version == schema, - "stored transaction schema differs from its envelope" - ); - ensure!( - schema != DEPLOYMENT_TRANSACTION_SCHEMA_V2 || value.post_fencing_abort.is_none(), - "a v2 transaction cannot carry post-fencing abort evidence" - ); - Ok(value.into_current()) -} - -/// Read one stored transaction, lifting a v2 or v3 record to the current shape. -/// -/// The control store refuses noncanonical bytes: a record must re-encode to -/// exactly what is stored. That check is what makes tampering visible, and it -/// also means adding a field is a schema change: an older record re-encodes -/// with extra keys and would be read as tampered. -/// -/// So an older record is decoded by its own layout and lifted. Byte-exactness -/// cannot apply across a version boundary -- the bytes are a different schema -/// by definition -- so the lift leans on the full semantic `validate()` -/// instead, and everything downstream sees only the current shape. -fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result { - match envelope.schema_id.as_deref() { - Some(DEPLOYMENT_TRANSACTION_SCHEMA) => { - let value: DeploymentTransaction = decode_record(envelope)?; - value.validate()?; - Ok(value) - } - _ => { - let mut value = lift_legacy_transaction(envelope)?; - cleanup_evidence::owe_legacy_continuity_projection(&mut value); - value.validate()?; - Ok(value) - } - } -} - -/// Read one stored admitted generation, lifting a v2 or v3 record to the -/// current shape by the same rule as `read_transaction_record`. -fn read_generation_record(envelope: &CultCacheEnvelope) -> Result { - match envelope.schema_id.as_deref() { - Some(ADMITTED_GENERATION_SCHEMA) => { - let value: AdmittedGeneration = decode_record(envelope)?; - value.validate()?; - Ok(value) - } - Some(ADMITTED_GENERATION_SCHEMA_V3) => { - let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload) - .context("decoding a v3 admitted generation")?; - ensure!( - legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V3, - "stored generation schema differs from its envelope" - ); - let value = legacy.into_current()?; - value.validate()?; - Ok(value) - } - Some(ADMITTED_GENERATION_SCHEMA_V2) => { - let legacy: LegacyAdmittedGeneration = rmp_serde::from_slice(&envelope.payload) - .context("decoding a v2 admitted generation")?; - ensure!( - legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V2, - "stored generation schema differs from its envelope" - ); - let value = legacy.into_current(); - value.validate()?; - Ok(value) - } - _ => bail!("Idunn control store contains an unsupported admitted generation"), - } -} - -/// The v1 target-supervision layout: the first seven keys of the current one. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.target_supervision", - schema = "idunn.target_supervision.v1" -)] -struct LegacyTargetSupervisionV1 { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - #[cultcache(key = 2)] - route_actuations: Vec, - #[cultcache(key = 3)] - continuity_restarts: Vec, - #[cultcache(key = 4)] - continuity_deferred_until: Option, - #[cultcache(key = 5)] - continuity_deferral_reason: Option, - #[cultcache(key = 6)] - route_survivals: Vec, -} - -impl LegacyTargetSupervisionV1 { - /// No streak, no anchor, no re-materialization: the same defaults a fresh - /// record starts with. - fn into_current(self) -> TargetSupervision { - TargetSupervision { - schema_version: TARGET_SUPERVISION_SCHEMA.into(), - target: self.target, - route_actuations: self.route_actuations, - continuity_restarts: self.continuity_restarts, - continuity_deferred_until: self.continuity_deferred_until, - continuity_deferral_reason: self.continuity_deferral_reason, - route_survivals: self.route_survivals, - continuity_attempt_streak: 0, - continuity_backoff_from_unix_millis: None, - continuity_rematerialized_at_unix_millis: None, - } - } -} - -/// Read one stored target supervision record, lifting a v1 record to the -/// current shape by the same rule as `read_generation_record`. One schema id -/// names one shape: a v1 record that carries ten keys, or a v2 record that -/// carries seven, is refused. -fn read_supervision_record(envelope: &CultCacheEnvelope) -> Result { - let value = match envelope.schema_id.as_deref() { - Some(TARGET_SUPERVISION_SCHEMA) => decode_record::(envelope)?, - Some(TARGET_SUPERVISION_SCHEMA_V1) => { - let legacy: LegacyTargetSupervisionV1 = decode_record(envelope) - .context("decoding a v1 target supervision record")?; - ensure!( - legacy.schema_version == TARGET_SUPERVISION_SCHEMA_V1, - "stored supervision schema differs from its envelope" - ); - legacy.into_current() - } - _ => bail!("Idunn control store contains an unsupported target supervision record"), - }; - value.validate()?; - ensure!( - envelope.key == value.target, - "target supervision key is not its target" - ); - Ok(value) -} - -/// Rewrite every older record as current, once, before the engine runs. -/// -/// The read path lifts older records on its own, so this is convergence rather -/// than correctness: without it the store keeps records in several shapes for -/// as long as the oldest one survives. Each rewrite is a compare-exchange -/// against the exact stored envelope, so a record that changed underneath is -/// left alone rather than clobbered. -fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { - if !store_path.exists() { - return Ok(0); - } - let store = SingleFileMessagePackBackingStore::new(store_path); - let stale = store - .pull_all_read_only_snapshot() - .context("reading Idunn control snapshot for migration")? - .into_iter() - .filter(|envelope| { - let schema = envelope.schema_id.as_deref(); - (envelope.r#type == DeploymentTransaction::TYPE - && matches!( - schema, - Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) - )) - || (envelope.r#type == AdmittedGeneration::TYPE - && matches!( - schema, - Some(ADMITTED_GENERATION_SCHEMA_V2 | ADMITTED_GENERATION_SCHEMA_V3) - )) - || (envelope.r#type == TargetSupervision::TYPE - && schema == Some(TARGET_SUPERVISION_SCHEMA_V1)) - }) - .collect::>(); - let mut migrated = 0; - for envelope in stale { - let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { - let (key, next) = cleanup_evidence::migrate_transaction_record(&envelope)?; - (DeploymentTransaction::TYPE, key, next) - } else if envelope.r#type == TargetSupervision::TYPE { - let value = read_supervision_record(&envelope)?; - let next = target_supervision_envelope(&value, now_millis()?)?; - (TargetSupervision::TYPE, value.target, next) - } else { - let value = read_generation_record(&envelope)?; - let next = admitted_envelope(&value, value.admitted_at_unix_millis)?; - (AdmittedGeneration::TYPE, value.target, next) - }; - next.stored_at = envelope.stored_at.clone(); - ensure!( - store.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: record_type.into(), - key, - current: Some(envelope.clone()), - }], - &[next], - )?, - "control record changed during schema migration" - ); - migrated += 1; - } - Ok(migrated) -} - -fn transaction_envelope(value: &DeploymentTransaction, now: u64) -> Result { - ensure!( - !value.carries_legacy_marker(), - "only the legacy lift may produce a legacy-marked record; it cannot be written as a new one" - ); - encode_transaction(value, now) -} - -fn encode_transaction(value: &DeploymentTransaction, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.transaction_id, - DeploymentTransaction::TYPE, - DEPLOYMENT_TRANSACTION_SCHEMA, - value, - now, - ) -} - -fn admitted_envelope(value: &AdmittedGeneration, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.target, - AdmittedGeneration::TYPE, - ADMITTED_GENERATION_SCHEMA, - value, - now, - ) -} - -fn target_supervision_envelope(value: &TargetSupervision, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.target, - TargetSupervision::TYPE, - TARGET_SUPERVISION_SCHEMA, - value, - now, - ) -} - -pub(crate) fn typed_envelope( - key: &str, - record_type: &str, - schema: &str, - value: &T, - now: u64, -) -> Result { - Ok(CultCacheEnvelope { - key: key.into(), - r#type: record_type.into(), - payload: rmp_serde::to_vec(value)?, - stored_at: rfc3339_millis(now)?, - schema_id: Some(schema.into()), - }) -} - -/// `history.cc`, beside the control store. Terminal transactions live here so -/// the control store holds only what still gates a decision. -fn history_store_path(state_store: &Path) -> PathBuf { - state_store.with_file_name("history.cc") -} - -/// Move one finished transaction out of the live set. -/// -/// History first, then the live copy: a crash between the two leaves the -/// record in both, which reads as still-resident and is archived again on the -/// next attempt. The other order can lose it, and a completion nobody can -/// observe is the failure that actually costs an operator something. -/// -/// `insert_entry_if_absent` makes the repeat a no-op, and the delete is a -/// compare-and-swap on the exact envelope archived, so a transaction that -/// changed underneath us is left alone rather than dropped. -fn archive_terminal_transaction(state_store: &Path, envelope: &CultCacheEnvelope) -> Result<()> { - SingleFileMessagePackBackingStore::new(&history_store_path(state_store)) - .insert_entry_if_absent(envelope.clone()) - .context("archiving a finished transaction to history")?; - SingleFileMessagePackBackingStore::new(state_store) - .delete_batch_if_unchanged(std::slice::from_ref(envelope)) - .context("retiring a finished transaction from the control store")?; - // The command goes with its last transaction. A command is consumed by - // being frozen once; left resident after its transaction retired, it read - // as queued again and was frozen again on the next tick, which is how one - // `idunn up` became an unbounded series of deployment attempts. - let transaction = read_transaction_record(envelope)?; - let snapshot = ControlSnapshot::read(state_store)?; - if !snapshot - .transaction_for_command(&transaction.command_id) - .is_empty() - { - return Ok(()); - } - if let Some(command) = snapshot - .commands - .iter() - .find(|stored| stored.value.command_id == transaction.command_id) - { - SingleFileMessagePackBackingStore::new(&history_store_path(state_store)) - .insert_entry_if_absent(command.envelope.clone()) - .context("archiving a consumed command to history")?; - SingleFileMessagePackBackingStore::new(state_store) - .delete_batch_if_unchanged(std::slice::from_ref(&command.envelope)) - .context("retiring a consumed command from the control store")?; - } - Ok(()) -} - -/// What `history.cc` yielded: the transactions it decoded and the entries it -/// could not, each with its key and reason. -struct HistoryRead { - transactions: Vec, - undecodable: Vec<(String, String)>, -} - -impl HistoryRead { - /// What is missing from `transactions`, or `None` when nothing is. - fn report(&self) -> Option { - if self.undecodable.is_empty() { - return None; - } - let keys = self - .undecodable - .iter() - .map(|(key, error)| format!("{key} ({error})")) - .collect::>() - .join("; "); - Some(format!( - "Idunn history: {} archived transaction(s) could not be decoded and are not counted \ - by status or continuity backoff: {keys}", - self.undecodable.len() - )) - } -} - -/// Finished transactions. `control.cc` keeps the byte-exact canonical check -/// because its records still gate decisions. History describes what already -/// happened, so an undecodable entry is reported and skipped rather than -/// allowed to refuse the read -- that asymmetry is the whole reason the two -/// files are separate. A file that cannot be read at all is an error: the -/// caller decides whether an empty answer is safe. -fn read_history(state_store: &Path) -> Result { - let path = history_store_path(state_store); - if !path.exists() { - return Ok(HistoryRead { - transactions: Vec::new(), - undecodable: Vec::new(), - }); - } - let envelopes = SingleFileMessagePackBackingStore::new(&path) - .pull_all_read_only_snapshot() - .with_context(|| format!("Idunn cannot read {}", path.display()))?; - let (transactions, undecodable) = decode_history_transactions(envelopes); - Ok(HistoryRead { - transactions, - undecodable, - }) -} - -/// History for display only (status). An unreadable file shows as empty and -/// says so; decisions use `Engine::history_for_decision`, which does not. -fn read_history_transactions(state_store: &Path) -> Vec { - match read_history(state_store) { - Ok(history) => { - if let Some(report) = history.report() { - eprintln!("{report}"); - } - history.transactions - } - Err(error) => { - eprintln!("{error:#}; every archived transaction is invisible to status"); - Vec::new() - } - } -} - -/// How long to wait after `failures` consecutive failures: one poll interval -/// after the first, doubling each time, never past the ceiling. -fn backoff_wait(base_millis: u64, failures: u32, ceiling_millis: u64) -> u64 { - base_millis - .saturating_mul(1u64 << failures.min(20)) - .min(ceiling_millis) -} - -/// Whether an attempt due at `not_before` must still wait at `now`. A due time -/// further ahead than any wait its owner can set means the clock stepped back, -/// so the attempt is due: a step stalls nothing longer than the wait itself. -fn is_waiting(now: u64, not_before: u64, longest_wait: u64) -> bool { - now < not_before && not_before - now <= longest_wait -} - -/// Whether a transaction moved between two reads. An error note (`last_error`) -/// and its timestamp are commentary on a stuck step, not a step. -fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) -> bool { - let mut after = after.clone(); - after.last_error.clone_from(&before.last_error); - after.updated_at_unix_millis = before.updated_at_unix_millis; - &after != before -} - -/// Says a fault once on stderr while it lasts. A scheduler tick that hits the same -/// unreadable file every half second must not print it every half second. -/// -/// It is a trace and owns nothing else: it is in memory, keyed by free-form -/// strings, and an Idunn restart forgets it. Whether the operator is told -/// belongs to the incident record (`incident`), never to this map. -#[derive(Default)] -struct ReportOnce { - last: Mutex>, -} - -impl ReportOnce { - /// The text to print now: `report` when it differs from the last one - /// offered, nothing while the same fault stands or once it has cleared. - fn offer(&self, report: Option) -> Option { - let mut last = self.last.lock().expect("report mutex"); - if *last == report { - return None; - } - last.clone_from(&report); - report - } -} - -/// Decode every archived transaction. An entry that fails is returned with its -/// key and reason instead of vanishing, so the reader can say how many records -/// its answer is missing. -/// -/// `LegacyDeploymentTransaction` decodes the v2 and v3 layouts. `history.cc` is -/// never migrated or pruned, so it dies only when history is pruned or -/// migrated, not at boot. -fn decode_history_transactions( - envelopes: Vec, -) -> (Vec, Vec<(String, String)>) { - let mut transactions = Vec::new(); - let mut undecodable = Vec::new(); - for envelope in envelopes - .into_iter() - .filter(|envelope| envelope.r#type == DeploymentTransaction::TYPE) - { - let decoded = match envelope.schema_id.as_deref() { - Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) => { - lift_legacy_transaction(&envelope) - } - _ => rmp_serde::from_slice::(&envelope.payload) - .map_err(anyhow::Error::from), - }; - match decoded { - Ok(transaction) => transactions.push(transaction), - Err(error) => undecodable.push((envelope.key, format!("{error:#}"))), - } - } - (transactions, undecodable) -} - -/// Consumed commands, read as leniently as their transactions. -fn read_history_commands(state_store: &Path) -> Vec { - let path = history_store_path(state_store); - if !path.exists() { - return Vec::new(); - } - let Ok(envelopes) = SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot() - else { - return Vec::new(); - }; - envelopes - .into_iter() - .filter(|envelope| envelope.r#type == DeploymentCommand::TYPE) - .filter_map(|envelope| rmp_serde::from_slice::(&envelope.payload).ok()) - .collect() -} - -fn replace_transaction( - store_path: &Path, - current: &Stored, - next: &DeploymentTransaction, -) -> Result<()> { - next.validate()?; - ensure!( - current.value.transaction_id == next.transaction_id, - "transaction replacement changes identity" - ); - let envelope = transaction_envelope(next, next.updated_at_unix_millis)?; - ensure!( - SingleFileMessagePackBackingStore::new(store_path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: current.value.transaction_id.clone(), - current: Some(current.envelope.clone()), - }], - std::slice::from_ref(&envelope), - )?, - "deployment transaction changed before its compare-exchange" - ); - if next.is_terminal() { - archive_terminal_transaction(store_path, &envelope)?; - } - Ok(()) -} - -#[derive(Clone)] -struct LoadedBinding { - binding: OperatorBinding, - bytes: Vec, -} - -fn load_bindings(directory: &Path) -> Result> { - let mut paths = fs::read_dir(directory) - .with_context(|| format!("reading Idunn binding directory {}", directory.display()))? - .map(|entry| entry.map(|entry| entry.path())) - .collect::>>()?; - paths.retain(|path| path.extension() == Some(std::ffi::OsStr::new("toml"))); - paths.sort(); - let mut bindings = BTreeMap::new(); - for path in paths { - let bytes = fs::read(&path) - .with_context(|| format!("reading operator binding {}", path.display()))?; - let text = std::str::from_utf8(&bytes) - .with_context(|| format!("operator binding {} is not UTF-8", path.display()))?; - let binding = OperatorBinding::parse(text) - .with_context(|| format!("validating operator binding {}", path.display()))?; - let target = binding.target.clone(); - ensure!( - bindings - .insert(target.clone(), LoadedBinding { binding, bytes }) - .is_none(), - "operator binding target {target} is duplicated" - ); - } - ensure!(!bindings.is_empty(), "Idunn binding directory is empty"); - validate_route_bindings(&bindings)?; - Ok(bindings) -} - -fn validate_route_bindings(bindings: &BTreeMap) -> Result<()> { - let routes = bindings - .iter() - .filter_map(|(target, loaded)| { - loaded - .binding - .route - .as_ref() - .map(|route| (target.as_str(), route)) - }) - .collect::>(); - validate_route_binding_set(&routes) -} - -fn validate_route_binding_set(routes: &[(&str, &RouteBinding)]) -> Result<()> { - for (index, (target, route)) in routes.iter().enumerate() { - let (stable_host, stable_port) = route.stable_socket()?; - let private_host = route - .private_host - .parse::() - .context("validated route private host stopped being an IP address")?; - ensure!( - stable_host != private_host - || !(route.private_port_start..=route.private_port_end).contains(&stable_port), - "route {target} stable socket overlaps its candidate port range" - ); - - for (other_target, other) in &routes[index + 1..] { - ensure!( - route.route_id != other.route_id, - "route id {} is shared by targets {target} and {other_target}", - route.route_id - ); - ensure!( - route.config_path != other.config_path, - "route fragment {} is shared by targets {target} and {other_target}", - route.config_path.display() - ); - let other_stable = other.stable_socket()?; - ensure!( - route.driver != other.driver || (stable_host, stable_port) != other_stable, - "stable route socket {}:{} is shared by targets {target} and {other_target}", - stable_host, - stable_port - ); - if route.driver != other.driver { - continue; - } - let other_private_host = other - .private_host - .parse::() - .context("validated route private host stopped being an IP address")?; - let ranges_overlap = route.private_host == other.private_host - && route.private_port_start <= other.private_port_end - && other.private_port_start <= route.private_port_end; - ensure!( - !ranges_overlap, - "candidate port ranges overlap for targets {target} and {other_target}" - ); - ensure!( - stable_host != other_private_host - || !(other.private_port_start..=other.private_port_end).contains(&stable_port), - "stable route socket for {target} overlaps {other_target}'s candidate range" - ); - ensure!( - other_stable.0 != private_host - || !(route.private_port_start..=route.private_port_end) - .contains(&other_stable.1), - "stable route socket for {other_target} overlaps {target}'s candidate range" - ); - } - } - Ok(()) -} - -fn resolve_selector( - bindings: &BTreeMap, - selector: &str, -) -> Result> { - let mut targets = if let Some(profile) = selector.strip_prefix("profile:") { - bindings - .values() - .filter(|binding| binding.binding.profiles.contains(profile)) - .map(|binding| binding.binding.target.clone()) - .collect::>() - } else { - ensure!( - bindings.contains_key(selector), - "unknown deployment target {selector}" - ); - vec![selector.to_owned()] - }; - ensure!( - !targets.is_empty(), - "unknown or empty deployment profile {selector}" - ); - targets.sort_by(|left, right| { - (left != "odin") - .cmp(&(right != "odin")) - .then_with(|| left.cmp(right)) - }); - Ok(targets) -} - -fn submit( - store_path: &Path, - selector: &str, - requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { - if let Some(parent) = store_path.parent() { - fs::create_dir_all(parent)?; - } - let now = now_millis()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), - kind: CommandKind::Deploy, - selector: selector.into(), - requested_by: requested_by.into(), - requested_at_unix_millis: now, - }; - command.validate()?; - ensure!( - SingleFileMessagePackBackingStore::new(store_path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, now)?], - )?, - "deployment command id collided" - ); - println!("{}", command.command_id); - if !wait { - return Ok(()); - } - let deadline = now.saturating_add(timeout_seconds.saturating_mul(1000)); - loop { - let snapshot = ControlSnapshot::read(store_path)?; - ensure!( - snapshot - .commands - .iter() - .any(|stored| stored.value.command_id == command.command_id), - "submitted deployment command disappeared" - ); - let transactions = snapshot.transaction_for_command(&command.command_id); - if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { - if let Some(error) = - transactions - .iter() - .find_map(|transaction| match &transaction.completion { - Some(TransactionCompletion::FailedBeforeFencing { error }) - | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => { - Some(error.as_str()) - } - _ => None, - }) - { - bail!("deployment failed: {error}") - } - println!("succeeded {} target(s)", transactions.len()); - return Ok(()); - } - if now_millis()? >= deadline { - bail!("deployment command timed out") - } - thread::sleep(Duration::from_millis(250)); - } -} - -fn status( - out: &mut impl Write, - store_path: &Path, - incident_store: &Path, - command_id: Option<&str>, -) -> Result<()> { - let snapshot = ControlSnapshot::read(store_path)?; - // Commands stay resident; their finished transactions do not. Without - // history a completed command would report as though it had never run, - // which is the operator surface R11 refused to trade away for a smaller - // live set. - let archived = read_history_transactions(store_path); - let archived_commands = read_history_commands(store_path); - let mut commands = snapshot - .commands - .iter() - .map(|stored| &stored.value) - .chain(archived_commands.iter()) - .collect::>(); - commands.sort_by_key(|command| command.requested_at_unix_millis); - if let Some(command_id) = command_id { - commands.retain(|command| command.command_id == command_id); - ensure!(!commands.is_empty(), "deployment command is unknown"); - } - for command in commands { - let mut transactions = snapshot.transaction_for_command(&command.command_id); - transactions.extend( - archived - .iter() - .filter(|value| value.command_id == command.command_id), - ); - transactions.sort_by_key(|value| value.ordinal); - let (state, detail) = derived_command_status(&transactions); - writeln!( - out, - "{} {} {} {}", - command.command_id, command.selector, state, detail - )?; - // Naming one command asks about that command, so print what a stuck - // transaction is actually waiting on. A gate reason lives in - // `last_error` and was never rendered anywhere, which left "Sealing" - // looking identical whether the brake had not authorized the - // transaction or the phase was simply slow. The release and deployment - // ids are here because they are exactly what a brake release must name. - if command_id.is_some() { - for transaction in transactions { - writeln!(out, " transaction {}", transaction.transaction_id)?; - writeln!( - out, - " target {} phase {:?}", - transaction.target, transaction.phase - )?; - if let Some(expected) = &transaction.expected { - writeln!(out, " runtime {}", expected.runtime_id)?; - writeln!(out, " release {}", expected.sealed_release_id)?; - } - writeln!( - out, - " odin publisher cursor {}", - transaction.odin_publisher_sequence_cursor - )?; - if let Some(evidence) = &transaction.latest_odin_observation { - writeln!( - out, - " latest odin observation sequence {}", - evidence.publisher_sequence - )?; - } - if let Some(reason) = &transaction.last_error { - writeln!(out, " waiting on {reason}")?; - } - // A Complete transaction that is not terminal still owns its - // target through unfinished cleanup. Without this an operator - // sees "Complete" and a stale reason and cannot tell what is - // holding the target. - if let Some(completion) = &transaction.completion { - writeln!(out, " completion {completion:?}")?; - } - if let Some(abort) = &transaction.pre_fencing_abort { - writeln!(out, " pre-fencing abort {abort:?}")?; - } - if let Some(abort) = &transaction.post_fencing_abort { - writeln!(out, " post-fencing abort {abort:?}")?; - } - if let Some(cleanup) = &transaction.post_commit_cleanup { - writeln!(out, " post-commit cleanup {cleanup:?}")?; - } - writeln!( - out, - " terminal {} owns-target {}", - transaction.is_terminal(), - transaction.blocks_new_target_mutation() - )?; - } - } - } - // What supervision holds for each target: the union of the admitted - // generations and the metered targets, so a target that has only been - // deployed to (no generation yet) still shows what it has used. - for line in render_targets(&snapshot, incident_store, now_millis()?) { - writeln!(out, "{line}")?; - } - Ok(()) -} - -/// Every supervised target with its meters and its incidents. An incident -/// store that cannot be read costs the incident lines and nothing else. -fn render_targets(snapshot: &ControlSnapshot, incident_store: &Path, now: u64) -> Vec { - let incidents = IncidentStore::new(incident_store).read(); - let mut lines = Vec::new(); - for target in supervised_targets(snapshot) { - lines.extend(render_supervision( - target, - snapshot.admitted_for(target).map(|stored| &stored.value), - snapshot.supervision_for(target).map(|stored| &stored.value), - now, - )); - if let Ok(records) = &incidents { - lines.extend(render_incidents(records, target, now)); - } - } - if let Err(error) = &incidents { - lines.push(format!("incidents unreadable: {}", error.root_cause())); - } - lines -} - -/// Every target supervision holds something for: the admitted generations -/// and the metered targets. -fn supervised_targets(snapshot: &ControlSnapshot) -> BTreeSet<&str> { - snapshot - .admitted - .iter() - .map(|stored| stored.value.target.as_str()) - .chain(snapshot.targets.iter().map(|stored| stored.value.target.as_str())) - .collect() -} - -/// One target's meters and route pacing, one line each, times in unix -/// milliseconds. The counts are what is still inside the window at `now`. -fn render_supervision( - target: &str, - generation: Option<&AdmittedGeneration>, - supervision: Option<&TargetSupervision>, - now: u64, -) -> Vec { - let at = |time: Option| time.map_or("none".to_owned(), |millis| millis.to_string()); - let meters = supervision.cloned().unwrap_or_else(|| TargetSupervision::new(target)); - let mut lines = vec![format!( - "target {target} {}", - generation.map_or("no-admitted-generation", |value| value.generation_id.as_str()) - )]; - if let Some(reason) = generation.and_then(|value| value.last_error.as_deref()) { - lines.push(format!(" held: {reason}")); - } - lines.push(format!( - " continuity attempts-in-hour {} pages-at {} streak {} next-attempt-at {}", - meters.restarts_used(now), - CONTINUITY_PAGING_RESTARTS, - meters.continuity_attempt_streak, - at(generation.and_then(|value| meters.backoff_due(value.admitted_at_unix_millis))) - )); - if let Some(rematerialized) = meters.continuity_rematerialized_at_unix_millis { - lines.push(format!(" continuity rematerialized-at {rematerialized}")); - } - if let (Some(until), Some(reason)) = ( - meters.continuity_deferred_until, - &meters.continuity_deferral_reason, - ) { - lines.push(format!(" continuity deferred until {until}: {reason}")); - } - if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { - lines.push(format!( - " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", - match route.degraded_since_unix_millis { - Some(since) => format!("degraded-since {since}"), - None => "healthy".to_owned(), - }, - route.consecutive_failures, - at(route.last_challenge_at_unix_millis), - at(route.next_challenge_at_unix_millis) - )); - } - if generation.is_none_or(|value| value.route_supervision.is_some()) - || !meters.route_actuations.is_empty() - || !meters.route_survivals.is_empty() - { - lines.push(format!( - " route actuations {}/{} in window, reopens-at {}", - meters.route_used(now), - ROUTE_ACTUATION_CEILING, - at(meters.route_reopens_at(now)) - )); - // A record, not a meter: only the newest are kept, so a saturated - // count says "at least". - let survivals = meters.survivals_in_window(now); - lines.push(format!( - " route survival actuations {}{} in window (uncounted), last-at {}", - if survivals == ROUTE_SURVIVAL_RECORD { "at least " } else { "" }, - survivals, - at(meters.route_survivals.last().copied()) - )); - } - lines -} - -fn derived_command_status(transactions: &[&DeploymentTransaction]) -> (&'static str, String) { - if transactions.is_empty() { - return ("queued", String::new()); - } - // A live transaction is the command's present tense. Reporting an older - // attempt's failure while a new one is sealing hid a running deployment - // behind the word "failed". - let current = transactions - .iter() - .filter(|transaction| !transaction.is_terminal()) - .map(|transaction| format!("{}:{:?}", transaction.target, transaction.phase)) - .collect::>(); - if !current.is_empty() { - return ("running", current.join(",")); - } - if let Some(error) = transactions - .iter() - .find_map(|transaction| match &transaction.completion { - Some(TransactionCompletion::FailedBeforeFencing { error }) - | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => Some(error.clone()), - _ => None, - }) - { - return ("failed", error); - } - if transactions.iter().all(|transaction| { - transaction.is_terminal() - && matches!( - transaction.completion, - Some(TransactionCompletion::Admitted { .. }) - ) - }) { - return ( - "succeeded", - transactions - .iter() - .map(|transaction| transaction.target.as_str()) - .collect::>() - .join(","), - ); - } - ("complete", String::new()) -} - -struct ProcessLock { - file: File, - #[cfg(not(unix))] - path: PathBuf, -} - -impl ProcessLock { - fn acquire(store_path: &Path) -> Result { - let path = sibling_path(store_path, ".daemon.lock"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - #[cfg(unix)] - { - use std::os::fd::AsRawFd; - let file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(&path) - .with_context(|| format!("opening Idunn daemon lock {}", path.display()))?; - let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; - ensure!(result == 0, "another Idunn daemon owns this control store"); - Ok(Self { file }) - } - #[cfg(not(unix))] - { - let file = OpenOptions::new() - .create_new(true) - .read(true) - .write(true) - .open(&path) - .with_context(|| format!("another Idunn daemon owns {}", path.display()))?; - Ok(Self { file, path }) - } - } -} - -impl Drop for ProcessLock { - fn drop(&mut self) { - #[cfg(unix)] - { - use std::os::fd::AsRawFd; - let _ = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; - } - #[cfg(not(unix))] - { - let _ = fs::remove_file(&self.path); - } - } -} - -/// One target's in-memory challenge wait: when it is due, and the stored -/// records of the target as they were when the record failed to land. -struct ChallengeWait { - due_unix_millis: u64, - stamp: Vec, -} - -struct Engine { - options: RuntimeOptions, - idunn_signer: ServiceIdentitySigner, - idunn_anchor: ServiceIdentityTrustAnchor, - bootstrap_odin_authority: AdmittedOdinAuthority, - source: Arc, - docker_runner: DockerRunnerDriver, - systemd_workload: Arc, - host_actuators: Option, - history_report: ReportOnce, - /// One `ReportOnce` per faulting record, so one wedged transaction says - /// its fault once and never drowns another's. - fault_reports: Mutex>, - /// Retry pacing for a transaction whose step keeps failing, by - /// transaction id: consecutive failures and the earliest next attempt. - /// Process-local on purpose: a restart retries at once, and no schema - /// carries a clock that only pacing reads. - resume_backoff: Mutex>, - /// When each routed target's next challenge is due, held only when the - /// stored `RouteSupervisionState` could not record the last one. The stored - /// value owns the schedule and survives restarts; this spaces attempts - /// while the record did not land. An entry ends the moment any write for - /// its target lands (its stamp no longer matches the store) or the target - /// is retired or withdrawn: it never outlives the episode that made it. - challenge_wait: Mutex>, -} - -/// The route driver's door to the actuation ceiling: every group of host -/// mutations a driver makes for `target` is charged through here. The gate -/// owns which charge a request is: a driver asks for the change it is making, -/// and the command that owns the transaction decides whether that change is -/// deployment (`Forward`, refusable) or the continuity of the admitted -/// generation (`Survival`, never refused). -struct EngineRouteGate<'a> { - engine: &'a Engine, - target: &'a str, - command: CommandKind, -} - -impl RouteActuationGate for EngineRouteGate<'_> { - fn admit(&self, requested: RouteActuation) -> Result<()> { - let kind = match self.command { - CommandKind::Deploy => requested, - CommandKind::Continuity => RouteActuation::Survival, - }; - self.engine.charge_route_actuation(self.target, kind) - } -} - -#[derive(Clone, Copy)] -struct ResumeBackoff { - failures: u32, - not_before_unix_millis: u64, -} - -/// Longest wait between attempts at a failing transaction step. -const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; - -impl Engine { - fn open(options: RuntimeOptions) -> Result { - Self::open_with_systemd_workload( - options, - Arc::new(SystemdTransientWorkloadDriver::default()), - ) - } - - /// `open` with the systemd workload port supplied. It is the seam that - /// lets the phase engine run past Fencing without a systemd to talk to. - fn open_with_systemd_workload( - options: RuntimeOptions, - systemd_workload: Arc, - ) -> Result { - let idunn_signer = - open_service_identity_at::(&options.idunn_identity_store) - .context("opening Idunn activation identity")?; - let idunn_anchor = idunn_signer.trust_anchor()?; - let odin_anchor = read_trust_anchor::(&options.odin_trust_anchor) - .context("reading bootstrap Odin topology anchor")?; - let bootstrap_odin_authority = AdmittedOdinAuthority::from_anchor(&odin_anchor)?; - let source: Arc = Arc::new(GitSourceDriver::new( - &options.source_root, - options.staging_root.join("frozen-sources"), - options.source_identity, - )); - let host_actuators = options - .host_actuator_bind - .map(|bind| HostActuatorHub::bind(bind).map(|hub| Arc::new(Mutex::new(hub)))) - .transpose()?; - if let Some(hub) = &host_actuators { - eprintln!( - "Idunn host actuator hub listening on {}", - hub.lock().expect("hub mutex").local_addr()? - ); - let service_signer = - open_service_identity_at::(&options.idunn_identity_store) - .context("opening Idunn identity for the host actuator hub thread")?; - let bindings_dir = options.bindings_dir.clone(); - spawn_hub_service(Arc::clone(hub), service_signer, move || { - host_anchors_from(&bindings_dir) - }); - } - Ok(Self { - options, - idunn_signer, - idunn_anchor, - bootstrap_odin_authority, - source, - docker_runner: DockerRunnerDriver::default(), - systemd_workload, - host_actuators, - history_report: ReportOnce::default(), - fault_reports: Mutex::default(), - resume_backoff: Mutex::default(), - challenge_wait: Mutex::default(), - }) - } - - fn host_anchors(&self) -> BTreeMap { - host_anchors_from(&self.options.bindings_dir) - } - - /// Every route driver comes from here, so the actuator paths are the - /// options' and nothing else's. - fn route_driver(&self, binding: RouteBinding) -> NginxRouteDriver { - NginxRouteDriver::with_actuators(binding, &self.options.route_actuators) - } - - fn route_gate<'a>(&'a self, target: &'a str, command: CommandKind) -> EngineRouteGate<'a> { - EngineRouteGate { - engine: self, - target, - command, - } - } - - /// Charge one route actuation to the target's `TargetSupervision` before - /// the driver acts. The record is created by the first charge, so a first - /// deployment is metered like any other. A `Forward` is counted against - /// the sliding ceiling; a refused one is the typed `RouteActuationRefused`, - /// and one that cannot be recorded is refused too: no charge, no - /// actuation. A `Survival` is never counted or refused, only recorded when - /// it can be, and runs regardless when it cannot: an unwritable store must - /// not stop a rollback, a withdrawal or a repair. - fn charge_route_actuation(&self, target: &str, kind: RouteActuation) -> Result<()> { - match self.record_route_actuation(target, kind) { - Ok(charged) => charged.map_err(anyhow::Error::new), - Err(error) if kind == RouteActuation::Survival => { - eprintln!( - "Idunn could not record a survival route actuation of {target}; it proceeds unrecorded: {error:#}" - ); - Ok(()) - } - Err(error) => Err(error), - } - } - - /// The outer error is the ledger failing to read or write; the inner one - /// is the ceiling refusing. - fn record_route_actuation( - &self, - target: &str, - kind: RouteActuation, - ) -> Result> { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let seen = snapshot.supervision_for(target); - let mut next = snapshot.supervision_or_new(target); - let now = now_millis()?; - let charged = next.charge_route(now, kind); - // A refusal writes nothing unless the clock had stepped back and the - // log needed settling. - if charged.is_ok() || seen.is_some_and(|stored| stored.value != next) { - self.write_target_supervision(seen, &next, now)?; - } - Ok(charged) - } - - /// Replace a target's meters, or create them when `seen` is `None`, by - /// compare-exchange against exactly what was read. - fn write_target_supervision( - &self, - seen: Option<&Stored>, - next: &TargetSupervision, - now: u64, - ) -> Result<()> { - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: TargetSupervision::TYPE.into(), - key: next.target.clone(), - current: seen.map(|stored| stored.envelope.clone()), - }], - &[target_supervision_envelope(next, now)?], - )?, - "target supervision changed before its meters were written" - ); - Ok(()) - } - - /// Replace an admitted generation by compare-exchange against the envelope - /// that was read. `Ok(false)` is a lost race: another writer moved the - /// record, so the caller's snapshot is stale. `Err` is a write the store - /// refused, which changed nothing. - fn replace_generation( - &self, - seen: &Stored, - next: &AdmittedGeneration, - now: u64, - ) -> Result { - Ok(SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: seen.value.target.clone(), - current: Some(seen.envelope.clone()), - }], - &[admitted_envelope(next, now)?], - )?) - } - - /// Say a fault once while it lasts, under `key`. - fn report_once(&self, key: &str, message: String) { - let offered = self - .fault_reports - .lock() - .expect("fault report mutex") - .entry(key.to_string()) - .or_default() - .offer(Some(message)); - if let Some(message) = offered { - eprintln!("{message}"); - } - } - - fn incidents(&self) -> IncidentStore { - IncidentStore::new(&self.options.incident_store) - } - - /// Settle the outcome of one incident operation. A failure never leaves - /// this function: it is traced once per episode, under the operation and - /// the subject it was decided for (`*` for a pass over the whole store), - /// and continuity decides exactly as it would have with a healthy store. - /// Each decision site has its own key, so one site's success never ends - /// another site's episode. The message names the operation and the root - /// cause and never a path or a value. Incident writes are never progress. - fn record_incident(&self, operation: &str, subject: &str, outcome: Result) { - let key = format!("incident-store:{operation}:{subject}"); - match outcome { - Ok(_) => self.clear_fault(&key), - Err(error) => self.report_once( - &key, - format!( - "Idunn could not {operation} an operator incident: {}", - error.root_cause() - ), - ), - } - } - - /// Close every open incident whose subject is a target that is no longer - /// admitted, then retire what has been closed past retention. - fn reconcile_incidents(&self, snapshot: &ControlSnapshot) { - let store = self.incidents(); - let close_unadmitted = || -> Result<()> { - let now = now_millis()?; - for record in store.read()? { - if record.is_open() - && record.condition.ends_with_admission() - && snapshot.admitted_for(&record.subject).is_none() - { - store.close( - record.condition, - &record.subject, - CloseReason::NoLongerAdmitted, - now, - )?; - } - } - Ok(()) - }; - self.record_incident("close", "*", close_unadmitted()); - self.record_incident("retire", "*", now_millis().and_then(|now| store.retire_closed(now))); - } - - fn host_access(&self) -> Result> { - let hub = self - .host_actuators - .as_ref() - .context("this Idunn serves no host actuators (no --host-actuator-bind)")?; - Ok(HostActuatorAccess { - hub: Arc::clone(hub), - anchors: self.host_anchors(), - signer: &self.idunn_signer, - }) - } -} - -/// The trust anchor of every host a binding names. A binding whose anchor -/// cannot be read leaves its host unattachable and is logged; the other -/// hosts are unaffected. -fn host_anchors_from(bindings_dir: &Path) -> BTreeMap { - { - let mut anchors = BTreeMap::new(); - let bindings = match load_bindings(bindings_dir) { - Ok(bindings) => bindings, - Err(error) => { - eprintln!("Idunn cannot read bindings for host anchors: {error:#}"); - return anchors; - } - }; - for loaded in bindings.values() { - let WorkloadBinding::HostActuator(workload) = &loaded.binding.workload else { - continue; - }; - match read_trust_anchor::(&workload.host_trust_anchor_store) - { - Ok(anchor) => { - anchors.insert(workload.host.clone(), anchor); - } - Err(error) => eprintln!( - "Idunn cannot read the actuator anchor for host {}: {error:#}", - workload.host - ), - } - } - anchors - } -} - -impl Engine { - /// The runner the plan's binding declares. Every runner in one binding - /// is of one kind; the binding validated that. - fn runner_for(&self, plan: &CompiledDeploymentPlan) -> Result> { - let (_, binding) = plan.parsed_inputs()?; - Ok(match &binding.workload { - WorkloadBinding::SystemdTransient(_) => Box::new(self.docker_runner.clone()), - WorkloadBinding::HostActuator(_) => Box::new(HostActuatorRunnerDriver { - access: self.host_access()?, - }), - }) - } - - /// The workload driver the plan's binding declares. - fn workload_for(&self, plan: &CompiledDeploymentPlan) -> Result> { - let (_, binding) = plan.parsed_inputs()?; - Ok(match &binding.workload { - WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), - WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { - access: self.host_access()?, - }), - }) - } - - /// History for a decision that would be wrong on an empty answer: the - /// continuity restart ceiling and the check that a command is consumed. - /// `None` means `history.cc` cannot be read, so the decision is not made; - /// a history that merely has no entries is `Some`. The fault is printed - /// once while it lasts, not on every tick. - fn history_for_decision(&self) -> Option> { - let (report, transactions) = match read_history(&self.options.state_store) { - Ok(history) => (history.report(), Some(history.transactions)), - Err(error) => ( - Some(format!( - "{error:#}; continuity and command retirement stop until it is readable" - )), - None, - ), - }; - if let Some(text) = self.history_report.offer(report) { - eprintln!("{text}"); - } - transactions - } - - fn topology(&self) -> CultCacheTopologyDriver { - CultCacheTopologyDriver { - projection_store: self.options.topology_store.clone(), - correlation_store: self.options.odin_correlation_store.clone(), - } - } - - fn current_odin_authority(&self, snapshot: &ControlSnapshot) -> Result { - // The bootstrap key stands only until Odin itself is admitted. Once it - // is, its own generation names the authority; an admitted Odin that - // carries none has no authority to name, and falling back to the - // bootstrap key would let a stale key vouch for the Verse. - let authority = match snapshot.admitted_odin() { - None => self.bootstrap_odin_authority.clone(), - Some(stored) => stored.value.odin_authority.clone().with_context(|| { - "admitted Odin generation carries no Odin authority (route-proof readiness)" - })?, - }; - authority.validate()?; - Ok(authority) - } - - fn trusted_topology_context(&self, now: u64) -> OdinTopologyAuthenticationContext { - OdinTopologyAuthenticationContext { - trusted_received_at_unix_millis: now, - maximum_age_millis: self.options.topology_maximum_age_millis, - maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, - } - } - - /// Re-prove the evidence that still gates a decision. - /// - /// Terminal transactions are skipped: they describe what already happened - /// and authorize nothing further, so re-proving them only creates ways for - /// history to refuse a boot. On yggdrasil that is 259 of 260 records. - /// - /// The operator anchor is read on demand rather than up front. Reading it - /// unconditionally meant an absent brake artifact refused startup against - /// an *empty* store -- a deployment brake gating Idunn itself, which - /// `F:\Projects\CLAUDE.md` forbids outright: a target brake "may never gate - /// Idunn itself or unrelated service lifecycle". - fn validate_durable_authority(&self, snapshot: &ControlSnapshot) -> Result<()> { - let mut operator_anchor = None; - for stored in &snapshot.transactions { - let transaction = &stored.value; - if transaction.is_terminal() { - continue; - } - if let Some(authorization) = &transaction.deployment_authorization { - authorization.validate_shape()?; - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; - if operator_anchor.is_none() { - operator_anchor = - Some(read_trust_anchor::( - &self.options.deployment_brake_operator_anchor, - )?); - } - verify_idunn_deployment_brake_authorization( - &record, - operator_anchor.as_ref().expect("read directly above"), - )?; - let expected = required(&transaction.expected, "authorized Expected projection")?; - ensure!( - record.authorized_release_id.as_deref() - == Some(expected.sealed_release_id.as_str()) - && record.authorized_deployment_id.as_deref() - == Some(transaction.transaction_id.as_str()) - && record.runtime_id == expected.runtime_id, - "durable deployment authorization names another release or transaction" - ); - } - let lease = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - if let Some(evidence) = &transaction.latest_odin_observation { - let authenticated = self.authenticate_topology_bytes( - snapshot, - transaction, - &evidence.canonical_bytes, - lease, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(evidence, &authenticated)?; - } - if let Some(evidence) = &transaction.warming { - match evidence { - WarmingEvidence::OdinTopology { evidence } => { - let authenticated = self.authenticate_topology_bytes( - snapshot, - transaction, - &evidence.canonical_bytes, - None, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(evidence, &authenticated)?; - let incumbent_lease_sha256 = - self.incumbent_lease_sha256_for_warming(snapshot, transaction)?; - ensure!( - is_semantic_warming( - required(&transaction.expected, "Warming Expected projection",)?, - required(&transaction.activation, "Warming activation")?, - incumbent_lease_sha256.as_deref(), - &authenticated, - )?, - "durable Warming gate is not supported by current runtime evidence" - ); - } - WarmingEvidence::FirstOdinDirect { evidence } => { - self.authenticate_first_odin_warming_presence( - transaction, - &evidence.message_id, - evidence.challenged_at_unix_millis, - evidence.admitted_at_unix_millis, - &evidence.canonical_bytes, - )?; - } - WarmingEvidence::RouteProofDirect { evidence } => { - self.reauthenticate_route_proof( - transaction, - evidence, - route_proof_warming_states(required( - &transaction.expected, - "Warming Expected projection", - )?), - None, - evidence.admitted_at_unix_millis, - )?; - } - } - } - if let Some(ReadinessEvidence::RouteProof { evidence }) = &transaction.ready { - self.reauthenticate_route_proof( - transaction, - evidence, - &["active"], - lease, - evidence.admitted_at_unix_millis, - )?; - } - if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) { - let authenticated = self.authenticate_topology_bytes( - snapshot, - transaction, - &evidence.canonical_bytes, - lease, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(evidence, &authenticated)?; - ensure!( - is_semantic_ready(&authenticated), - "durable Ready label is not exact semantic Ready" - ); - } - } - let odin_authority = self.current_odin_authority(snapshot)?; - for stored in &snapshot.admitted { - let generation = &stored.value; - // Only a generation whose own Expected says Odin vouches for it has - // Odin receipts to re-prove. One whose evidence disagrees with its - // class is held and reported by supervision, not re-proved here. - if !matches!(generation.readiness(), Ok(class) if class != ReadinessClass::RouteProof) - { - continue; - } - let odin = generation.odin_receipts()?; - let authority = self.runtime_authority_parts( - &generation.plan, - &generation.expected, - &generation.activation, - )?; - let latest = authenticate_odin_runtime_topology_correlation( - &odin.latest.canonical_bytes, - &authority, - generation.leasing.lease_sha256(), - &odin_authority.signer_public_key, - self.trusted_topology_context(odin.latest.admitted_at_unix_millis), - )?; - validate_authenticated_evidence(odin.latest, &latest)?; - let ready = authenticate_odin_runtime_topology_correlation( - &odin.ready.canonical_bytes, - &authority, - generation.leasing.lease_sha256(), - &odin_authority.signer_public_key, - self.trusted_topology_context(odin.ready.admitted_at_unix_millis), - )?; - validate_authenticated_evidence(odin.ready, &ready)?; - ensure!( - is_semantic_ready(&ready), - "admitted generation Ready label is not exact semantic Ready" - ); - } - Ok(()) - } -} - -/// What the boot reconciliation of failed continuity projections found. -#[derive(Debug, Default, PartialEq, Eq)] -struct ProjectionReconciliation { - /// Failed continuity transactions whose exact activation was demoted. - demoted: Vec, - /// Targets whose projection names an activation that neither the admitted - /// generation, a live transaction, nor a failed issuer accounts for. They - /// are reported and left alone. - unexplained: Vec, -} - -impl Engine { - /// Once at boot: demote the activation a failed continuity left standing, - /// where the projection still names exactly the activation that failed - /// transaction issued. A continuity abort before the single resolution - /// rule left such residue (and the lift owes the same demotion to any - /// abort still resident). The issuer is identified by exact activation, - /// from history and from resident terminal records not yet retired to it. - /// - /// Nothing is adopted: an activation with no failed issuer is never - /// demoted or kept on anyone's behalf, only reported. - fn reconcile_failed_continuity_projections(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut outcome = ProjectionReconciliation::default(); - // Resident issuers need no history; only the history issuers wait on it. - let history = self.history_for_decision(); - if history.is_none() { - eprintln!( - "Idunn cannot read history at boot: failed continuities archived there are not \ - reconciled, resident ones still are" - ); - } - let topology = self.topology(); - let issuers = history - .iter() - .flatten() - .chain(snapshot.transactions.iter().map(|stored| &stored.value)) - .filter(|transaction| { - transaction.command_kind == CommandKind::Continuity - && matches!( - transaction.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - | Some(TransactionCompletion::FailedAfterFencing { .. }) - ) - }); - for issuer in issuers { - let (Some(expected), Some(activation), Some(plan)) = - (&issuer.expected, &issuer.activation, &issuer.plan) - else { - continue; - }; - let demoted = (|| -> Result { - if topology.projected_activation(expected)?.as_ref() != Some(activation) { - return Ok(false); - } - let provider_anchor = self.provider_anchor_for_plan(plan)?; - topology.demote_to_expected_only(expected, &provider_anchor, activation, None)?; - Ok(true) - })(); - match demoted { - Ok(true) => outcome.demoted.push(issuer.transaction_id.clone()), - Ok(false) => {} - Err(error) => eprintln!( - "Idunn could not reconcile the projection of failed continuity {}: {error:#}", - issuer.transaction_id - ), - } - } - for generation in &snapshot.admitted { - let projected = match topology.projected_activation(&generation.value.expected) { - Ok(projected) => projected, - Err(error) => { - eprintln!( - "Idunn cannot read the projected activation of {}: {error:#}", - generation.value.target - ); - continue; - } - }; - let Some(projected) = projected else { - continue; - }; - let accounted = projected == generation.value.activation - || snapshot.transactions.iter().any(|stored| { - stored.value.target == generation.value.target - && !stored.value.is_terminal() - && stored.value.activation.as_ref() == Some(&projected) - }); - if !accounted { - outcome.unexplained.push(generation.value.target.clone()); - } - } - Ok(outcome) - } -} - -/// Everything the daemon does once, before its loop: lock, migrate, validate, -/// reconcile the projection. The lock is returned so the caller holds it for as -/// long as the engine runs. -fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { - for path in [ - &options.state_store, - &options.topology_store, - &options.staging_root, - ] { - let directory = if path.extension().is_some() { - path.parent() - .context("configured Idunn path has no parent")? - } else { - path.as_path() - }; - fs::create_dir_all(directory) - .with_context(|| format!("creating Idunn directory {}", directory.display()))?; - } - let lock = ProcessLock::acquire(&options.state_store)?; - let migrated = migrate_control_store_to_current_schema(&options.state_store) - .context("migrating Idunn control records to the current schema")?; - if migrated > 0 { - println!( - "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA}" - ); - } - ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; - let engine = Engine::open(options)?; - engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; - match engine.reconcile_failed_continuity_projections() { - Ok(outcome) => { - for transaction_id in &outcome.demoted { - println!("demoted the activation failed continuity {transaction_id} left projected"); - } - for target in &outcome.unexplained { - eprintln!( - "Idunn found an activation projected for {target} that no admitted generation, \ - live transaction or failed continuity accounts for; left as it is" - ); - } - } - Err(error) => eprintln!("Idunn boot projection reconciliation failed: {error:#}"), - } - Ok((lock, engine)) -} - -fn serve(options: RuntimeOptions) -> Result<()> { - let (_lock, engine) = boot(options)?; - serve_while(&engine, || true) -} - -/// The daemon's loop: tick, sleep, and go again while `again` says so. `serve` -/// says so forever; a test says so a few times. -fn serve_while(engine: &Engine, mut again: impl FnMut() -> bool) -> Result<()> { - while again() { - match engine.run_scheduler_tick() { - Ok(true) => continue, - Ok(false) => {} - // A fault in one transaction is that transaction's problem. Idunn - // runs under Restart=always, so propagating it here turns a single - // unreadable record into a crashloop that takes every unrelated - // target's continuity down with it -- the daemon-survival organ - // killed by the thing it exists to survive. Log against the tick - // and keep going; the fault recurs every poll until it is fixed, - // which is louder than a restart loop and cheaper than an outage. - Err(error) => eprintln!("Idunn scheduler tick failed: {error:#}"), - } - thread::sleep(Duration::from_millis(engine.options.poll_millis)); - } - Ok(()) -} - -impl Engine { - /// One resident terminal transaction goes to history per tick. - /// - /// Transactions that finished before finished transactions travelled to - /// history are still resident with their commands, gating nothing and - /// consuming their commands correctly, but growing the store every decision - /// reads. They leave the same way a transaction finishing today does. - fn retire_one_terminal_transaction(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - // A record that cannot be archived (history unreadable, a torn write) - // is that record's fault: the next terminal record is still tried, and - // the tick goes on to supervise and freeze. - for stored in snapshot - .transactions - .iter() - .filter(|stored| stored.value.is_terminal()) - { - match archive_terminal_transaction(&self.options.state_store, &stored.envelope) { - Ok(()) => { - eprintln!( - "Idunn retired resident terminal transaction {} to history", - stored.value.transaction_id - ); - return Ok(true); - } - Err(error) => self.note_fault( - "could not retire terminal transaction", - &stored.value.transaction_id, - &error, - ), - } - } - Ok(false) - } - - /// One pass of the scheduler. An unreadable control store is the tick's - /// fault and is returned; anything one transaction or one admitted - /// generation does wrong is logged against it and never stops the others. - fn run_scheduler_tick(&self) -> Result { - if self.retire_one_terminal_transaction()? { - return Ok(true); - } - let transaction_progress = self.resume_one_transaction()?; - let continuity_progress = self.supervise_one_admitted_generation()?; - // Freezing is not gated on the others' progress: a queued command - // whose target is free must not wait behind an unrelated target's - // wedge. `freeze_command` already refuses a busy target, so this is - // still at most one freeze per tick and never a second claimant. - let froze = self.freeze_one_queued_command()?; - Ok(transaction_progress || continuity_progress || froze) - } - - /// Say a per-record fault once while it lasts and leave it in the - /// record's `last_error`, which status renders. Recording is best effort: - /// a store too broken to take the note is already in stderr. - fn note_fault(&self, what: &str, transaction_id: &str, error: &anyhow::Error) { - let detail = truncate(&format!("{error:#}"), 2048); - let offered = { - let mut reports = self.fault_reports.lock().expect("fault report mutex"); - reports - .entry(transaction_id.to_string()) - .or_default() - .offer(Some(detail.clone())) - }; - let recorded = self.record_last_error(transaction_id, &detail); - if offered.is_some() { - eprintln!("Idunn {what} {transaction_id}: {detail}"); - if let Err(error) = recorded { - eprintln!("Idunn could not leave that in the record of {transaction_id}: {error:#}"); - } - } - } - - fn clear_fault(&self, transaction_id: &str) { - self.fault_reports - .lock() - .expect("fault report mutex") - .remove(transaction_id); - } - - /// Write `detail` as the record's `last_error`, touching nothing else and - /// not archiving: a terminal record whose archive is the fault stays put. - fn record_last_error(&self, transaction_id: &str, detail: &str) -> Result<()> { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let Some(stored) = snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == transaction_id) - else { - return Ok(()); - }; - if stored.value.last_error.as_deref() == Some(detail) { - return Ok(()); - } - let envelope = cleanup_evidence::last_error_envelope(&stored.value, detail)?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction_id.to_string(), - current: Some(stored.envelope.clone()), - }], - &[envelope], - )?, - "transaction changed before its fault note" - ); - Ok(()) - } - - /// Startup and every later loop use the same order: unfinished ownership - /// work first, admitted-body continuity second, new commands last. A - /// waiting transaction yields without relaxing ordinal order inside its - /// own command, so one target brake cannot suspend unrelated continuity. - fn resume_one_transaction(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; - let mut candidates = snapshot - .transactions - .iter() - .filter(|stored| !stored.value.is_terminal()) - .collect::>(); - candidates.sort_by_key(|stored| { - ( - stored.value.created_at_unix_millis, - stored.value.ordinal, - stored.value.transaction_id.as_str(), - ) - }); - for current in candidates { - if snapshot.has_earlier_authority_sibling(¤t.value) { - continue; - } - match self.resume_candidate(current) { - Ok(moved) => progressed |= moved, - Err(error) => { - self.note_fault( - "could not resume transaction", - ¤t.value.transaction_id, - &error, - ); - } - } - } - Ok(progressed) - } - - /// Advance one transaction, turning its failure into the durable record - /// its phase calls for. Returns whether its record changed. - /// - /// "Moved" means the transaction's state advanced: a phase change, a - /// completion, or a durable abort step. Rewriting `last_error` to say the - /// same step failed again in new words is not movement, or a wedged - /// transaction would report progress every tick. - fn resume_candidate(&self, current: &Stored) -> Result { - let id = current.value.transaction_id.as_str(); - if let Some(moved) = self.resolve_phase_deadline(current)? { - return Ok(moved); - } - if self - .resume_backoff - .lock() - .expect("resume backoff mutex") - .get(id) - .is_some_and(|backoff| { - now_millis().is_ok_and(|now| { - is_waiting(now, backoff.not_before_unix_millis, RESUME_BACKOFF_CEILING_MILLIS) - }) - }) - { - return Ok(false); - } - if let Some(disagreement) = current.value.held_disagreement() { - self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); - return Ok(false); - } - let advanced = self.advance_transaction(current); - if advanced.is_ok() { - // The step ran and succeeded: the fault, if any, has recovered, so - // a later failure is news. A skipped or failed attempt clears nothing. - self.resume_backoff - .lock() - .expect("resume backoff mutex") - .remove(id); - self.clear_fault(id); - } - if let Err(error) = advanced { - let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; - let latest = latest_snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while recording an execution error")?; - if latest.value.is_terminal() { - self.resume_backoff - .lock() - .expect("resume backoff mutex") - .remove(id); - return Ok(true); - } - self.back_off(id)?; - if latest.value.phase < DeploymentPhase::Fencing { - // Before the fence an abort that cannot finish a step is - // resumable, never a post-fence abort: that path refuses a - // pre-fence phase, so choosing it only wedged the tick. - // - // A publish that lost to a held lock is not a failed step: the - // holder lets go, so the transaction stays in its phase with - // the error recorded and the backoff above retries it. The - // phase deadline, not an abort, bounds the wait. - if latest.value.pre_fencing_abort.is_none() - && error.downcast_ref::().is_none() - { - self.begin_pre_fencing_abort(latest, error)?; - } else { - self.record_resumable_error(latest, &error)?; - } - } else if latest.value.post_fencing_abort.is_none() - && self.candidate_is_permanently_stopped(&latest.value)? - { - // Past the fence an error is resumable while the candidate - // can still recover. This one cannot: its transient unit - // has failed and carries Restart=no, so retrying would hold - // the target forever behind a transaction that can never - // finish. - self.begin_post_fencing_abort(latest, error)?; - } else { - // Resumable only until the phase deadline: the resolver ends - // the phase before the next step once it has passed. - self.record_resumable_error(latest, &error)?; - } - } - let after = ControlSnapshot::read(&self.options.state_store)?; - let live = after - .transactions - .iter() - .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while checking scheduler progress")?; - Ok(state_advanced(¤t.value, &live.value)) - } - - /// Wait twice as long after each consecutive failure of a transaction's - /// step, from one poll interval up to a ceiling. - fn back_off(&self, transaction_id: &str) -> Result<()> { - let now = now_millis()?; - let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); - let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); - let wait = backoff_wait(self.options.poll_millis, failures, RESUME_BACKOFF_CEILING_MILLIS); - all.insert( - transaction_id.to_string(), - ResumeBackoff { - failures: failures.saturating_add(1), - not_before_unix_millis: now.saturating_add(wait), - }, - ); - Ok(()) - } - - fn freeze_one_queued_command(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let live_commands = snapshot - .transactions - .iter() - .map(|stored| stored.value.command_id.clone()) - .collect::>(); - let mut candidates = snapshot - .commands - .iter() - .filter(|stored| { - stored.value.kind == CommandKind::Deploy - && !live_commands.contains(&stored.value.command_id) - }) - .collect::>(); - if candidates.is_empty() { - return Ok(false); - } - // A command is consumed by any transaction, live or retired. Commands - // whose transactions were retired to history before commands travelled - // with them are still resident; they are not queued, they are - // history that never moved. Retire them here and never freeze them. - // With history unreadable, whether a command was consumed is unknown: - // freezing it again could run a finished deployment twice. - let Some(history) = self.history_for_decision() else { - return Ok(false); - }; - let historical_commands = history - .into_iter() - .map(|transaction| transaction.command_id) - .collect::>(); - let mut retired = false; - candidates.retain(|stored| { - if !historical_commands.contains(&stored.value.command_id) { - return true; - } - match SingleFileMessagePackBackingStore::new(&history_store_path( - &self.options.state_store, - )) - .insert_entry_if_absent(stored.envelope.clone()) - .and_then(|_| { - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .delete_batch_if_unchanged(std::slice::from_ref(&stored.envelope)) - }) { - Ok(_) => { - eprintln!( - "Idunn retired consumed command {}: its transactions are history", - stored.value.command_id - ); - retired = true; - } - Err(error) => eprintln!( - "Idunn left consumed command {} resident: {error:#}", - stored.value.command_id - ), - } - false - }); - if retired { - return Ok(true); - } - candidates.sort_by_key(|stored| stored.value.requested_at_unix_millis); - // Oldest first, but a command whose target is busy does not block the - // ones behind it for other targets. - for command in candidates { - if self.freeze_command(&snapshot, command)? { - return Ok(true); - } - } - Ok(false) - } - - /// Freeze one queued command into its transactions. `Ok(false)` means its - /// target is busy and nothing was written. - fn freeze_command( - &self, - snapshot: &ControlSnapshot, - command: &Stored, - ) -> Result { - let bindings = match load_bindings(&self.options.bindings_dir) { - Ok(bindings) => bindings, - Err(error) => { - eprintln!( - "Idunn left command {} queued because operator bindings are invalid: {error:#}", - command.value.command_id - ); - return Ok(false); - } - }; - let targets = match resolve_selector(&bindings, &command.value.selector) { - Ok(targets) => targets, - Err(error) => { - let now = now_millis()?; - let rejected = DeploymentTransaction::rejected(&command.value, error, now)?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: rejected.transaction_id.clone(), - current: None, - }], - &[transaction_envelope(&rejected, now)?], - )?, - "bad selector changed before refusal was recorded" - ); - return Ok(true); - } - }; - let busy = snapshot - .transactions - .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) - .map(|stored| stored.value.target.as_str()) - .collect::>(); - if targets.iter().any(|target| busy.contains(target.as_str())) { - return Ok(false); - } - let now = now_millis()?; - let transactions = targets - .into_iter() - .enumerate() - .map(|(ordinal, target)| { - DeploymentTransaction::new( - &command.value, - target.clone(), - u32::try_from(ordinal)?, - snapshot.admitted_for(&target).map(|stored| &stored.value), - now, - ) - }) - .collect::>>()?; - let expected = transactions - .iter() - .map(|transaction| CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }) - .collect::>(); - let next = transactions - .iter() - .map(|transaction| transaction_envelope(transaction, now)) - .collect::>>()?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .compare_exchange(&expected, &next)?, - "queued command lost its ordered transaction creation CAS" - ); - Ok(true) - } - - fn supervise_one_admitted_generation(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - // A wait belongs to the failed record it spaces. Once any write for its - // target has landed, or the target is no longer admitted, the stored - // schedule owns the target again. - self.challenge_wait - .lock() - .expect("challenge wait mutex") - .retain(|target, wait| snapshot.target_stamp(target) == wait.stamp); - self.reconcile_incidents(&snapshot); - let mut progressed = false; - let mut admitted = snapshot.admitted.iter().collect::>(); - admitted.sort_by_key(|stored| stored.value.target.as_str()); - for current in admitted { - if let Err(disagreement) = current.value.readiness() { - self.note_fault( - "holds admitted generation", - &format!("generation:{}", current.value.target), - &anyhow::Error::new(disagreement), - ); - } - let blocker = snapshot.transactions.iter().find(|stored| { - stored.value.target == current.value.target - && stored.value.blocks_new_target_mutation() - }); - if blocker.is_some_and(|stored| { - let owns_from = if stored.value.rollout_stops_incumbent_first() { - // The incumbent is stopped at Starting by design; a - // continuity restart in that window would fight the - // candidate for the same host resources. - DeploymentPhase::Starting - } else { - DeploymentPhase::Fencing - }; - stored.value.phase >= owns_from && stored.value.phase < DeploymentPhase::Complete - }) { - continue; - } - match self.supervise_admitted_route(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn rejected admitted {} route continuity: {error:#}", - current.value.target - ), - } - match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved admitted {} after rejecting topology observation: {error:#}", - current.value.target - ), - } - match self.retire_stale_incarnations(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved a stale {} incarnation projection: {error:#}", - current.value.target - ), - } - // Restore the admitted Expected before anything is asked of the - // workload, and only the Expected. - // - // A target that cannot see its own Expected cannot start, and an - // aborted deployment leaves exactly that gap, so this cannot wait - // for an observation the dead process will never provide. The - // activation is deliberately not restored: it names one running - // incarnation, every restart issues a fresh one, and republishing - // the admitted one leaves the projection naming an incarnation that - // no longer exists -- which the workload rightly refuses to adopt. - // That record is published from observation once the process is up. - if blocker.is_none() - || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) - { - let repair = (|| -> Result<()> { - let topology = self.topology(); - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - if !topology.admitted_expected_projection_is_exact( - ¤t.value.expected, - &provider_anchor, - )? { - ensure!( - topology.publish_expected(¤t.value.expected, &provider_anchor)? - == current.value.expected.canonical_sha256()?, - "admitted Expected projection repair differs" - ); - } - Ok(()) - })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target - ); - } - } - let mut operational_error = None; - let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { - workload.observe( - ¤t.value.expected, - ¤t.value.activation, - ¤t.value.workload, - ) - }) { - Ok(observation) => { - let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { - let lease_health = (|| -> Result { - let lease_path = current - .value - .plan - .parsed_inputs()? - .1 - .process_write_lease - .context("admitted lease has no operator binding")? - .record_path; - let driver = - CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - if driver.observe_exact(lease)? { - return Ok(true); - } - if driver.observe_empty()? { - return Ok(false); - } - bail!("write-lease store contains unexpected authority") - })(); - match lease_health { - Ok(true) => false, - Ok(false) => { - operational_error = - Some(anyhow!("admitted physical write lease is missing")); - true - } - Err(error) => { - eprintln!( - "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", - current.value.target - ); - continue; - } - } - } else { - false - }; - if lease_is_missing { - None - } else { - Some(observation) - } - } - Err(error) if error.downcast_ref::().is_some() => { - // The host cannot be asked right now (its actuator is - // between sessions, or Idunn just restarted). That is - // silence, not a death; counting it as one burned every - // continuity attempt in the reattach window on - // 2026-09-11 and left a healthy Muninn demoted. - eprintln!( - "Idunn cannot observe admitted {} this tick: {error:#}", - current.value.target - ); - continue; - } - Err(error) => { - operational_error = Some(error); - None - } - }; - if observation.is_some() { - let seen = snapshot.supervision_for(¤t.value.target); - if let Some(stored) = seen.filter(|stored| { - stored.value.continuity_backoff_from_unix_millis.is_some() - }) { - let mut next = stored.value.clone(); - next.end_backoff(); - self.write_target_supervision(Some(stored), &next, now_millis()?)?; - } - // The admitted workload was seen running: the only evidence - // that ends an exhaustion incident, once the window no longer holds it. - // The window sliding alone is not health and never closes one. - self.record_incident( - "close", - ¤t.value.target, - now_millis().and_then(|now| { - // Running is not enough while the window still holds - // the restarts that exhausted it: a release that - // survives one poll after each restart is still the - // same incident. - if snapshot - .supervision_or_new(¤t.value.target) - .restarts_page(now) - { - return Ok(false); - } - self.incidents().close( - IncidentCondition::ContinuityExhausted, - ¤t.value.target, - CloseReason::Recovered, - now, - ) - }), - ); - continue; - } - let workload_error = operational_error - .context("admitted operational state has neither observation nor error")?; - // A held generation is a record the recipe never declared - // readiness for. Idunn reports it and does not decide for it: a - // continuity minted over it copies the undeclared Expected, is - // held at once, and owns the target forever, so the declaring - // redeploy that would clear the hold could never freeze. The dead - // held target stays down and free. - if current.value.readiness().is_err() { - self.note_fault( - "holds a dead admitted generation down; declare readiness in its recipe and redeploy", - &format!("generation-down:{}", current.value.target), - &anyhow!("admitted generation {} is not running", current.value.generation_id), - ); - continue; - } - // Continuity restarts a release; it cannot repair one, and it - // never stops (budget-never-stop). Every charged attempt is spaced - // by a doubling, capped wait, so a queued deployment always gets - // a free gap (map B10). The log outlives every generation and only - // pages. Nothing here reads history: an unreadable history file - // must not stop crash recovery. - let now = now_millis()?; - let supervision = snapshot.supervision_or_new(¤t.value.target); - // A stepped-back clock leaves entries in the future. Settle them - // before deciding from them; the write ends this target's pass. - let mut settled = supervision.clone(); - settled.settle(now); - if settled != supervision { - self.write_target_supervision( - snapshot.supervision_for(¤t.value.target), - &settled, - now, - )?; - progressed = true; - continue; - } - let continuity_key = format!("continuity:{}", current.value.target); - - // Continuity defers to any live transaction for its target and - // never aborts one (handover-phase-split). - if blocker.is_some() { - continue; - } - if supervision.restarts_page(now) { - // The record comes first and the trace after it: the trace is - // this record's shadow, never its substitute. - self.record_incident( - "open", - ¤t.value.target, - self.incidents().open( - IncidentCondition::ContinuityExhausted, - ¤t.value.target, - now, - ), - ); - self.report_once( - &continuity_key, - format!( - "Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff (streak {}): {workload_error:#}", - current.value.target, - supervision.restarts_used(now), - supervision.continuity_attempt_streak, - ), - ); - } else { - self.clear_fault(&continuity_key); - } - if supervision.continuity_deferred(now) { - continue; - } - // A unit the host lost to a reboot is re-materialized once per - // generation, uncharged and with no wait; the driver owns the - // evidence and supervision owns the budget. An unanswerable - // driver is not evidence of a reboot. - let admitted_at = current.value.admitted_at_unix_millis; - let rematerialize = !supervision.rematerialization_spent(admitted_at) - && match self.workload_for(¤t.value.plan).and_then(|workload| { - workload.lost_to_host_reboot(¤t.value.workload, admitted_at) - }) { - Ok(lost) => lost, - Err(error) => { - self.note_fault( - "could not tell whether the host rebooted under", - &format!("reboot:{}", current.value.target), - &error, - ); - false - } - }; - let mut streak = 0; - if !rematerialize { - match supervision.continuity_backoff(now, admitted_at) { - ContinuityBackoff::Waiting => continue, - ContinuityBackoff::Begin => { - let mut began = supervision.clone(); - began.begin_backoff(now); - self.write_target_supervision( - snapshot.supervision_for(¤t.value.target), - &began, - now, - )?; - progressed = true; - continue; - } - ContinuityBackoff::Due { streak: due } => streak = due, - } - } - - // The projected activation names an incarnation that is gone. Every - // restart is issued a fresh one, so leaving the old record standing - // makes the projection describe a process that no longer exists -- - // and a workload that reads it refuses to adopt an activation that - // is not its own, which is a restart loop rather than a recovery. - // Demote to Expected-only, the same shape an aborted deployment - // leaves, and let the restart publish its own activation once it is - // observed. - let demotion = (|| -> Result<()> { - let topology = self.topology(); - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - if !topology.projected_activation_is_present(¤t.value.expected)? { - return Ok(()); - } - topology.demote_to_expected_only( - ¤t.value.expected, - &provider_anchor, - ¤t.value.activation, - current.value.leasing.lease(), - )?; - Ok(()) - })(); - // Expected-only is the precondition of a restart: a continuity - // minted over a projection that still names an activation would - // fail between preparing and publishing its own, and its abort - // could not resolve. So a failed demotion mints nothing. The - // failure is written to the generation, and the next attempt is - // not before the deferral ends. - if let Err(error) = demotion { - eprintln!( - "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", - current.value.target - ); - let mut deferred = supervision.clone(); - deferred.defer_continuity(now, &truncate(&format!("{error:#}"), 2048)); - self.write_target_supervision( - snapshot.supervision_for(¤t.value.target), - &deferred, - now, - )?; - progressed = true; - continue; - } - // An engaged lifecycle brake means no continuity transaction is - // minted at all. A transaction that exists and waits on the brake - // owns the target, and a target owned by a parked restart accepts - // no deployment -- which is the one thing a lifecycle brake must - // never gate. The Sealing-phase check remains as the guard for a - // transaction minted just before the brake was engaged. - if !self.lifecycle_allows_generation(current, now)? { - continue; - } - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("continuity-{}", Uuid::new_v4()), - kind: CommandKind::Continuity, - selector: current.value.target.clone(), - requested_by: "idunn-continuity".into(), - requested_at_unix_millis: now, - }; - command.validate()?; - let transaction = - DeploymentTransaction::from_continuity(&command, ¤t.value, now)?; - // The attempt is charged, or recorded as the one re-materialization, - // in the same CAS that mints it, so a crash cannot schedule one the - // meters never saw. - let mut counted = supervision; - if rematerialize { - counted.record_rematerialization(now); - } else { - counted.record_restart(now, streak); - } - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: TargetSupervision::TYPE.into(), - key: counted.target.clone(), - current: snapshot - .supervision_for(&counted.target) - .map(|stored| stored.envelope.clone()), - }, - ], - &[ - command_envelope(&command, now)?, - transaction_envelope(&transaction, now)?, - target_supervision_envelope(&counted, now)?, - ], - )?, - "continuity scheduling lost its command/transaction/meter CAS" - ); - progressed = true; - } - Ok(progressed) - } - - fn supervise_admitted_route( - &self, - snapshot: &ControlSnapshot, - current: &Stored, - ) -> Result { - let Some(expected_route) = current.value.expected.route.as_ref() else { - ensure!( - matches!(¤t.value.routing, RoutingEvidence::SkippedUnrouted), - "unrouted admitted generation carries route authority" - ); - return Ok(false); - }; - let RoutingEvidence::Promoted { - observation, - promoted_at_unix_millis, - } = ¤t.value.routing - else { - bail!("routed admitted generation has no promoted route receipt") - }; - let now = now_millis()?; - // A route whose proofs keep failing is challenged on a widening - // schedule, not every tick. - if current - .value - .route_supervision - .as_ref() - .is_some_and(|state| state.is_waiting(now, self.options.topology_maximum_age_millis)) - { - return Ok(false); - } - if self - .challenge_wait - .lock() - .expect("challenge wait mutex") - .get(¤t.value.target) - .is_some_and(|wait| { - is_waiting( - now, - wait.due_unix_millis, - ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(self.options.topology_maximum_age_millis), - ) - }) - { - return Ok(false); - } - ensure!( - observation.route_id == expected_route.route_id - && observation.runtime_instance_id == current.value.activation.runtime_instance_id, - "admitted route receipt names another incarnation" - ); - let binding = current.value.plan.parsed_inputs()?.1; - let lease_driver = if let Some(lease) = current.value.leasing.lease() { - let lease_path = &binding - .process_write_lease - .as_ref() - .context("stateful admitted generation has no write-lease binding")? - .record_path; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - ensure!( - driver.observe_exact(lease)?, - "admitted process write lease is no longer exact" - ); - Some(driver) - } else { - None - }; - let route_binding = binding - .route - .context("routed admitted generation has no operator route binding")?; - let driver = self.route_driver(route_binding); - let gate = self.route_gate(¤t.value.target, CommandKind::Continuity); - - if driver.observe_membership(¤t.value.expected, &observation.membership_sha256)? - && route_observation_is_current( - observation.observed_at_unix_millis, - now, - self.options.topology_maximum_age_millis, - self.options.topology_maximum_future_skew_millis, - ) - { - return Ok(false); - } - let route_key = format!("route:{}", current.value.target); - // A failed repair or a failed proof changes observation state only: it - // marks the route degraded and widens the wait. That record is written - // like a proved one: when it lands the pass ends (`Ok(true)`), because - // the caller's snapshot no longer matches the generation; when the store - // refuses it the pass reports no progress and the in-memory wait below - // spaces the next attempt. - let challenge = (|| -> Result { - // Actuates only when the fragment on disk differs from the - // admitted membership; an exact fragment makes this a no-op. It is - // survival: restoring the admitted route is never refused. - driver.restore_admitted_membership( - ¤t.value.expected, - &observation.membership_sha256, - &gate, - )?; - let authority = self.runtime_authority_parts( - ¤t.value.plan, - ¤t.value.expected, - ¤t.value.activation, - )?; - let refreshed = self.prove_stable_route_against( - ¤t.value.expected, - ¤t.value.activation, - &authority, - current.value.leasing.lease_sha256(), - &driver, - observation.membership_sha256.clone(), - )?; - ensure!( - driver.observe_membership(¤t.value.expected, &refreshed.membership_sha256)?, - "admitted route membership changed during its continuity challenge" - ); - if let (Some(lease_driver), Some(lease)) = (&lease_driver, current.value.leasing.lease()) - { - ensure!( - lease_driver.observe_exact(lease)?, - "admitted process write lease changed during its continuity challenge" - ); - } - Ok(refreshed) - })(); - let mut next = current.value.clone(); - // The wait the record carries, also held in memory until a record lands. - let (recorded, wait_until) = match challenge { - Ok(refreshed) => { - next.routing = RoutingEvidence::Promoted { - observation: refreshed, - promoted_at_unix_millis: *promoted_at_unix_millis, - }; - if let Some(state) = next.route_supervision.as_mut() { - state.record_proved_challenge(now); - } - self.clear_fault(&route_key); - ( - "proved", - now.saturating_add(self.options.topology_maximum_age_millis), - ) - } - Err(error) => { - if let Some(state) = next.route_supervision.as_mut() { - state.record_failed_challenge(now, self.options.topology_maximum_age_millis); - } - self.report_once( - &route_key, - format!( - "Idunn could not prove admitted {} on its stable route: {error:#}", - current.value.target - ), - ); - let due = next - .route_supervision - .as_ref() - .and_then(|state| state.next_challenge_at_unix_millis) - .unwrap_or(now); - ("failed", due) - } - }; - // Progress is a store that changed: our record landed, or another - // writer beat it (the snapshot is stale either way, so the pass ends). - // A write the store refused changed nothing: no progress, the scheduler - // sleeps if nothing else moved, and the in-memory wait spaces the next - // challenge. - let record_key = format!("route-record:{}", current.value.target); - match self.replace_generation(current, &next, now) { - Ok(_) => { - self.challenge_wait - .lock() - .expect("challenge wait mutex") - .remove(¤t.value.target); - self.clear_fault(&record_key); - Ok(true) - } - Err(record) => { - self.challenge_wait - .lock() - .expect("challenge wait mutex") - .insert( - current.value.target.clone(), - ChallengeWait { - due_unix_millis: wait_until, - stamp: snapshot.target_stamp(¤t.value.target), - }, - ); - self.report_once( - &record_key, - format!( - "Idunn could not record the {recorded} route challenge of {}: {}", - current.value.target, - // The root cause only: the error chain names a fresh temp file each - // attempt, which would defeat once-per-fault. - record.root_cause() - ), - ); - Ok(false) - } - } - } - - fn refresh_admitted_topology( - &self, - snapshot: &ControlSnapshot, - current: &Stored, - ) -> Result { - if !matches!(current.value.readiness(), Ok(class) if class != ReadinessClass::RouteProof) { - return Ok(false); - } - let odin = current.value.odin_receipts()?; - let Some(received) = self.topology().receive( - ¤t.value.target, - ¤t.value.expected.canonical_sha256()?, - )? - else { - return Ok(false); - }; - ensure!( - received.target == current.value.target, - "topology transport substituted admitted target" - ); - let now = now_millis()?; - let authority = self.runtime_authority_parts( - ¤t.value.plan, - ¤t.value.expected, - ¤t.value.activation, - )?; - let odin_authority = self.current_odin_authority(snapshot)?; - let authenticated = match authenticate_odin_runtime_topology_correlation( - &received.canonical_bytes, - &authority, - current.value.leasing.lease_sha256(), - &odin_authority.signer_public_key, - self.trusted_topology_context(now), - ) { - Ok(authenticated) => authenticated, - // An aged-out correlation is nothing new about the admitted - // generation, not a rejected observation. - Err(error) if is_stale_observation(&error) => return Ok(false), - Err(error) => return Err(error), - }; - let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; - if !sequence_requires_admission( - Some(odin.latest), - snapshot.max_odin_sequence(¤t.value.target, &evidence.signer_identity_id), - &evidence, - )? { - return Ok(false); - } - let mut next = current.value.clone(); - next.latest_odin_observation = Some(evidence.clone()); - next.odin_publisher_sequence_cursor = evidence.publisher_sequence; - if is_semantic_ready(&authenticated) { - next.ready = ReadinessEvidence::OdinCorrelated { evidence }; - } - next.validate()?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: current.value.target.clone(), - current: Some(current.envelope.clone()), - }], - &[admitted_envelope(&next, now)?], - )?, - "admitted generation changed before topology sequence CAS" - ); - Ok(true) - } - - fn advance_transaction(&self, current: &Stored) -> Result<()> { - if current.value.post_fencing_abort.is_some() && current.value.completion.is_none() { - return self.advance_post_fencing_abort(current); - } - if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() { - return self.advance_pre_fencing_abort(current); - } - match current.value.phase { - DeploymentPhase::Sealing => self.advance_sealing(current), - DeploymentPhase::Starting => self.advance_starting(current), - DeploymentPhase::Warming => self.advance_warming(current), - DeploymentPhase::Fencing => self.advance_fencing(current), - DeploymentPhase::Leasing => self.advance_leasing(current), - DeploymentPhase::AwaitingReady => self.advance_awaiting_ready(current), - DeploymentPhase::Routing => self.advance_routing(current), - DeploymentPhase::Committing => self.advance_committing(current), - DeploymentPhase::Complete => self.advance_post_commit_cleanup(current), - } - } - - fn advance_sealing(&self, current: &Stored) -> Result<()> { - if current.value.plan.is_none() { - ensure!( - current.value.command_kind == CommandKind::Deploy, - "continuity transaction lost its admitted plan" - ); - let bindings = load_bindings(&self.options.bindings_dir)?; - let loaded = bindings - .get(¤t.value.target) - .context("operator binding disappeared before sealing")?; - let now = now_millis()?; - let resolved = - self.source - .resolve(&loaded.binding, ¤t.value.transaction_id, now)?; - let provider_snapshot = ControlSnapshot::read(&self.options.state_store)?; - let providers = self.current_ready_provider_tokens(&provider_snapshot)?; - let candidate_port = self.select_candidate_port(&loaded.binding)?; - let plan = compile_deployment_plan( - &resolved.recipe_bytes, - &loaded.bytes, - resolved.facts, - format!("incarnation-{}", current.value.transaction_id), - candidate_port, - now, - &providers, - )?; - // Idunn infers no way to prove readiness. A recipe that declares - // none is refused here, before anything is frozen, built or installed. - plan.readiness_class()?; - return self.persist_same_phase(current, |next| { - next.plan = Some(plan); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - - if current.value.command_kind == CommandKind::Deploy - && current.value.frozen_source.is_none() - { - let now = now_millis()?; - let plan = required(¤t.value.plan, "transaction plan")?; - let receipt = self.source.freeze(¤t.value.transaction_id, plan)?; - return self.persist_same_phase(current, |next| { - next.frozen_source = Some(receipt); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - - if current.value.sealed_release.is_none() - || current.value.installed_release.is_none() - || current.value.expected.is_none() - { - ensure!( - current.value.command_kind == CommandKind::Deploy, - "continuity transaction lost its admitted release evidence" - ); - let now = now_millis()?; - let plan = required(¤t.value.plan, "transaction plan")?; - let frozen_receipt = required(¤t.value.frozen_source, "frozen source")?; - let frozen = self.source.observe_frozen(plan, frozen_receipt)?; - let materialized = self.runner_for(plan)?.materialize( - &frozen, - plan, - &self.options.staging_root, - now, - )?; - let installed = self.workload_for(plan)?.install(plan, &materialized)?; - let expected = materialized.release.expected_projection(plan)?; - return self.persist_same_phase(current, |next| { - next.sealed_release = Some(materialized.release); - next.installed_release = Some(installed); - next.expected = Some(expected); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - - // The candidate's Expected is published in Starting, after the brake - // has admitted this exact transaction. Publishing it is the first - // Verse-visible change a deployment makes, and the brake gates changes. - let now = now_millis()?; - let mut next = current.value.clone(); - match current.value.command_kind { - CommandKind::Deploy => { - let Some(authorization) = self.deployment_authorization(current, now)? else { - return self.record_gate_wait( - current, - "deployment brake has not released this exact transaction", - ); - }; - next.deployment_authorization = Some(authorization); - } - CommandKind::Continuity => { - if !self.lifecycle_allows(current, now)? { - return self - .record_gate_wait(current, "lifecycle brake denies continuity restart"); - } - next.lifecycle_authorized_at_unix_millis = Some(now); - } - } - validate_live_providers_for_deploy(current.value.command_kind, || { - self.validate_selected_providers_current(required( - ¤t.value.plan, - "transaction plan", - )?) - })?; - next.enter_phase(DeploymentPhase::Starting, now); - next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) - } - - fn advance_starting(&self, current: &Stored) -> Result<()> { - let expected = required(¤t.value.expected, "Expected projection")?; - let plan = required(¤t.value.plan, "transaction plan")?; - let release = required(¤t.value.sealed_release, "sealed release")?; - let installed = required(¤t.value.installed_release, "installed release")?; - if current.value.expected_publication_sha256.is_none() { - // Published under the candidate's own incarnation key, beside - // whatever the target's admitted incarnation currently projects. - // Nothing the incumbent reads about itself changes here. - let now = now_millis()?; - let provider_anchor = self.provider_anchor_for_plan(plan)?; - let digest = self - .topology() - .publish_expected(expected, &provider_anchor)?; - return self.persist_same_phase(current, |next| { - next.expected_publication_sha256 = Some(digest); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - if current.value.activation.is_none() { - validate_live_providers_for_deploy(current.value.command_kind, || { - self.validate_selected_providers_current(plan) - })?; - let now = now_millis()?; - let runtime_instance_id = runtime_instance_id(¤t.value.transaction_id)?; - let launch = IdunnRuntimeActivationLaunch::issue( - expected, - runtime_instance_id, - now, - &self.idunn_signer, - )?; - let activation = self - .workload_for(plan)? - .prepare_activation(plan, expected, launch)?; - return self.persist_same_phase(current, |next| { - next.activation = Some(activation); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - if current.value.workload.is_none() { - let now = now_millis()?; - let activation = required(¤t.value.activation, "activation")?; - if current.value.rollout_stops_incumbent_first() { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? { - self.workload_for(&incumbent.value.plan)? - .stop(&incumbent.value.workload) - .context("stopping the incumbent before the candidate starts")?; - } - } - let observation = self - .workload_for(plan)? - .start_prepared(plan, release, installed, expected, activation)?; - return self.persist_same_phase(current, |next| { - next.workload = Some(observation); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - if current.value.activation_publication_sha256.is_none() { - let now = now_millis()?; - let activation = required(¤t.value.activation, "activation")?; - let workload = required(¤t.value.workload, "workload")?; - self.workload_for(required(¤t.value.plan, "transaction plan")?)? - .observe(expected, activation, workload)?; - let digest = self - .topology() - .publish_observed_activation(expected, activation, workload)?; - return self.persist_same_phase(current, |next| { - next.activation_publication_sha256 = Some(digest); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - let mut next = current.value.clone(); - next.enter_phase(DeploymentPhase::Warming, now_millis()?); - next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) - } - - fn advance_warming(&self, current: &Stored) -> Result<()> { - let expected = required(¤t.value.expected, "Expected projection")?; - let activation = required(¤t.value.activation, "activation")?; - let workload = required(¤t.value.workload, "workload")?; - self.workload_for(required(¤t.value.plan, "transaction plan")?)? - .observe(expected, activation, workload)?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent_lease_sha256 = - self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; - let class = ReadinessClass::of(expected)?; - if current.value.warming.is_none() && class == ReadinessClass::RouteProof { - // A route-proof target's warming is Idunn's own challenge to the - // candidate endpoint. No Odin is read: the target declared no - // reason to be aware of one. - return match self.challenge_candidate( - ¤t.value, - route_proof_warming_states(expected), - None, - )? { - CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), - CandidateAnswer::Answered { evidence, present } => { - let warming = WarmingEvidence::RouteProofDirect { evidence }; - let _token = SequenceAdmittedWarming::from_direct_presence( - current.value.transaction_id.clone(), - warming.clone(), - present, - )?; - self.persist_same_phase(current, |next| { - next.warming = Some(warming); - Ok(()) - }) - } - }; - } - if current.value.warming.is_none() { - // Odin's warming is observed directly when nothing else can observe - // it. That is the first bootstrap, and it is also every continuity - // restart of Odin: the incumbent that would do the observing is the - // very process being restarted, so it cannot report on its own - // return. A deployment of Odin is not included -- there the healthy - // incumbent observes the candidate, which is the point. - // Odin's warming is observed directly exactly when no Odin can - // observe it: the first bootstrap, a continuity restart of Odin - // itself, or any moment the admitted Odin has no usable correlation - // to report through. The rule is the same one that justified the - // bootstrap case -- "observed through Odin" is not available -- and - // it stays scoped to this one target. Every other target keeps - // waiting for Odin, which is what makes Odin the root of the chain. - // - // Asked once. Admitting the latest topology persists the sequence - // cursor, so asking twice with the same record fails the second - // time as "transaction changed before topology admission". - let is_odin = class == ReadinessClass::OdinSelf; - let odin_observation = if is_odin - && snapshot.admitted_odin().is_some() - && current.value.command_kind != CommandKind::Continuity - { - self.admit_latest_topology(current, None)? - } else { - None - }; - let odin_observes_itself = is_odin && odin_observation.is_none(); - if odin_observes_itself { - ensure!( - expected.write_lease_required, - "direct Odin warming must be a stateful incarnation" - ); - let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let warming = WarmingEvidence::FirstOdinDirect { evidence }; - let _token = SequenceAdmittedWarming::from_direct_presence( - current.value.transaction_id.clone(), - warming.clone(), - present, - )?; - return self.persist_same_phase(current, |next| { - next.warming = Some(warming); - Ok(()) - }); - } - let observation = match odin_observation { - Some(observation) => Some(observation), - None => self.admit_latest_topology(current, None)?, - }; - let Some((admitted, authenticated)) = observation else { - // Only the first-Odin bootstrap above observes presence - // directly. Every other target's warming presence arrives as - // Odin's authenticated runtime topology correlation, so until - // Odin is admitted and publishing, a candidate warms forever - // with nothing to say why. - return self.record_gate_wait( - current, - "no authenticated Odin topology correlation yet: warming presence for a \ - non-Odin target is observed through Odin, which must be admitted first", - ); - }; - if admitted.envelope != current.envelope { - return Ok(()); - } - let semantic_warming = is_semantic_warming( - expected, - activation, - incumbent_lease_sha256.as_deref(), - &authenticated, - )?; - let latest = required( - &admitted.value.latest_odin_observation, - "sequence-admitted warming evidence", - )?; - if semantic_warming { - let evidence = latest.clone(); - let _token = SequenceAdmittedWarming::from_topology( - admitted.value.transaction_id.clone(), - authenticated, - )?; - return self.persist_same_phase(&admitted, |next| { - next.warming = Some(WarmingEvidence::OdinTopology { evidence }); - Ok(()) - }); - } else { - return Ok(()); - } - } - let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; - ensure!( - warming.transaction_id() == current.value.transaction_id - && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), - "durable Warming evidence belongs to another candidate" - ); - - if expected.route.is_some() && current.value.route_preflight.is_none() { - let now = now_millis()?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; - let incumbent_route = - incumbent.and_then(|generation| generation.value.routing.observation().cloned()); - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let route_binding = binding - .route - .context("routed Expected lost route binding")?; - let driver = self.route_driver(route_binding); - let receipt = driver.preflight( - expected, - &activation.runtime_instance_id, - incumbent_route.as_ref(), - &self.route_gate(¤t.value.target, current.value.command_kind), - )?; - return self.persist_same_phase(current, |next| { - next.route_preflight = Some(receipt); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - - if current.value.isolation.is_none() { - let now = now_millis()?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; - // Isolation is a property of two processes running at once. A - // stopped incumbent has already released its DynamicUser UID, and - // systemd is free to hand that same UID to the candidate -- so - // comparing the candidate against the incumbent's *recorded* - // identity turns ordinary UID reuse into a permanent refusal to - // fence. There is nothing left to be isolated from. - let incumbent_workload = match incumbent { - Some(value) - if self - .workload_for(&value.value.plan)? - .is_permanently_stopped(&value.value.workload)? => - { - None - } - other => other.map(|value| &value.value.workload), - }; - let isolation = WorkloadObservation::prove_isolation(workload, incumbent_workload)?; - return self.persist_same_phase(current, |next| { - next.isolation = Some(isolation); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - - self.transition(current, DeploymentPhase::Fencing) - } - - fn advance_fencing(&self, current: &Stored) -> Result<()> { - if current.value.fencing.is_none() { - let now = now_millis()?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; - let incumbent_lease = incumbent.and_then(|generation| generation.value.leasing.lease()); - let expected = required(¤t.value.expected, "Expected projection")?; - let evidence = if expected.write_lease_required || incumbent_lease.is_some() { - let incumbent_lease_path = incumbent_lease - .map(|_| -> Result { - Ok(incumbent - .context("incumbent lease has no admitted generation")? - .value - .plan - .parsed_inputs()? - .1 - .process_write_lease - .context("incumbent lease has no admitted binding")? - .record_path) - }) - .transpose()?; - if let (Some(lease), Some(path)) = (incumbent_lease, &incumbent_lease_path) { - let incumbent = incumbent.context("incumbent lease lost its generation")?; - self.workload_for(&incumbent.value.plan)? - .stop(&incumbent.value.workload) - .context("stopping the exact incumbent before revoking its lifetime-held write lease")?; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); - driver.revoke_exact(Some(lease))?; - ensure!( - driver.observe_empty()?, - "incumbent write lease remained after exact fencing" - ); - self.topology().withdraw_process_write_lease( - &incumbent.value.expected, - &incumbent.value.activation, - Some(lease), - )?; - } - let candidate_lease_path = if expected.write_lease_required { - Some( - current - .value - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1 - .process_write_lease - .context("stateful candidate has no write-lease binding")? - .record_path, - ) - } else { - None - }; - if let Some(path) = &candidate_lease_path { - if incumbent_lease_path.as_ref() != Some(path) { - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); - driver.revoke_exact(None)?; - ensure!( - driver.observe_empty()?, - "candidate write-lease path was not empty before grant" - ); - } - } - self.topology().withdraw_process_write_lease( - expected, - required(¤t.value.activation, "candidate activation")?, - None, - )?; - FencingEvidence::Revoked { - incumbent_lease_sha256: incumbent_lease - .map(IdunnProcessWriteLeaseRecord::canonical_sha256) - .transpose()?, - candidate_lease_path_verified_empty: candidate_lease_path.is_some(), - } - } else { - FencingEvidence::SkippedStateless - }; - return self.persist_same_phase(current, |next| { - next.fencing = Some(evidence); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - self.transition(current, DeploymentPhase::Leasing) - } - - fn advance_leasing(&self, current: &Stored) -> Result<()> { - let expected = required(¤t.value.expected, "Expected projection")?; - if !expected.write_lease_required { - if current.value.leasing.is_none() { - let now = now_millis()?; - return self.persist_same_phase(current, |next| { - next.leasing = Some(LeasingEvidence::SkippedStateless); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - return self.transition(current, DeploymentPhase::AwaitingReady); - } - - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let lease_path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - - if matches!( - current.value.leasing.as_ref(), - Some(LeasingEvidence::Granted { .. }) - ) { - let activation = required(¤t.value.activation, "activation")?; - let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; - let lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease) - .context("granted Leasing evidence has no write lease")?; - let recorded_sha256 = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("granted Leasing evidence has no lease digest")?; - ensure!( - driver.observe_exact(lease)?, - "physical write lease disappeared after Granted became durable" - ); - self.workload_for(required(¤t.value.plan, "transaction plan")?)? - .observe( - expected, - activation, - required(¤t.value.workload, "candidate workload")?, - )?; - ensure!( - driver.grant(expected, activation, &warming, lease)? == recorded_sha256, - "replayed physical write lease differs from Granted evidence" - ); - ensure!( - self.topology() - .publish_process_write_lease(expected, activation, lease)? - == recorded_sha256, - "replayed write-lease projection differs from Granted evidence" - ); - return self.transition(current, DeploymentPhase::AwaitingReady); - } - - if let Some((lease, prepared_sha256)) = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::prepared_lease) - { - let now = now_millis()?; - let activation = required(¤t.value.activation, "activation")?; - let historical_warming = self.rehydrate_warming_token(¤t.value, now, false)?; - let physical_is_exact = driver.observe_exact(lease)?; - let warming = if physical_is_exact { - historical_warming - } else { - ensure!( - driver.observe_empty()?, - "candidate write-lease store contains authority other than its durable prepared lease" - ); - match self.rehydrate_warming_token(¤t.value, now, true) { - Ok(warming) => warming, - Err(_) => { - let Some((admitted, fresh_evidence, fresh_warming)) = - self.fresh_warming_for_lease(current, now)? - else { - return Ok(()); - }; - let replacement = self.prepare_candidate_write_lease( - &admitted.value, - &fresh_warming, - now, - )?; - let replacement_sha256 = replacement.canonical_sha256()?; - return self.persist_same_phase(&admitted, |next| { - next.warming = Some(fresh_evidence); - next.leasing = Some(LeasingEvidence::Prepared { - lease: replacement, - lease_sha256: replacement_sha256, - }); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - } - }; - self.workload_for(required(¤t.value.plan, "transaction plan")?)? - .observe( - expected, - activation, - required(¤t.value.workload, "candidate workload")?, - )?; - if !physical_is_exact { - ensure!( - driver.observe_empty()?, - "candidate write-lease store changed before physical grant" - ); - } - let granted_sha256 = driver.grant(expected, activation, &warming, lease)?; - ensure!( - granted_sha256 == prepared_sha256, - "granted write lease differs from the durable prepared lease" - ); - let projected_sha256 = self - .topology() - .publish_process_write_lease(expected, activation, lease)?; - ensure!( - projected_sha256 == granted_sha256, - "projected write lease differs from the granted process authority" - ); - let lease = lease.clone(); - return self.persist_same_phase(current, |next| { - next.leasing = Some(LeasingEvidence::Granted { - lease, - lease_sha256: projected_sha256, - }); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - - ensure!( - current.value.leasing.is_none(), - "stateful Leasing phase carries invalid lease evidence" - ); - ensure!( - driver.observe_empty()?, - "candidate write-lease store contains authority before lease preparation" - ); - let now = now_millis()?; - let Some((admitted, fresh_evidence, fresh_warming)) = - self.fresh_warming_for_lease(current, now)? - else { - return Ok(()); - }; - let lease = self.prepare_candidate_write_lease(&admitted.value, &fresh_warming, now)?; - let lease_sha256 = lease.canonical_sha256()?; - self.persist_same_phase(&admitted, |next| { - next.warming = Some(fresh_evidence); - next.leasing = Some(LeasingEvidence::Prepared { - lease, - lease_sha256, - }); - next.updated_at_unix_millis = now; - Ok(()) - }) - } - - fn advance_awaiting_ready(&self, current: &Stored) -> Result<()> { - let now = now_millis()?; - let expected = required(¤t.value.expected, "Expected projection")?; - self.observe_candidate_before_waiting(¤t.value)?; - if expected.write_lease_required { - let activation = required(¤t.value.activation, "activation")?; - let warming = self.rehydrate_warming_token(¤t.value, now, false)?; - let lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease) - .context("stateful transaction has no process write lease")?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - ensure!( - CultCacheWriteLeaseDriver::new(¤t.value.target, path) - .observe(expected, activation, &warming, lease)?, - "candidate process write lease is no longer exact" - ); - } - if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { - // Ready is the candidate answering Idunn's own challenge Active, - // holding the exact lease Idunn granted. Once recorded it is - // history; the stable route's challenge is what keeps proving it. - if matches!( - current.value.ready, - Some(ReadinessEvidence::RouteProof { .. }) - ) { - return self.transition(current, DeploymentPhase::Routing); - } - let lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - return match self.challenge_candidate(¤t.value, &["warming", "active"], lease)? { - CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), - CandidateAnswer::Answered { present, .. } if present.record().state == "warming" => { - self.record_gate_wait(current, "candidate is still warming") - } - CandidateAnswer::Answered { evidence, .. } => { - let adoption = LeaseAdoptionEvidence::with_ready( - lease, - &evidence.canonical_sha256, - AdoptionSource::Direct, - evidence.admitted_at_unix_millis, - ); - self.persist_same_phase(current, |next| { - next.ready = Some(ReadinessEvidence::RouteProof { evidence }); - next.lease_adoption = adoption; - Ok(()) - }) - } - }; - } - { - let current_lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let Some((admitted, authenticated)) = - self.admit_latest_topology(current, current_lease)? - else { - return Ok(()); - }; - if admitted.envelope != current.envelope { - return Ok(()); - } - let semantic_ready = is_semantic_ready(&authenticated); - let latest = required( - &admitted.value.latest_odin_observation, - "sequence-admitted Ready evidence", - )?; - if current.value.ready.as_ref().and_then(ReadinessEvidence::odin) == Some(latest) { - ensure!(semantic_ready, "stored Ready receipt changed meaning"); - } else if semantic_ready { - let evidence = latest.clone(); - // A Ready stateful correlation carries exactly the current - // lease (the authenticator refuses one that does not). - let adoption = authenticated - .record() - .signed_presence_sha256 - .as_deref() - .and_then(|presence| { - LeaseAdoptionEvidence::with_ready( - current_lease, - presence, - AdoptionSource::OdinTopology, - evidence.admitted_at_unix_millis, - ) - }); - let _token = SequenceAdmittedReady { - transaction_id: admitted.value.transaction_id.clone(), - evidence: evidence.clone(), - expected: expected.clone(), - authenticated, - }; - return self.persist_same_phase(&admitted, |next| { - next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence }); - next.lease_adoption = adoption; - Ok(()) - }); - } else { - return Ok(()); - } - } - self.transition(current, DeploymentPhase::Routing) - } - - fn advance_routing(&self, current: &Stored) -> Result<()> { - let expected = required(¤t.value.expected, "Expected projection")?; - let activation = required(¤t.value.activation, "activation")?; - self.observe_candidate_before_waiting(¤t.value)?; - if current.value.routing.is_none() { - let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { - // Ready is the candidate's own proof, recorded in AwaitingReady. - // Validation refuses Routing without a Ready receipt, and a - // receipt the other party vouched for is held before any step - // runs. No Odin correlation is read to admit a route. - current.clone() - } else { - let current_lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let Some((admitted, authenticated)) = - self.admit_latest_topology(current, current_lease)? - else { - return Ok(()); - }; - if admitted.envelope != current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at route admission" - ); - let latest = required( - &admitted.value.latest_odin_observation, - "current route-admission topology evidence", - )?; - if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) - != Some(latest) - { - let latest = latest.clone(); - return self.persist_same_phase(&admitted, |next| { - next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); - Ok(()) - }); - } - let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; - ensure!( - ready.transaction_id() == admitted.value.transaction_id, - "Ready token belongs to another transaction" - ); - admitted - }; - validate_live_providers_for_deploy(admitted.value.command_kind, || { - self.validate_selected_providers_current(required( - &admitted.value.plan, - "transaction plan", - )?) - })?; - self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; - let evidence = if expected.route.is_some() { - let binding = admitted.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let route_binding = binding.route.context("routed plan has no route binding")?; - let preflight = required(&admitted.value.route_preflight, "route preflight")?; - let driver = self.route_driver(route_binding); - let gate = self.route_gate(&admitted.value.target, admitted.value.command_kind); - ensure!( - preflight.candidate_runtime_instance_id == activation.runtime_instance_id, - "route preflight belongs to another runtime instance" - ); - let rollback_allowed = may_rollback_route_after_failed_proof(required( - &admitted.value.fencing, - "route fencing evidence", - )?); - let membership_sha256 = driver.install( - expected, - &activation.runtime_instance_id, - preflight, - rollback_allowed, - &gate, - )?; - let observation = match self.prove_stable_route( - &admitted.value, - &driver, - membership_sha256, - ) { - Ok(observation) => observation, - Err(proof_error) if !rollback_allowed => { - return Err(proof_error).context( - "incumbent authority was fenced; candidate route remains installed for fail-closed retry", - ); - } - Err(proof_error) => { - return match driver.rollback( - expected, - &activation.runtime_instance_id, - preflight, - &gate, - ) { - Ok(()) => Err(proof_error) - .context("candidate did not answer its stable route challenge"), - Err(rollback_error) => Err(proof_error).context(format!( - "candidate did not answer its stable route challenge; exact route rollback also failed: {rollback_error:#}" - )), - }; - } - }; - ensure!( - driver.observe_membership(expected, &observation.membership_sha256)?, - "route membership changed during its signed stable-listener observation" - ); - self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; - let promoted_at_unix_millis = observation.observed_at_unix_millis; - RoutingEvidence::Promoted { - observation, - promoted_at_unix_millis, - } - } else { - RoutingEvidence::SkippedUnrouted - }; - let now = now_millis()?; - return self.persist_same_phase(&admitted, |next| { - next.routing = Some(evidence); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - self.transition(current, DeploymentPhase::Committing) - } - - fn advance_committing(&self, current: &Stored) -> Result<()> { - self.observe_candidate_before_waiting(¤t.value)?; - let route_proof = ReadinessClass::of(required( - ¤t.value.expected, - "Expected projection", - )?)? == ReadinessClass::RouteProof; - let ready_current = if route_proof { - // The candidate's Ready proof is durable, and the stable route's - // challenge below is its currency. No Odin is read to commit. - current.clone() - } else { - let current_lease_sha256 = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - // A newer sequence admitted here changed the record, and commit - // goes on from that record rather than returning: returning let - // Odin's next re-stamp abort every attempt (F16). - let Some((ready_current, authenticated)) = - self.admit_latest_topology(current, current_lease_sha256.as_deref())? - else { - return Ok(()); - }; - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at admission commit" - ); - // Not require_current: the Ready receipt is durable evidence, so - // authenticating it against `now` asks a stored record to be live and - // refuses it once it ages past the 30s observation window. Route - // admission (:3906) refreshes the receipt to the latest observation and - // can therefore demand currency; by Committing the receipt is history. - // Currency here is answered by the latest observation, authenticated - // and checked for semantic readiness immediately above. - self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; - ready_current - }; - validate_live_providers_for_deploy(ready_current.value.command_kind, || { - self.validate_selected_providers_current(required( - &ready_current.value.plan, - "transaction plan", - )?) - })?; - - let expected = required(&ready_current.value.expected, "Expected projection")?; - let activation = required(&ready_current.value.activation, "activation")?; - let workload = required(&ready_current.value.workload, "workload")?; - self.workload_for(required(&ready_current.value.plan, "transaction plan")?)? - .observe(expected, activation, workload)?; - self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; - if let Some(route) = ready_current - .value - .routing - .as_ref() - .and_then(RoutingEvidence::observation) - { - let binding = ready_current - .value - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1; - let driver = - self.route_driver(binding.route.context("routed plan has no route binding")?); - ensure!( - driver.observe_membership(expected, &route.membership_sha256)?, - "route membership changed before admission commit" - ); - let current_route = self.prove_stable_route( - &ready_current.value, - &driver, - route.membership_sha256.clone(), - )?; - ensure!( - current_route.route_id == route.route_id - && current_route.runtime_instance_id == route.runtime_instance_id, - "stable route changed incarnation before admission commit" - ); - ensure!( - driver.observe_membership(expected, &route.membership_sha256)?, - "route membership changed during the final signed admission challenge" - ); - self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; - } - - let commit_current = if route_proof { - ready_current - } else { - let current_lease_sha256 = ready_current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - // The final proof's own presence makes Odin re-stamp, so this - // admission usually writes. Commit is from the record it wrote. - let Some((commit_current, authenticated)) = - self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? - else { - return Ok(()); - }; - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready after the final admission challenge" - ); - self.rehydrate_ready_token(&commit_current.value, now_millis()?, false)?; - commit_current - }; - let now = now_millis()?; - validate_live_providers_for_deploy(commit_current.value.command_kind, || { - self.validate_selected_providers_current(required( - &commit_current.value.plan, - "transaction plan", - )?) - })?; - self.workload_for(required(&commit_current.value.plan, "transaction plan")?)? - .observe( - required(&commit_current.value.expected, "Expected projection")?, - required(&commit_current.value.activation, "activation")?, - required(&commit_current.value.workload, "workload")?, - )?; - self.ensure_transaction_write_lease_current(&commit_current.value, now)?; - - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; - let odin_authority = (!route_proof) - .then(|| self.current_odin_authority(&snapshot)) - .transpose()?; - let generation = AdmittedGeneration::from_transaction( - &commit_current.value, - odin_authority, - now, - )?; - let post_commit_cleanup = PostCommitCleanup { - incumbent: match incumbent { - Some(incumbent) - if incumbent_was_stopped_during_fencing(required( - &commit_current.value.fencing, - "commit fencing evidence", - )?) => - { - IncumbentCleanupEvidence::Complete { - generation_id: incumbent.value.generation_id.clone(), - } - } - Some(incumbent) => IncumbentCleanupEvidence::Pending { - generation_id: incumbent.value.generation_id.clone(), - workload: incumbent.value.workload.clone(), - }, - None => IncumbentCleanupEvidence::SkippedNoIncumbent, - }, - source: match commit_current.value.command_kind { - CommandKind::Deploy => SourceCleanupEvidence::Pending, - CommandKind::Continuity => SourceCleanupEvidence::SkippedContinuity, - }, - }; - let mut complete = commit_current.value.clone(); - complete.enter_phase(DeploymentPhase::Complete, now); - complete.last_error = None; - complete.completion = Some(TransactionCompletion::Admitted { - generation_id: generation.generation_id.clone(), - }); - complete.post_commit_cleanup = Some(post_commit_cleanup); - complete.validate()?; - - let admitted_expected = CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: generation.target.clone(), - current: incumbent.map(|stored| stored.envelope.clone()), - }; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: commit_current.value.transaction_id.clone(), - current: Some(commit_current.envelope.clone()), - }, - admitted_expected, - ], - &[ - transaction_envelope(&complete, now)?, - admitted_envelope(&generation, now)?, - ], - )?, - "incumbent or transaction changed before atomic admission commit" - ); - Ok(()) - } - - fn exact_incumbent<'a>( - &self, - snapshot: &'a ControlSnapshot, - transaction: &DeploymentTransaction, - ) -> Result>> { - let current = snapshot.admitted_for(&transaction.target); - match (&transaction.incumbent_generation_id, current) { - (None, None) => Ok(None), - (Some(expected), Some(current)) if current.value.generation_id == *expected => { - Ok(Some(current)) - } - _ => bail!("target incumbent changed after transaction creation"), - } - } - - fn incumbent_lease_sha256_for_warming( - &self, - snapshot: &ControlSnapshot, - transaction: &DeploymentTransaction, - ) -> Result> { - if let Some(fencing) = &transaction.fencing { - return match fencing { - FencingEvidence::SkippedStateless => Ok(None), - FencingEvidence::Revoked { - incumbent_lease_sha256, - .. - } => Ok(incumbent_lease_sha256.clone()), - }; - } - self.exact_incumbent(snapshot, transaction)? - .and_then(|incumbent| incumbent.value.leasing.lease()) - .map(IdunnProcessWriteLeaseRecord::canonical_sha256) - .transpose() - } - - fn fresh_warming_for_lease( - &self, - current: &Stored, - now: u64, - ) -> Result< - Option<( - Stored, - WarmingEvidence, - SequenceAdmittedWarming, - )>, - > { - let prior = self.rehydrate_warming_token(¤t.value, now, false)?; - match required(¤t.value.warming, "pre-fence Warming evidence")?.clone() { - WarmingEvidence::FirstOdinDirect { .. } => { - // Whether this transaction may observe Odin directly was - // settled when the Warming evidence was recorded, and that - // decision is durable in the transaction. Re-deciding it here - // with the narrower "no admitted Odin" test contradicted the - // caller and failed the refresh. What the refresh owes is - // freshness, and the replay check below is what provides it. - // That the target is Odin was already required by the - // rehydration above, which authenticates the recorded answer. - let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let warming = WarmingEvidence::FirstOdinDirect { evidence }; - let token = SequenceAdmittedWarming::from_direct_presence( - current.value.transaction_id.clone(), - warming.clone(), - present, - )?; - ensure!( - token.signed_presence_sha256() != prior.signed_presence_sha256(), - "first Odin replayed its pre-fence Warming presence" - ); - Ok(Some((current.clone(), warming, token))) - } - WarmingEvidence::RouteProofDirect { .. } => { - // A stateful route-proof candidate is warming until it holds - // the lease this refresh is about to bind, so the fresh - // presence must be warming again, and must be a new one. - let CandidateAnswer::Answered { evidence, present } = - self.challenge_candidate(¤t.value, &["warming"], None)? - else { - return Ok(None); - }; - let warming = WarmingEvidence::RouteProofDirect { evidence }; - let token = SequenceAdmittedWarming::from_direct_presence( - current.value.transaction_id.clone(), - warming.clone(), - present, - )?; - ensure!( - token.signed_presence_sha256() != prior.signed_presence_sha256(), - "route-proof candidate replayed its pre-fence Warming presence" - ); - Ok(Some((current.clone(), warming, token))) - } - WarmingEvidence::OdinTopology { - evidence: prior_evidence, - } => { - let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)? - else { - return Ok(None); - }; - let fresh_evidence = required( - &admitted.value.latest_odin_observation, - "post-fence Odin Warming observation", - )? - .clone(); - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent_lease_sha256 = - self.incumbent_lease_sha256_for_warming(&snapshot, &admitted.value)?; - if !is_semantic_warming( - required(&admitted.value.expected, "Warming Expected projection")?, - required(&admitted.value.activation, "Warming activation")?, - incumbent_lease_sha256.as_deref(), - &authenticated, - )? { - return Ok(None); - } - let token = SequenceAdmittedWarming::from_topology( - admitted.value.transaction_id.clone(), - authenticated, - )?; - if !provider_warming_advanced( - prior_evidence.publisher_sequence, - prior.signed_presence_sha256(), - fresh_evidence.publisher_sequence, - token.signed_presence_sha256(), - ) { - return Ok(None); - } - Ok(Some(( - admitted, - WarmingEvidence::OdinTopology { - evidence: fresh_evidence, - }, - token, - ))) - } - } - } - - fn prepare_candidate_write_lease( - &self, - transaction: &DeploymentTransaction, - warming: &SequenceAdmittedWarming, - now: u64, - ) -> Result { - let expected = required(&transaction.expected, "Expected projection")?; - let activation = required(&transaction.activation, "activation")?; - ensure!( - expected.write_lease_required - && warming.transaction_id() == transaction.transaction_id - && warming.runtime_instance_id() == activation.runtime_instance_id, - "fresh Warming token does not own this stateful candidate" - ); - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let epoch = self - .exact_incumbent(&snapshot, transaction)? - .and_then(|generation| generation.value.leasing.lease()) - .map_or(1, |lease| lease.lease_epoch.saturating_add(1)); - let lease = IdunnProcessWriteLeaseRecord { - schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: expected.canonical_sha256()?, - plan_id: expected.plan_id.clone(), - incarnation_id: expected.incarnation_id.clone(), - sealed_release_id: expected.sealed_release_id.clone(), - activation_witness_sha256: activation.canonical_sha256()?, - state_schema_generation: expected - .state_schema_generation - .clone() - .context("stateful Expected has no schema generation")?, - state_contract_sha256: expected - .state_contract_sha256 - .clone() - .context("stateful Expected has no state contract")?, - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: activation.runtime_instance_id.clone(), - warming_presence_sha256: warming.signed_presence_sha256().to_owned(), - lease_epoch: epoch, - issued_at_unix_millis: now, - }; - lease.validate()?; - Ok(lease) - } - - fn ensure_transaction_write_lease_current( - &self, - transaction: &DeploymentTransaction, - now: u64, - ) -> Result<()> { - let expected = required(&transaction.expected, "Expected projection")?; - let lease = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease); - ensure!( - expected.write_lease_required == lease.is_some(), - "transaction write-lease disposition differs from Expected" - ); - let Some(lease) = lease else { - return Ok(()); - }; - let activation = required(&transaction.activation, "activation")?; - let warming = self.rehydrate_warming_token(transaction, now, false)?; - let lease_path = required(&transaction.plan, "transaction plan")? - .parsed_inputs()? - .1 - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - ensure!( - CultCacheWriteLeaseDriver::new(&transaction.target, lease_path) - .observe(expected, activation, &warming, lease,)?, - "process write lease changed across the authority boundary" - ); - Ok(()) - } - - fn select_candidate_port(&self, binding: &OperatorBinding) -> Result> { - let Some(route) = &binding.route else { - return Ok(None); - }; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut used = BTreeSet::new(); - for plan in snapshot - .admitted - .iter() - .map(|stored| &stored.value.plan) - .chain( - snapshot - .transactions - .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) - .filter_map(|stored| stored.value.plan.as_ref()), - ) - { - if let Some(port) = plan.candidate_port { - used.insert(port); - } - } - (route.private_port_start..=route.private_port_end) - .find(|port| !used.contains(port)) - .map(Some) - .context("no private candidate port remains in the operator range") - } - - fn deployment_authorization( - &self, - current: &Stored, - now: u64, - ) -> Result> { - let expected = required(¤t.value.expected, "Expected projection")?; - let plan = required(¤t.value.plan, "transaction plan")?; - let binding = plan.parsed_inputs()?.1; - let Some((record, canonical_bytes)) = - read_deployment_brake(&binding.brakes.deployment_store)? - else { - return Ok(None); - }; - let anchor = read_trust_anchor::( - &self.options.deployment_brake_operator_anchor, - )?; - if evaluate_idunn_deployment_brake( - IdunnDeploymentBrakeObservation::Present(&record), - &anchor, - &expected.runtime_id, - &expected.sealed_release_id, - ¤t.value.transaction_id, - now, - ) - .is_err() - { - return Ok(None); - } - let authorization_id = record - .authorization_id - .clone() - .context("released brake has no authorization id")?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - ensure!( - !snapshot.transactions.iter().any(|stored| { - stored - .value - .deployment_authorization - .as_ref() - .is_some_and(|used| used.authorization_id == authorization_id) - }), - "deployment authorization was already consumed" - ); - Ok(Some(DeploymentAuthorization { - authorization_id, - brake_sha256: sha256_id(&canonical_bytes), - canonical_brake_bytes: canonical_bytes, - authorized_at_unix_millis: now, - })) - } - - fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { - let plan = required(¤t.value.plan, "continuity plan")?; - let binding = plan.parsed_inputs()?.1; - let expected = required(¤t.value.expected, "continuity Expected")?; - match read_lifecycle_brake(&binding.brakes.lifecycle_store) { - Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart( - IdunnLifecycleBrakeObservation::Present(&record), - &expected.runtime_id, - ¤t.value.target, - now, - ) - .is_ok()), - Ok(None) => Ok(evaluate_idunn_continuity_restart( - IdunnLifecycleBrakeObservation::Missing, - &expected.runtime_id, - ¤t.value.target, - now, - ) - .is_ok()), - Err(_) => Ok(false), - } - } - - /// A phase that waits on Odin's evidence about the candidate must first - /// confirm the candidate still exists. Otherwise a candidate that died - /// while its correlation was pending is never observed again: the phase - /// returns early on "no evidence yet" every tick, the error that would have - /// triggered the post-fencing abort never happens, and the transaction - /// owns the target forever behind a process that is gone. - fn observe_candidate_before_waiting(&self, transaction: &DeploymentTransaction) -> Result<()> { - self.workload_for(required(&transaction.plan, "transaction plan")?)? - .observe( - required(&transaction.expected, "Expected projection")?, - required(&transaction.activation, "activation")?, - required(&transaction.workload, "candidate workload")?, - ) - .map(|_| ()) - } - - /// Withdraw one projected incarnation of this target that nothing owns: - /// not the admitted generation, not any live transaction. A replaced - /// incumbent, an aborted candidate whose reconciliation was skipped, or a - /// record left by the projection migration all end here. One per tick; - /// the projection is a shared file and each withdrawal is one CAS. - fn retire_stale_incarnations( - &self, - snapshot: &ControlSnapshot, - current: &Stored, - ) -> Result { - let topology = self.topology(); - let target = current.value.target.as_str(); - let owned = snapshot - .transactions - .iter() - .filter(|stored| stored.value.target == target && !stored.value.is_terminal()) - .filter_map(|stored| stored.value.expected.as_ref()) - .map(IdunnExpectedIncarnationRecord::canonical_sha256) - .collect::>>()?; - let admitted_sha256 = current.value.expected.canonical_sha256()?; - let Some(stale) = topology - .projected_incarnations(target)? - .into_iter() - .find(|expected| { - expected - .canonical_sha256() - .is_ok_and(|sha256| sha256 != admitted_sha256 && !owned.contains(&sha256)) - }) - else { - return Ok(false); - }; - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - topology.withdraw_stale_incarnation(&stale, &provider_anchor)?; - eprintln!( - "Idunn withdrew stale {} incarnation {} from the projection", - target, stale.incarnation_id - ); - Ok(true) - } - - /// The lifecycle brake as it applies to restarting one admitted - /// generation: the same evaluation `lifecycle_allows` makes for a - /// continuity transaction, asked before one is minted. - fn lifecycle_allows_generation( - &self, - current: &Stored, - now: u64, - ) -> Result { - let binding = current.value.plan.parsed_inputs()?.1; - let expected = ¤t.value.expected; - match read_lifecycle_brake(&binding.brakes.lifecycle_store) { - Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart( - IdunnLifecycleBrakeObservation::Present(&record), - &expected.runtime_id, - ¤t.value.target, - now, - ) - .is_ok()), - Ok(None) => Ok(evaluate_idunn_continuity_restart( - IdunnLifecycleBrakeObservation::Missing, - &expected.runtime_id, - ¤t.value.target, - now, - ) - .is_ok()), - Err(_) => Ok(false), - } - } - - fn record_gate_wait( - &self, - current: &Stored, - detail: &str, - ) -> Result<()> { - if current.value.last_error.as_deref() == Some(detail) { - return Ok(()); - } - let mut next = current.value.clone(); - next.last_error = Some(detail.into()); - next.updated_at_unix_millis = now_millis()?; - replace_transaction(&self.options.state_store, current, &next) - } - - fn runtime_authority( - &self, - transaction: &DeploymentTransaction, - ) -> Result { - self.runtime_authority_parts( - required(&transaction.plan, "transaction plan")?, - required(&transaction.expected, "Expected projection")?, - required(&transaction.activation, "activation")?, - ) - } - - /// The stable route's proof: a challenged presence that is Active. - fn authenticate_routed_presence( - &self, - authority: &cultnet_rs::VerifiedRuntimeAuthority, - current_write_lease_sha256: Option<&str>, - message_id: &str, - challenged_at_unix_millis: u64, - received_at_unix_millis: u64, - canonical_presence: &[u8], - ) -> Result<(String, u64)> { - let present = self.authenticate_challenged_presence( - authority, - &["active"], - current_write_lease_sha256, - message_id, - challenged_at_unix_millis, - received_at_unix_millis, - canonical_presence, - )?; - Ok(( - present.signed_presence_sha256().to_owned(), - received_at_unix_millis, - )) - } - - /// A presence Idunn challenged for itself, from the stable endpoint or the - /// candidate's: signed by the provider and the launch's activation key, - /// bound to current authority, minted after the challenge, answering that - /// exact challenge, in one of `states`. A warming presence holds no write - /// lease; a presence in any other state holds exactly the current one. - /// Every disagreement with authority is named, so a capacity below the - /// Expected minimum reads as that and not as a generic refusal. - fn authenticate_challenged_presence( - &self, - authority: &cultnet_rs::VerifiedRuntimeAuthority, - states: &[&str], - current_write_lease_sha256: Option<&str>, - message_id: &str, - challenged_at_unix_millis: u64, - received_at_unix_millis: u64, - canonical_presence: &[u8], - ) -> Result { - ensure!( - received_at_unix_millis >= challenged_at_unix_millis, - "route proof predates its challenge" - ); - let authenticated = authenticate_runtime_presence_claim( - canonical_presence, - authority, - RuntimePresenceAuthenticationContext { - trusted_received_at_unix_millis: received_at_unix_millis, - maximum_age_millis: self.options.topology_maximum_age_millis, - maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, - }, - )?; - let correlation = correlate_runtime_presence_claim(authenticated, authority)?; - if !correlation.disagreements().is_empty() { - return Err(PresenceDisagrees { - disagreements: correlation.disagreements().to_vec(), - } - .into()); - } - let present = correlation.into_undisputed_present()?; - let presence = present.record(); - ensure!( - presence.observed_at_unix_millis >= challenged_at_unix_millis, - "route proof returned a presence minted before its challenge" - ); - ensure!( - states.contains(&presence.state.as_str()), - "route proof runtime is {}, not {}", - presence.state, - states.join(" or ") - ); - ensure!( - presence.detail == format!("route-observation:{message_id}"), - "route proof response is not bound to the exact challenge" - ); - // A warming presence is exempt from the lease comparison: it is - // Idunn's own lease that a warming candidate cannot hold yet, and the - // authenticator above already refuses one that claims a lease. - let lease_is_current = presence.state == "warming" - || presence.write_lease_sha256.as_deref() == current_write_lease_sha256; - ensure!( - lease_is_current, - "route proof runtime does not hold the exact current process write lease" - ); - Ok(present) - } - - fn authenticate_first_odin_warming_presence( - &self, - transaction: &DeploymentTransaction, - message_id: &str, - challenged_at_unix_millis: u64, - received_at_unix_millis: u64, - canonical_presence: &[u8], - ) -> Result { - // Whether direct observation is warranted at all is decided in - // advance_warming, which asks the question that matters: can any Odin - // report on this one. Re-deciding it here with the narrower "has an - // Odin ever been admitted" test only contradicted that caller. What is - // still checked is what this function itself depends on -- the target - // is Odin, and the incarnation is stateful. - let expected = required(&transaction.expected, "Odin Expected")?; - ensure!( - ReadinessClass::of(expected)? == ReadinessClass::OdinSelf - && expected.write_lease_required, - "direct Warming presence is reserved for a stateful Odin incarnation" - ); - ensure!( - received_at_unix_millis >= challenged_at_unix_millis, - "first Odin Warming observation predates its challenge" - ); - let authority = self.runtime_authority(transaction)?; - let authenticated = authenticate_runtime_presence_claim( - canonical_presence, - &authority, - RuntimePresenceAuthenticationContext { - trusted_received_at_unix_millis: received_at_unix_millis, - maximum_age_millis: self.options.topology_maximum_age_millis, - maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, - }, - )?; - let correlation = correlate_runtime_presence_claim(authenticated, &authority)?; - ensure!( - correlation.disagreements().is_empty(), - "first Odin Warming presence disagrees with its Expected activation" - ); - let present = correlation.into_undisputed_present()?; - let presence = present.record(); - ensure!( - presence.observed_at_unix_millis >= challenged_at_unix_millis - && presence.state == "warming" - && presence.write_lease_sha256.is_none() - && presence.detail == format!("idunn-warming:{message_id}"), - "first Odin candidate did not return exact fresh pre-lease Warming evidence" - ); - Ok(present) - } - - fn observe_first_odin_warming( - &self, - transaction: &DeploymentTransaction, - ) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { - let expected = required(&transaction.expected, "first Odin Expected")?; - let binding = required(&transaction.plan, "first Odin plan")? - .parsed_inputs()? - .1; - let driver = self.route_driver( - binding - .route - .context("first Odin bootstrap has no candidate route binding")?, - ); - let message_id = format!("warming-{}", Uuid::new_v4().simple()); - let challenged_at_unix_millis = now_millis()?; - let response = driver.request_candidate_runtime_presence(expected, &message_id)?; - ensure!( - response.message_id == message_id, - "first Odin candidate transport substituted its challenge identity" - ); - let admitted_at_unix_millis = now_millis()?; - let present = self.authenticate_first_odin_warming_presence( - transaction, - &message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - &response.canonical_presence, - )?; - let evidence = RuntimePresenceEvidence::from_present( - &present, - message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - )?; - Ok((evidence, present)) - } - - fn prove_stable_route( - &self, - transaction: &DeploymentTransaction, - driver: &NginxRouteDriver, - membership_sha256: String, - ) -> Result { - let expected = required(&transaction.expected, "route Expected projection")?; - let activation = required(&transaction.activation, "route activation")?; - let authority = self.runtime_authority(transaction)?; - let current_write_lease_sha256 = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - self.prove_stable_route_against( - expected, - activation, - &authority, - current_write_lease_sha256, - driver, - membership_sha256, - ) - } - - fn prove_stable_route_against( - &self, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - authority: &cultnet_rs::VerifiedRuntimeAuthority, - current_write_lease_sha256: Option<&str>, - driver: &NginxRouteDriver, - membership_sha256: String, - ) -> Result { - let message_id = format!("route-{}", Uuid::new_v4().simple()); - let challenged_at_unix_millis = now_millis()?; - let response = driver.request_runtime_presence(expected, &message_id)?; - ensure!( - response.message_id == message_id, - "route transport substituted its challenge identity" - ); - let received_at_unix_millis = now_millis()?; - let (signed_presence_sha256, observed_at_unix_millis) = self.authenticate_routed_presence( - authority, - current_write_lease_sha256, - &message_id, - challenged_at_unix_millis, - received_at_unix_millis, - &response.canonical_presence, - )?; - let observation = RouteObservation { - route_id: driver.binding.route_id.clone(), - runtime_instance_id: activation.runtime_instance_id.clone(), - membership_sha256, - signed_presence_sha256, - observed_at_unix_millis, - }; - observation.validate()?; - Ok(observation) - } - - fn runtime_authority_parts( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - let provider_anchor = self.provider_anchor_for_plan(plan)?; - verify_runtime_authority( - expected, - activation, - &self.idunn_anchor, - &provider_anchor.public_key, - ) - } - - fn provider_anchor_for_plan( - &self, - plan: &CompiledDeploymentPlan, - ) -> Result { - let binding = plan.parsed_inputs()?.1; - let provider_anchor = read_trust_anchor::( - &binding.runtime_identity.trust_anchor_store, - )?; - ensure!( - provider_anchor.schema_version - == ::TRUST_ANCHOR_SCHEMA, - "provider runtime presence trust anchor schema is unsupported" - ); - Ok(provider_anchor) - } - - fn authenticate_topology_bytes( - &self, - snapshot: &ControlSnapshot, - transaction: &DeploymentTransaction, - canonical_bytes: &[u8], - current_write_lease_sha256: Option<&str>, - trusted_received_at: u64, - ) -> Result { - let authority = self.runtime_authority(transaction)?; - let odin_authority = self.current_odin_authority(snapshot)?; - authenticate_odin_runtime_topology_correlation( - canonical_bytes, - &authority, - current_write_lease_sha256, - &odin_authority.signer_public_key, - self.trusted_topology_context(trusted_received_at), - ) - } - - /// Persist every newly authenticated publisher sequence before deciding - /// whether it means Warming, Ready, degraded, or disagreement. - fn admit_latest_topology( - &self, - current: &Stored, - current_write_lease_sha256: Option<&str>, - ) -> Result< - Option<( - Stored, - AuthenticatedOdinRuntimeTopologyCorrelation, - )>, - > { - // Odin keeps one correlation per incarnation, so the transaction's own - // Expected selects exactly the evidence about it. A correlation about - // the incarnation this one is replacing lives under another key and is - // never handed back here. - let expected_sha256 = - required(¤t.value.expected, "Expected projection")?.canonical_sha256()?; - let Some(received) = self - .topology() - .receive(¤t.value.target, &expected_sha256)? - else { - return Ok(None); - }; - ensure!( - received.target == current.value.target, - "topology transport substituted target" - ); - let now = now_millis()?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let live = snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared before topology admission")?; - ensure!( - live.envelope == current.envelope, - "transaction changed before topology admission" - ); - let authenticated = match self.authenticate_topology_bytes( - &snapshot, - &live.value, - &received.canonical_bytes, - current_write_lease_sha256, - now, - ) { - Ok(authenticated) => authenticated, - // Odin re-stamps a correlation only when its facts change, and a - // live presence changes them every heartbeat. A correlation that - // has aged out therefore says one thing: nothing about this - // incarnation has arrived lately. That is absence to wait on, not - // evidence to refuse; it failed every Heimdall deployment and the - // first observed Odin deployment as a fault. - Err(error) if is_stale_observation(&error) => return Ok(None), - Err(error) => return Err(error), - }; - let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; - if !sequence_requires_admission( - live.value.latest_odin_observation.as_ref(), - snapshot.max_odin_sequence(&live.value.target, &evidence.signer_identity_id), - &evidence, - )? { - return Ok(Some((live.clone(), authenticated))); - } - let mut next = live.value.clone(); - next.latest_odin_observation = Some(evidence.clone()); - next.odin_publisher_sequence_cursor = evidence.publisher_sequence; - next.updated_at_unix_millis = now; - next.last_error = None; - replace_transaction(&self.options.state_store, live, &next)?; - let admitted = ControlSnapshot::read(&self.options.state_store)? - .transactions - .into_iter() - .find(|stored| stored.value.transaction_id == next.transaction_id) - .context("sequence-admitted transaction disappeared")?; - ensure!( - admitted.value.latest_odin_observation.as_ref() == Some(&evidence) - && admitted.value.odin_publisher_sequence_cursor == evidence.publisher_sequence, - "topology sequence was not durably admitted" - ); - Ok(Some((admitted, authenticated))) - } - - fn rehydrate_warming_token( - &self, - transaction: &DeploymentTransaction, - now: u64, - require_current: bool, - ) -> Result { - let evidence = required(&transaction.warming, "durable warming evidence")?.clone(); - match evidence { - WarmingEvidence::OdinTopology { evidence } => { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let authenticated = self.authenticate_topology_bytes( - &snapshot, - transaction, - &evidence.canonical_bytes, - None, - if require_current { - now - } else { - evidence.admitted_at_unix_millis - }, - )?; - validate_authenticated_evidence(&evidence, &authenticated)?; - let incumbent_lease_sha256 = - self.incumbent_lease_sha256_for_warming(&snapshot, transaction)?; - ensure!( - is_semantic_warming( - required(&transaction.expected, "Warming Expected projection")?, - required(&transaction.activation, "Warming activation")?, - incumbent_lease_sha256.as_deref(), - &authenticated, - )?, - "durable warming evidence no longer satisfies the Warming gate" - ); - SequenceAdmittedWarming::from_topology( - transaction.transaction_id.clone(), - authenticated, - ) - } - WarmingEvidence::FirstOdinDirect { evidence } => { - let present = self.authenticate_first_odin_warming_presence( - transaction, - &evidence.message_id, - evidence.challenged_at_unix_millis, - if require_current { - now - } else { - evidence.admitted_at_unix_millis - }, - &evidence.canonical_bytes, - )?; - SequenceAdmittedWarming::from_direct_presence( - transaction.transaction_id.clone(), - WarmingEvidence::FirstOdinDirect { evidence }, - present, - ) - } - WarmingEvidence::RouteProofDirect { evidence } => { - let present = self.reauthenticate_route_proof( - transaction, - &evidence, - route_proof_warming_states(required( - &transaction.expected, - "Warming Expected projection", - )?), - None, - if require_current { - now - } else { - evidence.admitted_at_unix_millis - }, - )?; - SequenceAdmittedWarming::from_direct_presence( - transaction.transaction_id.clone(), - WarmingEvidence::RouteProofDirect { evidence }, - present, - ) - } - } - } - - /// A route-proof presence Idunn recorded, authenticated again as of `at`. - fn reauthenticate_route_proof( - &self, - transaction: &DeploymentTransaction, - evidence: &RuntimePresenceEvidence, - states: &[&str], - current_write_lease_sha256: Option<&str>, - at_unix_millis: u64, - ) -> Result { - let authority = self.runtime_authority(transaction)?; - self.authenticate_challenged_presence( - &authority, - states, - current_write_lease_sha256, - &evidence.message_id, - evidence.challenged_at_unix_millis, - at_unix_millis, - &evidence.canonical_bytes, - ) - } - - /// One direct challenge to a route-proof target's candidate endpoint. - /// Failing to reach or hear the candidate is only silence: a process that - /// is still starting does not answer, and that is waiting, not a fault. - /// What does answer is authenticated, and a bad answer is an error, as is - /// a challenge Idunn could not make: those stay a `ChallengeFailure` in the - /// error's chain. - fn challenge_candidate( - &self, - transaction: &DeploymentTransaction, - states: &[&str], - current_write_lease_sha256: Option<&str>, - ) -> Result { - let expected = required(&transaction.expected, "candidate Expected")?; - let binding = required(&transaction.plan, "candidate plan")? - .parsed_inputs()? - .1; - let driver = self.route_driver( - binding - .route - .context("route-proof candidate has no route binding")?, - ); - let message_id = format!("candidate-{}", Uuid::new_v4().simple()); - let challenged_at_unix_millis = now_millis()?; - let response = match driver.request_candidate_runtime_presence(expected, &message_id) { - Ok(response) => response, - Err(ChallengeFailure::Silent(error)) => { - return Ok(CandidateAnswer::Silent(format!( - "candidate endpoint did not answer its challenge: {error:#}" - ))); - } - Err(bad @ ChallengeFailure::Refused(_)) => return Err(bad.into()), - }; - let admitted_at_unix_millis = now_millis()?; - let authority = self.runtime_authority(transaction)?; - let present = self.authenticate_challenged_presence( - &authority, - states, - current_write_lease_sha256, - &message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - &response.canonical_presence, - )?; - let evidence = RuntimePresenceEvidence::from_present( - &present, - message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - )?; - Ok(CandidateAnswer::Answered { evidence, present }) - } - - fn rehydrate_ready_token( - &self, - transaction: &DeploymentTransaction, - now: u64, - require_current: bool, - ) -> Result { - let evidence = required(&transaction.ready, "durable Ready evidence")? - .odin() - .context("durable Ready evidence is not an Odin receipt")? - .clone(); - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let current_lease = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let authenticated = self.authenticate_topology_bytes( - &snapshot, - transaction, - &evidence.canonical_bytes, - current_lease, - if require_current { - now - } else { - evidence.admitted_at_unix_millis - }, - )?; - validate_authenticated_evidence(&evidence, &authenticated)?; - ensure!( - is_semantic_ready(&authenticated), - "durable Ready evidence no longer authenticates as Ready" - ); - Ok(SequenceAdmittedReady { - transaction_id: transaction.transaction_id.clone(), - evidence, - expected: required(&transaction.expected, "Expected projection")?.clone(), - authenticated, - }) - } - - fn current_ready_provider_tokens( - &self, - snapshot: &ControlSnapshot, - ) -> Result> { - let mut providers = Vec::new(); - for stored in &snapshot.admitted { - // No pre-filter on ready == latest: a provider that published again - // after going ready is still ready. rehydrate_admitted_ready owns - // that judgment now, and reports why when it refuses. - match self.rehydrate_admitted_ready(snapshot, &stored.value) { - Ok(provider) => providers.push(provider), - Err(error) => eprintln!( - "Idunn excluded non-current provider {}: {error:#}", - stored.value.target - ), - } - } - Ok(providers) - } - - fn rehydrate_admitted_ready( - &self, - snapshot: &ControlSnapshot, - generation: &AdmittedGeneration, - ) -> Result { - let odin = generation.odin_receipts()?; - let authority = self.runtime_authority_parts( - &generation.plan, - &generation.expected, - &generation.activation, - )?; - let odin_authority = self.current_odin_authority(snapshot)?; - let current_lease = generation.leasing.lease_sha256(); - let authenticated = authenticate_odin_runtime_topology_correlation( - &odin.ready.canonical_bytes, - &authority, - current_lease, - &odin_authority.signer_public_key, - self.trusted_topology_context(odin.ready.admitted_at_unix_millis), - )?; - validate_authenticated_evidence(odin.ready, &authenticated)?; - ensure!( - is_semantic_ready(&authenticated), - "admitted provider's Ready receipt is not Ready at its admission time" - ); - // A provider that published after going ready is the ordinary case, not - // a stale one, so the cursor is authenticated on its own terms rather - // than compared to the receipt. Authentication is what binds it to this - // incarnation -- it requires the record's current_activation_sha256 and - // runtime_instance_id to match this generation's activation -- so a - // separate identity check here would only restate it more weakly. - // Note this is the cursor as frozen at admission, not a live read of - // Odin. Both receipts are historical proofs; current workload health - // is owned by continuity and route observation, not receipt age. - let authenticated_latest = authenticate_odin_runtime_topology_correlation( - &odin.latest.canonical_bytes, - &authority, - current_lease, - &odin_authority.signer_public_key, - self.trusted_topology_context(odin.latest.admitted_at_unix_millis), - )?; - validate_authenticated_evidence(odin.latest, &authenticated_latest)?; - ensure!( - is_semantic_ready(&authenticated_latest), - "admitted provider's latest receipt is not Ready at its admission time" - ); - Ok(SequenceAdmittedReady { - transaction_id: generation.transaction_id.clone(), - evidence: odin.ready.clone(), - expected: generation.expected.clone(), - authenticated, - }) - } - - fn validate_selected_providers_current( - &self, - plan: &CompiledDeploymentPlan, - ) -> Result<()> { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - for selection in &plan.dependencies { - let Some(provider) = &selection.provider else { - ensure!( - selection.requirement.kind == DependencyKind::Optional, - "required dependency has no provider" - ); - continue; - }; - let DependencyProviderAuthority::ManagedReady { - target, - incarnation_id, - plan_id, - sealed_release_id, - expected_projection_sha256, - odin_topology_correlation_sha256, - odin_topology_publisher_sequence, - } = &provider.authority - else { - continue; - }; - let admitted = snapshot - .admitted_for(target) - .context("selected managed dependency is no longer admitted")?; - let token = self.rehydrate_admitted_ready(&snapshot, &admitted.value)?; - ensure!( - admitted.value.expected.incarnation_id == *incarnation_id - && admitted.value.plan.plan_id == *plan_id - && admitted.value.sealed_release.sealed_release_id == *sealed_release_id - && admitted.value.expected.canonical_sha256()? == *expected_projection_sha256 - && token.publisher_sequence() >= *odin_topology_publisher_sequence, - "selected dependency authority changed before actuation" - ); - ensure!( - token.publisher_sequence() != *odin_topology_publisher_sequence - || token.evidence_sha256() == odin_topology_correlation_sha256, - "selected dependency Odin sequence changed evidence" - ); - ensure!( - token - .authenticated() - .record() - .observed_capabilities - .iter() - .any(|capability| { - capability_compatible( - &selection.requirement.capability, - &selection.requirement.schema, - &selection.requirement.compatibility, - &capability.capability, - &capability.schema, - &capability.compatibility, - ) && capability.capacity >= selection.requirement.minimum_capacity - }), - "selected dependency no longer provides its required capability" - ); - } - Ok(()) - } - - fn persist_same_phase( - &self, - current: &Stored, - mutation: F, - ) -> Result<()> - where - F: FnOnce(&mut DeploymentTransaction) -> Result<()>, - { - let mut next = current.value.clone(); - mutation(&mut next)?; - ensure!( - next.phase == current.value.phase, - "same-phase evidence update changed phase" - ); - next.updated_at_unix_millis = now_millis()?; - next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) - } - - fn transition( - &self, - current: &Stored, - next_phase: DeploymentPhase, - ) -> Result<()> { - ensure!( - next_phase as u8 == current.value.phase as u8 + 1, - "deployment phase transition is not adjacent" - ); - let mut next = current.value.clone(); - next.enter_phase(next_phase, now_millis()?); - next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) - } - - /// Every post-fencing phase ends. Before a live transaction past the - /// fence runs its step, a phase whose deadline has passed, or whose - /// command an operator asked to expire, is resolved as `phase_end` says. - /// Returns `Some(moved)` when it decided the tick, `None` to run the step. - /// - /// The deadline is written only by `enter_phase`, so nothing here or in a - /// resume path extends it. A record lifted from before deadlines existed - /// has none; its phase is timed once, from the first time this sees it. - fn resolve_phase_deadline( - &self, - current: &Stored, - ) -> Result> { - let transaction = ¤t.value; - if !transaction.resolves_by_deadline() { - return Ok(None); - } - let now = now_millis()?; - let Some(deadline) = transaction.phase_deadline else { - let mut next = transaction.clone(); - next.enter_phase(transaction.phase, now); - if next.phase_deadline.is_none() { - return Ok(None); - } - replace_transaction(&self.options.state_store, current, &next)?; - return Ok(Some(true)); - }; - let requested = ControlSnapshot::read(&self.options.state_store)? - .expiry_requests - .iter() - .any(|request| request.value.command_id == transaction.command_id); - let cause = if now >= deadline.deadline_at_unix_millis { - format!( - "the {:?} phase passed its deadline at unix ms {}", - transaction.phase, deadline.deadline_at_unix_millis - ) - } else if requested { - format!("the {:?} phase was expired by the operator", transaction.phase) - } else { - return Ok(None); - }; - match transaction.phase_end() { - PhaseEnd::Abort { lease_not_adopted } => { - let error = if lease_not_adopted { - anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") - } else { - anyhow!("{cause}") - }; - self.begin_post_fencing_abort(current, error)?; - Ok(Some(true)) - } - PhaseEnd::OperatorRequired => { - self.report_once( - &format!("deadline:{}", transaction.transaction_id), - format!( - "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", - transaction.transaction_id - ), - ); - Ok(None) - } - } - } - - /// Delete expiry requests whose command has no phase left to end: it was - /// resolved, it finished, or its command is gone. - fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { - let spent = snapshot - .expiry_requests - .iter() - .filter(|request| { - !snapshot.transactions.iter().any(|stored| { - stored.value.command_id == request.value.command_id - && stored.value.resolves_by_deadline() - }) - }) - .map(|request| request.envelope.clone()) - .collect::>(); - if spent.is_empty() { - return Ok(false); - } - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .delete_batch_if_unchanged(&spent) - } - - fn record_resumable_error( - &self, - current: &Stored, - error: &anyhow::Error, - ) -> Result<()> { - let detail = truncate(&format!("{error:#}"), 2048); - if current.value.last_error.as_deref() == Some(detail.as_str()) { - return Ok(()); - } - let mut next = current.value.clone(); - next.last_error = Some(detail); - next.updated_at_unix_millis = now_millis()?; - replace_transaction(&self.options.state_store, current, &next) - } - - fn begin_pre_fencing_abort( - &self, - current: &Stored, - error: anyhow::Error, - ) -> Result<()> { - ensure!( - current.value.phase < DeploymentPhase::Fencing, - "post-fence transaction cannot terminal-fail" - ); - ensure!( - current.value.pre_fencing_abort.is_none(), - "pre-fencing abort intent is already durable" - ); - let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); - self.persist_same_phase(current, |next| { - next.pre_fencing_abort = Some(abort); - Ok(()) - }) - } - - /// Whether this transaction's candidate can no longer run at all. - /// - /// A transaction that has never started one has no candidate to be dead, so - /// it is not permanently stopped -- it is simply not there yet. - fn candidate_is_permanently_stopped( - &self, - transaction: &DeploymentTransaction, - ) -> Result { - let Some(workload) = &transaction.workload else { - return Ok(false); - }; - self.workload_for(required(&transaction.plan, "transaction plan")?)? - .is_permanently_stopped(workload) - } - - fn begin_post_fencing_abort( - &self, - current: &Stored, - error: anyhow::Error, - ) -> Result<()> { - ensure!( - current.value.phase >= DeploymentPhase::Fencing, - "pre-fence transaction must abort through the pre-fencing path" - ); - ensure!( - current.value.post_fencing_abort.is_none(), - "post-fencing abort intent is already durable" - ); - let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); - self.persist_same_phase(current, |next| { - next.post_fencing_abort = Some(abort); - Ok(()) - }) - } - - /// Withdraw the candidate in the one order that never leaves two writers: - /// take away its route, then its write lease, then the process itself, and - /// only then reconcile the projection. - fn advance_post_fencing_abort(&self, current: &Stored) -> Result<()> { - let abort = required( - ¤t.value.post_fencing_abort, - "post-fencing abort intent", - )?; - if abort.route_restoration == CleanupEvidence::Pending { - let preflight = required(¤t.value.route_preflight, "route preflight receipt")?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let driver = - self.route_driver(binding.route.context("routed plan has no route binding")?); - driver - .withdraw_candidate_membership( - preflight, - &self.route_gate(¤t.value.target, current.value.command_kind), - ) - .context("restoring the route the candidate found")?; - return self.persist_same_phase(current, |next| { - next.post_fencing_abort.as_mut().unwrap().route_restoration = - CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.lease_withdrawal == CleanupEvidence::Pending { - let expected = required(¤t.value.expected, "Expected projection")?; - let activation = required(¤t.value.activation, "activation")?; - // A prepared lease is withdrawn too: its physical grant may have - // landed before `Granted` was durable. Revoking an absent lease, - // and withdrawing an absent projection, are no-ops. - let lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::issued_lease) - .context("post-fencing abort lost the lease it must withdraw")?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let lease_path = binding - .process_write_lease - .context("leased target has no write-lease binding")? - .record_path; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - driver - .revoke_exact(Some(lease)) - .context("revoking the abandoned candidate write lease")?; - ensure!( - driver.observe_empty()?, - "candidate write lease remained after post-fencing withdrawal" - ); - self.topology() - .withdraw_process_write_lease(expected, activation, Some(lease)) - .context("withdrawing the abandoned candidate write-lease projection")?; - return self.persist_same_phase(current, |next| { - next.post_fencing_abort.as_mut().unwrap().lease_withdrawal = - CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.candidate_cleanup == CleanupEvidence::Pending { - let plan = required(¤t.value.plan, "abandoned candidate plan")?; - if let Some(workload) = ¤t.value.workload { - self.workload_for(plan)? - .stop(workload) - .context("stopping the abandoned candidate")?; - } - self.workload_for(plan)? - .discard_prepared( - required(¤t.value.plan, "abandoned candidate plan")?, - required(¤t.value.expected, "abandoned Expected projection")?, - required(¤t.value.activation, "abandoned activation")?, - ) - .context("discarding the abandoned activation material")?; - return self.persist_same_phase(current, |next| { - next.post_fencing_abort.as_mut().unwrap().candidate_cleanup = - CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.topology_reconciliation == CleanupEvidence::Pending { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - // The failed candidate's projection is resolved first. For a - // deployment, fencing also stopped the incumbent and revoked its - // lease, so the incumbent is demoted to Expected-only under its - // key with its own exact activation, which is what lets - // continuity bring it back. A continuity candidate is the - // incumbent's key: it has nothing further to demote. - let topology = self.topology(); - let failed_provider_anchor = self.provider_anchor_for_plan(required( - ¤t.value.plan, - "failed transaction plan", - )?)?; - reconcile_failed_candidate_projection( - &topology, - ¤t.value, - &failed_provider_anchor, - )?; - if let Some(incumbent) = self - .exact_incumbent(&snapshot, ¤t.value)? - .filter(|_| current.value.command_kind == CommandKind::Deploy) - { - let admitted_provider_anchor = - self.provider_anchor_for_plan(&incumbent.value.plan)?; - let expected_sha256 = topology.demote_to_expected_only( - &incumbent.value.expected, - &admitted_provider_anchor, - &incumbent.value.activation, - incumbent.value.leasing.lease(), - )?; - ensure!( - expected_sha256 == incumbent.value.expected.canonical_sha256()?, - "restored incumbent Expected receipt differs" - ); - } - return self.persist_same_phase(current, |next| { - next.post_fencing_abort - .as_mut() - .unwrap() - .topology_reconciliation = CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.source_cleanup == CleanupEvidence::Pending { - self.source - .cleanup( - ¤t.value.transaction_id, - current.value.frozen_source.as_ref(), - ) - .context("cleaning the abandoned source")?; - return self.persist_same_phase(current, |next| { - next.post_fencing_abort.as_mut().unwrap().source_cleanup = - CleanupEvidence::Complete; - Ok(()) - }); - } - ensure!( - abort.is_complete(), - "post-fencing abort cleanup is incomplete" - ); - let mut next = current.value.clone(); - next.enter_phase(DeploymentPhase::Complete, now_millis()?); - next.last_error = Some(abort.error.clone()); - next.completion = Some(TransactionCompletion::FailedAfterFencing { - error: abort.error.clone(), - recovery: current.value.terminal_recovery(), - }); - replace_transaction(&self.options.state_store, current, &next) - } - - fn advance_pre_fencing_abort(&self, current: &Stored) -> Result<()> { - ensure!( - current.value.phase < DeploymentPhase::Fencing, - "pre-fencing abort crossed the fencing boundary" - ); - let abort = required(¤t.value.pre_fencing_abort, "pre-fencing abort intent")?; - if abort.candidate_cleanup == CleanupEvidence::Pending { - let plan = required(¤t.value.plan, "transaction plan")?; - if let Some(workload) = ¤t.value.workload { - self.workload_for(plan)? - .stop(workload) - .context("stopping exact pre-fence candidate")?; - } - self.workload_for(plan)? - .discard_prepared( - required(¤t.value.plan, "pre-fencing candidate plan")?, - required(¤t.value.expected, "pre-fencing Expected projection")?, - required(¤t.value.activation, "pre-fencing activation")?, - ) - .context("discarding exact pre-fence activation material")?; - return self.persist_same_phase(current, |next| { - next.pre_fencing_abort.as_mut().unwrap().candidate_cleanup = - CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.topology_reconciliation == CleanupEvidence::Pending { - // Before fencing a deployment never touched its incumbent: it is - // still running under its own key with its activation and lease, - // so only the candidate's records are withdrawn. A continuity - // candidate shares the admitted key and demotes what it issued. - let plan = required(¤t.value.plan, "failed transaction plan")?; - let provider_anchor = self.provider_anchor_for_plan(plan)?; - reconcile_failed_candidate_projection( - &self.topology(), - ¤t.value, - &provider_anchor, - )?; - return self.persist_same_phase(current, |next| { - next.pre_fencing_abort - .as_mut() - .unwrap() - .topology_reconciliation = CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.source_cleanup == CleanupEvidence::Pending { - self.source - .cleanup( - ¤t.value.transaction_id, - current.value.frozen_source.as_ref(), - ) - .context("cleaning failed pre-fence source")?; - return self.persist_same_phase(current, |next| { - next.pre_fencing_abort.as_mut().unwrap().source_cleanup = CleanupEvidence::Complete; - Ok(()) - }); - } - ensure!( - abort.is_complete(), - "pre-fencing abort cleanup is incomplete" - ); - let mut next = current.value.clone(); - next.enter_phase(DeploymentPhase::Complete, now_millis()?); - next.last_error = Some(abort.error.clone()); - next.completion = Some(TransactionCompletion::FailedBeforeFencing { - error: abort.error.clone(), - }); - replace_transaction(&self.options.state_store, current, &next) - } - - fn advance_post_commit_cleanup(&self, current: &Stored) -> Result<()> { - let cleanup = required( - ¤t.value.post_commit_cleanup, - "post-commit cleanup evidence", - )?; - if let IncumbentCleanupEvidence::Pending { - generation_id, - workload, - } = &cleanup.incumbent - { - let binding = required(¤t.value.plan, "committed transaction plan")? - .parsed_inputs()? - .1; - if let Some(promoted_at_unix_millis) = required( - ¤t.value.routing, - "committed transaction routing evidence", - )? - .promoted_at_unix_millis() - { - let retire_not_before = - route_drain_deadline(promoted_at_unix_millis, binding.rollout.drain_seconds)?; - if now_millis()? < retire_not_before { - return Ok(()); - } - } - // The incumbent is the same target under the same binding kind; - // an incumbent of another kind is refused by the driver, not - // guessed at. - self.workload_for(required(¤t.value.plan, "committed transaction plan")?)? - .stop(workload) - .with_context(|| format!("retiring admitted incumbent {generation_id}"))?; - let generation_id = generation_id.clone(); - return self.persist_same_phase(current, |next| { - next.post_commit_cleanup.as_mut().unwrap().incumbent = - IncumbentCleanupEvidence::Complete { generation_id }; - Ok(()) - }); - } - if cleanup.source == SourceCleanupEvidence::Pending { - self.source - .cleanup( - ¤t.value.transaction_id, - current.value.frozen_source.as_ref(), - ) - .context("cleaning committed deployment source")?; - return self.persist_same_phase(current, |next| { - next.post_commit_cleanup.as_mut().unwrap().source = SourceCleanupEvidence::Complete; - Ok(()) - }); - } - ensure!(cleanup.is_complete(), "post-commit cleanup is incomplete"); - Ok(()) - } -} - -fn route_drain_deadline(promoted_at_unix_millis: u64, drain_seconds: u32) -> Result { - promoted_at_unix_millis - .checked_add( - u64::from(drain_seconds) - .checked_mul(1_000) - .context("route drain duration overflows milliseconds")?, - ) - .context("route drain deadline overflows Unix milliseconds") -} - -/// CultLib refuses a correlation older than the trusted window with this -/// text. Idunn reads that one refusal as "nothing new", everywhere it waits -/// on Odin's evidence; every other refusal stays a refusal. -fn is_stale_observation(error: &anyhow::Error) -> bool { - format!("{error:#}").contains("outside the trusted observation window") -} - -fn route_observation_is_current( - observed_at_unix_millis: u64, - now_unix_millis: u64, - maximum_age_millis: u64, - maximum_future_skew_millis: u64, -) -> bool { - observed_at_unix_millis <= now_unix_millis.saturating_add(maximum_future_skew_millis) - && now_unix_millis.saturating_sub(observed_at_unix_millis) <= maximum_age_millis -} - -fn provider_warming_advanced( - prior_odin_sequence: u64, - prior_signed_presence_sha256: &str, - candidate_odin_sequence: u64, - candidate_signed_presence_sha256: &str, -) -> bool { - candidate_odin_sequence > prior_odin_sequence - && candidate_signed_presence_sha256 != prior_signed_presence_sha256 -} - -/// Resolve the projection a failed candidate left. A deployment candidate owns -/// its own incarnation key and withdraws it whole. A continuity candidate -/// shares the admitted key: the admitted Expected is never withdrawn, and the -/// only thing demoted is the activation this transaction issued, exactly. -fn reconcile_failed_candidate_projection( - topology: &CultCacheTopologyDriver, - transaction: &DeploymentTransaction, - provider_anchor: &ServiceIdentityTrustAnchor, -) -> Result<()> { - let expected = required(&transaction.expected, "failed Expected projection")?; - match transaction.command_kind { - CommandKind::Deploy => topology - .withdraw_incarnation( - expected, - provider_anchor, - transaction.activation.as_ref(), - None, - ) - .context("withdrawing the failed candidate projection"), - CommandKind::Continuity => topology - .demote_to_expected_only( - expected, - provider_anchor, - required(&transaction.activation, "failed continuity activation")?, - None, - ) - .map(drop) - .context("demoting the failed continuity activation"), - } -} - -fn may_rollback_route_after_failed_proof(fencing: &FencingEvidence) -> bool { - matches!(fencing, FencingEvidence::SkippedStateless) -} - -fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { - matches!( - fencing, - FencingEvidence::Revoked { - incumbent_lease_sha256: Some(_), - .. - } - ) -} - -fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { - PreFencingAbort { - error: truncate(error, 2048), - candidate_cleanup: transaction.candidate_cleanup_owed(), - topology_reconciliation: transaction.abort_topology_reconciliation(), - source_cleanup: if transaction.command_kind == CommandKind::Deploy { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - } -} - -fn post_fencing_abort_intent( - transaction: &DeploymentTransaction, - error: &str, -) -> PostFencingAbort { - PostFencingAbort { - error: truncate(error, 2048), - route_restoration: if transaction.route_preflight.is_some() { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - lease_withdrawal: if transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::issued_lease) - .is_some() - { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - candidate_cleanup: transaction.candidate_cleanup_owed(), - topology_reconciliation: transaction.abort_topology_reconciliation(), - source_cleanup: if transaction.command_kind == CommandKind::Deploy { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - } -} - -fn sequence_requires_admission( - latest: Option<&TopologyEvidence>, - maximum_admitted_sequence: u64, - candidate: &TopologyEvidence, -) -> Result { - candidate.validate_shape()?; - if latest.is_some_and(|existing| { - existing.canonical_bytes == candidate.canonical_bytes - && existing.canonical_sha256 == candidate.canonical_sha256 - && existing.signer_identity_id == candidate.signer_identity_id - && existing.publisher_sequence == candidate.publisher_sequence - }) { - return Ok(false); - } - ensure!( - candidate.publisher_sequence > maximum_admitted_sequence, - "Odin topology publisher sequence was replayed or reordered" - ); - Ok(true) -} - -fn validate_live_providers_for_deploy(command_kind: CommandKind, validation: F) -> Result<()> -where - F: FnOnce() -> Result<()>, -{ - match command_kind { - CommandKind::Deploy => validation(), - CommandKind::Continuity => Ok(()), - } -} - -fn validate_authenticated_evidence( - evidence: &TopologyEvidence, - authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, -) -> Result<()> { - let record = authenticated.record(); - ensure!( - authenticated.canonical_bytes() == evidence.canonical_bytes.as_slice() - && sha256_id(authenticated.canonical_bytes()) == evidence.canonical_sha256 - && record.signer_identity_id == evidence.signer_identity_id - && record.publisher_sequence == evidence.publisher_sequence, - "durable topology evidence differs from its authenticated receipt" - ); - Ok(()) -} - -fn is_semantic_warming( - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - incumbent_lease_sha256: Option<&str>, - authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, -) -> Result { - let record = authenticated.record(); - if !record.present || record.observed_write_lease_sha256.is_some() { - return Ok(false); - } - let expected_projection_detail = format!( - "expected:{};activation:{}", - expected.canonical_sha256()?, - activation.canonical_sha256()? - ); - if !warming_disagreements_match_incumbent( - &expected_projection_detail, - incumbent_lease_sha256, - &record.disagreements, - ) { - return Ok(false); - } - Ok(if expected.write_lease_required { - !record.ready && record.observed_presence_state.as_deref() == Some("warming") - } else { - matches!( - record.observed_presence_state.as_deref(), - Some("warming" | "active") - ) - }) -} - -fn warming_disagreements_match_incumbent( - expected_projection_detail: &str, - incumbent_lease_sha256: Option<&str>, - disagreements: &[OdinTopologyDisagreement], -) -> bool { - if disagreements.is_empty() { - true - } else if let (Some(incumbent_lease_sha256), [disagreement]) = - (incumbent_lease_sha256, disagreements) - { - disagreement.code == "projected-write-lease" - && disagreement.expected.as_deref() == Some(expected_projection_detail) - && disagreement.observed.as_deref() == Some(incumbent_lease_sha256) - } else { - false - } -} - -fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool { - let record = authenticated.record(); - record.present - && record.ready - && record.observed_presence_state.as_deref() == Some("active") - && record.disagreements.is_empty() -} - -fn read_deployment_brake(path: &Path) -> Result)>> { - let Some(envelope) = read_single_envelope(path)? else { - return Ok(None); - }; - ensure!( - envelope.r#type == IdunnDeploymentBrakeRecord::TYPE - && envelope.schema_id.as_deref() == Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA), - "deployment brake store contains a foreign record" - ); - let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&envelope.payload)?; - record.validate()?; - ensure!( - rmp_serde::to_vec(&record)? == envelope.payload && envelope.key == record.brake_id, - "deployment brake is noncanonical or keyed by another authority" - ); - Ok(Some((record, envelope.payload))) -} - -fn read_lifecycle_brake(path: &Path) -> Result> { - let Some(envelope) = read_single_envelope(path)? else { - return Ok(None); - }; - ensure!( - envelope.r#type == IdunnLifecycleBrakeRecord::TYPE - && envelope.schema_id.as_deref() == Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA), - "lifecycle brake store contains a foreign record" - ); - IdunnLifecycleBrakeRecord::decode_canonical(&envelope.payload).map(Some) -} - -fn read_single_envelope(path: &Path) -> Result> { - if !path.exists() { - return Ok(None); - } - let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - match entries.as_slice() { - [] => Ok(None), - [entry] => Ok(Some(entry.clone())), - _ => bail!("single-record authority store is ambiguous"), - } -} - -pub(crate) fn read_trust_anchor( - path: &Path, -) -> Result { - let envelope = read_single_envelope(path)? - .with_context(|| format!("service identity trust anchor {} is absent", path.display()))?; - ensure!( - envelope.r#type == P::TRUST_ANCHOR_TYPE - && envelope.key == P::TRUST_ANCHOR_KEY - && envelope.schema_id.as_deref() == Some(P::TRUST_ANCHOR_SCHEMA), - "service identity trust anchor belongs to another profile" - ); - let anchor: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&envelope.payload)?; - ensure!( - rmp_serde::to_vec(&anchor)? == envelope.payload - && derive_service_identity_id::

(&anchor.public_key)? == anchor.identity_id, - "service identity trust anchor is noncanonical or self-inconsistent" - ); - Ok(anchor) -} - -fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { - value.as_ref().with_context(|| format!("{label} is absent")) -} - -fn now_millis() -> Result { - Ok(SystemTime::now() - .duration_since(UNIX_EPOCH) - .context("system clock predates Unix epoch")? - .as_millis() - .try_into()?) -} - -fn rfc3339_millis(millis: u64) -> Result { - chrono::DateTime::from_timestamp_millis(i64::try_from(millis)?) - .context("timestamp is out of range") - .map(|value| value.to_rfc3339()) -} - -fn sibling_path(path: &Path, suffix: &str) -> PathBuf { - let mut value = path.as_os_str().to_os_string(); - value.push(suffix); - PathBuf::from(value) -} - -fn string_value(args: &mut impl Iterator, name: &str) -> Result { - args.next() - .ok_or_else(|| anyhow!("{name} requires a value")) -} - -fn path_value(args: &mut impl Iterator, name: &str) -> Result { - Ok(PathBuf::from(string_value(args, name)?)) -} - -fn u32_value(args: &mut impl Iterator, name: &str) -> Result { - string_value(args, name)? - .parse() - .with_context(|| format!("{name} requires a u32")) -} - -fn u64_value(args: &mut impl Iterator, name: &str) -> Result { - string_value(args, name)? - .parse() - .with_context(|| format!("{name} requires a u64")) -} - -fn require_selector(value: &str) -> Result<()> { - let value = value.strip_prefix("profile:").unwrap_or(value); - require_id(value, "deployment selector") -} - -pub(crate) fn require_id(value: &str, label: &str) -> Result<()> { - ensure!( - !value.is_empty() - && value.len() <= 256 - && value.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/') - }), - "{label} is invalid" - ); - Ok(()) -} - -fn require_value(value: &str, label: &str) -> Result<()> { - ensure!( - !value.trim().is_empty() - && value == value.trim() - && value.len() <= 1024 - && !value.contains('\0'), - "{label} is invalid" - ); - Ok(()) -} - -fn require_detail(value: &str, label: &str) -> Result<()> { - ensure!( - !value.is_empty() && value.len() <= 2048 && !value.contains('\0'), - "{label} is invalid" - ); - Ok(()) -} - -fn truncate(value: &str, length: usize) -> String { - value.chars().take(length).collect() -} - -fn sha256_id(bytes: &[u8]) -> String { - let digest = Sha256::digest(bytes); - let mut encoded = String::with_capacity(71); - encoded.push_str("sha256-"); - for byte in digest { - use std::fmt::Write as _; - write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); - } - encoded -} - -fn runtime_instance_id(transaction_id: &str) -> Result { - require_id(transaction_id, "runtime transaction id")?; - Ok(sha256_id( - format!("gamecult.idunn.runtime-instance.v1:{transaction_id}").as_bytes(), - )) -} - -fn usage() -> &'static str { - "Idunn deployment, admission, and continuity control plane\n\n\ - idunn serve [runtime options] [--host-actuator-bind ADDR]\n\ - idunn up [--state-store PATH] [--no-wait]\n\ - idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ - idunn cancel [--state-store PATH]\n\ - idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ - Recipes describe capability and process requirements. Idunn seals exact\n\ - source and artifacts, admits one incarnation, and delegates execution to\n\ - systemd and routing mechanics to the configured proxy." -} - -#[cfg(test)] -mod tests { - use cultnet_rs::{ - GameCultProviderHealthIdentity, OdinRuntimeTopologyCorrelationPurpose, - OdinRuntimeTopologyCorrelationRecord, enroll_service_identity_at, - export_service_identity_trust_anchor, - }; - use tempfile::TempDir; - - use super::*; - use crate::drivers::{SystemdWorkloadObservation, incarnation_key}; - use crate::incident::INCIDENT_RETENTION_MILLIS; - - /// Fixed clock for the signing fixture; correlations must land inside - /// DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS of it to authenticate. - const NOW: u64 = 1_700_000_000_000; - - fn digest(byte: char) -> String { - byte.to_string().repeat(64) - } - - fn route_binding( - route_id: &str, - stable_port: u16, - private_port_start: u16, - private_port_end: u16, - ) -> RouteBinding { - RouteBinding { - driver: crate::deployment::RouteDriver::NginxStreamTcp, - route_id: route_id.into(), - stable_endpoint: format!("tcp://127.0.0.1:{stable_port}"), - private_host: "127.0.0.1".into(), - private_port_start, - private_port_end, - config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), - reload_unit: "nginx.service".into(), - } - } - - fn command(kind: CommandKind) -> DeploymentCommand { - DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: match kind { - CommandKind::Deploy => "up-test".into(), - CommandKind::Continuity => "continuity-test".into(), - }, - kind, - selector: "ghostlight".into(), - requested_by: "operator".into(), - requested_at_unix_millis: 100, - } - } - - /// Drives the replacement gate against the real `control.cc`. The point is - /// not that the unit tests pass but that the record actually stuck on the - /// host would now commit. Run with: - /// IDUNN_LIVE_CONTROL=/path/to/control.cc cargo test live_committing -- --ignored --nocapture - #[test] - #[ignore] - fn live_committing_record_is_unstuck_by_the_replacement_gate() -> Result<()> { - let Ok(path) = std::env::var("IDUNN_LIVE_CONTROL") else { - eprintln!("IDUNN_LIVE_CONTROL unset"); - return Ok(()); - }; - let snapshot = ControlSnapshot::read(Path::new(&path))?; - for stored in &snapshot.transactions { - let value = &stored.value; - if value.completion.is_some() { - continue; - } - let (Some(ready), Some(latest)) = ( - value.ready.as_ref().and_then(ReadinessEvidence::odin), - &value.latest_odin_observation, - ) else { - continue; - }; - println!( - "LIVE {} target={} phase={:?} ready_seq={} latest_seq={}", - value.transaction_id, - value.target, - value.phase, - ready.publisher_sequence, - latest.publisher_sequence - ); - println!( - " old gate (receipt == cursor): {}", - if ready == latest { "PASS" } else { "REFUSE" } - ); - let (latest_record, _) = - OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( - &latest.canonical_bytes, - )?; - println!( - " cursor says: projection={} instance={:?} ready={} present={} disagreements={}", - latest_record.expected_projection_sha256, - latest_record.runtime_instance_id, - latest_record.ready, - latest_record.present, - latest_record.disagreements.len() - ); - let (Some(expected), Some(activation)) = (&value.expected, &value.activation) else { - println!(" transaction has not declared an incarnation yet"); - continue; - }; - println!( - " declared: projection={} instance={}", - expected.canonical_sha256()?, - activation.runtime_instance_id - ); - let binds = latest_record.expected_projection_sha256 == expected.canonical_sha256()? - && latest_record.runtime_instance_id.as_deref() - == Some(activation.runtime_instance_id.as_str()); - println!(" binds declared incarnation (what authentication enforces): {binds}"); - } - Ok(()) - } - - /// The smallest real `Engine` the scheduler needs. Idunn had no - /// Engine-level test at all, which is why "does one bad record kill the - /// daemon" was arguable rather than measurable. - struct EngineFixture { - _temp: TempDir, - engine: Engine, - state_store: PathBuf, - odin_signer: ServiceIdentitySigner, - root: PathBuf, - /// The host programs every route driver of this Engine actuates - /// through: logged stubs, so nothing touches the workstation's nginx. - #[cfg(unix)] - route_stubs: RouteStubs, - } - - /// Stub nginx, systemd-run, systemctl and ufw that append each call to one - /// log. `reload-fails` makes `systemctl reload` exit non-zero. - #[cfg(unix)] - struct RouteStubs { - calls: PathBuf, - reload_fails: PathBuf, - ufw_delete_fails: PathBuf, - } - - #[cfg(unix)] - impl RouteStubs { - fn new(root: &Path) -> Result<(RouteActuators, Self)> { - use std::os::unix::fs::PermissionsExt; - - let dir = root.join("route-stubs"); - std::fs::create_dir_all(&dir)?; - let calls = dir.join("calls"); - let reload_fails = dir.join("reload-fails"); - let ufw_delete_fails = dir.join("ufw-delete-fails"); - let program = |name: &str, refuse: &str| -> Result { - let path = dir.join(name); - std::fs::write( - &path, - format!( - "#!/bin/sh\necho \"{name} $*\" >> '{}'\n{refuse}exit 0\n", - calls.display() - ), - )?; - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))?; - Ok(path) - }; - let actuators = RouteActuators { - nginx: program("nginx", "")?, - systemd_run: program("systemd-run", "")?, - systemctl: program( - "systemctl", - &format!( - "if [ \"$1\" = reload ] && [ -e '{}' ]; then exit 1; fi\n", - reload_fails.display() - ), - )?, - ufw: program( - "ufw", - &format!( - "if [ \"$1\" = delete ] && [ -e '{}' ]; then exit 1; fi\n", - ufw_delete_fails.display() - ), - )?, - preflight_root: root.join("route-preflight"), - }; - Ok(( - actuators, - Self { - calls, - reload_fails, - ufw_delete_fails, - }, - )) - } - - fn refuse_reloads(&self) -> Result<()> { - std::fs::write(&self.reload_fails, b"x")?; - Ok(()) - } - - /// `ufw delete` exits non-zero with an error that is not "non-existent". - fn refuse_ufw_deletes(&self) -> Result<()> { - std::fs::write(&self.ufw_delete_fails, b"x")?; - Ok(()) - } - - /// How many logged calls begin with `prefix`, e.g. `systemctl reload`. - fn count(&self, prefix: &str) -> usize { - std::fs::read_to_string(&self.calls) - .unwrap_or_default() - .lines() - .filter(|line| line.starts_with(prefix)) - .count() - } - } - - impl EngineFixture { - fn new() -> Result { - Self::build(None) - } - - /// An Engine whose systemd workload port is `workload`, so the phase - /// machine can run without systemd. This is the whole seam: nginx, - /// systemctl and the network are never reached on an unrouted, - /// stateless target, and the Odin evidence is signed by the key this - /// fixture enrolled. - fn with_workload(workload: Arc) -> Result { - Self::build(Some(workload)) - } - - fn build(workload: Option>) -> Result { - let temp = TempDir::new()?; - let root = temp.path(); - std::fs::create_dir_all(root.join("identities"))?; - let idunn = root.join("identities/idunn.cc"); - enroll_service_identity_at::(&idunn)?; - let odin_private = root.join("identities/odin.cc"); - let odin_signer = enroll_service_identity_at::(&odin_private)?; - let odin_anchor = root.join("identities/odin-anchor.cc"); - export_service_identity_trust_anchor(&odin_signer, &odin_anchor)?; - - let state_store = root.join("control.cc"); - #[cfg(unix)] - let (route_actuators, route_stubs) = RouteStubs::new(root)?; - let options = RuntimeOptions { - state_store: state_store.clone(), - bindings_dir: root.join("bindings"), - source_root: root.join("sources"), - staging_root: root.join("staging"), - topology_store: root.join("topology.cc"), - incident_store: root.join("incidents.cc"), - odin_correlation_store: root.join("odin-correlation.cc"), - odin_trust_anchor: odin_anchor, - idunn_identity_store: idunn, - deployment_brake_operator_anchor: root.join("brake-anchor.cc"), - #[cfg(unix)] - route_actuators, - ..RuntimeOptions::default() - }; - let engine = match workload { - Some(workload) => Engine::open_with_systemd_workload(options, workload)?, - None => Engine::open(options)?, - }; - Ok(Self { - root: root.to_path_buf(), - _temp: temp, - engine, - state_store, - odin_signer, - #[cfg(unix)] - route_stubs, - }) - } - } - - fn terminal_transaction(target: &str) -> Result { - Ok(terminal_transaction_with_command(target)?.0) - } - - fn terminal_transaction_with_command( - target: &str, - ) -> Result<(DeploymentTransaction, DeploymentCommand)> { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{target}"), - kind: CommandKind::Deploy, - selector: target.into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let mut transaction = DeploymentTransaction::new(&command, target.into(), 0, None, 100)?; - transaction.phase = DeploymentPhase::Complete; - transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { - error: "sealed source vanished".into(), - }); - transaction.pre_fencing_abort = Some(PreFencingAbort { - error: "sealed source vanished".into(), - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Complete, - }); - assert!(transaction.is_terminal(), "fixture must be terminal"); - Ok((transaction, command)) - } - - #[test] - fn a_finished_transaction_leaves_the_live_set_and_stays_observable() -> Result<()> { - let world = EngineFixture::new()?; - let history = history_store_path(&world.state_store); - - let (live, command) = terminal_transaction_with_command("ghostlight")?; - // A transaction is only readable beside its immutable command. - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope( - &command, - command.requested_at_unix_millis - )?], - )? - ); - let mut opening = live.clone(); - opening.phase = DeploymentPhase::Sealing; - opening.completion = None; - opening.pre_fencing_abort = None; - let opening_envelope = transaction_envelope(&opening, opening.updated_at_unix_millis)?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: opening.transaction_id.clone(), - current: None, - }], - std::slice::from_ref(&opening_envelope), - )? - ); - let stored = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .next() - .context("seeded transaction")?; - - replace_transaction(&world.state_store, &stored, &live)?; - - // Gone from the set that still gates decisions... - assert!( - ControlSnapshot::read(&world.state_store)? - .transactions - .is_empty(), - "a finished transaction stayed resident" - ); - // ...and still answerable, which is what R11 refused to trade away. - let archived = read_history_transactions(&world.state_store); - assert_eq!(archived.len(), 1); - assert_eq!(archived[0].transaction_id, live.transaction_id); - assert!(history.exists()); - // The command went with it. Left resident it would read as queued and - // be frozen again next tick; consumed, it is history beside its - // transaction and `status` still answers for it. - assert!( - ControlSnapshot::read(&world.state_store)? - .commands - .is_empty(), - "a consumed command stayed resident" - ); - let archived_commands = read_history_commands(&world.state_store); - assert_eq!(archived_commands.len(), 1); - assert_eq!(archived_commands[0].command_id, command.command_id); - assert!(!world.engine.freeze_one_queued_command()?); - Ok(()) - } - - #[test] - fn a_resident_command_whose_transactions_are_history_is_retired_not_frozen() -> Result<()> { - // Before commands travelled to history with their transactions, a - // command outlived its retired transaction in the control store and - // read as queued again. Freeze must retire it, never re-freeze it. - let world = EngineFixture::new()?; - let (finished, command) = terminal_transaction_with_command("ghostlight")?; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( - command_envelope(&command, command.requested_at_unix_millis)?, - )?; - SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) - .insert_entry_if_absent(transaction_envelope( - &finished, - finished.updated_at_unix_millis, - )?)?; - assert_eq!(ControlSnapshot::read(&world.state_store)?.commands.len(), 1); - - assert!(world.engine.freeze_one_queued_command()?); - - assert!( - ControlSnapshot::read(&world.state_store)? - .commands - .is_empty(), - "consumed command stayed resident" - ); - assert_eq!(read_history_commands(&world.state_store).len(), 1); - assert!( - ControlSnapshot::read(&world.state_store)? - .transactions - .is_empty(), - "a consumed command was frozen again" - ); - assert!(!world.engine.freeze_one_queued_command()?); - Ok(()) - } - - #[test] - fn a_resident_terminal_transaction_is_retired_by_the_tick_with_its_command() -> Result<()> { - let world = EngineFixture::new()?; - let (finished, command) = terminal_transaction_with_command("ghostlight")?; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( - command_envelope(&command, command.requested_at_unix_millis)?, - )?; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( - transaction_envelope(&finished, finished.updated_at_unix_millis)?, - )?; - assert!(world.engine.retire_one_terminal_transaction()?); - let live = ControlSnapshot::read(&world.state_store)?; - assert!(live.transactions.is_empty() && live.commands.is_empty()); - assert_eq!(read_history_transactions(&world.state_store).len(), 1); - assert_eq!(read_history_commands(&world.state_store).len(), 1); - assert!(!world.engine.retire_one_terminal_transaction()?); - Ok(()) - } - - #[test] - fn re_archiving_after_a_crash_window_is_a_no_op() -> Result<()> { - // History is written before the live copy is removed, so a crash - // between the two leaves the record in both. The repeat must not - // duplicate it or fail. - let world = EngineFixture::new()?; - let live = terminal_transaction("odin")?; - let envelope = transaction_envelope(&live, live.updated_at_unix_millis)?; - - archive_terminal_transaction(&world.state_store, &envelope)?; - archive_terminal_transaction(&world.state_store, &envelope)?; - - let archived = read_history_transactions(&world.state_store); - assert_eq!(archived.len(), 1, "the repeat duplicated the record"); - Ok(()) - } - - #[test] - fn an_unreadable_history_entry_is_skipped_rather_than_fatal() -> Result<()> { - // The asymmetry that justifies two files: control.cc refuses a record - // it cannot re-encode byte for byte, because those records still gate - // decisions. History describes what already happened and must never be - // able to refuse a read. - let world = EngineFixture::new()?; - let live = terminal_transaction("voidbot")?; - let envelope = transaction_envelope(&live, live.updated_at_unix_millis)?; - archive_terminal_transaction(&world.state_store, &envelope)?; - - let history = history_store_path(&world.state_store); - assert!( - SingleFileMessagePackBackingStore::new(&history).insert_entry_if_absent( - CultCacheEnvelope { - key: "tx-corrupt".into(), - r#type: DeploymentTransaction::TYPE.into(), - payload: vec![0xc1], - stored_at: rfc3339_millis(1)?, - schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), - } - )? - ); - - let archived = read_history_transactions(&world.state_store); - assert_eq!(archived.len(), 1, "the readable record must survive"); - assert_eq!(archived[0].transaction_id, live.transaction_id); - Ok(()) - } - - #[test] - fn an_absent_brake_anchor_does_not_gate_an_empty_store() -> Result<()> { - // `F:\Projects\CLAUDE.md`: a target brake "may never gate Idunn itself - // or unrelated service lifecycle". Reading the operator anchor up front - // meant a missing brake artifact refused startup against a store with - // nothing in it -- the deployment authority held hostage by an artifact - // describing one target's consent. - let world = EngineFixture::new()?; - assert!( - !world - .engine - .options - .deployment_brake_operator_anchor - .exists(), - "fixture must not have written a brake anchor" - ); - let snapshot = ControlSnapshot::read(&world.state_store)?; - world.engine.validate_durable_authority(&snapshot)?; - Ok(()) - } - - #[test] - fn one_unreadable_record_faults_its_tick_without_touching_the_store() -> Result<()> { - let world = EngineFixture::new()?; - - // An empty store schedules nothing and must not fault. - assert!(!world.engine.run_scheduler_tick()?); - - // A record the control store cannot read. Idunn runs under - // Restart=always, so before run_scheduler_tick existed this - // propagated out of the loop and crashlooped the daemon -- taking - // every unrelated target's continuity with it. - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: "tx-unreadable".into(), - current: None, - }], - &[CultCacheEnvelope { - key: "tx-unreadable".into(), - r#type: DeploymentTransaction::TYPE.into(), - // 0xc1 is msgpack's never-used byte: decodable by nothing. - payload: vec![0xc1], - stored_at: rfc3339_millis(1)?, - schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), - }], - )? - ); - let poisoned = std::fs::read(&world.state_store)?; - - for _ in 0..3 { - assert!( - world.engine.run_scheduler_tick().is_err(), - "the tick must report the fault rather than swallow it" - ); - } - - // Faulting is not the same as flailing: a tick that cannot read must - // not rewrite, truncate or repair the store behind the operator. - assert_eq!( - std::fs::read(&world.state_store)?, - poisoned, - "a faulting tick rewrote the control store" - ); - Ok(()) - } - - /// A real signing chain for topology gates. Until this existed no test in - /// this file could construct a signed Odin correlation -- every topology - /// fixture was `vec![byte]` -- so every gate between Idunn and Odin was - /// asserted against bytes that could not be authenticated. That is how a - /// comparison against Idunn's own wall clock survived in the admission - /// path. Modelled on `odin-daemon`'s TestWorld, which already had to build - /// this chain to test the other side of the same contract. - struct TopologyFixture { - _temp: TempDir, - idunn_anchor: ServiceIdentityTrustAnchor, - odin_signer: ServiceIdentitySigner, - provider_public_key: Vec, - expected: IdunnExpectedIncarnationRecord, - activation: IdunnRuntimeActivationRecord, - } - - impl TopologyFixture { - fn new(target: &str) -> Result { - let temp = TempDir::new()?; - let root = temp.path().join("identities"); - std::fs::create_dir_all(&root)?; - let idunn_signer = - enroll_service_identity_at::(&root.join("idunn.cc"))?; - let idunn_anchor = idunn_signer.trust_anchor()?; - let odin_signer = - enroll_service_identity_at::(&root.join("odin.cc"))?; - let provider_signer = enroll_service_identity_at::( - &root.join("provider.cc"), - )?; - - let expected = IdunnExpectedIncarnationRecord { - schema_version: cultnet_rs::IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), - target: target.into(), - plan_id: sha256_id(b"plan"), - incarnation_id: format!("{target}/generation-1"), - sealed_release_id: sha256_id(b"release"), - source_repository: format!("github.com/GameCult/{target}"), - source_revision: "3".repeat(40), - recipe_sha256: sha256_id(b"recipe"), - runtime_id: format!("{target}-runtime"), - expected_signer_identity_id: provider_signer.entry().identity_id.clone(), - health_contract: format!("{target}.runtime-health.v1"), - artifact_sha256: sha256_id(b"artifact"), - state_schema_generation: None, - state_contract_sha256: None, - write_lease_required: false, - route: None, - capabilities: Vec::new(), - dependencies: Vec::new(), - }; - expected.validate()?; - - // Idunn-signed, exactly as the real activation path issues it: the - // authority chain is what makes a correlation authenticable at all. - let launch = IdunnRuntimeActivationLaunch::issue( - &expected, - sha256_id(b"artifact-witness"), - NOW - 20, - &idunn_signer, - )?; - let activation = launch.activation().clone(); - - Ok(Self { - _temp: temp, - idunn_anchor, - odin_signer, - provider_public_key: provider_signer.entry().public_key.clone(), - expected, - activation, - }) - } - - fn authority(&self) -> Result { - verify_runtime_authority( - &self.expected, - &self.activation, - &self.idunn_anchor, - &self.provider_public_key, - ) - } - - fn correlation( - &self, - sequence: u64, - ready: bool, - ) -> Result { - Ok(OdinRuntimeTopologyCorrelationRecord { - schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - target: self.expected.target.clone(), - expected_projection_sha256: self.expected.canonical_sha256()?, - expected: true, - current_activation_sha256: Some(self.activation.canonical_sha256()?), - signed_presence_sha256: Some(sha256_id(b"presence")), - observed_presence_state: Some("active".into()), - observed_presence_publisher_sequence: Some(sequence), - observed_write_lease_sha256: None, - observed_capabilities: Vec::new(), - runtime_id: self.expected.runtime_id.clone(), - runtime_instance_id: Some(self.activation.runtime_instance_id.clone()), - present: true, - ready, - dependencies: Vec::new(), - disagreements: Vec::new(), - signer_identity_id: self.odin_signer.entry().identity_id.clone(), - publisher_sequence: sequence, - observed_at_unix_millis: NOW, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - }) - } - - fn sign(&self, record: &mut OdinRuntimeTopologyCorrelationRecord) -> Result> { - record.signature = self - .odin_signer - .sign::( - &record.unsigned_signature_payload()?, - ) - .signature; - record.canonical_bytes() - } - - fn authenticate( - &self, - canonical: &[u8], - ) -> Result { - authenticate_odin_runtime_topology_correlation( - canonical, - &self.authority()?, - None, - &self.odin_signer.entry().public_key, - OdinTopologyAuthenticationContext { - trusted_received_at_unix_millis: NOW, - maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, - maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, - }, - ) - } - } - - #[test] - fn the_fixture_actually_authenticates() -> Result<()> { - // First: prove the chain can REFUSE. Without this the tests below - // establish nothing -- a fixture that authenticates anything would - // make every gate look satisfied. - let world = TopologyFixture::new("odin")?; - let mut record = world.correlation(297_765, true)?; - let canonical = world.sign(&mut record)?; - world.authenticate(&canonical)?; - - // Re-encoded with the old signature over changed content. - let mut forged = record.clone(); - forged.publisher_sequence = 297_766; - assert!(world.authenticate(&forged.canonical_bytes()?).is_err()); - - let mut unsigned = world.correlation(297_765, true)?; - unsigned.signature = vec![0; 64]; - assert!(world.authenticate(&unsigned.canonical_bytes()?).is_err()); - - // Stale beyond the trusted observation window. - let mut old = world.correlation(297_765, true)?; - old.observed_at_unix_millis = NOW - DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS - 1; - let old_canonical = world.sign(&mut old)?; - assert!(world.authenticate(&old_canonical).is_err()); - Ok(()) - } - - #[test] - fn a_signed_observation_far_past_the_receipt_admits_this_incarnation() -> Result<()> { - // The yggdrasil wedge, with a real signature rather than vec![byte]: - // the cursor has run hundreds of thousands of sequences past the Ready - // receipt and still describes the incarnation the transaction declared. - let world = TopologyFixture::new("odin")?; - let mut record = world.correlation(501_713, true)?; - let canonical = world.sign(&mut record)?; - let authenticated = world.authenticate(&canonical)?; - - // Authenticating at all is the incarnation check: the authority is - // built from the declared expected and activation, and the correlation - // had to bind both to get here. Nothing further to compare. - assert!(is_semantic_ready(&authenticated)); - assert_eq!(authenticated.record().publisher_sequence, 501_713); - Ok(()) - } - - #[test] - fn authentication_binds_a_correlation_to_the_declared_incarnation() -> Result<()> { - // The failure that matters: Odin's signature is valid and the record is - // fresh, but the process it describes is not the one we activated. - // Authentication already refuses this -- the authority is built from - // this transaction's own expected and activation, and the correlation - // must bind both. Pinned here because a redundant identity gate sat on - // top of this for want of a test that could reach it. - let world = TopologyFixture::new("odin")?; - let mut restarted = world.correlation(501_713, true)?; - restarted.runtime_instance_id = Some(sha256_id(b"another-instance")); - let canonical = world.sign(&mut restarted)?; - let error = world - .authenticate(&canonical) - .expect_err("correlation must bind the current activation"); - assert!( - format!("{error:#}").contains("does not bind the current activation"), - "{error:#}" - ); - - // Same for a correlation about another Expected projection: the - // authority carries this transaction's declaration, so a correlation - // that is not about it cannot authenticate against it. - let mut reprojected = world.correlation(501_714, true)?; - reprojected.expected_projection_sha256 = sha256_id(b"another-projection"); - let canonical = world.sign(&mut reprojected)?; - assert!(world.authenticate(&canonical).is_err()); - Ok(()) - } - - #[test] - fn a_signed_observation_that_is_not_ready_does_not_admit() -> Result<()> { - let world = TopologyFixture::new("odin")?; - let mut degraded = world.correlation(501_713, false)?; - let canonical = world.sign(&mut degraded)?; - assert!(!is_semantic_ready(&world.authenticate(&canonical)?)); - - // Ready alongside a disagreement is not merely "not ready" -- the - // correlation contract refuses to encode it at all, so a publisher - // cannot assert readiness over its own dissent. Worth pinning: it is - // why is_semantic_ready's disagreement clause is belt to the schema's - // braces rather than the only thing standing there. - let mut disputed = world.correlation(501_714, true)?; - disputed.disagreements.push(OdinTopologyDisagreement { - code: "route-membership-differs".into(), - expected: Some("a".into()), - observed: Some("b".into()), - }); - assert!(world.sign(&mut disputed).is_err()); - Ok(()) - } - - // --------------------------------------------------------------------- - // Admission audit (soul/admission-audit). Each test below is a negative - // check on one claim made by c09993c..8448f83; none asserts that the - // happy path works, the tests above already do that. - // --------------------------------------------------------------------- - - impl TopologyFixture { - /// Authenticate with an explicit trusted time, against this fixture's - /// own authority and Odin key. Mirrors `rehydrate_ready_token`'s two - /// callers: `now` when require_current, `admitted_at` otherwise. - fn authenticate_at( - &self, - canonical: &[u8], - trusted_received_at_unix_millis: u64, - ) -> Result { - authenticate_odin_runtime_topology_correlation( - canonical, - &self.authority()?, - None, - &self.odin_signer.entry().public_key, - OdinTopologyAuthenticationContext { - trusted_received_at_unix_millis, - maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, - maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, - }, - ) - } - } - - /// Claim 1: deleting `observation_describes_declared_incarnation` opened - /// nothing because authentication binds the same identity. The strongest - /// attack is a correlation that is internally perfect for ANOTHER - /// incarnation of the same target -- its own projection, its own activation - /// digest, its own instance id -- signed by OUR admitted Odin, so the only - /// thing left to refuse it is the incarnation binding itself. - #[test] - fn a_perfect_correlation_about_another_incarnation_of_this_target_is_refused() -> Result<()> { - let ours = TopologyFixture::new("odin")?; - let theirs = TopologyFixture::new("odin")?; - assert_ne!( - ours.expected.canonical_sha256()?, - theirs.expected.canonical_sha256()?, - "fixtures must declare distinct incarnations for this test to mean anything" - ); - // Sanity: it is a real, Ready correlation for the other incarnation. - let mut genuine = theirs.correlation(501_713, true)?; - let genuine = theirs.sign(&mut genuine)?; - assert!(is_semantic_ready(&theirs.authenticate(&genuine)?)); - - let mut record = theirs.correlation(501_713, true)?; - let canonical = ours.sign(&mut record)?; - let error = ours - .authenticate(&canonical) - .expect_err("another incarnation's correlation must not authenticate against ours"); - assert!( - format!("{error:#}").contains("substitutes or omits Expected authority"), - "{error:#}" - ); - Ok(()) - } - - /// Claim 1, second face: same Expected, different activation. This is the - /// case the deleted gate could not see at all -- it compared instance ids - /// only -- and the one that matters when a process is re-activated under - /// the same projection. Authentication compares the activation digest. - #[test] - fn a_correlation_bound_to_another_activation_of_the_same_expected_is_refused() -> Result<()> { - let world = TopologyFixture::new("odin")?; - let other = TopologyFixture::new("odin")?; - - let mut reactivated = world.correlation(501_713, true)?; - reactivated.current_activation_sha256 = Some(other.activation.canonical_sha256()?); - let canonical = world.sign(&mut reactivated)?; - let error = world - .authenticate(&canonical) - .expect_err("activation digest must bind"); - assert!( - format!("{error:#}").contains("does not bind the current activation"), - "{error:#}" - ); - - // A Present record with no activation cannot even be encoded: the - // schema refuses it upstream of authentication ("Present topology - // state lacks an authenticated runtime session"). Either refusal is - // the right answer; what must not happen is an absent field reading as - // a match for ours. - let mut anonymous = world.correlation(501_714, true)?; - anonymous.current_activation_sha256 = None; - let error = world - .sign(&mut anonymous) - .and_then(|canonical| world.authenticate(&canonical)) - .expect_err("absent activation must not read as ours"); - assert!( - format!("{error:#}").contains("lacks an authenticated runtime session"), - "{error:#}" - ); - - let mut nameless = world.correlation(501_715, true)?; - nameless.runtime_instance_id = None; - let error = world - .sign(&mut nameless) - .and_then(|canonical| world.authenticate(&canonical)) - .expect_err("absent instance id must not read as ours"); - // Likewise refused at encoding: a Present record must carry its - // instance id ("runtime instance identity and observation evidence are - // partial"). - assert!(format!("{error:#}").contains("are partial"), "{error:#}"); - Ok(()) - } - - /// Claim 1, third face: the deleted gate also compared `target`. The - /// projection digest already covers the target, but a record can carry a - /// foreign `target` field beside our projection digest; prove - /// `validate_against_expected` refuses that on its own. - #[test] - fn a_correlation_naming_another_target_beside_our_projection_is_refused() -> Result<()> { - let world = TopologyFixture::new("odin")?; - let mut stranger = world.correlation(501_713, true)?; - stranger.target = "ghostlight".into(); - let canonical = world.sign(&mut stranger)?; - let error = world - .authenticate(&canonical) - .expect_err("target must bind"); - assert!( - format!("{error:#}").contains("substitutes or omits Expected authority"), - "{error:#}" - ); - Ok(()) - } - - /// Claim 4: the fixture's chain can refuse at every link, not only at the - /// correlation signature. If any of these passed, the tests above would be - /// asserting against a rubber stamp. - #[test] - fn the_fixture_chain_refuses_at_every_link() -> Result<()> { - let ours = TopologyFixture::new("odin")?; - let theirs = TopologyFixture::new("odin")?; - - // Authority link: activation must bind this Expected ... - assert!( - verify_runtime_authority( - &ours.expected, - &theirs.activation, - &ours.idunn_anchor, - &ours.provider_public_key - ) - .is_err(), - "another incarnation's activation verified against our Expected" - ); - // ... be signed by the Idunn we trust ... - assert!( - verify_runtime_authority( - &ours.expected, - &ours.activation, - &theirs.idunn_anchor, - &ours.provider_public_key - ) - .is_err(), - "activation verified against a foreign Idunn anchor" - ); - // ... and name the provider key the Expected selected. - assert!( - verify_runtime_authority( - &ours.expected, - &ours.activation, - &ours.idunn_anchor, - &theirs.provider_public_key - ) - .is_err(), - "a foreign provider key was accepted as the Expected signer" - ); - - // Odin link: a correlation honestly signed by another Odin identity - // (its own id, its own signature) is refused against our admitted key. - let mut record = ours.correlation(501_713, true)?; - record.signer_identity_id = theirs.odin_signer.entry().identity_id.clone(); - let canonical = theirs.sign(&mut record)?; - let error = ours - .authenticate(&canonical) - .expect_err("foreign Odin signer accepted"); - assert!( - format!("{error:#}").contains("names a different identity"), - "{error:#}" - ); - - // Odin link: our identity id, their signature bytes. - let mut record = ours.correlation(501_713, true)?; - record.signature = theirs - .odin_signer - .sign::(&record.unsigned_signature_payload()?) - .signature; - let error = ours - .authenticate(&record.canonical_bytes()?) - .expect_err("forged signature accepted"); - assert!( - format!("{error:#}").contains("signature verification failed"), - "{error:#}" - ); - Ok(()) - } - - /// Claim 2: `rehydrate_ready_token(.., require_current = false)` replays the - /// receipt's authentication at its own `admitted_at`, so the window it - /// widens is exactly the one the receipt already passed at admission. Pin - /// both halves: at `admitted_at` the receipt authenticates however old it - /// is now; at `now` past the window it does not. An `admitted_at` earlier - /// than the observation is refused by the future-skew bound, so the field - /// cannot be pulled backwards to launder a later observation either. - #[test] - fn a_durable_receipt_replays_at_its_admission_time_and_not_at_now() -> Result<()> { - let world = TopologyFixture::new("odin")?; - let mut record = world.correlation(3_841, true)?; - let canonical = world.sign(&mut record)?; - let admitted_at = NOW + 5; - - world.authenticate_at(&canonical, admitted_at)?; - let much_later = admitted_at + DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS + 1; - assert!( - world.authenticate_at(&canonical, much_later).is_err(), - "require_current would refuse" - ); - world.authenticate_at(&canonical, admitted_at)?; - - let before_observation = NOW - DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS - 1; - assert!( - world - .authenticate_at(&canonical, before_observation) - .is_err(), - "an admitted_at earlier than the observation must not authenticate" - ); - Ok(()) - } - - /// Claim 5 companion: what the live store holds, not only whether the - /// stuck record would commit. Run with: - /// IDUNN_LIVE_CONTROL=/path/to/control.cc cargo test live_store_inventory -- --ignored --nocapture - #[test] - #[ignore] - fn live_store_inventory() -> Result<()> { - let Ok(path) = std::env::var("IDUNN_LIVE_CONTROL") else { - eprintln!("IDUNN_LIVE_CONTROL unset"); - return Ok(()); - }; - let snapshot = ControlSnapshot::read(Path::new(&path))?; - println!( - "INVENTORY commands={} transactions={} admitted={}", - snapshot.commands.len(), - snapshot.transactions.len(), - snapshot.admitted.len() - ); - for stored in &snapshot.transactions { - let value = &stored.value; - println!( - "TX {} target={} kind={:?} phase={:?} complete={} ready_seq={:?} latest_seq={:?} incumbent={:?}", - value.transaction_id, - value.target, - value.command_kind, - value.phase, - value.completion.is_some(), - value - .ready - .as_ref() - .and_then(ReadinessEvidence::odin) - .map(|evidence| evidence.publisher_sequence), - value - .latest_odin_observation - .as_ref() - .map(|evidence| evidence.publisher_sequence), - value.incumbent_generation_id, - ); - } - for stored in &snapshot.admitted { - let value = &stored.value; - println!( - "ADMITTED target={} generation={} tx={} admitted_at={} ready_seq={:?} ready_admitted_at={:?} latest_seq={:?} latest_admitted_at={:?} instance={} {}", - value.target, - value.generation_id, - value.transaction_id, - value.admitted_at_unix_millis, - value.ready.odin().map(|evidence| evidence.publisher_sequence), - value.ready.odin().map(|evidence| evidence.admitted_at_unix_millis), - value - .latest_odin_observation - .as_ref() - .map(|evidence| evidence.publisher_sequence), - value - .latest_odin_observation - .as_ref() - .map(|evidence| evidence.admitted_at_unix_millis), - value.activation.runtime_instance_id, - value.workload.describe(), - ); - } - Ok(()) - } - - fn topology(sequence: u64, byte: u8) -> TopologyEvidence { - let canonical_bytes = vec![byte]; - TopologyEvidence { - canonical_sha256: sha256_id(&canonical_bytes), - canonical_bytes, - signer_identity_id: "odin-signer".into(), - publisher_sequence: sequence, - admitted_at_unix_millis: 100, - } - } - - fn workload(uid: u32, pid_namespace_id: u64, mount_namespace_id: u64) -> WorkloadObservation { - WorkloadObservation::Systemd(SystemdWorkloadObservation { - unit: format!("idunn-{uid}.service"), - unit_description: format!("Idunn test {uid}"), - invocation_id: format!("invocation-{uid}"), - exec_main_start_timestamp_monotonic: 1, - service_type: "exec".into(), - restart_policy: "always".into(), - kill_mode: "control-group".into(), - dynamic_user: true, - systemd_user: format!("u{uid}"), - systemd_group: format!("u{uid}"), - supplementary_groups: String::new(), - capability_bounding_set: String::new(), - ambient_capabilities: String::new(), - private_mounts: true, - private_pids: true, - protect_proc: "invisible".into(), - proc_subset: "pid".into(), - no_new_privileges: true, - umask: "0077".into(), - inaccessible_paths: String::new(), - load_credential: String::new(), - main_pid: uid, - process_start_time: 1, - process_uids: [uid; 4], - process_gids: [uid; 4], - process_groups: vec![uid], - process_cap_inheritable: 0, - process_cap_permitted: 0, - process_cap_effective: 0, - process_cap_bounding: 0, - process_cap_ambient: 0, - process_no_new_privileges: true, - process_namespace_pids: vec![1], - mount_namespace_id, - pid_namespace_id, - executable: PathBuf::from("/opt/test/bin/service"), - executable_device: 1, - executable_inode: 1, - executable_sha256: sha256_id(&[1]), - runtime_instance_id: sha256_id(&uid.to_be_bytes()), - working_directory: PathBuf::from("/opt/test"), - runtime_bundle: PathBuf::from("/run/test"), - command_line_sha256: sha256_id(&[2]), - environment_names: Vec::new(), - environment_contract_sha256: sha256_id(&[3]), - control_group: format!("/system.slice/idunn-{uid}.service"), - credentials_directory: None, - parent_only_file_descriptors: Vec::new(), - activation_signer_identity_id: "activation".into(), - activation_signer_public_key: vec![1; 32], - service_credentials: Vec::new(), - }) - } - - fn write_lease() -> IdunnProcessWriteLeaseRecord { - IdunnProcessWriteLeaseRecord { - schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), - target: "ghostlight".into(), - expected_projection_sha256: sha256_id(&[1]), - plan_id: sha256_id(&[2]), - incarnation_id: "incarnation-test".into(), - sealed_release_id: sha256_id(&[3]), - activation_witness_sha256: sha256_id(&[4]), - state_schema_generation: "ghostlight-state-v1".into(), - state_contract_sha256: sha256_id(&[5]), - runtime_id: "ghostlight".into(), - runtime_instance_id: sha256_id(&[6]), - warming_presence_sha256: sha256_id(&[7]), - lease_epoch: 1, - issued_at_unix_millis: 100, - } - } - - #[test] - fn validate_is_offline_and_declarative() { - // It must accept a recipe alone, accept a recipe plus a binding, and - // refuse to be handed anything executable. It opens no store and - // actuates nothing, which is what makes it safe to run on a candidate - // binding before that binding is installed. - let parsed = parse( - ["validate", "--recipe", "/tmp/recipe.toml"] - .into_iter() - .map(str::to_owned), - ) - .unwrap(); - assert!(matches!(parsed, Command::Validate { binding: None, .. })); - - assert!( - parse( - ["validate", "--recipe", "/r.toml", "--binding", "/b.toml"] - .into_iter() - .map(str::to_owned) - ) - .is_ok() - ); - assert!(parse(["validate"].into_iter().map(str::to_owned)).is_err()); - for rejected in [ - vec![ - "validate", - "--recipe", - "/r.toml", - "--deploy-command", - "sh -c bad", - ], - vec!["validate", "--recipe", "/r.toml", "--state-store", "/s.cc"], - ] { - assert!(parse(rejected.into_iter().map(str::to_owned)).is_err()); - } - } - - #[test] - fn cli_exposes_only_declarative_commands() { - for arguments in [ - vec!["up", "ghostlight", "--deploy-command", "sh -c bad"], - vec!["serve", "--swarm-profile", "yggdrasil-local"], - vec!["serve", "--restart-command", "bad"], - vec!["validate-runtime-admission"], - ] { - assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); - } - } - - #[test] - fn garden_path_accepts_service_and_profile_selectors() { - for selector in ["ghostlight", "profile:aetheria", "profile:full-gamecult"] { - let parsed = - parse(["up", selector, "--no-wait"].into_iter().map(str::to_owned)).unwrap(); - let Command::Up { - selector: actual, .. - } = parsed - else { - panic!("expected up command") - }; - assert_eq!(actual, selector); - } - } - - #[test] - fn route_bindings_are_one_global_socket_and_candidate_authority_map() { - let first = route_binding("first", 4103, 14103, 14111); - let second = route_binding("second", 8831, 18831, 18839); - validate_route_binding_set(&[("first", &first), ("second", &second)]).unwrap(); - - let mut duplicate_id = second.clone(); - duplicate_id.route_id = first.route_id.clone(); - assert!( - validate_route_binding_set(&[("first", &first), ("second", &duplicate_id)]).is_err() - ); - - let mut overlapping_candidates = second.clone(); - overlapping_candidates.private_port_start = 14111; - overlapping_candidates.private_port_end = 14120; - assert!( - validate_route_binding_set(&[("first", &first), ("second", &overlapping_candidates),]) - .is_err() - ); - - let mut stable_inside_other_range = second; - stable_inside_other_range.stable_endpoint = "tcp://127.0.0.1:14105".into(); - assert!( - validate_route_binding_set(&[ - ("first", &first), - ("second", &stable_inside_other_range), - ]) - .is_err() - ); - } - - #[test] - fn deployment_command_is_immutable_positional_fact() -> Result<()> { - let value = command(CommandKind::Deploy); - value.validate()?; - let encoded = rmp_serde::to_vec(&value)?; - assert_eq!(encoded[0], 0x96); - assert!(!encoded.windows(7).any(|window| window == b"running")); - assert!(!encoded.windows(5).any(|window| window == b"owner")); - Ok(()) - } - - #[test] - fn phases_are_one_exact_forward_chain() { - let phases = [ - DeploymentPhase::Sealing, - DeploymentPhase::Starting, - DeploymentPhase::Warming, - DeploymentPhase::Fencing, - DeploymentPhase::Leasing, - DeploymentPhase::AwaitingReady, - DeploymentPhase::Routing, - DeploymentPhase::Committing, - DeploymentPhase::Complete, - ]; - assert!( - phases - .windows(2) - .all(|pair| pair[1] as u8 == pair[0] as u8 + 1) - ); - } - - #[test] - fn only_live_stateless_deployments_can_be_cancelled_after_fencing() { - let command = command(CommandKind::Deploy); - let mut stateless = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100).unwrap(); - stateless.phase = DeploymentPhase::Routing; - stateless.fencing = Some(FencingEvidence::SkippedStateless); - assert!(cancel_is_safe_for_live_stateless(&stateless)); - - let mut stateful = stateless.clone(); - stateful.fencing = Some(FencingEvidence::Revoked { - incumbent_lease_sha256: None, - candidate_lease_path_verified_empty: true, - }); - assert!(!cancel_is_safe_for_live_stateless(&stateful)); - - let mut unfenced = stateless.clone(); - unfenced.phase = DeploymentPhase::Sealing; - assert!(!cancel_is_safe_for_live_stateless(&unfenced)); - - let mut committed = stateless; - committed.completion = Some(TransactionCompletion::Admitted { - generation_id: "generation-test".into(), - }); - assert!(!cancel_is_safe_for_live_stateless(&committed)); - } - - #[test] - fn every_new_odin_sequence_must_advance_and_exact_retry_is_idempotent() -> Result<()> { - let first = topology(7, 1); - assert!(sequence_requires_admission(None, 6, &first)?); - assert!(!sequence_requires_admission( - Some(&first), - 7, - &topology(7, 1) - )?); - assert!(sequence_requires_admission( - Some(&first), - 7, - &topology(8, 2) - )?); - assert!(sequence_requires_admission(Some(&first), 7, &topology(7, 2)).is_err()); - assert!(sequence_requires_admission(Some(&first), 7, &topology(6, 3)).is_err()); - Ok(()) - } - - #[test] - fn odin_sequence_cursors_are_scoped_by_target_and_signer() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut ghostlight = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - ghostlight.latest_odin_observation = Some(topology(7, 1)); - ghostlight.odin_publisher_sequence_cursor = 7; - let mut odin = DeploymentTransaction::new(&command, "odin".into(), 1, None, 100)?; - odin.latest_odin_observation = Some(topology(41, 2)); - odin.odin_publisher_sequence_cursor = 41; - let envelope = |key: &str| CultCacheEnvelope { - key: key.into(), - r#type: DeploymentTransaction::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), - }; - let snapshot = ControlSnapshot { - expiry_requests: Vec::new(), - commands: Vec::new(), - transactions: vec![ - Stored { - envelope: envelope("ghostlight"), - value: ghostlight, - }, - Stored { - envelope: envelope("odin"), - value: odin, - }, - ], - admitted: Vec::new(), - targets: Vec::new(), - }; - - assert_eq!(snapshot.max_odin_sequence("ghostlight", "odin-signer"), 7); - assert_eq!(snapshot.max_odin_sequence("odin", "odin-signer"), 41); - assert_eq!(snapshot.max_odin_sequence("ghostlight", "other-signer"), 0); - - // A transaction that failed was never built upon, so its readings must - // not raise the bar for the next attempt. Counting them is how a target - // that has failed once becomes a target that can never deploy. - let mut abandoned = snapshot; - abandoned.transactions[0].value.completion = - Some(TransactionCompletion::FailedBeforeFencing { - error: "sealed source was rejected".into(), - }); - assert_eq!(abandoned.max_odin_sequence("ghostlight", "odin-signer"), 0); - abandoned.transactions[1].value.completion = - Some(TransactionCompletion::FailedAfterFencing { - error: "candidate died after the fence".into(), - recovery: TerminalRecovery::RestoreIncumbent, - }); - assert_eq!(abandoned.max_odin_sequence("odin", "odin-signer"), 0); - Ok(()) - } - - #[test] - fn stale_odin_provider_receipt_cannot_gate_continuity_restart() -> Result<()> { - let touched = std::cell::Cell::new(false); - validate_live_providers_for_deploy(CommandKind::Continuity, || { - touched.set(true); - bail!("Odin provider receipt is stale") - })?; - assert!(!touched.get()); - assert!( - validate_live_providers_for_deploy(CommandKind::Deploy, || { - bail!("Odin provider receipt is stale") - }) - .is_err() - ); - Ok(()) - } - - #[test] - fn prepared_write_lease_is_durable_but_not_admitted_authority() -> Result<()> { - let lease = write_lease(); - lease.validate()?; - let prepared = LeasingEvidence::Prepared { - lease: lease.clone(), - lease_sha256: lease.canonical_sha256()?, - }; - let encoded = rmp_serde::to_vec(&prepared)?; - let decoded: LeasingEvidence = rmp_serde::from_slice(&encoded)?; - assert_eq!(decoded, prepared); - assert!(decoded.lease().is_none()); - assert!(decoded.lease_sha256().is_none()); - assert_eq!(decoded.prepared_lease().unwrap().0, &lease); - Ok(()) - } - - #[test] - fn runtime_instance_identity_is_stable_for_activation_prepare_replay() -> Result<()> { - let first = runtime_instance_id("tx-one")?; - assert_eq!(runtime_instance_id("tx-one")?, first); - assert_ne!(runtime_instance_id("tx-two")?, first); - Ok(()) - } - - #[test] - fn pre_fencing_abort_is_durable_before_cleanup_and_cannot_resume_as_success() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut transaction = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - transaction.pre_fencing_abort = Some(PreFencingAbort { - error: "sealed source was rejected".into(), - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Pending, - }); - transaction.validate()?; - assert!(!transaction.is_terminal()); - - transaction - .pre_fencing_abort - .as_mut() - .unwrap() - .source_cleanup = CleanupEvidence::Complete; - transaction.phase = DeploymentPhase::Complete; - transaction.last_error = Some("sealed source was rejected".into()); - transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { - error: "sealed source was rejected".into(), - }); - transaction.validate()?; - assert!(transaction.is_terminal()); - Ok(()) - } - - #[test] - fn post_fencing_abort_is_terminal_only_when_every_cleanup_is_complete() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut transaction = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - transaction.phase = DeploymentPhase::Complete; - transaction.post_fencing_abort = Some(PostFencingAbort { - error: "candidate died after the fence".into(), - route_restoration: CleanupEvidence::Skipped, - lease_withdrawal: CleanupEvidence::Pending, - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Pending, - }); - transaction.last_error = Some("candidate died after the fence".into()); - transaction.completion = Some(TransactionCompletion::FailedAfterFencing { - error: "candidate died after the fence".into(), - recovery: TerminalRecovery::RestoreIncumbent, - }); - // The evidence exists so that a target is released only once the - // candidate's route, lease, process and projection are actually gone. - assert!(transaction.validate().is_err()); - assert!(!transaction.is_terminal()); - - let abort = transaction.post_fencing_abort.as_mut().unwrap(); - abort.lease_withdrawal = CleanupEvidence::Complete; - abort.source_cleanup = CleanupEvidence::Complete; - transaction.validate()?; - assert!(transaction.is_terminal()); - - // The recorded error and the completion must name the same failure. - transaction.completion = Some(TransactionCompletion::FailedAfterFencing { - error: "a different story".into(), - recovery: TerminalRecovery::RestoreIncumbent, - }); - assert!(transaction.validate().is_err()); - Ok(()) - } - - #[test] - fn a_pre_fence_transaction_cannot_carry_a_post_fencing_abort() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut transaction = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - assert!(transaction.phase < DeploymentPhase::Fencing); - transaction.post_fencing_abort = Some(PostFencingAbort { - error: "candidate died after the fence".into(), - route_restoration: CleanupEvidence::Skipped, - lease_withdrawal: CleanupEvidence::Skipped, - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Skipped, - }); - assert!(transaction.validate().is_err()); - Ok(()) - } - #[test] - fn candidate_cleanup_is_owed_when_an_activation_or_a_workload_exists() -> Result<()> { - // (activation issued, workload observed, expected cleanup) - let rows = [ - (false, false, CleanupEvidence::Skipped), - (true, false, CleanupEvidence::Pending), - (false, true, CleanupEvidence::Pending), - (true, true, CleanupEvidence::Pending), - ]; - for (has_activation, has_workload, expected) in rows { - let world = ContinuityProjection::after_candidate_observed()?; - let mut transaction = world.continuity_transaction()?; - if !has_activation { - transaction.activation = None; - } - transaction.workload = has_workload.then(|| workload(1000, 1, 2)); - assert_eq!( - transaction.candidate_cleanup_owed(), - expected, - "activation {has_activation}, workload {has_workload}" - ); - assert_eq!( - pre_fencing_abort_intent(&transaction, "x").candidate_cleanup, - expected, - "pre-fencing intent, activation {has_activation}, workload {has_workload}" - ); - assert_eq!( - post_fencing_abort_intent(&transaction, "x").candidate_cleanup, - expected, - "post-fencing intent, activation {has_activation}, workload {has_workload}" - ); - } - Ok(()) - } - - #[test] - fn a_stored_abort_whose_candidate_cleanup_disagrees_with_the_owed_rule_is_refused() -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - // (activation issued, recorded cleanup, accepted). - // Rows with a workload are omitted: a transaction holding one is refused on - // other grounds, so they could not speak for this rule. The workload operand - // of the owed rule is pinned by the method-level test above. - let rows = [ - (false, CleanupEvidence::Skipped, true), - (false, CleanupEvidence::Pending, false), - (false, CleanupEvidence::Complete, false), - (true, CleanupEvidence::Skipped, false), - (true, CleanupEvidence::Pending, true), - (true, CleanupEvidence::Complete, true), - ]; - for (has_activation, recorded, accepted) in rows { - let label = format!("activation {has_activation}, {recorded:?}"); - let shape = |mut transaction: DeploymentTransaction| -> DeploymentTransaction { - transaction.activation = has_activation.then(|| world.candidate_activation.clone()); - transaction - }; - - let mut pre = shape(world.continuity_transaction()?); - pre.pre_fencing_abort = Some(PreFencingAbort { - error: "candidate refused".into(), - candidate_cleanup: recorded, - topology_reconciliation: pre.abort_topology_reconciliation(), - source_cleanup: CleanupEvidence::Skipped, - }); - assert_eq!(pre.validate().is_ok(), accepted, "pre-fencing, {label}"); - - let mut post = shape(DeploymentTransaction::new( - &command(CommandKind::Deploy), - "ghostlight".into(), - 0, - None, - 100, - )?); - post.phase = DeploymentPhase::Complete; - post.post_fencing_abort = Some(PostFencingAbort { - error: "candidate died after the fence".into(), - route_restoration: CleanupEvidence::Skipped, - lease_withdrawal: CleanupEvidence::Skipped, - candidate_cleanup: recorded, - topology_reconciliation: post.abort_topology_reconciliation(), - source_cleanup: CleanupEvidence::Complete, - }); - post.last_error = Some("candidate died after the fence".into()); - post.completion = Some(TransactionCompletion::FailedAfterFencing { - error: "candidate died after the fence".into(), - recovery: TerminalRecovery::RestoreIncumbent, - }); - // A Pending cleanup is not terminal, so this completed shape is - // only judged for the recorded values a terminal abort can carry. - if recorded != CleanupEvidence::Pending { - assert_eq!(post.validate().is_ok(), accepted, "post-fencing, {label}"); - } - } - Ok(()) - } - - #[test] - fn bad_selector_becomes_one_terminal_refusal_record() -> Result<()> { - let command = command(CommandKind::Deploy); - let transaction = DeploymentTransaction::rejected( - &command, - anyhow!("deployment selector is unknown"), - 100, - )?; - assert!(transaction.is_terminal()); - assert_eq!(transaction.target, command.selector); - assert!(matches!( - transaction.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); - Ok(()) - } - - #[test] - fn post_commit_cleanup_keeps_exact_incumbent_work_owned_until_retired() -> Result<()> { - let incumbent = workload(1001, 40, 50); - let cleanup = PostCommitCleanup { - incumbent: IncumbentCleanupEvidence::Pending { - generation_id: "generation-old".into(), - workload: incumbent.clone(), - }, - source: SourceCleanupEvidence::Pending, - }; - let encoded = rmp_serde::to_vec(&cleanup)?; - let mut replayed: PostCommitCleanup = rmp_serde::from_slice(&encoded)?; - assert_eq!(replayed, cleanup); - assert!(!replayed.is_complete()); - replayed.incumbent = IncumbentCleanupEvidence::Complete { - generation_id: "generation-old".into(), - }; - assert!(!replayed.is_complete()); - replayed.source = SourceCleanupEvidence::Complete; - assert!(replayed.is_complete()); - Ok(()) - } - - #[test] - fn routed_incumbent_retirement_waits_for_the_declared_drain_deadline() -> Result<()> { - assert_eq!(route_drain_deadline(1_000, 30)?, 31_000); - assert!(route_drain_deadline(u64::MAX, 1).is_err()); - Ok(()) - } - - #[test] - fn admitted_route_receipt_refreshes_when_stale_or_implausibly_future_dated() { - assert!(route_observation_is_current(900, 1_000, 100, 10)); - assert!(route_observation_is_current(1_010, 1_000, 100, 10)); - assert!(!route_observation_is_current(899, 1_000, 100, 10)); - assert!(!route_observation_is_current(1_011, 1_000, 100, 10)); - } - - #[test] - fn lease_warming_requires_both_new_odin_and_new_provider_evidence() { - let old = digest('1'); - let fresh = digest('2'); - assert!(!provider_warming_advanced(7, &old, 7, &fresh)); - assert!(!provider_warming_advanced(7, &old, 8, &old)); - assert!(provider_warming_advanced(7, &old, 8, &fresh)); - } - - #[test] - fn failed_route_proof_never_restores_any_fenced_incumbent() { - assert!(!may_rollback_route_after_failed_proof( - &FencingEvidence::Revoked { - incumbent_lease_sha256: Some(digest('1')), - candidate_lease_path_verified_empty: false, - } - )); - assert!(!may_rollback_route_after_failed_proof( - &FencingEvidence::Revoked { - incumbent_lease_sha256: None, - candidate_lease_path_verified_empty: true, - } - )); - assert!(may_rollback_route_after_failed_proof( - &FencingEvidence::SkippedStateless - )); - } - - #[test] - fn fenced_writer_is_already_retired_before_post_commit_cleanup() { - assert!(incumbent_was_stopped_during_fencing( - &FencingEvidence::Revoked { - incumbent_lease_sha256: Some(digest('1')), - candidate_lease_path_verified_empty: false, - } - )); - assert!(!incumbent_was_stopped_during_fencing( - &FencingEvidence::Revoked { - incumbent_lease_sha256: None, - candidate_lease_path_verified_empty: true, - } - )); - assert!(!incumbent_was_stopped_during_fencing( - &FencingEvidence::SkippedStateless - )); - } - - #[test] - fn warming_accepts_only_the_exact_projected_incumbent_lease_disagreement() { - let expected_detail = format!("expected:{};activation:{}", digest('1'), digest('2')); - let incumbent = digest('3'); - let exact = OdinTopologyDisagreement { - code: "projected-write-lease".into(), - expected: Some(expected_detail.clone()), - observed: Some(incumbent.clone()), - }; - - assert!(warming_disagreements_match_incumbent( - &expected_detail, - Some(&incumbent), - &[exact.clone()], - )); - assert!(!warming_disagreements_match_incumbent( - &expected_detail, - None, - &[exact.clone()], - )); - - let mut substituted = exact.clone(); - substituted.observed = Some(digest('4')); - assert!(!warming_disagreements_match_incumbent( - &expected_detail, - Some(&incumbent), - &[substituted], - )); - assert!(!warming_disagreements_match_incumbent( - &expected_detail, - Some(&incumbent), - &[exact.clone(), exact], - )); - } - - #[test] - fn candidate_and_incumbent_must_differ_in_all_three_native_boundaries() { - let incumbent = workload(1001, 40, 50); - let prove = |candidate: &WorkloadObservation| { - WorkloadObservation::prove_isolation(candidate, Some(&incumbent)) - }; - assert!(prove(&workload(1002, 41, 51)).is_ok()); - assert!(prove(&workload(1001, 41, 51)).is_err()); - assert!(prove(&workload(1002, 40, 51)).is_err()); - assert!(prove(&workload(1002, 41, 50)).is_err()); - } - - #[test] - fn exactly_one_inflight_transaction_may_own_a_target() -> Result<()> { - let command = command(CommandKind::Deploy); - let one = DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - let two = DeploymentTransaction::new(&command, "ghostlight".into(), 1, None, 100)?; - let snapshot = ControlSnapshot { - expiry_requests: Vec::new(), - commands: vec![Stored { - envelope: command_envelope(&command, 100)?, - value: command, - }], - transactions: vec![ - Stored { - envelope: transaction_envelope(&one, 100)?, - value: one, - }, - Stored { - envelope: transaction_envelope(&two, 100)?, - value: two, - }, - ], - admitted: Vec::new(), - targets: Vec::new(), - }; - assert!(snapshot.validate_relations().is_err()); - Ok(()) - } - - #[test] - fn command_ordinals_do_not_turn_one_target_wait_into_a_global_brake() -> Result<()> { - let first_command = command(CommandKind::Deploy); - let first = DeploymentTransaction::new(&first_command, "odin".into(), 0, None, 100)?; - let second = DeploymentTransaction::new(&first_command, "ghostlight".into(), 1, None, 100)?; - let mut unrelated_command = command(CommandKind::Deploy); - unrelated_command.command_id = "up-unrelated".into(); - let unrelated = - DeploymentTransaction::new(&unrelated_command, "huginn".into(), 0, None, 101)?; - let snapshot = ControlSnapshot { - expiry_requests: Vec::new(), - commands: Vec::new(), - transactions: [&first, &second, &unrelated] - .into_iter() - .map(|value| Stored { - envelope: transaction_envelope(value, 100).unwrap(), - value: value.clone(), - }) - .collect(), - admitted: Vec::new(), - targets: Vec::new(), - }; - assert!(!snapshot.has_earlier_authority_sibling(&first)); - assert!(snapshot.has_earlier_authority_sibling(&second)); - assert!(!snapshot.has_earlier_authority_sibling(&unrelated)); - Ok(()) - } - - #[test] - fn complete_cleanup_remains_retryable_without_owning_the_current_incarnation() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut transaction = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - transaction.phase = DeploymentPhase::Complete; - transaction.completion = Some(TransactionCompletion::Admitted { - generation_id: format!("generation-{}", transaction.transaction_id), - }); - transaction.post_commit_cleanup = Some(PostCommitCleanup { - incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, - source: SourceCleanupEvidence::Pending, - }); - - assert!(!transaction.is_terminal()); - assert!(!transaction.owns_target_authority()); - assert!(transaction.blocks_new_target_mutation()); - Ok(()) - } - - #[test] - fn legacy_mutable_command_schema_is_rejected() -> Result<()> { - let temporary = tempfile::tempdir()?; - let path = temporary.path().join("control.cc"); - let legacy = CultCacheEnvelope { - key: "up-legacy".into(), - r#type: DeploymentCommand::TYPE.into(), - payload: rmp_serde::to_vec(&(DEPLOYMENT_COMMAND_SCHEMA, "up-legacy", "running"))?, - stored_at: rfc3339_millis(100)?, - schema_id: Some("idunn.deployment_command.v1".into()), - }; - assert!( - SingleFileMessagePackBackingStore::new(&path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: "up-legacy".into(), - current: None, - }], - &[legacy], - )? - ); - assert!(ControlSnapshot::read(&path).is_err()); - Ok(()) - } - - #[test] - fn loadcredential_era_control_schemas_are_rejected_before_rehydration() -> Result<()> { - for (record_type, key, schema) in [ - ( - DeploymentTransaction::TYPE, - "tx-legacy-workload-observation", - "idunn.deployment_transaction.v1", - ), - ( - AdmittedGeneration::TYPE, - "legacy-admitted-target", - "idunn.admitted_generation.v1", - ), - ] { - let temporary = tempfile::tempdir()?; - let path = temporary.path().join("control.cc"); - let legacy = CultCacheEnvelope { - key: key.into(), - r#type: record_type.into(), - payload: Vec::new(), - stored_at: rfc3339_millis(100)?, - schema_id: Some(schema.into()), - }; - assert!( - SingleFileMessagePackBackingStore::new(&path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: record_type.into(), - key: key.into(), - current: None, - }], - &[legacy], - )? - ); - assert!(ControlSnapshot::read(&path).is_err()); - } - Ok(()) - } - - #[test] - fn source_identity_is_explicit_and_atomic() { - let partial = parse( - ["serve", "--source-uid", "1001"] - .into_iter() - .map(str::to_owned), - ) - .unwrap_err() - .to_string(); - assert!(partial.contains("supplied together")); - let Command::Serve(options) = parse( - ["serve", "--source-uid", "1001", "--source-gid", "1002"] - .into_iter() - .map(str::to_owned), - ) - .unwrap() else { - panic!("expected serve") - }; - assert_eq!( - options.source_identity, - Some(ProcessIdentity { - uid: 1001, - gid: 1002 - }) - ); - } - - // ---- v2/v3 control records lifted to the v3 generation / v4 transaction ---- - // - // The bytes under tests/fixtures/idunn-control-legacy were written by the - // encoder at Idunn 9001b58 (see the README there). Nothing below encodes a - // legacy record: the legacy layout is decoded from those bytes, and the - // lifted record is compared with what the decoded legacy record says. - - const FIXTURE_GENERATION: &str = - include_str!("../tests/fixtures/idunn-control-legacy/generation-with-receipts.hex"); - const FIXTURE_GENERATION_REPAIRING: &str = include_str!( - "../tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex" - ); - const FIXTURE_GENERATION_V3_ODIN: &str = - include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-odin.hex"); - const FIXTURE_GENERATION_V3_ROUTE_PROOF: &str = - include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex"); - const FIXTURE_TRANSACTIONS: [(&str, &str); 6] = [ - ( - "fencing", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-fencing.hex"), - ), - ( - "leasing", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-leasing.hex"), - ), - ( - "awaiting-ready", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex"), - ), - ( - "routing", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-routing.hex"), - ), - ( - "committing", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing.hex"), - ), - ( - "failed-after-fencing", - include_str!( - "../tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex" - ), - ), - ]; - const FIXTURE_TRANSACTION_V2: &str = - include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex"); - - fn fixture_envelope(text: &str, record_type: &str) -> Result { - let mut lines = text.lines(); - let schema = lines.next().context("fixture has no schema line")?; - let hex = lines.next().context("fixture has no payload line")?.trim(); - let payload = (0..hex.len()) - .step_by(2) - .map(|index| u8::from_str_radix(&hex[index..index + 2], 16)) - .collect::, _>>()?; - let key = if record_type == AdmittedGeneration::TYPE && schema == ADMITTED_GENERATION_SCHEMA_V3 { - rmp_serde::from_slice::(&payload)?.target - } else if record_type == AdmittedGeneration::TYPE { - rmp_serde::from_slice::(&payload)?.target - } else { - rmp_serde::from_slice::(&payload)?.transaction_id - }; - Ok(CultCacheEnvelope { - key, - r#type: record_type.into(), - payload, - stored_at: rfc3339_millis(1_700_000_123_456)?, - schema_id: Some(schema.into()), - }) - } - - fn fixture_generation(text: &str) -> Result<(LegacyAdmittedGeneration, AdmittedGeneration)> { - let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; - let legacy = rmp_serde::from_slice(&envelope.payload)?; - Ok((legacy, read_generation_record(&envelope)?)) - } - - fn fixture_transaction( - text: &str, - ) -> Result<(LegacyDeploymentTransaction, DeploymentTransaction)> { - let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; - let legacy = rmp_serde::from_slice(&envelope.payload)?; - Ok((legacy, read_transaction_record(&envelope)?)) - } - - #[test] - fn a_v2_generation_lifts_to_the_decisions_it_encoded() -> Result<()> { - for text in [FIXTURE_GENERATION, FIXTURE_GENERATION_REPAIRING] { - let (legacy, lifted) = fixture_generation(text)?; - // Every receipt the old decisions read is the same receipt, now - // tagged Odin-correlated. - let odin = lifted.odin_receipts()?; - assert_eq!(odin.ready, &legacy.ready); - assert_eq!(odin.latest, &legacy.latest_odin_observation); - assert_eq!(lifted.odin_authority.as_ref(), Some(&legacy.odin_authority)); - assert_eq!( - lifted.odin_publisher_sequence_cursor, - legacy.odin_publisher_sequence_cursor - ); - assert_eq!(lifted.ready.voucher(), Voucher::Odin); - assert_eq!(lifted.routing, legacy.routing); - assert_eq!(lifted.leasing, legacy.leasing); - assert_eq!(lifted.plan, legacy.plan); - assert_eq!(lifted.expected, legacy.expected); - assert_eq!(lifted.activation, legacy.activation); - assert_eq!(lifted.workload, legacy.workload); - assert_eq!(lifted.generation_id, legacy.generation_id); - // The new state starts empty and the plan is still a v2 plan. - assert_eq!(lifted.route_supervision, Some(RouteSupervisionState::default())); - assert_eq!(lifted.last_error, None); - assert_eq!(lifted.plan.schema, crate::deployment_plan::COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2); - assert_eq!( - lifted.plan.phase_deadlines(), - crate::deployment_plan::PhaseDeadlines::IDUNN_DEFAULTS - ); - } - Ok(()) - } - - #[test] - fn v2_and_v3_transactions_lift_to_the_decisions_they_encoded() -> Result<()> { - let mut phases = Vec::new(); - for (name, text) in FIXTURE_TRANSACTIONS { - let (legacy, lifted) = fixture_transaction(text)?; - phases.push((name, lifted.phase)); - assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); - assert_eq!(lifted.phase, legacy.phase); - assert_eq!(lifted.completion, legacy.completion); - assert_eq!(lifted.plan, legacy.plan); - assert_eq!(lifted.expected, legacy.expected); - assert_eq!(lifted.leasing, legacy.leasing); - assert_eq!(lifted.fencing, legacy.fencing); - assert_eq!(lifted.routing, legacy.routing); - assert_eq!(lifted.warming, legacy.warming); - assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); - assert_eq!( - lifted.odin_publisher_sequence_cursor, - legacy.odin_publisher_sequence_cursor - ); - assert_eq!( - lifted.ready.as_ref().and_then(ReadinessEvidence::odin), - legacy.ready.as_ref() - ); - assert_eq!(lifted.phase_deadline, None); - assert_eq!(lifted.lease_adoption, None); - assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); - } - assert_eq!( - phases.iter().map(|(_, phase)| *phase).collect::>(), - [ - DeploymentPhase::Fencing, - DeploymentPhase::Leasing, - DeploymentPhase::AwaitingReady, - DeploymentPhase::Routing, - DeploymentPhase::Committing, - DeploymentPhase::Complete, - ] - ); - - // The genuine v2 layout lifts to the same record as the v3 one. - let (_, from_v2) = fixture_transaction(FIXTURE_TRANSACTION_V2)?; - let (_, from_v3) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; - assert_eq!(from_v2, from_v3); - - // A terminal failure predating recovery names the recovery it always did. - let (_, failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; - assert!(matches!( - failed.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - )); - assert!(failed.is_terminal()); - Ok(()) - } - - #[test] - fn the_boot_migration_rewrites_every_legacy_record_once() -> Result<()> { - // The fixtures were cut from one build, so their records share keys - // and each migrates in a store of its own. - let mut fixtures = vec![ - fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?, - fixture_envelope(FIXTURE_GENERATION_REPAIRING, AdmittedGeneration::TYPE)?, - fixture_envelope(FIXTURE_TRANSACTION_V2, DeploymentTransaction::TYPE)?, - ]; - for (_, text) in FIXTURE_TRANSACTIONS { - fixtures.push(fixture_envelope(text, DeploymentTransaction::TYPE)?); - } - for envelope in fixtures { - let temporary = tempfile::tempdir()?; - let path = temporary.path().join("control.cc"); - let store = SingleFileMessagePackBackingStore::new(&path); - assert!(store.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: envelope.r#type.clone(), - key: envelope.key.clone(), - current: None, - }], - std::slice::from_ref(&envelope), - )?); - let debug_of = |envelope: &CultCacheEnvelope| -> Result { - Ok(if envelope.r#type == AdmittedGeneration::TYPE { - format!("{:?}", read_generation_record(envelope)?) - } else { - format!("{:?}", read_transaction_record(envelope)?) - }) - }; - let before = debug_of(&envelope)?; - - assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); - assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); - - let after = store.pull_all_read_only_snapshot()?; - assert_eq!(after.len(), 1); - let expected_schema = if envelope.r#type == AdmittedGeneration::TYPE { - ADMITTED_GENERATION_SCHEMA - } else { - DEPLOYMENT_TRANSACTION_SCHEMA - }; - assert_eq!(after[0].schema_id.as_deref(), Some(expected_schema)); - assert_eq!(after[0].stored_at, envelope.stored_at); - // The rewritten record is canonical and means what the legacy one meant. - assert_eq!(debug_of(&after[0])?, before); - } - Ok(()) - } - - #[test] - fn history_still_reads_legacy_transactions() -> Result<()> { - let temporary = tempfile::tempdir()?; - let state_store = temporary.path().join("control.cc"); - let envelope = fixture_envelope(FIXTURE_TRANSACTIONS[5].1, DeploymentTransaction::TYPE)?; - SingleFileMessagePackBackingStore::new(&history_store_path(&state_store)) - .compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: envelope.key.clone(), - current: None, - }], - std::slice::from_ref(&envelope), - )?; - let archived = read_history_transactions(&state_store); - assert_eq!(archived.len(), 1); - assert_eq!(archived[0].transaction_id, envelope.key); - assert_eq!(archived[0].schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); - Ok(()) - } - - #[test] - fn readiness_class_follows_the_expected_odin_dependency() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - let mut expected = generation.expected.clone(); - assert!(expected.dependencies.iter().any(|dependency| { - dependency.kind == "shared-infrastructure" - && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY - })); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); - for dependency in &mut expected.dependencies { - // Same capability, other kind: only the shared-infrastructure - // declaration makes a target Odin-correlated. - dependency.kind = "required".into(); - } - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); - expected.dependencies.clear(); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); - // Nothing to challenge and no Odin dependency: the recipe has declared - // no way to prove readiness, and Idunn does not pick one. - expected.route = None; - assert_eq!( - ReadinessClass::of(&expected), - Err(UndeclaredReadiness { - target: expected.target.clone() - }) - ); - // Declaring the Odin dependency is the declaration, routed or not: a - // routed target that only publishes to Odin is Odin-correlated. - let odin_dependency = generation.expected.dependencies.clone(); - expected.dependencies = odin_dependency.clone(); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); - expected.route = generation.expected.route.clone(); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); - // Providing the rendezvous is being Odin, whatever else is declared. - expected.dependencies.clear(); - expected.route = None; - provide_odin(&mut expected); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); - Ok(()) - } - - #[test] - fn warming_evidence_is_shape_checked_whatever_its_source() { - let good = |evidence: &ReadinessEvidence| match evidence { - ReadinessEvidence::RouteProof { evidence } => evidence.clone(), - ReadinessEvidence::OdinCorrelated { .. } => unreachable!("a route-proof fixture"), - }; - let presence = good(&route_proof_evidence()); - for warming in [ - WarmingEvidence::FirstOdinDirect { evidence: presence.clone() }, - WarmingEvidence::RouteProofDirect { evidence: presence.clone() }, - WarmingEvidence::OdinTopology { evidence: topology(3, 1) }, - ] { - warming.validate_shape().expect("well-formed evidence"); - } - let mut damaged = presence; - damaged.canonical_bytes.push(1); - for warming in [ - WarmingEvidence::FirstOdinDirect { evidence: damaged.clone() }, - WarmingEvidence::RouteProofDirect { evidence: damaged }, - WarmingEvidence::OdinTopology { - evidence: TopologyEvidence { - canonical_sha256: "sha256:not-the-digest".into(), - ..topology(3, 1) - }, - }, - ] { - assert!(warming.validate_shape().is_err(), "{warming:?}"); - } - } - - #[test] - fn a_presence_disagreement_names_each_disagreement() { - let shortfall = PresenceDisagrees { - disagreements: vec![ - OdinTopologyDisagreement { - code: "capacity-below-minimum".into(), - expected: Some("2".into()), - observed: Some("1".into()), - }, - OdinTopologyDisagreement { - code: "state".into(), - expected: None, - observed: None, - }, - ], - }; - assert_eq!( - shortfall.to_string(), - "route proof answered with a runtime that disagrees with current authority: \ - capacity-below-minimum (expected 2, observed 1); state (expected none, observed none)" - ); - } - - fn route_proof_evidence() -> ReadinessEvidence { - let canonical_bytes = vec![7, 7, 7]; - ReadinessEvidence::RouteProof { - evidence: RuntimePresenceEvidence { - canonical_sha256: sha256_id(&canonical_bytes), - canonical_bytes, - message_id: "challenge-1".into(), - challenged_at_unix_millis: 10, - admitted_at_unix_millis: 11, - }, - } - } - - #[test] - fn route_proof_readiness_is_refused_for_a_target_that_declares_odin() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - let mut proof = generation.clone(); - proof.ready = route_proof_evidence(); - proof.latest_odin_observation = None; - proof.odin_authority = None; - // The fixture's Expected declares Odin, so route proof is not its class. - assert!(proof.validate().is_err()); - - // Mixed shapes are refused whatever the class. - let mut mixed = generation.clone(); - mixed.latest_odin_observation = None; - assert!(mixed.validate().is_err()); - let mut mixed = generation; - mixed.odin_authority = None; - assert!(mixed.validate().is_err()); - - let mut transaction = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?.1; - transaction.validate()?; - transaction.ready = Some(route_proof_evidence()); - assert!(transaction.validate().is_err()); - Ok(()) - } - - #[test] - fn route_supervision_state_exists_exactly_for_a_promoted_route() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - let mut missing = generation.clone(); - missing.route_supervision = None; - assert!(missing.validate().is_err()); - - let mut unrouted = generation; - unrouted.route_supervision = None; - unrouted.routing = RoutingEvidence::SkippedUnrouted; - // Unrouted while Expected still names a route: refused for that reason - // too, so the test above pins the supervision rule on its own. - assert!(unrouted.validate().is_err()); - Ok(()) - } - - #[test] - fn a_new_generation_starts_unobserved_and_undegraded() -> Result<()> { - let committing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1; - let (_, incumbent) = fixture_generation(FIXTURE_GENERATION)?; - let authority = incumbent - .odin_authority - .clone() - .context("fixture generation has no Odin authority")?; - let next = AdmittedGeneration::from_transaction( - &committing, - Some(authority), - 1_700_000_200_000, - )?; - assert_eq!(next.route_supervision, Some(RouteSupervisionState::default())); - assert_eq!(next.last_error, None); - Ok(()) - } - - #[test] - fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - let plan = &generation.plan; - let defaults = plan.phase_deadlines(); - for (phase, seconds) in [ - (DeploymentPhase::Fencing, defaults.fencing_seconds), - (DeploymentPhase::Leasing, defaults.leasing_seconds), - (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), - (DeploymentPhase::Routing, defaults.routing_seconds), - (DeploymentPhase::Committing, defaults.committing_seconds), - ] { - let deadline = PhaseDeadline::entering(phase, plan, 1_000).context("no deadline")?; - assert_eq!(deadline.phase, phase); - assert_eq!(deadline.entered_at_unix_millis, 1_000); - assert_eq!(deadline.deadline_at_unix_millis, 1_000 + u64::from(seconds) * 1000); - } - for phase in [ - DeploymentPhase::Sealing, - DeploymentPhase::Starting, - DeploymentPhase::Warming, - DeploymentPhase::Complete, - ] { - assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); - } - Ok(()) - } - - #[test] - fn transition_stamps_the_deadline_of_the_phase_it_enters() -> Result<()> { - let world = EngineFixture::new()?; - // Routing, already holding the route receipt Committing requires. - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?; - assert_eq!(transaction.phase, DeploymentPhase::Routing); - transaction.routing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1.routing; - let envelope = transaction_envelope(&transaction, transaction.updated_at_unix_millis)?; - let command = command_envelope(&command(CommandKind::Continuity), 100)?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.key.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - ], - &[command, envelope], - )? - ); - let stored = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .next() - .context("stored transaction vanished")?; - world.engine.transition(&stored, DeploymentPhase::Committing)?; - let advanced = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .next() - .context("advanced transaction vanished")?; - let deadline = advanced.value.phase_deadline.context("no deadline stamped")?; - assert_eq!(deadline.phase, DeploymentPhase::Committing); - assert_eq!( - deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, - u64::from(advanced.value.plan.as_ref().unwrap().phase_deadlines().committing_seconds) - * 1000 - ); - assert_eq!(deadline.entered_at_unix_millis, advanced.value.updated_at_unix_millis); - Ok(()) - } - - #[test] - fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; - let plan = transaction.plan.clone().unwrap(); - let now = transaction.updated_at_unix_millis; - transaction.phase_deadline = - PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now); - transaction.validate()?; - let mut wrong_phase = transaction.clone(); - wrong_phase.phase_deadline = - PhaseDeadline::entering(DeploymentPhase::Routing, &plan, now); - assert!(wrong_phase.validate().is_err()); - let mut backwards = transaction.clone(); - backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; - assert!(backwards.validate().is_err()); - // Entry is not ordered against the wall-clock stamp of later writes. - let mut entered_after_the_stamp = transaction; - entered_after_the_stamp.phase_deadline = - PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); - entered_after_the_stamp.validate()?; - Ok(()) - } - - // --------------------------------------------------------------------- - // The phase machine, driven by the real Engine past Fencing. - // --------------------------------------------------------------------- - - /// A workload port with no systemd behind it: observing a workload - /// returns what was observed, stopping and discarding succeed, and - /// anything that would launch a process refuses. - struct StillWorkload; - - impl WorkloadPort for StillWorkload { - fn install( - &self, - _: &CompiledDeploymentPlan, - _: &crate::drivers::MaterializedRelease, - ) -> Result { - bail!("StillWorkload launches nothing") - } - fn prepare_activation( - &self, - _: &CompiledDeploymentPlan, - _: &IdunnExpectedIncarnationRecord, - _: IdunnRuntimeActivationLaunch, - ) -> Result { - bail!("StillWorkload launches nothing") - } - fn start_prepared( - &self, - _: &CompiledDeploymentPlan, - _: &SealedRelease, - _: &crate::drivers::InstalledReleaseObservation, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - ) -> Result { - bail!("StillWorkload launches nothing") - } - fn discard_prepared( - &self, - _: &CompiledDeploymentPlan, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - Ok(()) - } - fn observe( - &self, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - prior: &WorkloadObservation, - ) -> Result { - Ok(prior.clone()) - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - Ok(()) - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(false) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(false) - } - } - - /// A Continuity transaction for an unrouted, stateless target with no - /// incumbent: the shape most Engine tests need. - fn transaction_at( - world: &EngineFixture, - phase: DeploymentPhase, - ) -> Result { - seeded_transaction(world, phase, CommandKind::Continuity, None) - } - - /// The Odin publisher sequence a transaction seeded over `incumbent` - /// warms at; its Ready is the next one. Above anything an earlier - /// generation of the world admitted. - fn seeded_sequence(incumbent: Option<&AdmittedGeneration>) -> u64 { - if incumbent.is_some() { 14 } else { 4 } - } - - /// A released deployment brake receipt, signed by an operator identity - /// enrolled in the world: what a Deploy's `deployment_authorization` holds. - fn deployment_authorization( - world: &EngineFixture, - expected: &IdunnExpectedIncarnationRecord, - transaction_id: &str, - now: u64, - ) -> Result { - use cultnet_rs::{ - IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, - IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, - IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, - }; - let signer = enroll_service_identity_at::( - &world.root.join("identities/brake-operator.cc"), - )?; - export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; - let mut record = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: expected.runtime_id.clone(), - status: "released".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: "test rollout".into(), - observed_at_unix_millis: now, - expires_at_unix_millis: Some(now + 600_000), - authorization_id: Some("authorization-test".into()), - authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), - authorized_release_id: Some(expected.sealed_release_id.clone()), - authorized_deployment_id: Some(transaction_id.to_owned()), - authorized_by: Some(signer.trust_anchor()?.identity_id), - authorization_issued_at_unix_millis: Some(now), - authorization_expires_at_unix_millis: Some(now + 600_000), - signature_algorithm: Some("ed25519".into()), - signature: None, - private_state_exposed: false, - updated_by: "operator/test".into(), - }; - record.signature = Some( - signer - .sign::(&rmp_serde::to_vec(&record)?) - .signature, - ); - record.validate()?; - let canonical_brake_bytes = rmp_serde::to_vec(&record)?; - Ok(DeploymentAuthorization { - authorization_id: "authorization-test".into(), - brake_sha256: sha256_id(&canonical_brake_bytes), - canonical_brake_bytes, - authorized_at_unix_millis: now, - }) - } - - /// What makes a target Odin: it provides the rendezvous. - fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { - expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { - capability: ODIN_RENDEZVOUS_CAPABILITY.into(), - schema: "odin.verse-topology.v1".into(), - compatibility: "v1".into(), - minimum_capacity: 1, - }); - } - - /// The declaration that makes an unrouted fixture target Odin-correlated. - /// Idunn infers no way to prove readiness, and a target with no route has - /// nothing for Idunn to challenge, so the world says outright that Odin - /// reports on it. The plan selects no provider: these worlds never plan - /// against Odin, they only read its correlation, which names the provider - /// Expected does. - fn declare_odin(expected: &mut IdunnExpectedIncarnationRecord) { - expected.dependencies.push(cultnet_rs::IdunnExpectedDependency { - kind: "shared-infrastructure".into(), - capability: ODIN_RENDEZVOUS_CAPABILITY.into(), - schema: "odin.verse-topology.v1".into(), - compatibility: "v1".into(), - minimum_capacity: 1, - startup: "before-promotion".into(), - provider_id: Some("odin".into()), - provider_authority: Some("managed-incarnation".into()), - provider_expected_projection_sha256: Some(sha256_id(b"odin expected projection")), - provider_endpoint: None, - }); - } - - /// A transaction for an unrouted, stateless target, sitting at `phase` - /// (Warming or later) with every earlier phase's evidence in place. The - /// plan, sealed release, Expected, activation and Warming evidence are the - /// real constructions; only the workload and isolation observations are - /// borrowed from a recorded transaction. - /// - /// `incumbent` is an admitted generation already in the world's store. A - /// Continuity over it restarts that generation's own release, so it shares - /// the incumbent's Expected key. A Deploy over it builds a second - /// incarnation. A Deploy carries what phases past Fencing never read but - /// validation requires: a three-field frozen-source receipt and a - /// well-formed release receipt from the brake. - fn seeded_transaction( - world: &EngineFixture, - phase: DeploymentPhase, - kind: CommandKind, - incumbent: Option<&AdmittedGeneration>, - ) -> Result { - use crate::deployment_plan::tests::{ - BINDING, RECIPE, artifact_receipt, external_input_receipt, source, - }; - let provider_path = world.root.join("identities/provider.cc"); - let provider_anchor = world.root.join("identities/provider-anchor.cc"); - let provider = if provider_path.exists() { - open_service_identity_at::(&provider_path)? - } else { - let provider = - enroll_service_identity_at::(&provider_path)?; - export_service_identity_trust_anchor(&provider, &provider_anchor)?; - provider - }; - - let (binding_head, binding_tail) = BINDING - .split_once("[route]") - .context("binding has no route table")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail - .split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ) - .replace( - "/etc/gamecult/trust/service.cc", - &provider_anchor.display().to_string(), - ) - .replace("service-runtime-signer", &provider.entry().identity_id); - let recipe = RECIPE - .split("[[dependencies]]") - .next() - .context("recipe is empty")? - .replace("route_required = true", "route_required = false") - .replace( - r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, - r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, - ); - let (plan, release) = match (kind, incumbent) { - (CommandKind::Continuity, Some(incumbent)) => ( - incumbent.plan.clone(), - incumbent.sealed_release.clone(), - ), - _ => { - let incarnation = if incumbent.is_some() { - "service-incarnation-2" - } else { - "service-incarnation-1" - }; - let plan = compile_deployment_plan( - recipe.as_bytes(), - binding.as_bytes(), - source(&recipe), - incarnation, - None, - 110, - &[], - )?; - let release = SealedRelease::new( - &plan, - vec![artifact_receipt()], - vec![external_input_receipt()], - 120, - )?; - (plan, release) - } - }; - let mut expected = release.expected_projection(&plan)?; - assert!(!expected.write_lease_required && expected.route.is_none()); - declare_odin(&mut expected); - - let now = now_millis()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!( - "{}-service{}", - match kind { - CommandKind::Deploy => "up", - CommandKind::Continuity => "continuity", - }, - if incumbent.is_some() { "-again" } else { "" } - ), - kind, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let mut transaction = match (kind, incumbent) { - (CommandKind::Continuity, Some(incumbent)) => { - DeploymentTransaction::from_continuity(&command, incumbent, now)? - } - _ => DeploymentTransaction::new(&command, "service".into(), 0, incumbent, now)?, - }; - let activation = IdunnRuntimeActivationLaunch::issue( - &expected, - runtime_instance_id(&transaction.transaction_id)?, - now, - &world.engine.idunn_signer, - )? - .activation() - .clone(); - let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? - .1 - .workload - .context("recorded transaction has no workload")?; - match kind { - CommandKind::Continuity => { - transaction.lifecycle_authorized_at_unix_millis = Some(now); - } - CommandKind::Deploy => { - transaction.frozen_source = Some(FrozenSourceReceipt { - transaction_id: transaction.transaction_id.clone(), - plan_id: plan.plan_id.clone(), - snapshot_sha256: sha256_id(b"frozen source snapshot"), - }); - transaction.deployment_authorization = Some(deployment_authorization( - world, - &expected, - &transaction.transaction_id, - now, - )?); - } - } - transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); - transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); - match &mut workload { - WorkloadObservation::Systemd(observed) => { - observed.runtime_instance_id = activation.runtime_instance_id.clone(); - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), - } - let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; - transaction.isolation = recorded.isolation; - transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { - sealed_release_id: release.sealed_release_id.clone(), - root: PathBuf::from("/srv/service/releases/test"), - }); - transaction.sealed_release = Some(release); - transaction.expected = Some(expected); - transaction.activation = Some(activation); - transaction.workload = Some(workload); - transaction.plan = Some(plan); - // Odin's first word about the candidate, before it was Ready. - let sequence = seeded_sequence(incumbent); - let warming = signed_correlation(world, &transaction, sequence, false)?; - let authenticated = world.engine.authenticate_topology_bytes( - &ControlSnapshot::read(&world.state_store)?, - &transaction, - &warming, - None, - now, - )?; - transaction.warming = Some(WarmingEvidence::OdinTopology { - evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, - }); - transaction.odin_publisher_sequence_cursor = sequence; - transaction.enter_phase(phase, now); - transaction.validate()?; - - let store = SingleFileMessagePackBackingStore::new(&world.state_store); - assert!(store.compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - ], - &[ - command_envelope(&command, command.requested_at_unix_millis)?, - transaction_envelope(&transaction, now)?, - ], - )?); - Ok(transaction) - } - - /// Odin's correlation for the transaction's incarnation, signed by the key - /// the Engine trusts and stamped now. - fn signed_correlation( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ready: bool, - ) -> Result> { - signed_correlation_as(world, transaction, sequence, ready, "active", None, b"presence") - } - - /// Odin's correlation about the transaction's incarnation, observing a - /// presence in `state` that holds `lease`, identified by `presence`. - fn signed_correlation_as( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ready: bool, - state: &str, - lease: Option<&str>, - presence: &[u8], - ) -> Result> { - let expected = transaction.expected.as_ref().context("no Expected")?; - let activation = transaction.activation.as_ref().context("no activation")?; - let mut record = OdinRuntimeTopologyCorrelationRecord { - schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: expected.canonical_sha256()?, - expected: true, - current_activation_sha256: Some(activation.canonical_sha256()?), - signed_presence_sha256: Some(sha256_id(presence)), - observed_presence_state: Some(state.into()), - observed_presence_publisher_sequence: Some(sequence), - observed_write_lease_sha256: lease.map(str::to_owned), - observed_capabilities: expected - .capabilities - .iter() - .map(|capability| cultnet_rs::GameCultRuntimeCapability { - capability: capability.capability.clone(), - schema: capability.schema.clone(), - compatibility: capability.compatibility.clone(), - capacity: 1, - }) - .collect(), - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: Some(activation.runtime_instance_id.clone()), - present: true, - ready, - dependencies: expected - .dependencies - .iter() - .map(|dependency| cultnet_rs::OdinRuntimeDependencyEvidence { - kind: dependency.kind.clone(), - capability: dependency.capability.clone(), - schema: dependency.schema.clone(), - compatibility: dependency.compatibility.clone(), - provider_id: dependency.provider_id.clone(), - provider_authority: dependency.provider_authority.clone(), - provider_expected_projection_sha256: dependency - .provider_expected_projection_sha256 - .clone(), - provider_endpoint: dependency.provider_endpoint.clone(), - observed_capacity: Some(dependency.minimum_capacity), - provider_evidence_sha256: Some(sha256_id(b"odin provider evidence")), - ready: true, - }) - .collect(), - disagreements: Vec::new(), - signer_identity_id: world.odin_signer.entry().identity_id.clone(), - publisher_sequence: sequence, - observed_at_unix_millis: now_millis()?, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - }; - record.signature = world - .odin_signer - .sign::(&record.unsigned_signature_payload()?) - .signature; - record.canonical_bytes() - } - - /// Odin publishes its Ready correlation for the transaction incarnation. - fn odin_reports_ready( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ) -> Result<()> { - let expected = transaction.expected.as_ref().context("no Expected")?; - SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store) - .insert_entry_if_absent(CultCacheEnvelope { - key: crate::drivers::incarnation_key_of( - &expected.target, - &expected.canonical_sha256()?, - ), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: signed_correlation(world, transaction, sequence, true)?, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), - })?; - Ok(()) - } - - fn resident(world: &EngineFixture) -> Result> { - ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .next() - .context("the transaction left the live set") - } - - /// The transaction wherever it now lives: live, or archived once terminal. - fn latest(world: &EngineFixture) -> Result { - match resident(world) { - Ok(stored) => Ok(stored.value), - Err(_) => read_history_transactions(&world.state_store) - .into_iter() - .next() - .context("the transaction is in neither the live set nor history"), - } - } - - /// One engine step at a time until `stop` holds, recording each phase - /// entered and checking after every step that the deadline on the record - /// is exactly the current post-fencing phase's, and absent outside one. - fn drive( - world: &EngineFixture, - stop: impl Fn(&DeploymentTransaction) -> bool, - ) -> Result> { - let mut phases = Vec::new(); - for _ in 0..40 { - let Ok(current) = resident(world) else { - return Ok(phases); - }; - if stop(¤t.value) { - return Ok(phases); - } - world.engine.advance_transaction(¤t)?; - let after = latest(world)?; - match &after.phase_deadline { - Some(deadline) => assert_eq!(deadline.phase, after.phase), - None => assert!( - !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&after.phase), - "a post-fencing record lost its deadline" - ), - } - if phases.last() != Some(&after.phase) { - phases.push(after.phase); - } - } - bail!("the transaction did not reach the stop condition in 40 steps") - } - - #[test] - fn an_admission_runs_from_fencing_to_complete_and_commits() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - - let phases = drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Complete - })?; - assert_eq!( - phases, - [ - DeploymentPhase::Fencing, - DeploymentPhase::Leasing, - DeploymentPhase::AwaitingReady, - DeploymentPhase::Routing, - DeploymentPhase::Committing, - DeploymentPhase::Complete, - ] - ); - let finished = latest(&world)?; - assert_eq!(finished.phase_deadline, None); - assert!(matches!( - finished.completion, - Some(TransactionCompletion::Admitted { .. }) - )); - let snapshot = ControlSnapshot::read(&world.state_store)?; - let admitted = snapshot - .admitted_for("service") - .context("commit wrote no admitted generation")?; - assert_eq!(admitted.value.transaction_id, finished.transaction_id); - assert_eq!(Some(&admitted.value.expected), seeded.expected.as_ref()); - Ok(()) - } - - #[test] - fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - transaction_at(&world, DeploymentPhase::Warming)?; - let warming = resident(&world)?; - assert_eq!(warming.value.phase_deadline, None); - - world - .engine - .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - let finished = latest(&world)?; - assert_eq!(finished.phase, DeploymentPhase::Complete); - assert_eq!(finished.phase_deadline, None); - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); - assert!(finished.is_terminal()); - Ok(()) - } - - #[test] - fn a_post_fence_abort_runs_to_complete() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Routing - })?; - let routing = resident(&world)?; - assert!(routing.value.phase_deadline.is_some()); - - world - .engine - .begin_post_fencing_abort(&routing, anyhow!("candidate refused"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - let finished = latest(&world)?; - assert_eq!(finished.phase, DeploymentPhase::Complete); - assert_eq!(finished.phase_deadline, None); - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { .. }) - )); - assert!( - ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .is_none(), - "an aborted candidate was admitted" - ); - Ok(()) - } - - #[test] - fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; - let plan = transaction.plan.clone().unwrap(); - let base = transaction.created_at_unix_millis; - transaction.enter_phase(DeploymentPhase::Routing, base + 1_000); - let deadline = transaction.phase_deadline.context("no deadline on entry")?; - assert_eq!(deadline.phase, DeploymentPhase::Routing); - assert_eq!( - deadline.deadline_at_unix_millis, - base + 1_000 + u64::from(plan.phase_deadlines().routing_seconds) * 1000 - ); - // A same-phase write after the wall clock stepped backwards is still a - // valid record: the deadline is fixed at entry, not ordered against - // later stamps, and neither is the creation time. - transaction.updated_at_unix_millis = deadline.entered_at_unix_millis - 1; - transaction.validate()?; - transaction.updated_at_unix_millis = base - 1; - transaction.validate()?; - transaction.enter_phase(DeploymentPhase::Committing, base + 2_000); - assert_eq!( - transaction.phase_deadline.unwrap().phase, - DeploymentPhase::Committing - ); - transaction.enter_phase(DeploymentPhase::Complete, base + 3_000); - assert_eq!(transaction.phase_deadline, None); - Ok(()) - } - - #[test] - fn an_undecodable_history_record_is_counted_and_never_dropped_silently() -> Result<()> { - let (finished, _) = terminal_transaction_with_command("ghostlight")?; - let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; - let mut torn = good.clone(); - torn.key = "tx-torn".into(); - // 0xc1 is msgpack's never-used byte: decodable by nothing. - torn.payload = vec![0xc1]; - let mut foreign = good.clone(); - foreign.key = "tx-foreign".into(); - foreign.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA_V3.into()); - foreign.payload = vec![0xc1]; - - let (decoded, undecodable) = - decode_history_transactions(vec![good, torn, foreign]); - assert_eq!(decoded, [finished]); - assert_eq!( - undecodable.iter().map(|(key, _)| key.as_str()).collect::>(), - ["tx-torn", "tx-foreign"] - ); - assert!(undecodable.iter().all(|(_, error)| !error.is_empty())); - Ok(()) - } - - /// The layout `IsolationEvidence` had before c848459 turned it into an - /// untagged Linux/Host enum. That commit claimed old records decode - /// unchanged; this is the check with the old shape itself. - #[test] - fn isolation_evidence_from_before_the_host_variant_still_decodes() -> Result<()> { - #[derive(Serialize)] - struct BeforeTheHostVariant { - candidate_uid: u32, - candidate_pid_namespace_id: u64, - candidate_mount_namespace_id: u64, - incumbent_uid: Option, - incumbent_pid_namespace_id: Option, - incumbent_mount_namespace_id: Option, - } - for incumbent in [None, Some(7)] { - let old = BeforeTheHostVariant { - candidate_uid: 61_000, - candidate_pid_namespace_id: 4_026_531_836, - candidate_mount_namespace_id: 4_026_531_841, - incumbent_uid: incumbent, - incumbent_pid_namespace_id: incumbent.map(|_| 8), - incumbent_mount_namespace_id: incumbent.map(|_| 9), - }; - for bytes in [rmp_serde::to_vec(&old)?, rmp_serde::to_vec_named(&old)?] { - let decoded: IsolationEvidence = rmp_serde::from_slice(&bytes)?; - let IsolationEvidence::Linux(linux) = decoded else { - bail!("an old Linux record decoded as a host record"); - }; - assert_eq!(linux.candidate_uid, 61_000); - assert_eq!(linux.incumbent_uid, incumbent); - } - } - Ok(()) - } - - #[test] - fn an_admitted_odin_without_odin_authority_is_refused_not_replaced_by_the_bootstrap_key() -> Result<()> { - let world = EngineFixture::new()?; - let empty = ControlSnapshot::read(&world.state_store)?; - assert_eq!( - world.engine.current_odin_authority(&empty)?, - world.engine.bootstrap_odin_authority - ); - - let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; - odin.target = "odin".into(); - // Odin is the target that provides the rendezvous, whatever it is named. - odin.expected.target = "odin".into(); - odin.expected.dependencies.clear(); - provide_odin(&mut odin.expected); - odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( - &world.odin_signer.trust_anchor()?, - )?); - let mut with_authority = ControlSnapshot::default(); - with_authority.admitted.push(Stored { - value: odin.clone(), - envelope: CultCacheEnvelope { - key: "odin".into(), - r#type: AdmittedGeneration::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), - }, - }); - assert_eq!( - world.engine.current_odin_authority(&with_authority)?, - odin.odin_authority.clone().unwrap() - ); - - odin.odin_authority = None; - let mut without = ControlSnapshot::default(); - without.admitted.push(Stored { - value: odin.clone(), - envelope: CultCacheEnvelope { - key: "odin".into(), - r#type: AdmittedGeneration::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), - }, - }); - let error = world - .engine - .current_odin_authority(&without) - .expect_err("a route-proof Odin must not fall back to the bootstrap key"); - assert!(format!("{error:#}").contains("no Odin authority"), "{error:#}"); - Ok(()) - } - - fn error_text(result: Result<()>) -> String { - format!("{:#}", result.expect_err("the record must be refused")) - } - - /// A route-proof generation from a real admission: the world's Expected - /// declares no Odin dependency, so route proof is its class. - fn route_proof_generation() -> Result { - route_proof::committed_generation() - } - - #[test] - fn a_route_proof_generation_is_admitted_only_in_its_exact_shape() -> Result<()> { - let generation = route_proof_generation()?; - assert_eq!(generation.ready.voucher(), Voucher::Candidate); - assert!(generation.odin_authority.is_none()); - assert!(generation.latest_odin_observation.is_none()); - assert!(generation.odin_receipts().is_err()); - - // Route proof carries no Odin receipt and no Odin authority. - let mut with_receipt = generation.clone(); - with_receipt.latest_odin_observation = Some(topology(1, 1)); - assert!(with_receipt.validate().is_err()); - let mut with_authority = generation.clone(); - with_authority.odin_authority = Some(AdmittedOdinAuthority { - signer_identity_id: "odin-signer".into(), - signer_public_key: vec![1; 32], - }); - assert!(with_authority.validate().is_err()); - - // Its readiness evidence is checked by shape. - let ReadinessEvidence::RouteProof { evidence } = &generation.ready else { - bail!("not route proof"); - }; - let mut tampered = generation.clone(); - tampered.ready = ReadinessEvidence::RouteProof { - evidence: RuntimePresenceEvidence { - canonical_sha256: sha256_id(b"another"), - ..evidence.clone() - }, - }; - assert!(error_text(tampered.validate()).contains("runtime presence evidence")); - let mut untimed = generation.clone(); - untimed.ready = ReadinessEvidence::RouteProof { - evidence: RuntimePresenceEvidence { - admitted_at_unix_millis: evidence.challenged_at_unix_millis - 1, - ..evidence.clone() - }, - }; - assert!(error_text(untimed.validate()).contains("timeline")); - Ok(()) - } - - #[test] - fn odin_correlated_readiness_evidence_is_checked_by_shape() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.validate()?; - let ReadinessEvidence::OdinCorrelated { evidence } = &generation.ready else { - bail!("fixture is not Odin-correlated"); - }; - for broken in [ - TopologyEvidence { - canonical_sha256: sha256_id(b"another"), - ..evidence.clone() - }, - TopologyEvidence { - publisher_sequence: 0, - ..evidence.clone() - }, - TopologyEvidence { - admitted_at_unix_millis: 0, - ..evidence.clone() - }, - ] { - let mut tampered = generation.clone(); - tampered.ready = ReadinessEvidence::OdinCorrelated { evidence: broken }; - assert!(tampered.validate().is_err()); - } - Ok(()) - } - - #[test] - fn lease_adoption_evidence_is_checked_by_shape_on_the_transaction() -> Result<()> { - let (_, transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; - let adoption = LeaseAdoptionEvidence { - write_lease_sha256: sha256_id(b"lease"), - signed_presence_sha256: sha256_id(b"presence"), - source: AdoptionSource::Direct, - observed_at_unix_millis: 100, - }; - let refused = |change: fn(&mut LeaseAdoptionEvidence)| { - let mut adopting = transaction.clone(); - let mut evidence = adoption.clone(); - change(&mut evidence); - adopting.lease_adoption = Some(evidence); - error_text(adopting.validate()) - }; - assert!(refused(|evidence| evidence.write_lease_sha256.clear()) - .contains("adopted lease digest")); - assert!(refused(|evidence| evidence.signed_presence_sha256 = "no spaces allowed".into()) - .contains("presence digest")); - assert!(refused(|evidence| evidence.observed_at_unix_millis = 0) - .contains("no observation time")); - // Well-formed but naming no granted lease: the record is refused for - // that reason and no other. - assert!(refused(|_| {}).contains("does not name the granted lease")); - Ok(()) - } - - #[test] - fn route_supervision_is_checked_on_the_generation_and_the_meters_on_their_own_record() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.validate()?; - let supervised = |change: fn(&mut RouteSupervisionState)| { - let mut tampered = generation.clone(); - change(tampered.route_supervision.as_mut().unwrap()); - error_text(tampered.validate()) - }; - assert!(supervised(|state| state.degraded_since_unix_millis = Some(0)) - .contains("no start time")); - - let meters = |change: fn(&mut TargetSupervision)| { - let mut meters = TargetSupervision::new("service"); - change(&mut meters); - error_text(meters.validate()) - }; - TargetSupervision::new("service").validate()?; - assert!(meters(|m| m.route_actuations = vec![0]).contains("ascending")); - assert!(meters(|m| m.route_actuations = vec![9, 5]).contains("ascending")); - assert!( - meters(|m| m.route_actuations = (1..=13).collect()).contains("exceeds its bound") - ); - assert!(meters(|m| m.continuity_restarts = (1..=7).collect()).contains("exceeds its bound")); - assert!(meters(|m| m.route_survivals = (1..=13).collect()).contains("exceeds its bound")); - assert!(meters(|m| m.route_survivals = vec![9, 5]).contains("ascending")); - assert!(meters(|m| m.continuity_deferred_until = Some(5)).contains("time or a reason")); - assert!(meters(|m| m.continuity_deferral_reason = Some("x".into())).contains("time or a reason")); - assert!(meters(|m| m.schema_version = "idunn.target_supervision.v0".into()) - .contains("unsupported")); - Ok(()) - } - - #[test] - fn max_odin_sequence_counts_admitted_generations_by_target_and_signer() -> Result<()> { - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - let authority = generation.odin_authority.clone().context("no authority")?; - generation.odin_publisher_sequence_cursor = 88; - let target = generation.target.clone(); - let snapshot = ControlSnapshot { - expiry_requests: Vec::new(), - commands: Vec::new(), - transactions: Vec::new(), - admitted: vec![Stored { - envelope: CultCacheEnvelope { - key: target.clone(), - r#type: AdmittedGeneration::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), - }, - value: generation, - }], - targets: Vec::new(), - }; - assert_eq!(snapshot.max_odin_sequence(&target, &authority.signer_identity_id), 88); - assert_eq!(snapshot.max_odin_sequence(&target, "another-signer"), 0); - assert_eq!( - snapshot.max_odin_sequence("another-target", &authority.signer_identity_id), - 0 - ); - Ok(()) - } - - #[test] - fn a_store_holding_a_lifted_legacy_generation_reads_and_keeps_its_key_honest() -> Result<()> { - let temp = TempDir::new()?; - let store = temp.path().join("control.cc"); - let envelope = fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?; - let backing = SingleFileMessagePackBackingStore::new(&store); - backing.insert_entry_if_absent(envelope.clone())?; - - let snapshot = ControlSnapshot::read(&store)?; - assert_eq!(snapshot.admitted.len(), 1); - let lifted = &snapshot.admitted[0].value; - assert_eq!(lifted.target, envelope.key); - assert_eq!(lifted.ready.voucher(), Voucher::Odin); - assert!(snapshot.admitted_for(&envelope.key).is_some()); - - // The lifted record is held to the same key rule as a current one. - let other = temp.path().join("misfiled.cc"); - let mut misfiled = envelope; - misfiled.key = "not-its-target".into(); - SingleFileMessagePackBackingStore::new(&other).insert_entry_if_absent(misfiled)?; - assert!(error_text(ControlSnapshot::read(&other).map(|_| ())) - .contains("admitted generation key is not its target")); - Ok(()) - } - - /// Replace Odin's one correlation for the incarnation with a newer one. - fn odin_republishes( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ) -> Result<()> { - let path = &world.engine.options.odin_correlation_store; - let store = SingleFileMessagePackBackingStore::new(path); - let existing = store.pull_all_read_only_snapshot()?; - assert!(store.delete_batch_if_unchanged(&existing)?); - odin_reports_ready(world, transaction, sequence) - } - - fn ready_sequence(transaction: &DeploymentTransaction) -> Option { - transaction - .ready - .as_ref() - .and_then(ReadinessEvidence::odin) - .map(|evidence| evidence.publisher_sequence) - } - - #[test] - fn a_newer_odin_reading_refreshes_the_receipt_before_awaiting_ready_or_routing_advance() - -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - - // AwaitingReady: the receipt is written, then confirmed, then the - // phase advances only on a receipt equal to the latest reading. - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::AwaitingReady - && ready_sequence(transaction) == Some(5) - })?; - odin_republishes(&world, &seeded, 6)?; - world.engine.advance_transaction(&resident(&world)?)?; // admits 6 - let admitted = resident(&world)?.value; - assert_eq!(admitted.phase, DeploymentPhase::AwaitingReady); - assert_eq!(admitted.latest_odin_observation.as_ref().unwrap().publisher_sequence, 6); - assert_eq!(ready_sequence(&admitted), Some(5)); - world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 6 - let refreshed = resident(&world)?.value; - assert_eq!(refreshed.phase, DeploymentPhase::AwaitingReady); - assert_eq!(ready_sequence(&refreshed), Some(6)); - world.engine.advance_transaction(&resident(&world)?)?; - assert_eq!(resident(&world)?.value.phase, DeploymentPhase::Routing); - - // Routing: route admission demands a receipt equal to the latest - // reading, so a newer one is adopted before anything is routed. - odin_republishes(&world, &seeded, 7)?; - world.engine.advance_transaction(&resident(&world)?)?; // admits 7 - assert_eq!(ready_sequence(&resident(&world)?.value), Some(6)); - world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 7 - let routing = resident(&world)?.value; - assert_eq!(routing.phase, DeploymentPhase::Routing); - assert_eq!(ready_sequence(&routing), Some(7)); - assert!(routing.routing.is_none()); - world.engine.advance_transaction(&resident(&world)?)?; - assert!(resident(&world)?.value.routing.is_some()); - Ok(()) - } - - #[test] - fn lease_adoption_names_only_the_granted_lease() -> Result<()> { - let lease = write_lease(); - let lease_sha256 = lease.canonical_sha256()?; - let adoption = LeaseAdoptionEvidence { - write_lease_sha256: lease_sha256.clone(), - signed_presence_sha256: sha256_id(b"adopting-presence"), - source: AdoptionSource::Direct, - observed_at_unix_millis: 100, - }; - adoption.validate_shape()?; - let granted = LeasingEvidence::Granted { - lease: lease.clone(), - lease_sha256: lease_sha256.clone(), - }; - assert!(adoption.names(&granted)); - // Prepared is not granted, stateless has nothing to adopt, and another - // lease's digest is not this lease. - assert!(!adoption.names(&LeasingEvidence::Prepared { - lease, - lease_sha256, - })); - assert!(!adoption.names(&LeasingEvidence::SkippedStateless)); - let mut other = adoption.clone(); - other.write_lease_sha256 = sha256_id(b"another-lease"); - assert!(!other.names(&granted)); - - // On a transaction, adoption without a granted lease is refused. - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; - transaction.lease_adoption = Some(adoption); - assert!(transaction.validate().is_err()); - Ok(()) - } - - /// Q3 as a table: which ended phases abort, which name the unadopted - /// lease, which are left to the operator, and which recovery the terminal - /// record names. - #[test] - fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; - let lease = write_lease(); - let sha256 = lease.canonical_sha256()?; - let granted = LeasingEvidence::Granted { - lease: lease.clone(), - lease_sha256: sha256.clone(), - }; - let prepared = LeasingEvidence::Prepared { - lease, - lease_sha256: sha256.clone(), - }; - let adoption = LeaseAdoptionEvidence::with_ready( - Some(&sha256), - &sha256_id(b"adopting-presence"), - AdoptionSource::Direct, - 1, - ); - assert!(adoption.is_some()); - assert_eq!( - LeaseAdoptionEvidence::with_ready(None, &sha256_id(b"p"), AdoptionSource::Direct, 1), - None - ); - let abort = |lease_not_adopted| PhaseEnd::Abort { lease_not_adopted }; - use CommandKind::{Continuity, Deploy}; - let cases = [ - (Deploy, None, None, abort(false)), - (Deploy, Some(LeasingEvidence::SkippedStateless), None, abort(false)), - (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), - (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), - (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), - (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), - (Continuity, Some(prepared), None, abort(false)), - (Continuity, Some(granted.clone()), None, abort(true)), - (Continuity, Some(granted), adoption.clone(), abort(false)), - ]; - for (kind, leasing, lease_adoption, expected) in cases { - transaction.command_kind = kind; - transaction.leasing = leasing.clone(); - transaction.lease_adoption = lease_adoption.clone(); - assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); - let recovery = if kind == Continuity && lease_adoption.is_some() { - TerminalRecovery::RestartAdmitted - } else { - TerminalRecovery::RestoreIncumbent - }; - assert_eq!(transaction.terminal_recovery(), recovery, "{kind:?} {lease_adoption:?}"); - } - Ok(()) - } - - #[test] - fn an_operator_required_recovery_must_say_why() -> Result<()> { - let (_, mut failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; - let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = - failed.completion.as_mut() - else { - panic!("fixture is not a post-fencing failure") - }; - *recovery = TerminalRecovery::OperatorRequired { - reason: "adopted-lease".into(), - }; - failed.validate()?; - assert!(failed.is_terminal()); - let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = - failed.completion.as_mut() - else { - unreachable!() - }; - *recovery = TerminalRecovery::OperatorRequired { - reason: String::new(), - }; - assert!(failed.validate().is_err()); - Ok(()) - } - /// The persisted projection a continuity restart leaves behind, read at - /// the file the daemon writes. - struct ContinuityProjection { - _temp: TempDir, - topology: CultCacheTopologyDriver, - provider_anchor: ServiceIdentityTrustAnchor, - expected: IdunnExpectedIncarnationRecord, - admitted_activation: IdunnRuntimeActivationRecord, - candidate_activation: IdunnRuntimeActivationRecord, - } - - impl ContinuityProjection { - /// The production sequence: the admitted incarnation dies and is - /// demoted to Expected-only, then the continuity candidate publishes - /// its own activation under the same key once it is observed. - fn after_candidate_observed() -> Result { - let temp = TempDir::new()?; - let root = temp.path(); - let idunn = enroll_service_identity_at::(&root.join("idunn.cc"))?; - let provider = enroll_service_identity_at::( - &root.join("provider.cc"), - )?; - let provider_anchor = provider.trust_anchor()?; - let mut expected = TopologyFixture::new("ghostlight")?.expected; - expected.expected_signer_identity_id = provider.entry().identity_id.clone(); - expected.artifact_sha256 = sha256_id(&[1]); - expected.validate()?; - let topology = CultCacheTopologyDriver { - projection_store: root.join("topology.cc"), - correlation_store: root.join("correlation.cc"), - }; - let admitted = workload(1, 40, 50); - let candidate = workload(2, 41, 51); - let issue = |observation: &WorkloadObservation| -> Result<_> { - Ok(IdunnRuntimeActivationLaunch::issue( - &expected, - observation.runtime_instance_id().to_owned(), - NOW, - &idunn, - )? - .activation() - .clone()) - }; - let admitted_activation = issue(&admitted)?; - let candidate_activation = issue(&candidate)?; - assert_ne!(admitted_activation, candidate_activation); - - topology.publish_expected(&expected, &provider_anchor)?; - topology.publish_observed_activation(&expected, &admitted_activation, &admitted)?; - topology.demote_to_expected_only( - &expected, - &provider_anchor, - &admitted_activation, - None, - )?; - topology.publish_expected(&expected, &provider_anchor)?; - topology.publish_observed_activation(&expected, &candidate_activation, &candidate)?; - Ok(Self { - _temp: temp, - topology, - provider_anchor, - expected, - admitted_activation, - candidate_activation, - }) - } - - fn continuity_transaction(&self) -> Result { - let mut transaction = DeploymentTransaction::new( - &command(CommandKind::Continuity), - "ghostlight".into(), - 0, - None, - 100, - )?; - transaction.expected = Some(self.expected.clone()); - transaction.activation = Some(self.candidate_activation.clone()); - Ok(transaction) - } - - /// Types of the records projected under the shared incarnation key. - fn projected_types(&self) -> Result> { - let key = incarnation_key(&self.expected)?; - Ok( - SingleFileMessagePackBackingStore::new(&self.topology.projection_store) - .pull_all_read_only_snapshot()? - .into_iter() - .filter(|entry| entry.key == key) - .map(|entry| entry.r#type) - .collect(), - ) - } - } - - #[test] - fn a_failed_continuity_demotes_its_own_activation_and_keeps_the_expected() -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - assert!( - world - .projected_types()? - .contains(&IdunnRuntimeActivationRecord::TYPE.to_owned()) - ); - let transaction = world.continuity_transaction()?; - assert_eq!( - transaction.abort_topology_reconciliation(), - CleanupEvidence::Pending - ); - assert_eq!( - post_fencing_abort_intent(&transaction, "candidate died").topology_reconciliation, - CleanupEvidence::Pending - ); - - reconcile_failed_candidate_projection( - &world.topology, - &transaction, - &world.provider_anchor, - )?; - - assert_eq!( - world.projected_types()?, - vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()], - "the shared key must keep its Expected and nothing else" - ); - Ok(()) - } - - #[test] - fn a_continuity_that_issued_no_activation_owes_no_projection_change() -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - let mut transaction = world.continuity_transaction()?; - transaction.activation = None; - assert_eq!( - transaction.abort_topology_reconciliation(), - CleanupEvidence::Skipped - ); - assert_eq!( - post_fencing_abort_intent(&transaction, "candidate never started") - .topology_reconciliation, - CleanupEvidence::Skipped - ); - Ok(()) - } - - #[test] - fn a_deploy_fencing_after_a_failed_continuity_resolves_with_the_admitted_activation() - -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - reconcile_failed_candidate_projection( - &world.topology, - &world.continuity_transaction()?, - &world.provider_anchor, - )?; - let settled = std::fs::read(&world.topology.projection_store)?; - - // The deploy's post-fence rollback demotes the admitted generation's - // exact activation. The failed continuity left nothing standing, so it - // resolves instead of refusing as substituted. - world.topology.demote_to_expected_only( - &world.expected, - &world.provider_anchor, - &world.admitted_activation, - None, - )?; - assert_eq!(std::fs::read(&world.topology.projection_store)?, settled); - Ok(()) - } - - #[test] - fn no_path_adopts_an_activation_the_transaction_did_not_issue() -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - let before = std::fs::read(&world.topology.projection_store)?; - - // The candidate's activation is still projected. The admitted - // generation's exact activation is not what stands there, so its - // demotion refuses rather than taking the newer one. - assert!( - world - .topology - .demote_to_expected_only( - &world.expected, - &world.provider_anchor, - &world.admitted_activation, - None, - ) - .is_err() - ); - - // Nor may a continuity transaction demote an activation it did not - // issue. - let mut foreign = world.continuity_transaction()?; - foreign.activation = Some(world.admitted_activation.clone()); - assert!( - reconcile_failed_candidate_projection( - &world.topology, - &foreign, - &world.provider_anchor - ) - .is_err() - ); - assert_eq!(std::fs::read(&world.topology.projection_store)?, before); - Ok(()) - } - - // --------------------------------------------------------------------- - // B1 fix batch: the transition, the precondition, the boot reconciliation, - // history that fails closed, and the abort paths through the Engine. - // --------------------------------------------------------------------- - - /// A workload port whose process can be killed: observing it succeeds - /// until `kill`, then fails the way a dead unit does. Everything else is - /// `StillWorkload`'s. - struct SwitchWorkload { - alive: std::sync::atomic::AtomicBool, - rebooted: std::sync::atomic::AtomicBool, - } - - impl SwitchWorkload { - fn new() -> Arc { - Arc::new(Self { - alive: std::sync::atomic::AtomicBool::new(true), - rebooted: std::sync::atomic::AtomicBool::new(false), - }) - } - - fn revive(&self) { - self.alive - .store(true, std::sync::atomic::Ordering::SeqCst); - self.rebooted - .store(false, std::sync::atomic::Ordering::SeqCst); - } - - /// The host rebooted under the unit: it is gone and the driver can tell. - fn reboot(&self) { - self.rebooted - .store(true, std::sync::atomic::Ordering::SeqCst); - self.kill(); - } - - fn kill(&self) { - self.alive - .store(false, std::sync::atomic::Ordering::SeqCst); - } - } - - impl WorkloadPort for SwitchWorkload { - fn install( - &self, - plan: &CompiledDeploymentPlan, - release: &crate::drivers::MaterializedRelease, - ) -> Result { - StillWorkload.install(plan, release) - } - fn prepare_activation( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - launch: IdunnRuntimeActivationLaunch, - ) -> Result { - StillWorkload.prepare_activation(plan, expected, launch) - } - fn start_prepared( - &self, - plan: &CompiledDeploymentPlan, - release: &SealedRelease, - installed: &crate::drivers::InstalledReleaseObservation, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) - } - fn discard_prepared( - &self, - _: &CompiledDeploymentPlan, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - Ok(()) - } - fn observe( - &self, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - prior: &WorkloadObservation, - ) -> Result { - ensure!( - self.alive.load(std::sync::atomic::Ordering::SeqCst), - "the workload is gone" - ); - Ok(prior.clone()) - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - Ok(()) - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(false) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) - } - } - - /// Admit a generation for "service" through the real phase machine and - /// retire its transaction, leaving the world with an incumbent. - fn admit_incumbent(world: &EngineFixture) -> Result { - let seeded = transaction_at(world, DeploymentPhase::Fencing)?; - odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; - drive(world, |transaction| { - transaction.phase == DeploymentPhase::Complete - })?; - assert!(world.engine.retire_one_terminal_transaction()?); - Ok(ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("commit wrote no admitted generation")? - .value - .clone()) - } - - #[test] - fn an_admitted_odin_correlated_generation_takes_a_newer_odin_reading_once() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - let first = seeded_sequence(None) + 1; - odin_reports_ready(&world, &seeded, first)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - assert!(world.engine.retire_one_terminal_transaction()?); - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("commit wrote no admitted generation")? - .value - .clone(); - assert_eq!(admitted.readiness(), Ok(ReadinessClass::OdinCorrelated)); - assert_eq!(admitted.odin_publisher_sequence_cursor, first); - - // Nothing newer than what is held: nothing is written. - let snapshot = ControlSnapshot::read(&world.state_store)?; - let current = snapshot.admitted_for("service").context("no generation")?; - let envelope = current.envelope.clone(); - assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); - assert_eq!(incumbent_envelope(&world)?, envelope); - - // A newer reading is adopted, once. - odin_republishes(&world, &seeded, first + 4)?; - let snapshot = ControlSnapshot::read(&world.state_store)?; - let current = snapshot.admitted_for("service").context("no generation")?; - assert!(world.engine.refresh_admitted_topology(&snapshot, current)?); - let refreshed = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no generation")? - .value - .clone(); - assert_eq!(refreshed.odin_publisher_sequence_cursor, first + 4); - assert_eq!( - refreshed.latest_odin_observation.as_ref().map(|latest| latest.publisher_sequence), - Some(first + 4) - ); - assert_eq!(refreshed.ready.odin().map(|ready| ready.publisher_sequence), Some(first + 4)); - let snapshot = ControlSnapshot::read(&world.state_store)?; - let current = snapshot.admitted_for("service").context("no generation")?; - assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); - Ok(()) - } - - /// Change the incumbent in the store, compare-and-swap. - fn edit_incumbent( - world: &EngineFixture, - edit: impl FnOnce(&mut AdmittedGeneration), - ) -> Result { - let snapshot = ControlSnapshot::read(&world.state_store)?; - let stored = snapshot - .admitted_for("service") - .context("no admitted generation")?; - let mut next = stored.value.clone(); - edit(&mut next); - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: "service".into(), - current: Some(stored.envelope.clone()), - }], - &[admitted_envelope(&next, now_millis()?)?], - )? - ); - Ok(next) - } - - /// Replace or create a target's meters in the store. - fn set_meters(world: &EngineFixture, meters: &TargetSupervision) -> Result<()> { - let snapshot = ControlSnapshot::read(&world.state_store)?; - world.engine.write_target_supervision( - snapshot.supervision_for(&meters.target), - meters, - now_millis()?, - ) - } - - fn meters_of(world: &EngineFixture, target: &str) -> Result { - Ok(ControlSnapshot::read(&world.state_store)?.supervision_or_new(target)) - } - - /// Meters for "service" whose restarts happened these many milliseconds ago. - fn restarted_ago(ages: &[u64]) -> Result { - let now = now_millis()?; - let mut meters = TargetSupervision::new("service"); - meters.continuity_restarts = ages.iter().map(|age| now - age).collect(); - meters.continuity_restarts.sort_unstable(); - // Every logged attempt belongs to one streak. - meters.continuity_attempt_streak = ages.len() as u32; - Ok(meters) - } - - /// Drop every live transaction and command: what a finished restart leaves. - fn clear_live_transactions(world: &EngineFixture) -> Result<()> { - let snapshot = ControlSnapshot::read(&world.state_store)?; - let envelopes = snapshot - .transactions - .iter() - .map(|stored| stored.envelope.clone()) - .chain(snapshot.commands.iter().map(|stored| stored.envelope.clone())) - .collect::>(); - if envelopes.is_empty() { - return Ok(()); - } - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store) - .delete_batch_if_unchanged(&envelopes)? - ); - Ok(()) - } - - fn incumbent_envelope(world: &EngineFixture) -> Result { - Ok(ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no admitted generation")? - .envelope - .clone()) - } - - /// Publish what a transaction's candidate projects: its Expected and, - /// when asked, its observed activation. - fn project_candidate( - world: &EngineFixture, - transaction: &DeploymentTransaction, - with_activation: bool, - ) -> Result<()> { - let expected = transaction.expected.as_ref().context("no Expected")?; - let anchor = world - .engine - .provider_anchor_for_plan(transaction.plan.as_ref().context("no plan")?)?; - let topology = world.engine.topology(); - topology.publish_expected(expected, &anchor)?; - if with_activation { - // The observation must name the release the Expected declares; - // the borrowed workload is patched to. - let mut workload = transaction.workload.clone().context("no workload")?; - if let WorkloadObservation::Systemd(observed) = &mut workload { - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - topology.publish_observed_activation( - expected, - transaction.activation.as_ref().context("no activation")?, - &workload, - )?; - } - Ok(()) - } - - /// Publish what an admitted generation projects while it runs. - fn project_admitted(world: &EngineFixture, generation: &AdmittedGeneration) -> Result<()> { - let anchor = world.engine.provider_anchor_for_plan(&generation.plan)?; - let topology = world.engine.topology(); - topology.publish_expected(&generation.expected, &anchor)?; - topology.publish_observed_activation( - &generation.expected, - &generation.activation, - &generation.workload, - )?; - Ok(()) - } - - /// The types of the records projected under one incarnation key, sorted. - fn projected_under( - world: &EngineFixture, - expected: &IdunnExpectedIncarnationRecord, - ) -> Result> { - let path = &world.engine.options.topology_store; - if !path.exists() { - return Ok(Vec::new()); - } - let key = incarnation_key(expected)?; - let mut types = SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot()? - .into_iter() - .filter(|entry| entry.key == key) - .map(|entry| entry.r#type) - .collect::>(); - types.sort(); - Ok(types) - } - - /// One transaction by id, wherever it now lives. A failed transaction is - /// archived the moment it finishes, so it is in history, not resident. - fn record_of(world: &EngineFixture, transaction_id: &str) -> Result { - if let Some(stored) = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .find(|stored| stored.value.transaction_id == transaction_id) - { - return Ok(stored.value); - } - read_history_transactions(&world.state_store) - .into_iter() - .find(|transaction| transaction.transaction_id == transaction_id) - .context("the transaction is in neither the live set nor history") - } - - fn expected_only() -> Vec { - vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()] - } - - // ---- the Engine's abort paths, Deploy and incumbent ---- - - #[test] - fn a_deploy_that_fences_commits_and_retires_the_incumbent_it_replaced() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - admit_incumbent(&world)?; - let meters = restarted_ago(&[600_000, 60_000])?; - set_meters(&world, &meters)?; - let incumbent = edit_incumbent(&world, |_| {})?; - - let candidate = seeded_transaction( - &world, - DeploymentPhase::Fencing, - CommandKind::Deploy, - Some(&incumbent), - )?; - odin_reports_ready(&world, &candidate, seeded_sequence(Some(&incumbent)) + 1)?; - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Complete - })?; - - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("the deploy admitted nothing")? - .value - .clone(); - assert_eq!(admitted.transaction_id, candidate.transaction_id); - assert_ne!(admitted.generation_id, incumbent.generation_id); - // The restart log belongs to the target: a commit must not reset it. - assert_eq!(meters_of(&world, "service")?, meters); - assert_eq!(admitted.route_supervision, incumbent.route_supervision); - - // The commit did not stop the incumbent: it is retired after. - let committed = record_of(&world, &candidate.transaction_id)?; - let cleanup = committed.post_commit_cleanup.clone().context("no cleanup")?; - assert_eq!( - cleanup.incumbent, - IncumbentCleanupEvidence::Pending { - generation_id: incumbent.generation_id.clone(), - workload: incumbent.workload.clone(), - } - ); - assert_eq!(cleanup.source, SourceCleanupEvidence::Pending); - let frozen = world - .engine - .options - .staging_root - .join("frozen-sources") - .join(&candidate.transaction_id); - std::fs::create_dir_all(&frozen)?; - std::fs::write(frozen.join("tree"), b"frozen")?; - - drive(&world, |transaction| transaction.is_terminal())?; - let retired = record_of(&world, &candidate.transaction_id)? - .post_commit_cleanup - .context("no cleanup")?; - assert_eq!( - retired.incumbent, - IncumbentCleanupEvidence::Complete { - generation_id: incumbent.generation_id - } - ); - assert_eq!(retired.source, SourceCleanupEvidence::Complete); - assert!(!frozen.exists(), "the frozen source outlived its deploy"); - Ok(()) - } - - #[test] - fn a_deploy_post_fence_abort_cleans_up_and_restores_its_incumbent_exactly() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - project_admitted(&world, &incumbent)?; - let candidate = seeded_transaction( - &world, - DeploymentPhase::Fencing, - CommandKind::Deploy, - Some(&incumbent), - )?; - project_candidate(&world, &candidate, true)?; - let candidate_expected = candidate.expected.clone().context("no Expected")?; - assert_eq!( - projected_under(&world, &candidate_expected)?.len(), - 2, - "the candidate did not project" - ); - let frozen = world - .engine - .options - .staging_root - .join("frozen-sources") - .join(&candidate.transaction_id); - std::fs::create_dir_all(&frozen)?; - std::fs::write(frozen.join("tree"), b"frozen")?; - let before = incumbent_envelope(&world)?; - - world - .engine - .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - assert!(resident(&world)?.value.post_fencing_abort.is_some()); - drive(&world, |transaction| transaction.completion.is_some())?; - - let finished = record_of(&world, &candidate.transaction_id)?; - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - )); - let abort = finished.post_fencing_abort.context("no abort evidence")?; - assert_eq!(abort.topology_reconciliation, CleanupEvidence::Complete); - assert_eq!(abort.source_cleanup, CleanupEvidence::Complete); - assert!(!frozen.exists(), "the abandoned source was not cleaned"); - assert!( - projected_under(&world, &candidate_expected)?.is_empty(), - "the abandoned candidate is still projected" - ); - // Fencing stopped the incumbent: it stands as Expected-only, with the - // exact activation it ran under withdrawn, ready for continuity. - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - assert_eq!(incumbent_envelope(&world)?, before); - Ok(()) - } - - #[test] - fn a_deploy_pre_fence_abort_cleans_up_and_leaves_its_incumbent_running() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - project_admitted(&world, &incumbent)?; - let candidate = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Deploy, - Some(&incumbent), - )?; - project_candidate(&world, &candidate, true)?; - let candidate_expected = candidate.expected.clone().context("no Expected")?; - let frozen = world - .engine - .options - .staging_root - .join("frozen-sources") - .join(&candidate.transaction_id); - std::fs::create_dir_all(&frozen)?; - std::fs::write(frozen.join("tree"), b"frozen")?; - let before = incumbent_envelope(&world)?; - - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let intent = resident(&world)?.value.pre_fencing_abort.context("no intent")?; - assert_eq!(intent.topology_reconciliation, CleanupEvidence::Pending); - assert_eq!(intent.source_cleanup, CleanupEvidence::Pending); - drive(&world, |transaction| transaction.completion.is_some())?; - - let finished = record_of(&world, &candidate.transaction_id)?; - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); - assert!(finished.is_terminal()); - assert!(!frozen.exists(), "the abandoned source was not cleaned"); - assert!(projected_under(&world, &candidate_expected)?.is_empty()); - // Before fencing a deployment never touched its incumbent. - assert_eq!( - projected_under(&world, &incumbent.expected)?, - { - let mut types = vec![ - IdunnExpectedIncarnationRecord::TYPE.to_owned(), - IdunnRuntimeActivationRecord::TYPE.to_owned(), - ]; - types.sort(); - types - } - ); - assert_eq!(incumbent_envelope(&world)?, before); - Ok(()) - } - - #[test] - fn a_continuity_abort_over_its_incumbent_demotes_only_the_activation_it_issued() -> Result<()> { - for post_fence in [false, true] { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - let candidate = seeded_transaction( - &world, - if post_fence { - DeploymentPhase::Fencing - } else { - DeploymentPhase::Warming - }, - CommandKind::Continuity, - Some(&incumbent), - )?; - // The candidate shares the incumbent's key: same Expected, its own - // activation. - assert_eq!(candidate.expected.as_ref(), Some(&incumbent.expected)); - assert_ne!(candidate.activation.as_ref(), Some(&incumbent.activation)); - project_candidate(&world, &candidate, true)?; - let before = incumbent_envelope(&world)?; - - let stored = resident(&world)?; - if post_fence { - world - .engine - .begin_post_fencing_abort(&stored, anyhow!("candidate died"))?; - assert_eq!( - resident(&world)? - .value - .post_fencing_abort - .context("no intent")? - .topology_reconciliation, - CleanupEvidence::Pending - ); - } else { - world - .engine - .begin_pre_fencing_abort(&stored, anyhow!("candidate died"))?; - assert_eq!( - resident(&world)? - .value - .pre_fencing_abort - .context("no intent")? - .topology_reconciliation, - CleanupEvidence::Pending - ); - } - drive(&world, |transaction| transaction.completion.is_some())?; - - assert_eq!( - projected_under(&world, &incumbent.expected)?, - expected_only(), - "post_fence={post_fence}: the shared key must keep its Expected and nothing else" - ); - let finished = record_of(&world, &candidate.transaction_id)?; - assert!(finished.is_terminal()); - assert!(matches!( - finished.completion, - Some( - TransactionCompletion::FailedBeforeFencing { .. } - | TransactionCompletion::FailedAfterFencing { .. } - ) - )); - assert_eq!(incumbent_envelope(&world)?, before); - } - Ok(()) - } - - // ---- fix 1: the legacy lift owns the transition ---- - - const FIXTURE_PRE_B1_ABORTS: [(&str, &str); 3] = [ - ( - "pre-fence abort in flight", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex"), - ), - ( - "pre-fence abort already terminal", - include_str!( - "../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex" - ), - ), - ( - "post-fence abort in flight", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex"), - ), - ]; - const FIXTURE_PROVIDER_ANCHOR: &str = - include_str!("../tests/fixtures/idunn-control-legacy/provider-anchor.hex"); - /// Where the fixtures' plan binding names the provider trust anchor. - const FIXTURE_ANCHOR_PATH: &str = "/tmp/idunn-b1-legacy/provider-anchor.cc"; - - fn hex_bytes(text: &str) -> Vec { - let hex = text.trim(); - (0..hex.len()) - .step_by(2) - .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("hex fixture")) - .collect() - } - - fn write_fixture_anchor() -> Result<()> { - let path = Path::new(FIXTURE_ANCHOR_PATH); - std::fs::create_dir_all(path.parent().context("anchor path has no parent")?)?; - std::fs::write(path, hex_bytes(FIXTURE_PROVIDER_ANCHOR))?; - Ok(()) - } - - fn abort_owed(transaction: &DeploymentTransaction) -> Option { - transaction - .pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - .or_else(|| { - transaction - .post_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - }) - } - - /// Put a continuity's record and its command in the control store as they - /// stood: `envelope` is written as given, in whatever schema it is in. - fn make_resident( - world: &EngineFixture, - transaction: &DeploymentTransaction, - envelope: CultCacheEnvelope, - ) -> Result<()> { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: transaction.command_id.clone(), - kind: CommandKind::Continuity, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: envelope.key.clone(), - current: None, - }, - ], - &[command_envelope(&command, 100)?, envelope], - )? - ); - Ok(()) - } - - #[test] - fn a_pre_b1_continuity_abort_lifts_without_reopening_and_boot_resolves_its_residue() - -> Result<()> { - write_fixture_anchor()?; - for (name, text) in FIXTURE_PRE_B1_ABORTS { - let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; - - // The record is what the old rule wrote: an activation was issued - // and the abort says it owes the projection nothing. - let legacy: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload)?; - assert_eq!(legacy.command_kind, CommandKind::Continuity, "{name}"); - assert!(legacy.activation.is_some(), "{name}"); - - // History describes what happened, so it lifts the record as written. - let archived = lift_legacy_transaction(&envelope)?; - assert_eq!(abort_owed(&archived), Some(CleanupEvidence::Skipped), "{name}"); - - // The control store lifts it without reopening anything. - let lifted = read_transaction_record(&envelope)?; - let terminal = archived.completion.is_some(); - if terminal { - assert!( - abort_owed(&lifted).is_some_and(CleanupEvidence::is_legacy_marker), - "{name}" - ); - assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); - assert_eq!(lifted.phase, archived.phase, "{name}"); - assert_eq!(lifted.completion, archived.completion, "{name}"); - assert!(!lifted.owns_target_authority(), "{name}"); - } else { - assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); - assert!(!lifted.is_terminal(), "{name}"); - assert_eq!(lifted.completion, None, "{name}"); - } - - // Boot: the store reads. - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - make_resident(&world, &lifted, envelope.clone())?; - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert_eq!(snapshot.transactions.len(), 1, "{name}"); - project_candidate(&world, &lifted, true)?; - let expected = lifted.expected.clone().context("no Expected")?; - assert_eq!(projected_under(&world, &expected)?.len(), 2, "{name}"); - - if terminal { - // A finished record owes nothing and is not touched: boot - // reconciliation demotes the residue by its exact activation. - let before = resident(&world)?.envelope; - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!(outcome.demoted, vec![lifted.transaction_id.clone()], "{name}"); - assert_eq!(resident(&world)?.envelope, before, "{name}"); - } else { - // A record in flight resolves through its own abort. - for _ in 0..12 { - let Ok(current) = resident(&world) else { break }; - if current.value.completion.is_some() { - break; - } - world.engine.advance_transaction(¤t)?; - } - assert!( - record_of(&world, &envelope.key)?.completion.is_some(), - "{name}: the abort never resolved" - ); - } - assert_eq!( - projected_under(&world, &expected)?, - expected_only(), - "{name}: the projection was not demoted to Expected-only" - ); - } - Ok(()) - } - - #[test] - fn resident_terminal_pre_b1_aborts_never_claim_target_authority() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; - let first = read_transaction_record(&envelope)?; - assert_eq!(first.target, live.target); - make_resident(&world, &first, envelope)?; - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - - // A second resident abort of the same shape for the same target, - // already written as the marker the migration persists. - let mut second = first.clone(); - second.transaction_id = "tx-second-legacy".into(); - second.command_id = "continuity-second-legacy".into(); - make_resident( - &world, - &second, - cleanup_evidence::last_error_envelope(&second, "second")?, - )?; - - // The store reads: three records, one of them live, none of the - // finished ones claiming the target. - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert_eq!(snapshot.transactions.len(), 3); - let live_before = snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == live.transaction_id) - .context("live transaction")? - .envelope - .clone(); - - // Reconciliation cleans the residue once, by exact activation. - let expected = first.expected.clone().context("no Expected")?; - project_candidate(&world, &first, true)?; - assert_eq!(projected_under(&world, &expected)?.len(), 2); - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!(outcome.demoted.len(), 1); - assert_eq!(projected_under(&world, &expected)?, expected_only()); - let after = ControlSnapshot::read(&world.state_store)?; - assert_eq!( - after - .transactions - .iter() - .find(|stored| stored.value.transaction_id == live.transaction_id) - .context("live transaction")? - .envelope, - live_before - ); - Ok(()) - } - - #[test] - fn boot_never_republishes_a_withdrawn_incarnation_and_history_keeps_one_copy() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let lifted = read_transaction_record(&envelope)?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - make_resident(&world, &lifted, envelope.clone())?; - // A crash between the two archive steps: history already holds it. - assert!( - SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) - .insert_entry_if_absent(envelope)? - ); - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - - // The incarnation was withdrawn: nothing is projected for it. - let expected = lifted.expected.clone().context("no Expected")?; - assert!(projected_under(&world, &expected)?.is_empty()); - for _ in 0..3 { - world.engine.run_scheduler_tick()?; - } - assert!( - world - .engine - .reconcile_failed_continuity_projections()? - .demoted - .is_empty() - ); - assert!( - projected_under(&world, &expected)?.is_empty(), - "a withdrawn incarnation's Expected was published again" - ); - - // One copy in history, finished, and nothing resident. - let history = read_history(&world.state_store)?; - assert_eq!(history.transactions.len(), 1); - assert!(history.transactions[0].is_terminal()); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - Ok(()) - } - - #[test] - fn boot_runs_the_projection_reconciliation_serve_starts_from() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - let failed = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - project_candidate(&world, &failed, true)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - // A pre-B1 abort's residue: the failed candidate's activation stands. - project_candidate(&world, &failed, true)?; - assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); - - let (_lock, _engine) = boot(world.engine.options.clone())?; - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - Ok(()) - } - - /// A workload whose candidate can neither be stopped nor recover: an abort - /// that cannot finish its first step. - struct WedgedWorkload; - - impl WorkloadPort for WedgedWorkload { - fn install( - &self, - plan: &CompiledDeploymentPlan, - release: &crate::drivers::MaterializedRelease, - ) -> Result { - StillWorkload.install(plan, release) - } - fn prepare_activation( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - launch: IdunnRuntimeActivationLaunch, - ) -> Result { - StillWorkload.prepare_activation(plan, expected, launch) - } - fn start_prepared( - &self, - plan: &CompiledDeploymentPlan, - release: &SealedRelease, - installed: &crate::drivers::InstalledReleaseObservation, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) - } - fn discard_prepared( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - StillWorkload.discard_prepared(plan, expected, activation) - } - fn observe( - &self, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - prior: &WorkloadObservation, - ) -> Result { - StillWorkload.observe(expected, activation, prior) - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - bail!("the unit will not stop") - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(true) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(false) - } - } - - #[test] - fn one_wedged_pre_fence_abort_does_not_stop_the_scheduler() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; - let wedged = transaction_at(&world, DeploymentPhase::Warming)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-nowhere".into(), - kind: CommandKind::Deploy, - selector: "nowhere".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, 100)?], - )? - ); - - // Recording the error is not progress, and neither is repeating it. - assert!(!world.engine.resume_one_transaction()?); - assert!(!world.engine.resume_one_transaction()?); - // Every tick answers: the wedged abort records its error and waits, - // and the rest of the scheduler goes on to freeze the queued command. - write_service_binding(&world)?; - for _ in 0..6 { - world.engine.run_scheduler_tick()?; - } - let stuck = record_of(&world, &wedged.transaction_id)?; - assert!(!stuck.is_terminal()); - assert!(stuck.post_fencing_abort.is_none()); - assert!(stuck.last_error.is_some()); - assert!( - read_history_transactions(&world.state_store) - .iter() - .any(|transaction| transaction.command_id == command.command_id), - "the queued command was never frozen" - ); - Ok(()) - } - - #[test] - fn a_wedged_post_fence_abort_only_waits() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; - let dead = transaction_at(&world, DeploymentPhase::Fencing)?; - world - .engine - .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; - - // The abort cannot stop the unit. The abort is already durable, so - // that is a resumable error, and the tick answers every time. - for _ in 0..3 { - world.engine.run_scheduler_tick()?; - } - let stuck = record_of(&world, &dead.transaction_id)?; - assert!(!stuck.is_terminal()); - assert!(stuck.last_error.is_some()); - Ok(()) - } - - /// A wedged abort whose `stop()` error carries a changing counter, as - /// systemctl output with a PID or timestamp would. - struct ChangingErrorWorkload(std::sync::atomic::AtomicU32, std::sync::atomic::AtomicBool); - - impl WorkloadPort for ChangingErrorWorkload { - fn install( - &self, - plan: &CompiledDeploymentPlan, - release: &crate::drivers::MaterializedRelease, - ) -> Result { - WedgedWorkload.install(plan, release) - } - fn prepare_activation( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - launch: IdunnRuntimeActivationLaunch, - ) -> Result { - WedgedWorkload.prepare_activation(plan, expected, launch) - } - fn start_prepared( - &self, - plan: &CompiledDeploymentPlan, - release: &SealedRelease, - installed: &crate::drivers::InstalledReleaseObservation, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - WedgedWorkload.start_prepared(plan, release, installed, expected, activation) - } - fn discard_prepared( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - WedgedWorkload.discard_prepared(plan, expected, activation) - } - fn observe( - &self, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - prior: &WorkloadObservation, - ) -> Result { - WedgedWorkload.observe(expected, activation, prior) - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - let n = self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - if self.1.load(std::sync::atomic::Ordering::SeqCst) { - return Ok(()); - } - bail!("the unit will not stop (main pid {n})") - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(true) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(false) - } - } - - fn queue_deploy_for_nowhere(world: &EngineFixture) -> Result { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-nowhere".into(), - kind: CommandKind::Deploy, - selector: "nowhere".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, 100)?], - )? - ); - Ok(command) - } - - #[test] - fn a_wedge_whose_error_text_changes_every_tick_does_not_starve_other_targets() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(ChangingErrorWorkload( - std::sync::atomic::AtomicU32::new(0), - std::sync::atomic::AtomicBool::new(false), - )))?; - transaction_at(&world, DeploymentPhase::Warming)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let command = queue_deploy_for_nowhere(&world)?; - write_service_binding(&world)?; - for _ in 0..20 { - world.engine.run_scheduler_tick()?; - } - assert!( - read_history_transactions(&world.state_store) - .iter() - .any(|transaction| transaction.command_id == command.command_id), - "the queued command was starved by another target's wedge" - ); - Ok(()) - } - - #[test] - fn rewriting_last_error_is_not_progress() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let before = transaction_at(&world, DeploymentPhase::Warming)?; - let mut noted = before.clone(); - noted.last_error = Some("the unit will not stop (main pid 7)".into()); - noted.updated_at_unix_millis += 1; - assert!(!state_advanced(&before, ¬ed)); - let mut moved = noted.clone(); - moved.phase = DeploymentPhase::AwaitingReady; - assert!(state_advanced(&before, &moved)); - Ok(()) - } - - #[test] - fn a_failing_step_is_retried_after_a_backoff_not_every_tick() -> Result<()> { - let workload = Arc::new(ChangingErrorWorkload( - std::sync::atomic::AtomicU32::new(0), - std::sync::atomic::AtomicBool::new(false), - )); - let world = EngineFixture::with_workload(workload.clone())?; - transaction_at(&world, DeploymentPhase::Warming)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let attempts = || workload.0.load(std::sync::atomic::Ordering::SeqCst); - let before = now_millis()?; - world.engine.run_scheduler_tick()?; - let after = now_millis()?; - assert_eq!(attempts(), 1, "the step did not run"); - // The first failure waits one poll interval: the wait back_off wrote. - let poll = world.engine.options.poll_millis; - let written = world - .engine - .resume_backoff - .lock() - .unwrap() - .get(&resident(&world)?.value.transaction_id) - .map(|backoff| backoff.not_before_unix_millis) - .expect("the failure set no backoff"); - assert!( - (before + poll..=after + poll).contains(&written), - "back_off wrote {written}, not one poll ({poll} ms) after the failure" - ); - // The wait is set from the wall clock, and a loaded host can spend it - // inside a few ticks. So the test owns the wait: held open, no tick - // retries; once due, the next tick does. - let id = resident(&world)?.value.transaction_id; - let set_due = |due: u64| { - world - .engine - .resume_backoff - .lock() - .unwrap() - .get_mut(&id) - .expect("the failure set no backoff") - .not_before_unix_millis = due; - }; - set_due(now_millis()? + RESUME_BACKOFF_CEILING_MILLIS / 2); - for _ in 0..10 { - world.engine.run_scheduler_tick()?; - } - assert_eq!(attempts(), 1, "the wedged step was retried inside its backoff"); - // Each further failure doubles the wait back_off writes. - for (attempt, polls) in [(2, 2), (3, 4)] { - set_due(1); - let before = now_millis()?; - world.engine.run_scheduler_tick()?; - let after = now_millis()?; - assert_eq!(attempts(), attempt, "the step was not retried once its backoff was due"); - let written = world - .engine - .resume_backoff - .lock() - .unwrap() - .get(&id) - .map(|backoff| backoff.not_before_unix_millis) - .expect("the failure set no backoff"); - assert!( - (before + polls * poll..=after + polls * poll).contains(&written), - "failure {attempt} waits {written}, not {polls} polls ({poll} ms each) after it" - ); - } - Ok(()) - } - - /// A candidate that fails to advance and is dead for good. - struct DeadCandidateWorkload; - - impl WorkloadPort for DeadCandidateWorkload { - fn install( - &self, - plan: &CompiledDeploymentPlan, - release: &crate::drivers::MaterializedRelease, - ) -> Result { - StillWorkload.install(plan, release) - } - fn prepare_activation( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - launch: IdunnRuntimeActivationLaunch, - ) -> Result { - StillWorkload.prepare_activation(plan, expected, launch) - } - fn start_prepared( - &self, - plan: &CompiledDeploymentPlan, - release: &SealedRelease, - installed: &crate::drivers::InstalledReleaseObservation, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) - } - fn discard_prepared( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - StillWorkload.discard_prepared(plan, expected, activation) - } - fn observe( - &self, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - _: &WorkloadObservation, - ) -> Result { - bail!("the candidate is gone") - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - Ok(()) - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(true) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(false) - } - } - - #[test] - fn a_permanently_dead_candidate_past_the_fence_begins_its_post_fence_abort() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(DeadCandidateWorkload))?; - let dead = transaction_at(&world, DeploymentPhase::Fencing)?; - assert!(record_of(&world, &dead.transaction_id)?.post_fencing_abort.is_none()); - for _ in 0..8 { - world.engine.resume_one_transaction()?; - } - assert!( - record_of(&world, &dead.transaction_id)? - .post_fencing_abort - .is_some(), - "a dead candidate past the fence was retried instead of aborted" - ); - Ok(()) - } - - #[test] - fn a_resume_fault_is_reported_once_and_left_in_the_record() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let live = transaction_at(&world, DeploymentPhase::Warming)?; - let error = anyhow!("store hiccup 1"); - world - .engine - .note_fault("could not resume transaction", &live.transaction_id, &error); - assert_eq!( - record_of(&world, &live.transaction_id)?.last_error.as_deref(), - Some("store hiccup 1") - ); - let reports = world.engine.fault_reports.lock().unwrap(); - assert_eq!(reports.len(), 1); - assert!(reports[&live.transaction_id].offer(Some("store hiccup 1".into())).is_none()); - Ok(()) - } - - #[test] - fn a_new_record_cannot_carry_the_legacy_marker() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let lifted = read_transaction_record(&envelope)?; - assert!(lifted.carries_legacy_marker()); - let error = error_text(transaction_envelope(&lifted, 1).map(|_| ())); - assert!(error.contains("legacy lift"), "{error}"); - // The migration's own path still writes it, and it reads back. - let (_, migrated) = cleanup_evidence::migrate_transaction_record(&envelope)?; - assert!(read_transaction_record(&migrated)?.carries_legacy_marker()); - Ok(()) - } - - #[test] - fn unreadable_history_does_not_skip_resident_residue_at_boot() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let first = read_transaction_record(&envelope)?; - make_resident(&world, &first, envelope)?; - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - let expected = first.expected.clone().context("no Expected")?; - project_candidate(&world, &first, true)?; - assert_eq!(projected_under(&world, &expected)?.len(), 2); - corrupt_history(&world)?; - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!(outcome.demoted.len(), 1); - assert_eq!(projected_under(&world, &expected)?, expected_only()); - Ok(()) - } - - #[test] - fn backoff_doubles_caps_and_waits_exactly() { - let ceiling = RESUME_BACKOFF_CEILING_MILLIS; - assert_eq!(backoff_wait(500, 0, ceiling), 500); - assert_eq!(backoff_wait(500, 1, ceiling), 1000); - assert_eq!(backoff_wait(500, 2, ceiling), 2000); - assert_eq!(backoff_wait(500, 3, ceiling), 4000); - assert_eq!(backoff_wait(500, 30, ceiling), ceiling); - assert_eq!(backoff_wait(u64::MAX, 5, ceiling), ceiling); - // The ceiling is the caller's: continuity's is lower than the resume path's. - assert_eq!(backoff_wait(5_000, 5, 300_000), 160_000); - assert_eq!(backoff_wait(5_000, 6, 300_000), 300_000); - assert_eq!(backoff_wait(5_000, 20, 300_000), 300_000); - assert!(is_waiting(9, 10, 5)); - assert!(!is_waiting(10, 10, 5)); - assert!(!is_waiting(11, 10, 5)); - // Due further ahead than the longest wait the owner can set: the clock - // stepped back, so the attempt is due, not stalled for the step. - assert!(is_waiting(5, 10, 5)); - assert!(!is_waiting(4, 10, 5)); - } - - #[test] - fn a_resume_backoff_set_by_a_stepped_back_clock_is_due_and_a_real_one_waits() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let id = transaction_at(&world, DeploymentPhase::Warming)?.transaction_id; - let not_before = |world: &EngineFixture| { - world - .engine - .resume_backoff - .lock() - .unwrap() - .get(&id) - .map(|backoff| backoff.not_before_unix_millis) - }; - let set = |world: &EngineFixture, not_before_unix_millis: u64| { - world.engine.resume_backoff.lock().unwrap().insert( - id.clone(), - ResumeBackoff { - failures: 3, - not_before_unix_millis, - }, - ); - }; - - // Half a minute ahead is a wait the backoff can set: the step is left alone. - let waiting = now_millis()? + 30_000; - set(&world, waiting); - let _ = world.engine.resume_candidate(&resident(&world)?); - assert_eq!(not_before(&world), Some(waiting)); - - // A day ahead is a clock that stepped back: the step is attempted, and - // whatever it does, it replaces the backoff. - let ahead = now_millis()? + 86_400_000; - set(&world, ahead); - let _ = world.engine.resume_candidate(&resident(&world)?); - assert!(not_before(&world).is_none_or(|due| due < ahead)); - Ok(()) - } - - #[test] - fn a_persistent_fault_is_reported_once_and_again_only_after_recovery() -> Result<()> { - let workload = Arc::new(ChangingErrorWorkload( - std::sync::atomic::AtomicU32::new(0), - std::sync::atomic::AtomicBool::new(false), - )); - let world = EngineFixture::with_workload(workload.clone())?; - let wedged = transaction_at(&world, DeploymentPhase::Warming)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let id = wedged.transaction_id; - let expire = || { - world.engine.resume_backoff.lock().unwrap().clear(); - }; - let reported = || world.engine.fault_reports.lock().unwrap().contains_key(&id); - // A fault already reported for this transaction (an Err from the - // resume itself); a handled, still-failing step must not re-arm it. - world - .engine - .note_fault("could not resume transaction", &id, &anyhow!("store hiccup")); - assert!(reported()); - world.engine.resume_one_transaction()?; - assert!(reported(), "a failing handled step re-armed the report"); - // Skipped inside the backoff: the fault is still the same fault. - world.engine.resume_one_transaction()?; - assert!(reported(), "a backoff skip re-armed the report"); - // Still failing after the backoff expires: still the same fault. - expire(); - world.engine.resume_one_transaction()?; - assert!(reported()); - // A real recovery clears it. - workload.1.store(true, std::sync::atomic::Ordering::SeqCst); - expire(); - world.engine.resume_one_transaction()?; - assert!(!reported(), "a step that ran and succeeded left the fault armed"); - Ok(()) - } - - #[test] - fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let first = read_transaction_record(&envelope)?; - make_resident(&world, &first, envelope)?; - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - assert!(record_of(&world, &first.transaction_id)?.carries_legacy_marker()); - corrupt_history(&world)?; - assert!(!world.engine.retire_one_terminal_transaction()?); - let noted = record_of(&world, &first.transaction_id)?; - assert!( - noted.last_error.is_some() && noted.last_error != first.last_error, - "the retire fault never reached the record" - ); - assert!(noted.carries_legacy_marker()); - Ok(()) - } - - #[test] - fn a_tick_that_only_freezes_or_only_advances_reports_progress() -> Result<()> { - // Only a freeze. - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - queue_deploy_for_nowhere(&world)?; - write_service_binding(&world)?; - assert!(world.engine.run_scheduler_tick()?, "a freeze-only tick reported no progress"); - - // Only a phase advance. - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - transaction_at(&world, DeploymentPhase::Fencing)?; - assert!(world.engine.resume_one_transaction()?, "a real advance reported none"); - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - transaction_at(&world, DeploymentPhase::Fencing)?; - assert!(world.engine.run_scheduler_tick()?); - Ok(()) - } - - #[test] - fn a_terminal_transaction_that_cannot_be_archived_does_not_stop_the_scheduler() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let live = transaction_at(&world, DeploymentPhase::Warming)?; - odin_reports_ready(&world, &live, seeded_sequence(None) + 1)?; - let (finished, command) = terminal_transaction_with_command("ghostlight")?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: finished.transaction_id.clone(), - current: None, - }, - ], - &[ - command_envelope(&command, 100)?, - transaction_envelope(&finished, finished.updated_at_unix_millis)?, - ], - )? - ); - corrupt_history(&world)?; - let before = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .find(|stored| stored.value.transaction_id == live.transaction_id) - .context("live transaction")? - .envelope; - - // The finished record cannot be archived, and the live one still moves. - world.engine.run_scheduler_tick()?; - let after = ControlSnapshot::read(&world.state_store)?; - assert!( - after - .transactions - .iter() - .any(|stored| stored.value.transaction_id == finished.transaction_id) - ); - assert_ne!( - after - .transactions - .iter() - .find(|stored| stored.value.transaction_id == live.transaction_id) - .context("live transaction")? - .envelope, - before, - "the live transaction was not resumed" - ); - Ok(()) - } - - // ---- fix 2: supervision owns the Expected-only precondition ---- - - #[test] - fn continuity_mints_no_restart_over_a_projection_it_cannot_demote() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - - // The projection names an activation that is not the incumbent's. - let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; - let topology = world.engine.topology(); - topology.publish_expected(&incumbent.expected, &anchor)?; - let stranger = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - topology.publish_observed_activation( - &incumbent.expected, - stranger.activation.as_ref().context("no activation")?, - stranger.workload.as_ref().context("no workload")?, - )?; - // Remove the seeded stranger transaction: only the projection matters. - let seeded = resident(&world)?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged( - &[ - seeded.envelope.clone(), - ControlSnapshot::read(&world.state_store)? - .commands - .into_iter() - .find(|stored| stored.value.command_id == seeded.value.command_id) - .context("seeded command")? - .envelope, - ] - )? - ); - workload.kill(); - - let before = now_millis()?; - assert!(world.engine.supervise_one_admitted_generation()?); - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert!( - snapshot.transactions.is_empty(), - "continuity minted a restart over a projection it could not demote" - ); - let recorded = snapshot - .supervision_for("service") - .context("the failed demotion was not recorded")? - .value - .clone(); - let deferred = recorded.continuity_deferred_until.context("no deferral time")?; - assert!(recorded.continuity_deferral_reason.is_some()); - assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); - let after = now_millis()?; - assert!( - (before + CONTINUITY_DEFERRAL_MILLIS..=after + CONTINUITY_DEFERRAL_MILLIS) - .contains(&deferred) - ); - - // Deferred: the next tick neither retries nor writes. - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(meters_of(&world, "service")?, recorded); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - - // The projection is repaired and the deferral has run out: the restart - // proceeds, from an Expected-only projection. - topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; - project_admitted(&world, &incumbent)?; - let mut ran_out = recorded.clone(); - ran_out.continuity_deferred_until = Some(1); - set_meters(&world, &ran_out)?; - assert!(world.engine.supervise_one_admitted_generation()?); - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert_eq!(snapshot.transactions.len(), 1); - let minted = snapshot.supervision_for("service").context("no meters")?; - assert_eq!(minted.value.continuity_restarts.len(), 1); - assert_eq!(minted.value.continuity_deferred_until, None); - assert_eq!(minted.value.continuity_deferral_reason, None); - assert_eq!(snapshot.transactions[0].value.command_kind, CommandKind::Continuity); - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - Ok(()) - } - - // ---- fix 3: one-time boot reconciliation, by exact issuer ---- - - #[test] - fn boot_demotes_the_activation_a_failed_continuity_left_and_only_that() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - let failed = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - project_candidate(&world, &failed, true)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - // A failed transaction is archived the moment it finishes. - let aborted = record_of(&world, &failed.transaction_id)?; - assert!(aborted.is_terminal()); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - - // Drift a pre-B1 abort left: the failed candidate's activation stands. - project_candidate(&world, &failed, true)?; - assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); - - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!( - outcome, - ProjectionReconciliation { - demoted: vec![failed.transaction_id.clone()], - unexplained: Vec::new(), - } - ); - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - - // Once: nothing is left to demote. - assert_eq!( - world.engine.reconcile_failed_continuity_projections()?, - ProjectionReconciliation::default() - ); - Ok(()) - } - - #[test] - fn boot_never_adopts_an_activation_no_failed_transaction_issued() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - - // The incumbent's own activation is accounted for. - project_admitted(&world, &incumbent)?; - assert_eq!( - world.engine.reconcile_failed_continuity_projections()?, - ProjectionReconciliation::default() - ); - - // Some other activation, issued by nothing in history, is reported and - // left exactly as it is. - let stranger = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; - let topology = world.engine.topology(); - topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; - project_candidate(&world, &stranger, true)?; - // A live transaction accounts for its own activation... - assert_eq!( - world.engine.reconcile_failed_continuity_projections()?, - ProjectionReconciliation::default() - ); - // ...and for nobody else's. - topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; - let mut other = stranger.clone(); - let other_activation = IdunnRuntimeActivationLaunch::issue( - &incumbent.expected, - runtime_instance_id("tx-someone-else")?, - now_millis()?, - &world.engine.idunn_signer, - )? - .activation() - .clone(); - if let Some(WorkloadObservation::Systemd(observed)) = other.workload.as_mut() { - observed.runtime_instance_id = other_activation.runtime_instance_id.clone(); - } - other.activation = Some(other_activation); - project_candidate(&world, &other, true)?; - assert_eq!( - world.engine.reconcile_failed_continuity_projections()?.unexplained, - vec!["service".to_owned()] - ); - topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; - project_candidate(&world, &stranger, true)?; - // ...and once it is gone, nothing does. - let seeded = resident(&world)?; - let command = ControlSnapshot::read(&world.state_store)? - .commands - .into_iter() - .find(|stored| stored.value.command_id == seeded.value.command_id) - .context("seeded command")?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store) - .delete_batch_if_unchanged(&[seeded.envelope.clone(), command.envelope])? - ); - let projection = std::fs::read(&world.engine.options.topology_store)?; - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!( - outcome, - ProjectionReconciliation { - demoted: Vec::new(), - unexplained: vec!["service".into()], - } - ); - assert_eq!(std::fs::read(&world.engine.options.topology_store)?, projection); - Ok(()) - } - - // ---- fix 4: history that cannot be read stops actuation ---- - - /// An operator binding for the target "service", so the command queue can - /// resolve selectors. - fn write_service_binding(world: &EngineFixture) -> Result<()> { - use crate::deployment_plan::tests::BINDING; - let (head, tail) = BINDING - .split_once("[route]") - .context("binding has no route table")?; - let binding = format!( - "{head}[brakes]{}", - tail.split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ); - std::fs::create_dir_all(&world.engine.options.bindings_dir)?; - std::fs::write(world.engine.options.bindings_dir.join("service.toml"), binding)?; - Ok(()) - } - - fn corrupt_history(world: &EngineFixture) -> Result<()> { - std::fs::write(history_store_path(&world.state_store), b"\xc1\xc1 torn")?; - Ok(()) - } - - #[test] - fn history_that_cannot_be_read_is_an_error_and_missing_history_is_empty() -> Result<()> { - let world = EngineFixture::new()?; - let history = read_history(&world.state_store)?; - assert!(history.transactions.is_empty() && history.report().is_none()); - - corrupt_history(&world)?; - let error = read_history(&world.state_store).err().context("torn history read")?; - assert!(format!("{error:#}").contains("history"), "{error:#}"); - assert!(world.engine.history_for_decision().is_none()); - // The display path still answers, with nothing. - assert!(read_history_transactions(&world.state_store).is_empty()); - Ok(()) - } - - #[test] - fn undecodable_history_names_what_it_is_missing() -> Result<()> { - let (finished, _) = terminal_transaction_with_command("ghostlight")?; - let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; - let mut torn = good.clone(); - torn.key = "tx-torn".into(); - torn.payload = vec![0xc1]; - let (transactions, undecodable) = decode_history_transactions(vec![good, torn]); - let history = HistoryRead { - transactions, - undecodable, - }; - let report = history.report().context("nothing reported")?; - assert!(report.contains("1 archived transaction"), "{report}"); - assert!(report.contains("tx-torn"), "{report}"); - - let complete = HistoryRead { - transactions: history.transactions, - undecodable: Vec::new(), - }; - assert_eq!(complete.report(), None); - Ok(()) - } - - #[test] - fn a_fault_is_said_once_while_it_lasts() { - let once = ReportOnce::default(); - assert_eq!(once.offer(None), None); - assert_eq!(once.offer(Some("torn".into())), Some("torn".into())); - assert_eq!(once.offer(Some("torn".into())), None); - assert_eq!(once.offer(Some("torn".into())), None); - assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); - assert_eq!(once.offer(None), None); - assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); - } - - #[test] - fn unreadable_history_does_not_stop_continuity() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - corrupt_history(&world)?; - - // Crash recovery reads the target's own log, never history: the dead - // workload is restarted, and the restart is counted there. - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); - assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); - Ok(()) - } - - #[test] - fn unreadable_history_stops_command_retirement_and_freezing() -> Result<()> { - let world = EngineFixture::new()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-nowhere".into(), - kind: CommandKind::Deploy, - selector: "nowhere".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, 100)?], - )? - ); - write_service_binding(&world)?; - corrupt_history(&world)?; - - // Whether the command was consumed is unknown, so it is not acted on. - assert!(!world.engine.freeze_one_queued_command()?); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - - // Readable, it is acted on: the unknown selector is refused. - std::fs::remove_file(history_store_path(&world.state_store))?; - assert!(world.engine.freeze_one_queued_command()?); - assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); - Ok(()) - } - - #[test] - fn the_legacy_lift_owes_only_a_continuity_that_issued_an_activation() -> Result<()> { - let envelope = fixture_envelope( - FIXTURE_PRE_B1_ABORTS[0].1, - DeploymentTransaction::TYPE, - )?; - let written = lift_legacy_transaction(&envelope)?; - let owed = |transaction: &DeploymentTransaction| { - transaction - .pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - }; - assert_eq!(owed(&written), Some(CleanupEvidence::Skipped)); - - let mut lifted = written.clone(); - cleanup_evidence::owe_legacy_continuity_projection(&mut lifted); - assert_eq!(owed(&lifted), Some(CleanupEvidence::Pending)); - - // A deployment published its own key and recorded what it owed. - let mut deploy = written.clone(); - deploy.command_kind = CommandKind::Deploy; - cleanup_evidence::owe_legacy_continuity_projection(&mut deploy); - assert_eq!(owed(&deploy), Some(CleanupEvidence::Skipped)); - - // A continuity that issued nothing owes nothing. - let mut unissued = written; - unissued.activation = None; - cleanup_evidence::owe_legacy_continuity_projection(&mut unissued); - assert_eq!(owed(&unissued), Some(CleanupEvidence::Skipped)); - Ok(()) - } - - // ---- operator incidents ---------------------------------------------- - - fn incident_path(world: &EngineFixture) -> PathBuf { - world.root.join("incidents.cc") - } - - fn incidents_of(world: &EngineFixture) -> Result> { - IncidentStore::new(&incident_path(world)).read() - } - - /// The control store never holds an incident: it would stop every older - /// binary that reads it. - fn assert_control_store_holds_no_incident(world: &EngineFixture) -> Result<()> { - ControlSnapshot::read(&world.state_store)?; - for envelope in SingleFileMessagePackBackingStore::new(&world.state_store) - .pull_all_read_only_snapshot()? - { - assert_ne!(envelope.r#type, "idunn.operator_incident"); - } - Ok(()) - } - - /// Six restarts inside the window, the last long enough ago that its wait is - /// over: only the ceiling holds. - fn spend_the_window(world: &EngineFixture) -> Result<()> { - set_meters( - world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - ) - } - - /// One entry has left the window, and the newest restart is recent enough - /// that continuity is still waiting: exhaustion is over, nothing restarts. - fn slide_the_window(world: &EngineFixture) -> Result<()> { - set_meters( - world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 10_000])?, - ) - } - - #[test] - fn an_exhausted_target_opens_one_incident_across_ticks_and_restarts() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - assert!(!incident_path(&world).exists()); - - // The first pass over the dead target starts its wait: progress, no mint. - assert!(world.engine.supervise_one_admitted_generation()?); - let opened = incidents_of(&world)?; - assert_eq!(opened.len(), 1); - assert_eq!(opened[0].subject, "service"); - assert_eq!(opened[0].condition, IncidentCondition::ContinuityExhausted); - assert!(opened[0].is_open()); - let written = std::fs::read(incident_path(&world))?; - - for _ in 0..3 { - assert!(!world.engine.supervise_one_admitted_generation()?); - } - // An Idunn restart forgets its in-memory trace; the durable record - // keeps the decision from being written again. - let restarted = - Engine::open_with_systemd_workload(world.engine.options.clone(), workload.clone())?; - assert!(restarted.fault_reports.lock().unwrap().is_empty()); - assert!(!restarted.supervise_one_admitted_generation()?); - assert_eq!(std::fs::read(incident_path(&world))?, written); - assert_eq!(incidents_of(&world)?, opened); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - assert_control_store_holds_no_incident(&world) - } - - #[test] - fn an_exhaustion_incident_closes_on_observed_health_not_on_the_window() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - let first = incidents_of(&world)?; - assert_eq!(first.len(), 1); - - // The window slides while the workload is still dead: exhaustion is - // over, the target is not healthy, and the incident stays open. - slide_the_window(&world)?; - let _ = world.engine.supervise_one_admitted_generation()?; - assert_eq!(incidents_of(&world)?, first); - - // Seen running, it closes as recovered. - workload.revive(); - assert!(!world.engine.supervise_one_admitted_generation()?); - let closed = incidents_of(&world)?; - assert_eq!(closed.len(), 1); - assert_eq!(closed[0].incident_key, first[0].incident_key); - assert!( - closed[0] - .closed_at_unix_millis - .is_some_and(|at| at >= closed[0].opened_at_unix_millis) - ); - assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); - - // A later exhaustion is a new record with a new opening time. - std::thread::sleep(Duration::from_millis(5)); - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - let again = incidents_of(&world)?; - assert_eq!(again.len(), 2); - let reopened = again - .iter() - .find(|record| record.is_open()) - .context("no new incident")?; - assert!(reopened.opened_at_unix_millis > first[0].opened_at_unix_millis); - assert_ne!(reopened.incident_key, first[0].incident_key); - assert_control_store_holds_no_incident(&world) - } - - #[test] - fn an_incident_whose_target_is_no_longer_admitted_closes() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - assert!(incidents_of(&world)?[0].is_open()); - - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no admitted generation")? - .envelope - .clone(); - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store) - .delete_batch_if_unchanged(&[admitted])? - ); - world.engine.supervise_one_admitted_generation()?; - let records = incidents_of(&world)?; - assert_eq!(records.len(), 1); - assert_eq!(records[0].close_reason, Some(CloseReason::NoLongerAdmitted)); - assert_control_store_holds_no_incident(&world) - } - - #[test] - fn a_tick_retires_incidents_closed_past_retention() -> Result<()> { - let world = EngineFixture::with_workload(SwitchWorkload::new())?; - let store = IncidentStore::new(&incident_path(&world)); - let condition = IncidentCondition::ContinuityExhausted; - let long_ago = now_millis()? - INCIDENT_RETENTION_MILLIS - 10_000; - assert!(store.open(condition, "gone", long_ago)?); - assert!(store.close(condition, "gone", CloseReason::Recovered, long_ago + 1)?); - assert!(store.open(condition, "recent", now_millis()?)?); - assert!(store.close(condition, "recent", CloseReason::Recovered, now_millis()?)?); - - world.engine.supervise_one_admitted_generation()?; - let live = store.read()?; - assert_eq!(live.len(), 1); - assert_eq!(live[0].subject, "recent"); - assert!(world.root.join("incident-history.cc").exists()); - Ok(()) - } - - /// The same exhaustion and recovery sequence, reduced to what continuity - /// decided: each tick's return, and the transactions and restart log it left. - fn continuity_decisions( - world: &EngineFixture, - engine: &Engine, - workload: &SwitchWorkload, - ) -> Result> { - let mut decided = Vec::new(); - let tick = |decided: &mut Vec<(bool, usize, usize)>| -> Result<()> { - let moved = engine.supervise_one_admitted_generation()?; - let snapshot = ControlSnapshot::read(&world.state_store)?; - decided.push(( - moved, - snapshot.transactions.len(), - snapshot.supervision_or_new("service").continuity_restarts.len(), - )); - Ok(()) - }; - workload.kill(); - spend_the_window(world)?; - tick(&mut decided)?; - tick(&mut decided)?; - slide_the_window(world)?; - tick(&mut decided)?; - workload.revive(); - tick(&mut decided)?; - workload.kill(); - spend_the_window(world)?; - tick(&mut decided)?; - Ok(decided) - } - - #[test] - fn an_unwritable_incident_store_changes_nothing_continuity_decides() -> Result<()> { - const CANARY: &str = "CANARY-6b1d-secret-segment"; - let healthy_workload = SwitchWorkload::new(); - let healthy = EngineFixture::with_workload(healthy_workload.clone())?; - admit_incumbent(&healthy)?; - let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; - assert!(incidents_of(&healthy)?.iter().any(|record| record.is_open())); - - // Not permission-dependent: a store path beneath a regular file cannot - // be created for any uid, root included. - let broken_workload = SwitchWorkload::new(); - let mut broken = EngineFixture::with_workload(broken_workload.clone())?; - admit_incumbent(&broken)?; - let blocker = broken.root.join(CANARY); - std::fs::write(&blocker, b"a file, not a directory")?; - let mut options = broken.engine.options.clone(); - options.incident_store = blocker.join("incidents.cc"); - broken.engine = Engine::open_with_systemd_workload(options, broken_workload.clone())?; - - let actual = continuity_decisions(&broken, &broken.engine, &broken_workload)?; - assert_eq!(actual, expected, "a broken incident store changed a continuity decision"); - assert!(!blocker.join("incidents.cc").exists()); - - // The fault is traced once, under its operation, and echoes no path. - let reports = broken.engine.fault_reports.lock().unwrap(); - let faults = reports - .iter() - .filter(|(key, _)| key.starts_with("incident-store:")) - .collect::>(); - assert_eq!( - faults.len(), - 1, - "{:?}", - faults.iter().map(|(key, _)| key).collect::>() - ); - let message = faults[0].1.last.lock().unwrap().clone().context("fault cleared")?; - assert!(message.contains("open an operator incident"), "{message}"); - assert!(!message.contains(CANARY), "{message}"); - drop(reports); - assert_control_store_holds_no_incident(&broken) - } - - #[test] - fn status_renders_incidents_and_survives_an_unreadable_store() -> Result<()> { - const CANARY: &str = "CANARY-90c2-secret-segment"; - let temp = TempDir::new()?; - let now = now_millis()?; - let mut snapshot = ControlSnapshot::default(); - let meters = TargetSupervision::new("service"); - snapshot.targets.push(Stored { - envelope: target_supervision_envelope(&meters, now)?, - value: meters, - }); - - let readable = temp.path().join("incidents.cc"); - IncidentStore::new(&readable).open(IncidentCondition::ContinuityExhausted, "service", now)?; - let lines = render_targets(&snapshot, &readable, now); - assert!(lines[0].starts_with("target service "), "{lines:?}"); - assert_eq!( - lines.last().map(String::as_str), - Some(format!(" incident continuity-exhausted opened-at {now}").as_str()) - ); - - let unreadable = temp.path().join(CANARY); - std::fs::write(&unreadable, b"not a cultcache file")?; - let lines = render_targets(&snapshot, &unreadable, now); - assert!(lines[0].starts_with("target service "), "{lines:?}"); - let last = lines.last().context("no lines")?; - assert!(last.starts_with("incidents unreadable: "), "{last}"); - assert!(!lines.iter().any(|line| line.contains(CANARY)), "{lines:?}"); - Ok(()) - } - - #[test] - fn the_incident_store_is_named_by_serve_and_status() { - let Command::Serve(options) = - parse(["serve", "--incident-store", "/tmp/i.cc"].into_iter().map(str::to_owned)) - .unwrap() - else { - panic!("expected serve") - }; - assert_eq!(options.incident_store, PathBuf::from("/tmp/i.cc")); - assert!(usage().contains("idunn status [--state-store PATH] [--incident-store PATH]")); - let Command::Status { incident_store, .. } = - parse(["status", "--incident-store", "/tmp/j.cc"].into_iter().map(str::to_owned)) - .unwrap() - else { - panic!("expected status") - }; - assert_eq!(incident_store, PathBuf::from("/tmp/j.cc")); - } - - /// A standing fault is traced once. A key is cleared only by an operation - /// that succeeded, so another site's short-circuit must not end it: a - /// target that is running but still exhausted closes nothing, and that - /// used to clear the reconcile pass's fault and print it again every pass. - #[test] - fn a_failing_reconcile_is_traced_once_while_a_running_target_stays_exhausted() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - std::fs::write(incident_path(&world), b"not an incident store")?; - spend_the_window(&world)?; - - for pass in 0..6 { - world.engine.supervise_one_admitted_generation()?; - let reports = world.engine.fault_reports.lock().unwrap(); - assert!( - reports.contains_key("incident-store:close:*"), - "pass {pass}: the standing reconcile fault was cleared, so it prints again" - ); - } - Ok(()) - } - - #[test] - fn a_release_that_runs_one_pass_after_each_restart_keeps_one_incident() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - - // Each cycle: the restarts are spent, the release dies, is seen running - // for one pass (its last restart took), and dies again. - for _ in 0..4 { - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - workload.revive(); - world.engine.supervise_one_admitted_generation()?; - let records = incidents_of(&world)?; - assert_eq!(records.len(), 1, "a flapping release opened another incident"); - assert!(records[0].is_open(), "running while still exhausted closed the incident"); - } - - // Real recovery: running, and the window no longer holds the restarts. - slide_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - let records = incidents_of(&world)?; - assert_eq!(records.len(), 1); - assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); - assert_control_store_holds_no_incident(&world) - } - - #[test] - fn a_target_leaving_admission_closes_only_its_own_incident() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let service = admit_incumbent(&world)?; - let store = IncidentStore::new(&incident_path(&world)); - let condition = IncidentCondition::ContinuityExhausted; - let now = now_millis()?; - for subject in ["service", "neighbour", "gone"] { - assert!(store.open(condition, subject, now)?); - } - - // Two targets are admitted: service, and a neighbour. Only the target - // that is not admitted has its incident closed. - let mut snapshot = ControlSnapshot::read(&world.state_store)?; - let envelope = snapshot - .admitted_for("service") - .context("no admitted generation")? - .envelope - .clone(); - snapshot.admitted.push(Stored { - envelope, - value: AdmittedGeneration { - target: "neighbour".into(), - ..service - }, - }); - world.engine.reconcile_incidents(&snapshot); - - let closed = |subject: &str| -> Result> { - Ok(incidents_of(&world)? - .into_iter() - .find(|record| record.subject == subject) - .context("no such incident")? - .close_reason) - }; - assert_eq!(closed("service")?, None); - assert_eq!(closed("neighbour")?, None); - assert_eq!(closed("gone")?, Some(CloseReason::NoLongerAdmitted)); - Ok(()) - } - - #[cfg(target_os = "linux")] - #[test] - fn a_held_incident_lock_changes_nothing_continuity_decides() -> Result<()> { - use crate::incident::tests::{STUCK_AFTER, hold_lock_of, run_while_held}; - - let healthy_workload = SwitchWorkload::new(); - let healthy = EngineFixture::with_workload(healthy_workload.clone())?; - admit_incumbent(&healthy)?; - let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; - - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - let path = incident_path(&world); - // Three ticks reach the open of an exhausted target, one exchange each. - let actual = run_while_held(hold_lock_of(&path), 3, || { - continuity_decisions(&world, &world.engine, &workload) - })?; - assert_eq!(actual, expected, "a held incident lock changed a continuity decision"); - assert!(!path.exists(), "a contended write landed"); - - // Every refused write is a traced fault naming no path. - { - let reports = world.engine.fault_reports.lock().unwrap(); - let faults = reports - .iter() - .filter(|(key, _)| key.starts_with("incident-store:")) - .map(|(_, report)| report.last.lock().unwrap().clone().unwrap_or_default()) - .collect::>(); - assert!(!faults.is_empty()); - let root = world.root.display().to_string(); - assert!(faults.iter().all(|fault| !fault.contains(&root)), "{faults:?}"); - } - - // The lock is free again: the next pass retries and the incident lands. - // (A child another test forked while it was held can keep it a moment.) - let deadline = std::time::Instant::now() + STUCK_AFTER; - while !incidents_of(&world)?.iter().any(|record| record.is_open()) { - assert!(std::time::Instant::now() < deadline, "the incident never landed"); - world.engine.supervise_one_admitted_generation()?; - std::thread::sleep(Duration::from_millis(20)); - } - assert_control_store_holds_no_incident(&world) - } - - /// A publish that could not land is not done, at the tick: the demotion - /// continuity needs before a restart is one exchange on the topology lock, - /// and another holder of it defers the restart instead of waiting for it. - #[cfg(target_os = "linux")] - #[test] - fn a_held_topology_lock_defers_continuity_and_mints_nothing() -> Result<()> { - use crate::incident::tests::{hold_lock_of, run_while_held}; - - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - project_admitted(&world, &incumbent)?; - let store = world.engine.options.topology_store.clone(); - let projected = projected_under(&world, &incumbent.expected)?; - assert!(projected.contains(&IdunnRuntimeActivationRecord::TYPE.to_owned())); - let before = std::fs::read(&store)?; - workload.kill(); - - // One exchange: the demotion. The window is not spent, so continuity - // would restart if the projection could be demoted. - let moved = run_while_held(hold_lock_of(&store), 1, || { - world.engine.supervise_one_admitted_generation() - })?; - assert!(moved, "the deferral is a write the tick made"); - assert_eq!(std::fs::read(&store)?, before, "a contended publish changed the projection"); - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert!(snapshot.transactions.is_empty(), "continuity minted over a projection it could not demote"); - let recorded = snapshot.supervision_for("service").context("no deferral recorded")?.value.clone(); - assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); - let reason = recorded.continuity_deferral_reason.clone().context("no deferral reason")?; - assert!(reason.contains("locked by another holder"), "{reason}"); - assert!(!reason.contains(&world.root.display().to_string()), "{reason}"); - - // The holder is gone and the deferral has run out: the demotion lands - // and the restart proceeds. - let mut ran_out = recorded; - ran_out.continuity_deferred_until = Some(1); - set_meters(&world, &ran_out)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - Ok(()) - } - - /// A deployment phase records a publication digest only after the publish - /// returned: the digest is the claim that it landed. - #[cfg(target_os = "linux")] - #[test] - fn a_contended_expected_publish_is_not_recorded_as_published() -> Result<()> { - use crate::incident::tests::{hold_lock_of, run_while_held}; - - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Warming)?; - let stored = resident(&world)?; - let mut starting = seeded.clone(); - starting.phase = DeploymentPhase::Starting; - starting.expected_publication_sha256 = None; - starting.activation = None; - starting.workload = None; - starting.activation_publication_sha256 = None; - starting.warming = None; - replace_transaction(&world.state_store, &stored, &starting)?; - let store = world.engine.options.topology_store.clone(); - let stored = resident(&world)?; - - let error = run_while_held(hold_lock_of(&store), 1, || { - world.engine.advance_transaction(&stored) - }) - .expect_err("a contended publish must not advance the phase"); - assert!(format!("{error:#}").contains("locked by another holder"), "{error:#}"); - assert!(!store.exists(), "a contended publish wrote the projection"); - let after = resident(&world)?.value; - assert_eq!(after.phase, DeploymentPhase::Starting); - assert_eq!(after.expected_publication_sha256, None); - - // Free again: the same step publishes and records what it published. - world.engine.advance_transaction(&resident(&world)?)?; - let expected = seeded.expected.context("seeded transaction has no Expected")?; - assert_eq!( - resident(&world)?.value.expected_publication_sha256, - Some(expected.canonical_sha256()?) - ); - Ok(()) - } - - /// A Continuity transaction parked in Starting with nothing published yet, - /// plus the stored record the tick will see. - #[cfg(target_os = "linux")] - fn parked_in_starting(world: &EngineFixture) -> Result { - let seeded = transaction_at(world, DeploymentPhase::Warming)?; - let stored = resident(world)?; - let mut starting = seeded.clone(); - starting.phase = DeploymentPhase::Starting; - starting.expected_publication_sha256 = None; - starting.activation = None; - starting.workload = None; - starting.activation_publication_sha256 = None; - starting.warming = None; - replace_transaction(&world.state_store, &stored, &starting)?; - Ok(seeded) - } - - /// A publish that lost to a held lock is transient: before the fence the - /// transaction stays in its phase with the error recorded, no abort is - /// begun, and the next tick after the release lands the publish. A - /// continuity restart is a Continuity transaction on this same path, so - /// its attempt is minted once and a contended publish spends no second one. - #[cfg(target_os = "linux")] - #[test] - fn a_contended_starting_publish_is_retried_on_a_later_tick_not_aborted() -> Result<()> { - use crate::incident::tests::{hold_lock_of, run_while_held}; - - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = parked_in_starting(&world)?; - let mut meters = TargetSupervision::new("service"); - meters.continuity_restarts = vec![now_millis()?]; - set_meters(&world, &meters)?; - let store = world.engine.options.topology_store.clone(); - - for tick in 0..2 { - run_while_held(hold_lock_of(&store), 1, || world.engine.resume_one_transaction())?; - world.engine.resume_backoff.lock().unwrap().clear(); - let held = resident(&world)?.value; - assert_eq!(held.phase, DeploymentPhase::Starting, "tick {tick}"); - assert!(held.pre_fencing_abort.is_none(), "tick {tick}: a held lock aborted the transaction"); - assert!(!held.is_terminal(), "tick {tick}"); - assert_eq!(held.expected_publication_sha256, None, "tick {tick}"); - let error = held.last_error.context("a contended publish left no last_error")?; - assert!(error.contains("locked by another holder"), "{error}"); - assert!(!store.exists(), "a contended publish wrote the projection"); - } - - // Released: the next tick lands the publish and records its digest. - world.engine.resume_one_transaction()?; - let landed = resident(&world)?.value; - assert_eq!(landed.phase, DeploymentPhase::Starting); - assert!(landed.pre_fencing_abort.is_none()); - assert_eq!( - landed.expected_publication_sha256, - Some(seeded.expected.context("seeded transaction has no Expected")?.canonical_sha256()?) - ); - assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); - Ok(()) - } - - /// Only contention is transient: a publish that fails for any other reason - /// still ends the transaction before the fence, as it always did. - #[cfg(target_os = "linux")] - #[test] - fn a_non_contention_publish_error_still_aborts_before_the_fence() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - parked_in_starting(&world)?; - // A directory where the store should be: the snapshot cannot be read. - std::fs::create_dir_all(&world.engine.options.topology_store)?; - world.engine.resume_one_transaction()?; - let after = resident(&world)?.value; - assert!(after.pre_fencing_abort.is_some(), "a non-contention error did not abort"); - assert!( - !after.last_error.as_deref().unwrap_or_default().contains("locked by another holder"), - "{:?}", - after.last_error - ); - Ok(()) - } - - #[test] - fn status_prints_an_open_incident_under_its_target() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - let opened = incidents_of(&world)?[0].opened_at_unix_millis; - - let mut printed = Vec::new(); - status(&mut printed, &world.state_store, &incident_path(&world), None)?; - let printed = String::from_utf8(printed)?; - let target = printed.find("target service ").context("no target line")?; - let incident = printed - .find(&format!(" incident continuity-exhausted opened-at {opened}")) - .context("status printed no incident")?; - assert!(target < incident, "{printed}"); - Ok(()) - } - - #[test] - fn serve_ticks_until_told_to_stop() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - let mut options = world.engine.options.clone(); - options.poll_millis = 1; - let engine = Engine::open_with_systemd_workload(options, workload.clone())?; - - let mut asked = 0; - serve_while(&engine, || { - asked += 1; - asked <= 3 - })?; - assert_eq!(asked, 4); - assert!(incidents_of(&world)?.iter().any(|record| record.is_open()), "no tick ran"); - Ok(()) - } - - #[test] - fn run_dispatches_serve_and_status() { - let temp = TempDir::new().unwrap(); - let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); - // A control store beneath a regular file: serve's boot cannot make it, - // and a serve that returned at once would say Ok instead. - let file = temp.path().join("not-a-directory"); - std::fs::write(&file, b"a file").unwrap(); - let store = file.join("control.cc"); - assert!( - run(args(&["serve", "--state-store", store.to_str().unwrap()]).into_iter()).is_err() - ); - // A control store that is not one: status reads it and refuses. - let junk = temp.path().join("junk.cc"); - std::fs::write(&junk, b"not a cultcache file").unwrap(); - assert!( - run(args(&["status", "--state-store", junk.to_str().unwrap()]).into_iter()).is_err() - ); - } - - /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment - /// waiting before Fencing. Continuity never stops and never yields a - /// deployment (budget-never-stop, handover-phase-split), so the window - /// sliding must not turn the deployment into a casualty. - #[test] - fn a_deployment_before_fencing_over_a_dead_incumbent_that_paged_is_never_yielded_and_lands() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - workload.kill(); - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - world.engine.supervise_one_admitted_generation()?; - // The oldest leaves the window: base would now yield the deployment. - set_meters( - &world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - for _ in 0..3 { - let _ = world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!( - after.pre_fencing_abort.is_none(), - "the deployment was yielded: {:?}", - after.pre_fencing_abort - ); - assert_eq!( - continuity_mints(&world)?, - 0, - "continuity minted beside a pre-fencing deployment" - ); - } - workload.revive(); - odin_reports_ready(&world, &redeploy, seeded_sequence(Some(&incumbent)) + 1)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - let landed = latest(&world)?; - assert_eq!(landed.transaction_id, redeploy.transaction_id); - assert_eq!(landed.phase, DeploymentPhase::Complete); - assert!(landed.pre_fencing_abort.is_none()); - Ok(()) - } - - /// The incumbent was admitted this long ago, so an anchor written "in the - /// past" is not older than its generation. - fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { - edit_incumbent(world, |generation| generation.admitted_at_unix_millis -= millis)?; - Ok(()) - } - - /// Put the target's backoff anchor `ago` milliseconds back. - fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { - let mut meters = meters_of(world, "service")?; - meters.continuity_backoff_from_unix_millis = Some(now_millis()? - ago); - set_meters(world, &meters) - } - - /// Continuity transactions minted over an incumbent (the one that admitted - /// it was seeded over nothing). - fn continuity_mints(world: &EngineFixture) -> Result { - Ok(every_transaction(world)? - .iter() - .filter(|transaction| { - transaction.command_kind == CommandKind::Continuity - && transaction.incumbent_generation_id.is_some() - }) - .count()) - } - - #[test] - fn continuity_keeps_restarting_past_the_paging_threshold() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - // Six charged attempts in the hour: the incident pages, nothing stops. - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); - let opened = incidents_of(&world)?; - assert_eq!(opened.len(), 1); - assert!(opened[0].is_open()); - - // The streak-6 wait is 160 s. - anchor_ago(&world, 159_000)?; - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - anchor_ago(&world, 160_001)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 1); - let after = meters_of(&world, "service")?; - assert_eq!(after.continuity_restarts.len(), 6); - assert!(after.continuity_restarts.last().is_some_and(|&last| now_millis().unwrap() - last < 60_000)); - assert_eq!(after.continuity_attempt_streak, 7); - assert_eq!(after.continuity_backoff_from_unix_millis, None); - assert_eq!(incidents_of(&world)?, opened); - Ok(()) - } - - #[test] - fn continuity_backoff_is_a_capped_doubling_of_the_attempt_streak() { - let from = 10_000_000_u64; - for (streak, seconds) in [ - (1_u32, 5_u64), - (2, 10), - (3, 20), - (4, 40), - (5, 80), - (6, 160), - (7, 300), - (8, 300), - (9, 300), - ] { - let mut meters = TargetSupervision::new("service"); - meters.continuity_attempt_streak = streak; - meters.continuity_backoff_from_unix_millis = Some(from); - // The paging log disagrees with the streak on purpose: the wait - // is a function of the streak, not of the window's count. - meters.continuity_restarts = vec![from - 10; if streak == 1 { 6 } else { 2 }]; - let due = from + seconds * 1_000; - assert_eq!(meters.backoff_due(0), Some(due), "streak {streak}"); - assert_eq!( - meters.continuity_backoff(due - 1, 0), - ContinuityBackoff::Waiting, - "streak {streak}" - ); - assert_eq!( - meters.continuity_backoff(due, 0), - ContinuityBackoff::Due { streak }, - "streak {streak}" - ); - } - // No streak, no wait. - let mut fresh = TargetSupervision::new("service"); - fresh.continuity_backoff_from_unix_millis = Some(from); - assert_eq!(fresh.continuity_backoff(from, 0), ContinuityBackoff::Due { streak: 0 }); - } - - #[test] - fn a_sustained_run_resets_the_streak_and_only_a_sustained_run() { - let now = 100_000_000_u64; - let mut meters = TargetSupervision::new("service"); - meters.continuity_attempt_streak = 7; - meters.continuity_restarts = vec![1_000]; - // Lived the sustained time, and no attempt since admission. - assert_eq!( - meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), - ContinuityBackoff::Due { streak: 0 } - ); - // A moment short of it. - assert_eq!( - meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + 10_000), - ContinuityBackoff::Begin - ); - // Attempts after admission failed on this generation: it has not run well. - meters.continuity_restarts = vec![now - 100_000]; - assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); - // Never attempted over this log, long lived: reset. - meters.continuity_restarts = Vec::new(); - assert_eq!( - meters.continuity_backoff(now, now - 700_000), - ContinuityBackoff::Due { streak: 0 } - ); - assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); - // An anchor older than its generation decides nothing. - meters.continuity_backoff_from_unix_millis = Some(now - 800_000); - assert_eq!(meters.backoff_due(now - 700_000), None); - assert_eq!( - meters.continuity_backoff(now, now - 700_000), - ContinuityBackoff::Due { streak: 0 } - ); - assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); - // Streak zero with no history is simply due. - let fresh = TargetSupervision::new("service"); - assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); - } - - #[test] - fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - let mut meters = restarted_ago(&[1_000_000])?; - meters.continuity_attempt_streak = 3; - set_meters(&world, &meters)?; - - // The last attempt was long ago, but the wait starts now: a wait - // anchored on the mint would already be over. - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - - // Streak 3 waits 20 s. - anchor_ago(&world, 19_000)?; - assert!(!world.engine.supervise_one_admitted_generation()?); - anchor_ago(&world, 20_001)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 1); - assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 4); - Ok(()) - } - - #[test] - fn a_queued_deployment_freezes_in_the_gap_after_a_continuity_attempt() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - let mut meters = restarted_ago(&[3_000_000])?; - meters.continuity_attempt_streak = 2; - set_meters(&world, &meters)?; - queue_declaring_redeploy(&world, "up-service-queued")?; - - for _ in 0..4 { - let _ = world.engine.run_scheduler_tick(); - } - let all = every_transaction(&world)?; - assert!( - all.iter().any(|transaction| transaction.command_id == "up-service-queued"), - "the queued deployment never froze" - ); - assert_eq!(continuity_mints(&world)?, 0); - for _ in 0..3 { - let _ = world.engine.run_scheduler_tick(); - assert_eq!(continuity_mints(&world)?, 0); - let aborts = every_transaction(&world)? - .into_iter() - .filter_map(|transaction| transaction.pre_fencing_abort) - .collect::>(); - // The deployment may fail on its own account (a root test runner - // cannot resolve its source); continuity must never be the reason. - assert!( - aborts.iter().all(|abort| !abort.error.contains("continuity")), - "the deployment was yielded: {aborts:?}" - ); - } - Ok(()) - } - - #[test] - fn a_unit_lost_to_a_host_reboot_is_rematerialized_once_with_no_backoff_step() -> Result<()> { - let fresh_world = |workload: &Arc| -> Result { - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - let mut meters = restarted_ago(&[1_000_000])?; - meters.continuity_attempt_streak = 3; - meters.continuity_backoff_from_unix_millis = Some(now_millis()?); - set_meters(&world, &meters)?; - Ok(world) - }; - let workload = SwitchWorkload::new(); - let world = fresh_world(&workload)?; - let before = meters_of(&world, "service")?; - - // Control: dead but not rebooted, the ordinary 20 s wait holds. - workload.kill(); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - - // The host rebooted under the unit: one mint, charged to nothing. - workload.reboot(); - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 1); - let after = meters_of(&world, "service")?; - assert_eq!(after.continuity_restarts, before.continuity_restarts); - assert_eq!(after.continuity_attempt_streak, 3); - assert!(after.continuity_rematerialized_at_unix_millis.is_some()); - assert_eq!(after.continuity_backoff_from_unix_millis, None); - - // The bound is spent: the next death of this generation waits like any other. - clear_live_transactions(&world)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - - // Control: a deferred projection spaces a re-materialization too. - let workload = SwitchWorkload::new(); - let world = fresh_world(&workload)?; - let mut deferred = meters_of(&world, "service")?; - deferred.defer_continuity(now_millis()?, "demotion failed"); - set_meters(&world, &deferred)?; - workload.reboot(); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - Ok(()) - } - - fn supervision_envelope_raw(payload: Vec, schema: &str) -> Result { - Ok(CultCacheEnvelope { - key: "service".into(), - r#type: TargetSupervision::TYPE.into(), - payload, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(schema.into()), - }) - } - - #[test] - fn a_v1_target_supervision_record_is_lifted_and_migrated_once() -> Result<()> { - let world = EngineFixture::new()?; - let meters = restarted_ago(&[200_000])?; - // A record as every one on a running host was written: seven keys, v1. - let payload = rmp_serde::to_vec(&( - TARGET_SUPERVISION_SCHEMA_V1, - &meters.target, - &meters.route_actuations, - &meters.continuity_restarts, - &meters.continuity_deferred_until, - &meters.continuity_deferral_reason, - &meters.route_survivals, - ))?; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( - supervision_envelope_raw(payload, TARGET_SUPERVISION_SCHEMA_V1)?, - )?; - let read = meters_of(&world, "service")?; - assert_eq!(read.continuity_restarts, meters.continuity_restarts); - assert_eq!(read.continuity_attempt_streak, 0); - assert_eq!(read.continuity_backoff_from_unix_millis, None); - assert_eq!(read.continuity_rematerialized_at_unix_millis, None); - - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 0); - let stored = ControlSnapshot::read(&world.state_store)?; - let envelope = &stored.supervision_for("service").unwrap().envelope; - assert_eq!(envelope.schema_id.as_deref(), Some(TARGET_SUPERVISION_SCHEMA)); - assert_eq!(meters_of(&world, "service")?.continuity_restarts, meters.continuity_restarts); - - let mut anchored = TargetSupervision::new("service"); - anchored.continuity_backoff_from_unix_millis = Some(0); - assert!(anchored.validate().is_err()); - anchored.continuity_backoff_from_unix_millis = None; - anchored.continuity_rematerialized_at_unix_millis = Some(0); - assert!(anchored.validate().is_err()); - Ok(()) - } - - #[test] - fn one_target_supervision_schema_id_names_one_shape() -> Result<()> { - let meters = restarted_ago(&[200_000])?; - let ten = rmp_serde::to_vec(&meters)?; - let seven = rmp_serde::to_vec(&( - TARGET_SUPERVISION_SCHEMA, - &meters.target, - &meters.route_actuations, - &meters.continuity_restarts, - &meters.continuity_deferred_until, - &meters.continuity_deferral_reason, - &meters.route_survivals, - ))?; - // The current id refuses the old shape and the old id refuses the new one. - assert!( - read_supervision_record(&supervision_envelope_raw(seven, TARGET_SUPERVISION_SCHEMA)?) - .is_err() - ); - assert!( - read_supervision_record(&supervision_envelope_raw( - ten.clone(), - TARGET_SUPERVISION_SCHEMA_V1 - )?) - .is_err() - ); - assert_eq!( - read_supervision_record(&supervision_envelope_raw(ten, TARGET_SUPERVISION_SCHEMA)?)?, - meters - ); - Ok(()) - } - - #[test] - fn continuity_keeps_minting_at_every_streak_and_log_length() -> Result<()> { - // No stop at any streak, however long, with the paging log full or not. - let full = [3_000_000_u64, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000]; - for streak in 1..=12_u32 { - for log in [&[1_000_000_u64][..], &full[..]] { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - let mut meters = restarted_ago(log)?; - meters.continuity_attempt_streak = streak; - set_meters(&world, &meters)?; - world.engine.supervise_one_admitted_generation()?; - assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the wait begins"); - anchor_ago(&world, 300_001)?; - world.engine.supervise_one_admitted_generation()?; - assert_eq!(continuity_mints(&world)?, 1, "streak {streak} log {} stopped", log.len()); - assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, streak + 1); - } - } - Ok(()) - } - - #[test] - fn a_mint_after_a_sustained_run_stores_the_reset_streak() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 700_000)?; - workload.kill(); - let mut meters = restarted_ago(&[3_000_000])?; - meters.continuity_attempt_streak = 7; - set_meters(&world, &meters)?; - // Ran 700 s since the last attempt: no wait, and the streak restarts at 1. - world.engine.supervise_one_admitted_generation()?; - assert_eq!(continuity_mints(&world)?, 1); - assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); - Ok(()) - } - - #[test] - fn a_re_materialization_counts_as_the_last_attempt_for_the_sustained_reset() { - let admitted_at = 10_000_000_u64; - let now = admitted_at + 700_000; - let mut meters = TargetSupervision::new("service"); - meters.continuity_attempt_streak = 5; - meters.continuity_restarts = vec![admitted_at - 1_000]; - // Before admission: a sustained run, reset. - meters.continuity_rematerialized_at_unix_millis = Some(admitted_at - 1); - assert_eq!( - meters.continuity_backoff(now, admitted_at), - ContinuityBackoff::Due { streak: 0 } - ); - // After admission: that attempt failed on this generation. - meters.continuity_rematerialized_at_unix_millis = Some(admitted_at + 1); - assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); - } - - mod after_a_reboot { - use super::*; - - fn resident_continuity(world: &EngineFixture) -> Result { - Ok(ControlSnapshot::read(&world.state_store)? - .transactions - .iter() - .filter(|s| s.value.command_kind == CommandKind::Continuity) - .count()) - } - - /// The attempt failed and was retired: drop continuity transactions - /// and their commands only. - fn drop_continuity(world: &EngineFixture) -> Result<()> { - let snapshot = ControlSnapshot::read(&world.state_store)?; - let ids = snapshot - .transactions - .iter() - .filter(|s| s.value.command_kind == CommandKind::Continuity) - .map(|s| s.value.command_id.clone()) - .collect::>(); - let envelopes = snapshot - .transactions - .iter() - .filter(|s| s.value.command_kind == CommandKind::Continuity) - .map(|s| s.envelope.clone()) - .chain( - snapshot - .commands - .iter() - .filter(|s| ids.contains(&s.value.command_id)) - .map(|s| s.envelope.clone()), - ) - .collect::>(); - if !envelopes.is_empty() { - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store) - .delete_batch_if_unchanged(&envelopes)? - ); - } - Ok(()) - } - - /// (attempts minted on tick 1, on tick 2, whether the queued deploy froze by then). - fn gap_run(reboot: bool) -> Result<(usize, usize, bool)> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 700_000)?; - if reboot { - workload.reboot(); - } else { - workload.kill(); - } - queue_declaring_redeploy(&world, "up-gap")?; - // One tick, in the tick's order: supervise, then freeze. - world.engine.supervise_one_admitted_generation()?; - let first = resident_continuity(&world)?; - world.engine.freeze_one_queued_command()?; - // That attempt failed and was retired. - drop_continuity(&world)?; - world.engine.supervise_one_admitted_generation()?; - let second = resident_continuity(&world)?; - world.engine.freeze_one_queued_command()?; - let frozen = every_transaction(&world)?.iter().any(|t| t.command_id == "up-gap"); - Ok((first, second, frozen)) - } - - #[test] - fn a_failed_re_materialization_is_followed_by_a_gap_a_queued_deployment_can_take( - ) -> Result<()> { - // Control: an ordinary attempt after a sustained run leaves a gap. - assert_eq!(gap_run(false)?, (1, 0, true)); - // The re-materialization is an attempt too: the next pass waits. - assert_eq!(gap_run(true)?, (1, 0, true)); - Ok(()) - } - - #[test] - fn a_running_pass_ends_the_wait_so_a_later_death_after_a_sustained_run_resets( - ) -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - let mut meters = restarted_ago(&[1_000_000])?; - meters.continuity_attempt_streak = 3; - set_meters(&world, &meters)?; - world.engine.supervise_one_admitted_generation()?; - assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); - workload.revive(); - world.engine.supervise_one_admitted_generation()?; - assert_eq!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis, None); - - // It runs on for days, then dies: no wait, and the streak restarts. - age_incumbent(&world, 200_000)?; - workload.kill(); - world.engine.supervise_one_admitted_generation()?; - assert_eq!(continuity_mints(&world)?, 1); - assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); - Ok(()) - } - } - - /// A source that resolves one fixed recipe and never freezes anything. - struct FixedSource { - recipe: String, - } - - impl SourcePort for FixedSource { - fn resolve( - &self, - _binding: &OperatorBinding, - _resolution_id: &str, - _selected_at_unix_millis: u64, - ) -> Result { - Ok(crate::drivers::ResolvedSource { - facts: crate::deployment_plan::tests::source(&self.recipe), - recipe_bytes: self.recipe.clone().into_bytes(), - }) - } - - fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { - bail!("this source freezes nothing") - } - - fn observe_frozen( - &self, - _: &CompiledDeploymentPlan, - _: &FrozenSourceReceipt, - ) -> Result { - bail!("this source freezes nothing") - } - - fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { - Ok(()) - } - } - - /// A Deploy transaction for an unrouted target, still Sealing, whose - /// source resolves `recipe`. One scheduler tick admits its plan or refuses it. - fn sealing_world(recipe: &str) -> Result { - use crate::deployment_plan::tests::BINDING; - let mut world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let (binding_head, binding_tail) = BINDING - .split_once("[route]") - .context("binding has no route table")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail - .split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ); - std::fs::create_dir_all(world.root.join("bindings"))?; - std::fs::write(world.root.join("bindings/service.toml"), binding)?; - world.engine.source = Arc::new(FixedSource { - recipe: recipe.to_owned(), - }); - let now = now_millis()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-service".into(), - kind: CommandKind::Deploy, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - ], - &[ - command_envelope(&command, command.requested_at_unix_millis)?, - transaction_envelope(&transaction, now)?, - ], - )? - ); - Ok(world) - } - - fn unrouted_recipe() -> String { - crate::deployment_plan::tests::RECIPE - .split("[[dependencies]]") - .next() - .expect("the recipe has a body") - .replace("route_required = true", "route_required = false") - .replace( - r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, - r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, - ) - } - - #[test] - fn admission_refuses_a_recipe_that_declares_no_way_to_prove_readiness() -> Result<()> { - let recipe = unrouted_recipe(); - let world = sealing_world(&recipe)?; - world.engine.run_scheduler_tick()?; - let refused = latest(&world)?; - assert!(refused.plan.is_none(), "the refused recipe was planned"); - assert!(refused.frozen_source.is_none(), "the refused recipe was frozen"); - let abort = refused - .pre_fencing_abort - .context("a recipe with no declared proof of readiness must be refused")?; - assert!( - abort.error.contains("target service declares no way to prove readiness"), - "{}", - abort.error - ); - Ok(()) - } - - #[test] - fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { - let recipe = unrouted_recipe().replace( - "capability = \"service.runtime\"", - &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), - ); - let world = sealing_world(&recipe)?; - world.engine.run_scheduler_tick()?; - let planned = latest(&world)?; - assert!(planned.pre_fencing_abort.is_none(), "{:?}", planned.pre_fencing_abort); - assert_eq!( - planned.plan.as_ref().map(CompiledDeploymentPlan::readiness_class).transpose()?, - Some(ReadinessClass::OdinSelf) - ); - Ok(()) - } - - // --------------------------------------------------------------------- - // A held record is reported and never decided for. Supervision mints no - // continuity over a held generation, and nothing yields to a held record. - // --------------------------------------------------------------------- - - /// Rewrite the world's admitted generation into what a pre-B3 binary left - /// behind: no Odin declaration, admitted on Odin receipts. - fn undeclare_incumbent(world: &EngineFixture) -> Result { - let now = now_millis()?; - let mut failure = None; - let next = edit_incumbent(world, |generation| { - generation.expected.dependencies.clear(); - match IdunnRuntimeActivationLaunch::issue( - &generation.expected, - generation.activation.runtime_instance_id.clone(), - now, - &world.engine.idunn_signer, - ) { - Ok(launch) => generation.activation = launch.activation().clone(), - Err(error) => failure = Some(error), - } - })?; - if let Some(error) = failure { - return Err(error); - } - ControlSnapshot::read(&world.state_store).context("store with the undeclared generation")?; - assert!(matches!(next.readiness(), Err(ReadinessDisagreement::Undeclared(_)))); - Ok(next) - } - - /// Queue the operator's declaring redeploy of `service`. - fn queue_declaring_redeploy(world: &EngineFixture, command_id: &str) -> Result<()> { - use crate::deployment_plan::tests::BINDING; - let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail.split_once("[brakes]").context("brakes")?.1 - ); - std::fs::create_dir_all(world.root.join("bindings"))?; - std::fs::write(world.root.join("bindings/service.toml"), binding)?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: command_id.into(), - kind: CommandKind::Deploy, - selector: "service".into(), - requested_by: "operator".into(), - requested_at_unix_millis: now_millis()?, - }; - assert!(SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, command.requested_at_unix_millis)?], - )?); - Ok(()) - } - - /// Every transaction the store knows: live, then archived. - fn every_transaction(world: &EngineFixture) -> Result> { - let mut all = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .map(|stored| stored.value) - .collect::>(); - all.extend(read_history_transactions(&world.state_store)); - Ok(all) - } - - fn is_continuity_over(transaction: &DeploymentTransaction, held: &AdmittedGeneration) -> bool { - transaction.command_kind == CommandKind::Continuity - && transaction.incumbent_generation_id.as_deref() == Some(held.generation_id.as_str()) - } - - #[test] - fn a_held_generation_that_dies_mints_no_continuity_and_leaves_its_target_free() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - let held = undeclare_incumbent(&world)?; - world.engine.validate_durable_authority(&ControlSnapshot::read(&world.state_store)?)?; - - workload.kill(); - for _ in 0..4 { - assert!(!world.engine.supervise_one_admitted_generation()?); - } - assert!( - ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), - "supervision minted a transaction over a held generation" - ); - // Reported, once per state change: the same text is not offered again. - let reports = world.engine.fault_reports.lock().unwrap(); - let down = reports.get("generation-down:service").context("the dead hold is not reported")?; - let last = down.last.lock().unwrap().clone(); - assert!(last.is_some()); - assert!(down.offer(last).is_none()); - drop(reports); - - // The target is free: the declaring redeploy freezes and is not - // displaced by a continuity. - queue_declaring_redeploy(&world, "up-service-declared")?; - for _ in 0..8 { - let _ = world.engine.run_scheduler_tick(); - } - let all = every_transaction(&world)?; - assert!( - all.iter().all(|transaction| !is_continuity_over(transaction, &held)), - "a continuity was minted over the held generation" - ); - assert!( - all.iter().any(|transaction| transaction.command_id == "up-service-declared"), - "the declaring redeploy never froze" - ); - Ok(()) - } - - #[test] - fn a_declaring_redeploy_is_not_yielded_when_the_held_incumbent_dies() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - let held = undeclare_incumbent(&world)?; - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; - assert_eq!(redeploy.held_disagreement(), None); - workload.kill(); - for _ in 0..4 { - let _ = world.engine.supervise_one_admitted_generation()?; - } - let all = every_transaction(&world)?; - assert!( - all.iter().all(|transaction| !is_continuity_over(transaction, &held)), - "a continuity was minted over the held incumbent" - ); - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!( - after.pre_fencing_abort.is_none(), - "the redeploy yielded: {:?}", - after.pre_fencing_abort - ); - Ok(()) - } - - /// `seeded` with its own Expected undeclared, so Idunn holds it. - fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { - let mut bare = seeded.clone(); - // Undeclare the Expected and re-derive what is bound to its digest. - let expected = bare.expected.as_mut().context("no expected")?; - expected.dependencies.clear(); - bare.expected_publication_sha256 = Some(expected.canonical_sha256()?); - let activation = IdunnRuntimeActivationLaunch::issue( - expected, - seeded.activation.as_ref().context("no activation")?.runtime_instance_id.clone(), - now_millis()?, - &world.engine.idunn_signer, - )? - .activation() - .clone(); - bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); - bare.activation = Some(activation); - bare.validate()?; - assert!(bare.held_disagreement().is_some()); - Ok(bare) - } - - /// A pre-fence Deploy whose own Expected is undeclared, stored in place. - fn held_predeploy( - world: &EngineFixture, - incumbent: &AdmittedGeneration, - ) -> Result { - let seeded = - seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(incumbent))?; - let bare = undeclared(world, &seeded)?; - replace_transaction(&world.state_store, &resident(world)?, &bare)?; - Ok(bare) - } - - #[test] - fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - // The incumbent is declared; the Deploy in front of it is the held one. - let held = held_predeploy(&world, &incumbent)?; - - workload.kill(); - for _ in 0..4 { - let _ = world.engine.supervise_one_admitted_generation()?; - } - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, held.transaction_id); - assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); - assert!(after.completion.is_none()); - Ok(()) - } - - #[test] - fn cancel_withdraws_a_held_predeploy_and_leaves_the_incumbent_untouched() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let held = held_predeploy(&world, &incumbent)?; - let before = incumbent_envelope(&world)?; - - cancel(&world.state_store, &held.command_id, "operator")?; - let cancelling = resident(&world)?.value; - assert_eq!( - cancelling.pre_fencing_abort.as_ref().map(|abort| abort.error.as_str()), - Some("cancelled by operator") - ); - assert!(cancelling.post_fencing_abort.is_none()); - - drive(&world, |transaction| transaction.completion.is_some())?; - // History also holds the incumbent's own admission: find ours by id. - let finished = every_transaction(&world)? - .into_iter() - .find(|transaction| transaction.transaction_id == held.transaction_id) - .context("the cancelled redeploy is nowhere")?; - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); - assert!(finished.is_terminal()); - // The target is released and nothing of the incumbent was touched. - assert!(!finished.blocks_new_target_mutation()); - assert_eq!(incumbent_envelope(&world)?, before); - assert!( - every_transaction(&world)? - .iter() - .all(|transaction| !is_continuity_over(transaction, &incumbent)), - "cancelling the redeploy minted a continuity" - ); - Ok(()) - } - - #[test] - fn cancel_refuses_a_predeploy_that_is_not_held_and_a_held_one_past_the_fence() -> Result<()> { - let world = EngineFixture::with_workload(SwitchWorkload::new())?; - let incumbent = admit_incumbent(&world)?; - let seeded = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - assert!(seeded.held_disagreement().is_none()); - assert!(cancel(&world.state_store, &seeded.command_id, "operator").is_err()); - assert!(resident(&world)?.value.pre_fencing_abort.is_none()); - - let mut past_the_fence = undeclared(&world, &seeded)?; - past_the_fence.enter_phase(DeploymentPhase::Fencing, now_millis()?); - assert!(past_the_fence.held_disagreement().is_some()); - assert!(!cancel_is_safe_for_held_prefence(&past_the_fence)); - Ok(()) - } - - // --------------------------------------------------------------------- - // B2: the meters, the v4 lift, and what `status` shows. - // --------------------------------------------------------------------- - - #[test] - fn the_actuation_log_slides() -> Result<()> { - let window = ROUTE_ACTUATION_WINDOW_MILLIS; - let t = 1_000_000_000_u64; - let mut meters = TargetSupervision::new("service"); - for offset in 0..12 { - meters.charge_route(t + offset, RouteActuation::Forward)?; - } - let refused = meters.charge_route(t + window - 1, RouteActuation::Forward).unwrap_err(); - assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + window)); - // Only the first entry has left the window: exactly one slot reopens. - meters.charge_route(t + window, RouteActuation::Forward)?; - let refused = meters.charge_route(t + window, RouteActuation::Forward).unwrap_err(); - assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + 1 + window)); - meters.validate()?; - Ok(()) - } - - #[test] - fn a_survival_actuation_is_recorded_but_never_counted_or_refused() -> Result<()> { - let t = 1_000_000_000_u64; - let mut meters = TargetSupervision::new("service"); - // Survivals first, past the ceiling: every one is admitted and - // recorded, and none takes a Forward slot. - for offset in 0..20 { - meters.charge_route(t + offset, RouteActuation::Survival)?; - } - assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); - assert_eq!(meters.route_survivals.last(), Some(&(t + 19))); - assert!(meters.route_actuations.is_empty()); - // Twelve Forward changes interleaved with more Survivals: the twelve - // are admitted, and the thirteenth is refused with a reopen time that - // only the oldest Forward entry decides. - for offset in 0..12 { - meters.charge_route(t + 100 + 2 * offset, RouteActuation::Forward)?; - meters.charge_route(t + 101 + 2 * offset, RouteActuation::Survival)?; - } - let refused = meters.charge_route(t + 200, RouteActuation::Forward).unwrap_err(); - assert_eq!( - (refused.used, refused.reopens_at_unix_millis), - (ROUTE_ACTUATION_CEILING, t + 100 + ROUTE_ACTUATION_WINDOW_MILLIS) - ); - // A full Forward ceiling still admits and records a Survival. - meters.charge_route(t + 201, RouteActuation::Survival)?; - assert_eq!(meters.route_survivals.last(), Some(&(t + 201))); - meters.validate()?; - Ok(()) - } - - #[test] - fn a_clock_stepped_back_stalls_nothing_past_one_window() -> Result<()> { - let t = 5_000_000_000_u64; - let now = t - 86_400_000; - let mut meters = TargetSupervision::new("service"); - meters.route_actuations = vec![t; ROUTE_ACTUATION_CEILING]; - meters.continuity_restarts = vec![t; CONTINUITY_PAGING_RESTARTS]; - meters.continuity_deferred_until = Some(t); - meters.continuity_deferral_reason = Some("demotion failed".into()); - meters.validate()?; - - // Entries from the future count as now: the meters hold for one window. - let read_only = meters.clone(); - assert!(read_only.restarts_page(now)); - let refused = meters.charge_route(now, RouteActuation::Forward).unwrap_err(); - assert!(refused.reopens_at_unix_millis <= now + ROUTE_ACTUATION_WINDOW_MILLIS); - // The clamp is written, so the window runs from the step and no longer. - assert_eq!(meters.route_actuations, vec![now; ROUTE_ACTUATION_CEILING]); - assert_eq!(meters.continuity_restarts, vec![now; CONTINUITY_PAGING_RESTARTS]); - assert!(!meters.restarts_page(now + CONTINUITY_RESTART_WINDOW_MILLIS)); - meters.charge_route(now + ROUTE_ACTUATION_WINDOW_MILLIS, RouteActuation::Forward)?; - - // Waits set for the future are due, not stalled for the length of the step. - assert!(!read_only.continuity_deferred(now)); - let mut anchored = TargetSupervision::new("service"); - anchored.continuity_attempt_streak = 7; - anchored.continuity_backoff_from_unix_millis = Some(t); - assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Due { streak: 7 }); - anchored.continuity_attempt_streak = 1; - anchored.continuity_backoff_from_unix_millis = Some(now + 10_000); - assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Waiting); - let route = RouteSupervisionState { - next_challenge_at_unix_millis: Some(t), - ..RouteSupervisionState::default() - }; - assert!(!route.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); - assert!(!is_waiting(now, t, RESUME_BACKOFF_CEILING_MILLIS)); - // The same waits, set for a moment ahead, still wait. - let ahead = RouteSupervisionState { - next_challenge_at_unix_millis: Some(now + 5_000), - ..RouteSupervisionState::default() - }; - assert!(ahead.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); - Ok(()) - } - - #[test] - fn a_route_challenge_wait_widens_to_a_cap_and_a_proof_clears_it() { - let age = DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS; - let mut state = RouteSupervisionState::default(); - let mut waits = Vec::new(); - for _ in 0..8 { - state.record_failed_challenge(1_000, age); - waits.push(state.next_challenge_at_unix_millis.unwrap() - 1_000); - } - assert_eq!(&waits[..3], &[age, age * 2, age * 4]); - assert_eq!(waits[7], ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); - assert_eq!(state.degraded_since_unix_millis, Some(1_000)); - state.record_proved_challenge(9_000); - assert_eq!(state.consecutive_failures, 0); - assert_eq!(state.next_challenge_at_unix_millis, None); - assert_eq!(state.degraded_since_unix_millis, None); - } - - #[test] - fn a_v3_generation_lifts_to_v4_once_and_reencodes_canonically() -> Result<()> { - for text in [FIXTURE_GENERATION_V3_ODIN, FIXTURE_GENERATION_V3_ROUTE_PROOF] { - let temp = TempDir::new()?; - let store = temp.path().join("control.cc"); - let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; - let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; - SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(envelope.clone())?; - // The v3 bytes are not v4 bytes: only the typed lift may read them. - assert!(decode_record::(&envelope).is_err()); - - assert_eq!(migrate_control_store_to_current_schema(&store)?, 1); - assert_eq!(migrate_control_store_to_current_schema(&store)?, 0); - let snapshot = ControlSnapshot::read(&store)?; - let migrated = snapshot.admitted.first().context("the lift lost the record")?; - assert_eq!(migrated.envelope.schema_id.as_deref(), Some(ADMITTED_GENERATION_SCHEMA)); - // After the lift the canonical re-encode accepts its own bytes. - let lifted: AdmittedGeneration = decode_record(&migrated.envelope)?; - assert_eq!(lifted, migrated.value); - - // The decisions are the ones the v3 record encoded; only the retired - // fields are gone, and no meters were invented. - assert_eq!(lifted.target, legacy.target); - assert_eq!(lifted.generation_id, legacy.generation_id); - assert_eq!(lifted.plan, legacy.plan); - assert_eq!(lifted.expected, legacy.expected); - assert_eq!(lifted.activation, legacy.activation); - assert_eq!(lifted.workload, legacy.workload); - assert_eq!(lifted.leasing, legacy.leasing); - assert_eq!(lifted.ready, legacy.ready); - assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); - assert_eq!(lifted.routing, legacy.routing); - assert_eq!(lifted.odin_authority, legacy.odin_authority); - assert_eq!( - lifted.odin_publisher_sequence_cursor, - legacy.odin_publisher_sequence_cursor - ); - assert_eq!( - lifted.route_supervision, - legacy.route_supervision.map(|state| RouteSupervisionState { - last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, - consecutive_failures: state.consecutive_failures, - next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, - degraded_since_unix_millis: state.degraded_since_unix_millis, - }) - ); - assert_eq!(lifted.last_error, None); - assert!(snapshot.targets.is_empty()); - } - Ok(()) - } - - #[test] - fn a_v3_generation_that_shows_written_meters_is_refused_not_dropped() -> Result<()> { - let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; - let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; - assert!(legacy.route_supervision.is_some(), "the fixture is a routed generation"); - let refused = |change: fn(&mut LegacyAdmittedGenerationV3)| -> Result { - let mut edited = legacy.clone(); - change(&mut edited); - let mut tampered = envelope.clone(); - tampered.payload = rmp_serde::to_vec(&edited)?; - Ok(error_text(read_generation_record(&tampered).map(|_| ()))) - }; - let restarts = refused(|edited| edited.continuity_backoff.attempts = 2)?; - assert!(restarts.contains("refusing to drop"), "{restarts}"); - assert!(restarts.contains(&legacy.target), "the error names the record: {restarts}"); - let actuations = - refused(|edited| edited.route_supervision.as_mut().unwrap().actuations.count = 1)?; - assert!(actuations.contains("refusing to drop"), "{actuations}"); - - // A whole store carrying one refuses to boot, naming the record. - let temp = TempDir::new()?; - let store = temp.path().join("control.cc"); - let mut edited = legacy.clone(); - edited.continuity_backoff.attempts = 2; - let mut tampered = envelope.clone(); - tampered.payload = rmp_serde::to_vec(&edited)?; - SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(tampered)?; - assert!(migrate_control_store_to_current_schema(&store).is_err()); - Ok(()) - } - - #[test] - fn a_v3_generation_lifts_each_route_supervision_field_to_its_own_place() -> Result<()> { - let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; - let mut legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; - // Four distinct non-default values, so a swap cannot land on itself. - legacy.route_supervision = Some(LegacyRouteSupervisionStateV3 { - last_challenge_at_unix_millis: Some(1_111), - consecutive_failures: 3, - next_challenge_at_unix_millis: Some(2_222), - degraded_since_unix_millis: Some(3_333), - actuations: LegacyActuationWindowV3::default(), - }); - let mut edited = envelope.clone(); - edited.payload = rmp_serde::to_vec(&legacy)?; - let lifted = read_generation_record(&edited)?; - assert_eq!( - lifted.route_supervision, - Some(RouteSupervisionState { - last_challenge_at_unix_millis: Some(1_111), - consecutive_failures: 3, - next_challenge_at_unix_millis: Some(2_222), - degraded_since_unix_millis: Some(3_333), - }) - ); - Ok(()) - } - - #[test] - fn status_covers_the_admitted_targets_and_the_metered_ones() -> Result<()> { - // "service" is admitted with no meters at all; "metered" has meters and - // no generation. Neither set contains the other. - let world = EngineFixture::with_workload(SwitchWorkload::new())?; - admit_incumbent(&world)?; - assert!(ControlSnapshot::read(&world.state_store)?.targets.is_empty()); - set_meters(&world, &TargetSupervision::new("metered"))?; - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert_eq!( - supervised_targets(&snapshot).into_iter().collect::>(), - ["metered", "service"] - ); - Ok(()) - } - - #[test] - fn target_supervision_reads_back_and_its_key_must_be_its_target() -> Result<()> { - let world = EngineFixture::new()?; - let meters = restarted_ago(&[5_000])?; - set_meters(&world, &meters)?; - assert_eq!(meters_of(&world, "service")?, meters); - assert_eq!(ControlSnapshot::read(&world.state_store)?.targets.len(), 1); - - let mut elsewhere = target_supervision_envelope(&meters, 5)?; - elsewhere.key = "other".into(); - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent(elsewhere)?; - let error = error_text(ControlSnapshot::read(&world.state_store).map(|_| ())); - assert!(error.contains("not its target"), "{error}"); - Ok(()) - } - - #[test] - fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { - let now = 10_000_000_u64; - let mut meters = TargetSupervision::new("service"); - meters.route_actuations = (0..12).map(|offset| now - 5_000 + offset).collect(); - meters.continuity_restarts = vec![now - 9_000, now - 4_000]; - meters.continuity_deferred_until = Some(now + 500); - meters.continuity_deferral_reason = Some("projection would not demote".into()); - meters.continuity_attempt_streak = 2; - meters.continuity_backoff_from_unix_millis = Some(now - 4_000); - - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.last_error = Some("held".into()); - generation.admitted_at_unix_millis = 1; - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); - assert!(lines.contains(&format!("target service {}", generation.generation_id)), "{lines}"); - assert!(lines.contains("held: held"), "{lines}"); - assert!(lines.contains(&format!( - "continuity attempts-in-hour 2 pages-at 6 streak 2 next-attempt-at {}", - now - 4_000 + 10_000 - )), "{lines}"); - assert!(!lines.contains("rematerialized-at"), "{lines}"); - meters.continuity_rematerialized_at_unix_millis = Some(now - 1_000); - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" -"); - assert!(lines.contains(&format!("continuity rematerialized-at {}", now - 1_000)), "{lines}"); - // An anchor older than the generation is not shown as a pending wait. - generation.admitted_at_unix_millis = now; - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" -"); - assert!(lines.contains("streak 2 next-attempt-at none"), "{lines}"); - generation.admitted_at_unix_millis = 1; - assert!(lines.contains("continuity deferred until 10000500: projection would not demote"), "{lines}"); - assert!(lines.contains(&format!( - "route actuations 12/12 in window, reopens-at {}", - now - 5_000 + ROUTE_ACTUATION_WINDOW_MILLIS - )), "{lines}"); - assert!(lines.contains("route healthy consecutive-failures 0"), "{lines}"); - assert!( - lines.contains("route survival actuations 0 in window (uncounted), last-at none"), - "{lines}" - ); - meters.route_survivals = vec![now - 7_000, now - 3_000]; - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); - assert!(lines.contains(&format!( - "route survival actuations 2 in window (uncounted), last-at {}", - now - 3_000 - )), "{lines}"); - meters.route_survivals = (0..ROUTE_SURVIVAL_RECORD as u64).map(|n| now - 100 + n).collect(); - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); - assert!(lines.contains("route survival actuations at least 12 in window"), "{lines}"); - // Survivals never move the ceiling's count. - assert!(lines.contains("route actuations 12/12 in window"), "{lines}"); - - // A first deployment has meters and no generation: it shows them alone. - let mut first = TargetSupervision::new("fresh"); - first.route_actuations = vec![now - 1]; - let lines = render_supervision("fresh", None, Some(&first), now).join("\n"); - assert!(lines.contains("target fresh no-admitted-generation"), "{lines}"); - assert!(lines.contains("route actuations 1/12 in window, reopens-at none"), "{lines}"); - // An unrouted, unmetered target shows no route line at all. - let lines = render_supervision("plain", Some(&unrouted_generation()?), None, now).join("\n"); - assert!(!lines.contains("route actuations"), "{lines}"); - // A generation that no longer routes, whose meters hold only survival - // actuations, still shows the record. - let mut survived = TargetSupervision::new("plain"); - survived.route_survivals = vec![now - 1]; - let lines = - render_supervision("plain", Some(&unrouted_generation()?), Some(&survived), now).join("\n"); - assert!(lines.contains("route survival actuations 1 in window"), "{lines}"); - Ok(()) - } - - fn unrouted_generation() -> Result { - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.route_supervision = None; - Ok(generation) - } - - // --------------------------------------------------------------------- - // Route-proof readiness: a routed target that declares no Odin dependency - // is admitted by Idunn's own challenges. Odin is never read. - // --------------------------------------------------------------------- - mod route_proof { - use super::*; - use std::io::{Read, Write}; - use std::net::{TcpListener, TcpStream}; - use std::sync::Mutex; - use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; - use std::time::Duration; - - /// The runtime behind both endpoints, answering Idunn's snapshot - /// challenge the way the TypeScript signer does: locally, in the - /// reported health, with `route-observation:` as the detail. - struct RuntimeStub { - expected: IdunnExpectedIncarnationRecord, - activation: IdunnRuntimeActivationRecord, - provider: cultnet_rs::ServiceIdentitySigner, - activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, - state: Mutex<&'static str>, - capacity: AtomicU32, - sequence: AtomicU64, - candidate_hits: AtomicUsize, - stable_hits: AtomicUsize, - stop: AtomicBool, - /// Accept the connection and close it without a word: a process - /// that is up but not answering. Unlike a closed port, no other - /// test's listener can be handed the address. - hang_up: AtomicBool, - /// The write lease its presence claims to hold: what a candidate - /// that picked up Idunn's grant reports. - lease: Mutex>, - reply: Mutex, - /// Runs when the stable listener accepts, before it answers. - on_stable_hit: Mutex>>, - } - - /// How the runtime answers a well-formed challenge. - #[derive(Clone, Copy, PartialEq, Eq)] - enum Reply { - Honest, - /// An answer to some other challenge. - ForeignChallengeId, - /// An HTTP error where the snapshot should be. - HttpFailure, - /// A first Odin candidate answering its Warming challenge: the - /// detail names the challenge as `idunn-warming:`. - FirstOdinWarming, - } - - impl RuntimeStub { - fn answer(&self, message_id: &str) -> Result> { - let expected = &self.expected; - let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { - schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: self.activation.expected_projection_sha256.clone(), - plan_id: expected.plan_id.clone(), - incarnation_id: expected.incarnation_id.clone(), - sealed_release_id: expected.sealed_release_id.clone(), - activation_witness_sha256: self.activation.canonical_sha256()?, - state_schema_generation: expected.state_schema_generation.clone(), - state_contract_sha256: expected.state_contract_sha256.clone(), - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: self.activation.runtime_instance_id.clone(), - bound_endpoint: expected - .route - .as_ref() - .map(|route| route.candidate_endpoint.clone()), - capabilities: expected - .capabilities - .iter() - .map(|capability| cultnet_rs::GameCultRuntimeCapability { - capability: capability.capability.clone(), - schema: capability.schema.clone(), - compatibility: capability.compatibility.clone(), - capacity: self.capacity.load(Ordering::SeqCst), - }) - .collect(), - health_contract: expected.health_contract.clone(), - state: (*self.state.lock().unwrap()).into(), - detail: match *self.reply.lock().unwrap() { - Reply::FirstOdinWarming => format!("idunn-warming:{message_id}"), - _ => format!("route-observation:{message_id}"), - }, - write_lease_sha256: self.lease.lock().unwrap().clone(), - signer_identity_id: self.provider.entry().identity_id.clone(), - publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, - observed_at_unix_millis: now_millis()?, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - activation_signer_identity_id: self.activation_signer.identity_id(), - activation_signature: Vec::new(), - }; - let proof = presence.canonical_proof_payload()?; - presence.signature = self - .provider - .sign::(&proof) - .signature; - presence.activation_signature = - self.activation_signer.sign_presence_proof(&presence)?; - Ok(rmp_serde::to_vec(&presence)?) - } - - fn set_state(&self, state: &'static str) { - *self.state.lock().unwrap() = state; - } - - fn serve_one(&self, mut stream: TcpStream) -> Result<()> { - stream.set_read_timeout(Some(Duration::from_secs(3)))?; - let mut request = Vec::new(); - let mut chunk = [0_u8; 4096]; - let (header_end, length) = loop { - let read = stream.read(&mut chunk)?; - ensure!(read > 0, "stub client closed early"); - request.extend_from_slice(&chunk[..read]); - if let Some(index) = request.windows(4).position(|w| w == b"\r\n\r\n") { - let head = String::from_utf8_lossy(&request[..index]).to_lowercase(); - let length = head - .lines() - .find_map(|line| line.strip_prefix("content-length:")) - .context("stub request has no Content-Length")? - .trim() - .parse::()?; - if request.len() >= index + 4 + length { - break (index + 4, length); - } - } - }; - let message = cultnet_rs::decode_cultnet_message_from_slice( - &request[header_end..header_end + length], - cultnet_rs::CultNetWireContract::CultNetSchemaV0, - )?; - let cultnet_rs::CultNetMessage::SnapshotRequest { message_id, .. } = message - else { - bail!("stub received something other than a snapshot request"); - }; - let reply = *self.reply.lock().unwrap(); - if reply == Reply::HttpFailure { - stream.write_all( - b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - )?; - return Ok(()); - } - let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { - message_id: if reply == Reply::ForeignChallengeId { - "candidate-foreign".to_owned() - } else { - message_id.clone() - }, - documents: vec![cultnet_rs::CultNetRawDocumentRecord { - schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), - record_key: self.expected.target.clone(), - stored_at: rfc3339_millis(now_millis()?)?, - payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, - payload: self.answer(&message_id)?, - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - }], - }; - let body = cultnet_rs::encode_cultnet_message_to_vec( - &response, - cultnet_rs::CultNetWireContract::CultNetSchemaV0, - )?; - stream.write_all( - format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ) - .as_bytes(), - )?; - stream.write_all(&body)?; - Ok(()) - } - - fn listen(self: &Arc, listener: TcpListener, candidate: bool) { - listener.set_nonblocking(true).expect("nonblocking listener"); - let stub = Arc::clone(self); - std::thread::spawn(move || { - while !stub.stop.load(Ordering::SeqCst) { - match listener.accept() { - Ok((stream, _)) => { - stream.set_nonblocking(false).ok(); - let hits = if candidate { - &stub.candidate_hits - } else { - &stub.stable_hits - }; - hits.fetch_add(1, Ordering::SeqCst); - if !candidate - && let Some(hook) = stub.on_stable_hit.lock().unwrap().as_ref() - { - hook(); - } - if !stub.hang_up.load(Ordering::SeqCst) { - let _ = stub.serve_one(stream); - } - } - Err(_) => std::thread::sleep(Duration::from_millis(5)), - } - } - }); - } - } - - impl Drop for RuntimeStub { - fn drop(&mut self) { - self.stop.store(true, Ordering::SeqCst); - } - } - - /// Odin, as the world it runs in sees it. - #[derive(Clone, Copy, PartialEq, Eq)] - enum Odin { - /// Every read of the correlation store fails. - Unreachable, - /// Odin publishes a correlation that says the target is not Ready. - NotReady, - } - - struct RoutedWorld { - world: EngineFixture, - workload: Arc, - stub: Arc, - odin: Odin, - poisoned: Vec, - } - - /// A stateless, routed target that declares no Odin dependency, over - /// `minimum_capacity`, seeded at `phase` (Warming or Fencing): its - /// Warming evidence, when the phase needs it, is the candidate's own - /// answer to a real challenge. Listeners stand at both endpoints. - fn routed_world( - odin: Odin, - minimum_capacity: u32, - phase: DeploymentPhase, - provides_odin: bool, - ) -> Result { - build_routed_world( - odin, - minimum_capacity, - phase, - provides_odin, - false, - CommandKind::Continuity, - ) - } - - /// The same world owned by a deployment command instead of a - /// continuity: its route actuations are `Forward` and can be refused. - fn routed_deploy_world( - odin: Odin, - minimum_capacity: u32, - phase: DeploymentPhase, - ) -> Result { - build_routed_world(odin, minimum_capacity, phase, false, false, CommandKind::Deploy) - } - - /// The same world for a target with process-writable state: its - /// candidate is granted a write lease and Ready must be the candidate - /// reporting exactly that lease. - fn stateful_routed_world(odin: Odin, phase: DeploymentPhase) -> Result { - build_routed_world(odin, 1, phase, false, true, CommandKind::Continuity) - } - - fn build_routed_world( - odin: Odin, - minimum_capacity: u32, - phase: DeploymentPhase, - provides_odin: bool, - stateful: bool, - kind: CommandKind, - ) -> Result { - use crate::deployment_plan::tests::{ - BINDING, RECIPE, artifact_receipt, external_input_receipt, source, - }; - let switch = SwitchWorkload::new(); - let world = EngineFixture::with_workload(switch.clone())?; - let candidate_listener = TcpListener::bind("127.0.0.1:0")?; - let stable_listener = TcpListener::bind("127.0.0.1:0")?; - let candidate_port = candidate_listener.local_addr()?.port(); - let stable_port = stable_listener.local_addr()?.port(); - - let provider_path = world.root.join("identities/provider.cc"); - let provider_anchor = world.root.join("identities/provider-anchor.cc"); - let provider = - enroll_service_identity_at::(&provider_path)?; - export_service_identity_trust_anchor(&provider, &provider_anchor)?; - let config = world.root.join("service.conf"); - let binding = BINDING - .replace( - "/etc/gamecult/trust/service.cc", - &provider_anchor.display().to_string(), - ) - .replace("service-runtime-signer", &provider.entry().identity_id) - .replace("127.0.0.1:17999", &format!("127.0.0.1:{stable_port}")) - .replace( - "private_port_start = 18000", - &format!("private_port_start = {candidate_port}"), - ) - .replace( - "private_port_end = 18009", - &format!("private_port_end = {}", candidate_port + 1), - ) - .replace( - "/etc/nginx/idunn-stream-routes/service.conf", - &config.display().to_string().replace('\\', "/"), - ); - let (binding, recipe_source) = if stateful { - std::fs::create_dir_all(world.root.join("lease"))?; - ( - binding.replace( - "[brakes]", - &format!( - "[process_write_lease]\nrecord_path = \"{}\"\n\n[brakes]", - world.root.join("lease/process-write-lease.cc").display() - ), - ), - RECIPE - .replace( - "writer = \"none\"\nrecovery = \"rebuildable\"\nstartup = \"open-at-start\"", - "writer = \"process-bound-single-writer\"\nrecovery = \"preserve\"\nstartup = \"create-or-open-after-write-lease\"", - ) - .replace( - "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", - "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_PROCESS_WRITE_LEASE\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", - ), - ) - } else { - (binding, RECIPE.to_owned()) - }; - let recipe = recipe_source - .split("[[dependencies]]") - .next() - .context("recipe is empty")? - .replace( - "capability = \"service.runtime\"", - &format!( - "capability = \"{}\"\ncapacity = {minimum_capacity}", - if provides_odin { - ODIN_RENDEZVOUS_CAPABILITY - } else { - "service.runtime" - } - ), - ); - let plan = compile_deployment_plan( - recipe.as_bytes(), - binding.as_bytes(), - source(&recipe), - "service-incarnation-1", - Some(candidate_port), - 110, - &[], - )?; - let release = SealedRelease::new( - &plan, - vec![artifact_receipt()], - vec![external_input_receipt()], - 120, - )?; - let expected = release.expected_projection(&plan)?; - assert!(expected.route.is_some() && expected.dependencies.is_empty()); - assert_eq!(expected.write_lease_required, stateful); - assert_eq!( - ReadinessClass::of(&expected), - Ok(if provides_odin { - ReadinessClass::OdinSelf - } else { - ReadinessClass::RouteProof - }) - ); - - let now = now_millis()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: match kind { - CommandKind::Deploy => "up-service", - CommandKind::Continuity => "continuity-service", - } - .into(), - kind, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let mut transaction = - DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; - match kind { - CommandKind::Continuity => { - transaction.lifecycle_authorized_at_unix_millis = Some(now); - } - CommandKind::Deploy => { - transaction.frozen_source = Some(FrozenSourceReceipt { - transaction_id: transaction.transaction_id.clone(), - plan_id: plan.plan_id.clone(), - snapshot_sha256: sha256_id(b"frozen source snapshot"), - }); - transaction.deployment_authorization = Some(deployment_authorization( - &world, - &expected, - &transaction.transaction_id, - now, - )?); - } - } - let launch = IdunnRuntimeActivationLaunch::issue( - &expected, - runtime_instance_id(&transaction.transaction_id)?, - now, - &world.engine.idunn_signer, - )?; - let mut seed = Vec::new(); - let activation = launch.write_credential(&mut seed)?; - let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; - let mut workload = recorded.workload.clone().context("no recorded workload")?; - match &mut workload { - WorkloadObservation::Systemd(observed) => { - observed.runtime_instance_id = activation.runtime_instance_id.clone(); - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), - } - transaction.isolation = recorded.isolation; - transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { - sealed_release_id: release.sealed_release_id.clone(), - root: PathBuf::from("/srv/service/releases/test"), - }); - transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); - transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); - transaction.sealed_release = Some(release); - transaction.expected = Some(expected.clone()); - transaction.activation = Some(activation.clone()); - transaction.workload = Some(workload); - transaction.plan = Some(plan); - - let stub = Arc::new(RuntimeStub { - expected, - activation, - provider: open_service_identity_at::( - &provider_path, - )?, - activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( - &seed[..], - )?, - state: Mutex::new("warming"), - capacity: AtomicU32::new(minimum_capacity), - sequence: AtomicU64::new(0), - candidate_hits: AtomicUsize::new(0), - stable_hits: AtomicUsize::new(0), - stop: AtomicBool::new(false), - hang_up: AtomicBool::new(false), - lease: Mutex::new(None), - reply: Mutex::new(Reply::Honest), - on_stable_hit: Mutex::new(None), - }); - stub.listen(candidate_listener, true); - stub.listen(stable_listener, false); - - if phase >= DeploymentPhase::Fencing && provides_odin { - // Odin's own first word about the candidate, as for any - // Odin-correlated target. - // A stateful candidate is warming, not active, until it holds - // the lease Idunn grants after the fence. - let warming = signed_correlation_as( - &world, - &transaction, - 4, - false, - if stateful { "warming" } else { "active" }, - None, - b"presence", - )?; - let authenticated = world.engine.authenticate_topology_bytes( - &ControlSnapshot::read(&world.state_store)?, - &transaction, - &warming, - None, - now, - )?; - transaction.warming = Some(WarmingEvidence::OdinTopology { - evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, - }); - transaction.odin_publisher_sequence_cursor = 4; - } else if phase >= DeploymentPhase::Fencing { - let CandidateAnswer::Answered { evidence, .. } = - world - .engine - .challenge_candidate(&transaction, &["warming"], None)? - else { - bail!("the stub candidate did not answer"); - }; - transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); - } - if phase >= DeploymentPhase::Fencing { - transaction.route_preflight = Some(candidate_preflight(&transaction)?); - } - if stateful { - // The lease driver reads the Expected and the observed - // activation from the topology store, as Starting left them. - let plan = transaction.plan.as_ref().unwrap(); - let anchor = world.engine.provider_anchor_for_plan(plan)?; - let topology = world.engine.topology(); - topology.publish_expected(transaction.expected.as_ref().unwrap(), &anchor)?; - topology.publish_observed_activation( - transaction.expected.as_ref().unwrap(), - transaction.activation.as_ref().unwrap(), - transaction.workload.as_ref().unwrap(), - )?; - } - transaction.enter_phase(phase, now); - transaction.validate()?; - let store = SingleFileMessagePackBackingStore::new(&world.state_store); - assert!(store.compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - ], - &[ - command_envelope(&command, command.requested_at_unix_millis)?, - transaction_envelope(&transaction, now)?, - ], - )?); - - // Odin's world. Unreachable: the correlation store is bytes no - // reader can decode, so any read of it is an error. NotReady: a - // valid correlation that says the target is not Ready. - let store = &world.engine.options.odin_correlation_store; - let poisoned = match odin { - _ if provides_odin && stateful => { - // A new warming word after the fence: what the lease - // grant waits for. - SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( - CultCacheEnvelope { - key: crate::drivers::incarnation_key_of( - &transaction.target, - &transaction.expected.as_ref().unwrap().canonical_sha256()?, - ), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: signed_correlation_as( - &world, - &transaction, - 5, - false, - "warming", - None, - b"presence-5", - )?, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some( - cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - ), - }, - )?; - Vec::new() - } - _ if provides_odin => { - odin_reports_ready(&world, &transaction, 5)?; - Vec::new() - } - Odin::Unreachable => { - let garbage = b"odin is not a cache".to_vec(); - std::fs::write(store, &garbage)?; - garbage - } - Odin::NotReady => { - let wrongly = signed_correlation(&world, &transaction, 9, false)?; - SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( - CultCacheEnvelope { - key: crate::drivers::incarnation_key_of( - &transaction.target, - &transaction.expected.as_ref().unwrap().canonical_sha256()?, - ), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: wrongly, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some( - cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - ), - }, - )?; - Vec::new() - } - }; - Ok(RoutedWorld { - world, - workload: switch, - stub, - odin, - poisoned, - }) - } - - /// The preflight receipt of a candidate whose route was never - /// installed: what Warming records before Fencing, without running the - /// host's nginx. - fn candidate_preflight( - transaction: &DeploymentTransaction, - ) -> Result { - let rendered = NginxRouteDriver::new( - transaction - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1 - .route - .context("no route binding")?, - ) - .render(transaction.expected.as_ref().unwrap())?; - Ok(crate::drivers::RoutePreflightReceipt { - route_id: "service".into(), - candidate_runtime_instance_id: transaction - .activation - .as_ref() - .unwrap() - .runtime_instance_id - .clone(), - candidate_membership_sha256: sha256_id(&rendered), - incumbent_runtime_instance_id: None, - incumbent_membership_sha256: None, - incumbent_configuration: None, - }) - } - - impl RoutedWorld { - fn step(&self) -> Result<()> { - self.world.engine.advance_transaction(&resident(&self.world)?) - } - - fn run_to_routing(&self) -> Result<()> { - for _ in 0..12 { - if self.transaction()?.phase == DeploymentPhase::Routing { - return Ok(()); - } - self.step()?; - } - bail!("the transaction never reached Routing") - } - - fn transaction(&self) -> Result { - Ok(resident(&self.world)?.value) - } - - /// Odin was never read: the poisoned store is byte for byte what - /// it was, and no step failed on it. - fn assert_odin_untouched(&self) -> Result<()> { - if self.odin == Odin::Unreachable { - assert_eq!( - std::fs::read(&self.world.engine.options.odin_correlation_store)?, - self.poisoned - ); - } - Ok(()) - } - - /// Routing runs for real: it installs the fragment through the - /// world's stub host programs and proves the stable route against - /// the stub runtime. The world only drives the steps until the - /// transaction enters Committing. - fn promote(&self) -> Result<()> { - assert_eq!(self.transaction()?.phase, DeploymentPhase::Routing); - for _ in 0..4 { - if self.transaction()?.phase == DeploymentPhase::Committing { - return Ok(()); - } - self.step()?; - } - bail!("the transaction never reached Committing") - } - } - - /// A route-proof generation from a real admission. - pub(super) fn committed_generation() -> Result { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - routed.step()?; - admitted(&routed) - } - - fn admitted(world: &RoutedWorld) -> Result { - Ok(ControlSnapshot::read(&world.world.state_store)? - .admitted - .into_iter() - .next() - .context("nothing was admitted")? - .value) - } - - #[test] - fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - // Fencing, Leasing, AwaitingReady: the candidate is still warming. - for _ in 0..12 { - routed.step()?; - if routed.transaction()?.last_error.is_some() { - break; - } - } - let waiting = routed.transaction()?; - assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); - assert!(waiting.ready.is_none()); - assert_eq!( - waiting.last_error.as_deref(), - Some("candidate is still warming") - ); - // The candidate finishes warming: Ready is its own Active answer. - routed.stub.set_state("active"); - routed.step()?; - assert!(matches!( - routed.transaction()?.ready, - Some(ReadinessEvidence::RouteProof { .. }) - )); - routed.step()?; - assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); - routed.promote()?; - routed.step()?; - - let generation = admitted(&routed)?; - generation.validate()?; - assert_eq!(generation.ready.voucher(), Voucher::Candidate); - assert!(generation.odin_authority.is_none()); - assert!(generation.latest_odin_observation.is_none()); - assert!(generation.route_supervision.is_some()); - // Idunn's own challenges carried every proof: the seed's warming, - // two Ready polls; the stable endpoint proved the route twice. - assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 3); - assert!(routed.stub.stable_hits.load(Ordering::SeqCst) >= 2); - routed.assert_odin_untouched()?; - - // Boot and supervision read no Odin either: the topology refresh - // returns before it opens Odin's store, and durable authority - // skips a route-proof generation's receipts. - let stored = ControlSnapshot::read(&routed.world.state_store)?; - let current = stored.admitted.first().context("no admitted generation")?; - assert!(!routed.world.engine.refresh_admitted_topology(&stored, current)?); - routed.world.engine.validate_durable_authority(&stored)?; - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { - let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - routed.step()?; - let generation = admitted(&routed)?; - assert_eq!(generation.ready.voucher(), Voucher::Candidate); - // The valid, not-Ready correlation was neither admitted as a - // receipt nor consulted: the record carries no Odin cursor. - assert_eq!(generation.odin_publisher_sequence_cursor, 0); - Ok(()) - } - - #[test] - fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - routed.step()?; - let warmed = routed.transaction()?; - assert_eq!(warmed.phase, DeploymentPhase::Warming); - let Some(WarmingEvidence::RouteProofDirect { evidence }) = &warmed.warming else { - bail!("Warming evidence is not the candidate's own presence"); - }; - assert!(evidence.message_id.starts_with("candidate-")); - assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); - routed.assert_odin_untouched()?; - - // Nothing answers: the same step waits, and says why. - let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - silent.stub.hang_up.store(true, Ordering::SeqCst); - silent.step()?; - let waiting = silent.transaction()?; - assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); - assert!( - waiting - .last_error - .as_deref() - .is_some_and(|error| error.contains("candidate endpoint did not answer")), - "{:?}", - waiting.last_error - ); - Ok(()) - } - - #[test] - fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - routed.stub.capacity.store(1, Ordering::SeqCst); - let failure = routed.step().expect_err("a shortfall must be refused"); - // The shortfall is a typed value, not a sentence to search. - let disagrees = failure - .downcast_ref::() - .with_context(|| format!("the refusal is not typed: {failure:#}"))?; - let shortfall = disagrees - .disagreements - .iter() - .find(|disagreement| disagreement.code == "expected-capability-000-capacity") - .context("the capacity shortfall is not named")?; - assert!( - shortfall.expected.as_deref().is_some_and(|text| text.ends_with("capacity>=2")), - "{shortfall:?}" - ); - assert!(shortfall.observed.is_some(), "{shortfall:?}"); - assert!(admitted(&routed).is_err()); - Ok(()) - } - - #[test] - fn odin_itself_is_never_route_proof() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let mut expected = routed.stub.expected.clone(); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); - // Odin provides the rendezvous capability and declares no dependency. - expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); - assert!(expected.dependencies.is_empty()); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); - - // Route-proof evidence for it is refused: its admitted generation - // is what names the Odin authority, so it must carry one. - let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; - odin.target = "odin".into(); - odin.expected = expected; - odin.ready = route_proof_evidence(); - odin.latest_odin_observation = None; - odin.odin_authority = None; - assert!(error_text(odin.validate()).contains("not route-proof")); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn route_admission_installs_through_the_host_programs_without_reading_odin() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - // One real Routing step: the fragment goes in through the world's - // stub nginx, ufw and systemctl, and the stable listener proves it. - routed.step()?; - assert!(matches!( - routed.transaction()?.routing, - Some(RoutingEvidence::Promoted { .. }) - )); - assert_eq!(routed.world.route_stubs.count("systemctl reload"), 1); - assert_eq!(routed.world.route_stubs.count("ufw allow"), 1); - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - routed.stub.set_state("active"); - routed.step()?; - assert!(matches!( - routed.transaction()?.warming, - Some(WarmingEvidence::RouteProofDirect { .. }) - )); - Ok(()) - } - - #[test] - fn the_lease_refresh_asks_the_candidate_for_a_new_warming_presence() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let current = resident(&routed.world)?; - let prior = routed - .world - .engine - .rehydrate_warming_token(¤t.value, now_millis()?, false)?; - let (_, evidence, token) = routed - .world - .engine - .fresh_warming_for_lease(¤t, now_millis()?)? - .context("a warming candidate must refresh")?; - assert!(matches!(evidence, WarmingEvidence::RouteProofDirect { .. })); - assert_ne!(token.signed_presence_sha256(), prior.signed_presence_sha256()); - // A candidate that went Active is not warming: nothing to bind. - routed.stub.set_state("active"); - let failure = routed - .world - .engine - .fresh_warming_for_lease(¤t, now_millis()?) - .map(|_| ()) - .expect_err("an active candidate is not a warming one"); - assert!(format!("{failure:#}").contains("not warming"), "{failure:#}"); - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn boot_reauthenticates_a_live_route_proof_transactions_evidence() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.validate_durable_authority(&snapshot)?; - - // Warming evidence that does not answer the challenge it names. - let current = resident(&routed.world)?; - let mut forged = current.value.clone(); - let Some(WarmingEvidence::RouteProofDirect { evidence }) = &mut forged.warming else { - bail!("no route-proof Warming evidence"); - }; - evidence.message_id = "candidate-another".into(); - replace_transaction(&routed.world.state_store, ¤t, &forged)?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - let failure = routed - .world - .engine - .validate_durable_authority(&snapshot) - .expect_err("Warming evidence for another challenge"); - assert!(format!("{failure:#}").contains("exact challenge"), "{failure:#}"); - - // A Ready label carried by a presence that was only warming. - let current = resident(&routed.world)?; - let CandidateAnswer::Answered { evidence, .. } = routed - .world - .engine - .challenge_candidate(¤t.value, &["warming"], None)? - else { - bail!("the stub candidate did not answer"); - }; - let mut forged = current.value.clone(); - forged.warming = Some(WarmingEvidence::RouteProofDirect { - evidence: evidence.clone(), - }); - forged.ready = Some(ReadinessEvidence::RouteProof { evidence }); - replace_transaction(&routed.world.state_store, ¤t, &forged)?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - let failure = routed - .world - .engine - .validate_durable_authority(&snapshot) - .expect_err("Ready carried by a warming presence"); - assert!(format!("{failure:#}").contains("not active"), "{failure:#}"); - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() - -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let ready = routed.transaction()?; - assert!(ready.ready.as_ref().and_then(ReadinessEvidence::odin).is_some()); - routed.promote()?; - routed.step()?; - let generation = admitted(&routed)?; - generation.validate()?; - assert_eq!(generation.ready.voucher(), Voucher::Odin); - assert_eq!( - generation.odin_authority, - Some(routed.world.engine.bootstrap_odin_authority.clone()) - ); - assert!(generation.latest_odin_observation.is_some()); - // Boot re-authenticates every Odin receipt this generation carries. - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.validate_durable_authority(&snapshot)?; - Ok(()) - } - - #[test] - fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() - -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - routed.step()?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.validate_durable_authority(&snapshot)?; - assert_eq!( - routed.world.engine.current_odin_authority(&snapshot)?, - routed.world.engine.bootstrap_odin_authority - ); - Ok(()) - } - - impl RoutedWorld { - /// Boot would accept this store, and Odin was never read. - fn assert_boots(&self) -> Result<()> { - let snapshot = ControlSnapshot::read(&self.world.state_store)?; - self.world.engine.validate_durable_authority(&snapshot)?; - self.assert_odin_untouched() - } - - /// Step until `done`, checking after every step that boot would - /// still accept the store. - fn drive_until(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { - for _ in 0..20 { - if done(&self.transaction()?) { - return Ok(()); - } - self.step()?; - self.assert_boots()?; - } - bail!("the transaction never reached the awaited state") - } - - fn answer_now(&self) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { - match self.world.engine.challenge_candidate( - &self.transaction()?, - &["warming", "active"], - None, - )? { - CandidateAnswer::Answered { evidence, present } => Ok((evidence, present)), - CandidateAnswer::Silent(reason) => bail!("the candidate was silent: {reason}"), - } - } - } - - // ----------------------------------------------------------------- - // The stateful path, end to end through the Engine. - // ----------------------------------------------------------------- - - #[test] - fn a_stateful_route_proof_target_holds_its_lease_from_fencing_to_commit() -> Result<()> { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - routed.assert_boots()?; - - routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Prepared { .. })))?; - assert_eq!(routed.transaction()?.phase, DeploymentPhase::Leasing); - routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; - routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; - let lease_sha256 = routed - .transaction()? - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("no granted lease")? - .to_owned(); - - // The candidate has not picked the lease up: still warming. - routed.step()?; - assert_eq!( - routed.transaction()?.last_error.as_deref(), - Some("candidate is still warming") - ); - assert!(routed.transaction()?.ready.is_none()); - // Active without the lease Idunn granted is not Ready. - routed.stub.set_state("active"); - let refusal = routed - .step() - .expect_err("an active candidate without the granted lease is refused"); - assert!( - format!("{refusal:#}").contains("exact current process write lease"), - "{refusal:#}" - ); - assert!(routed.transaction()?.ready.is_none()); - // Active holding exactly that lease is. - *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); - routed.step()?; - assert!(matches!( - routed.transaction()?.ready, - Some(ReadinessEvidence::RouteProof { .. }) - )); - routed.assert_boots()?; - - routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; - routed.promote()?; - routed.assert_boots()?; - routed.step()?; - - let generation = admitted(&routed)?; - generation.validate()?; - assert_eq!(generation.ready.voucher(), Voucher::Candidate); - assert!(generation.expected.write_lease_required); - assert_eq!(generation.leasing.lease_sha256(), Some(lease_sha256.as_str())); - assert!(generation.odin_authority.is_none()); - routed.assert_boots()?; - Ok(()) - } - - // ----------------------------------------------------------------- - // Warming is one challenge, then the phase moves. - // ----------------------------------------------------------------- - - #[test] - fn warming_challenges_once_and_then_advances_on_every_tick() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - routed.step()?; - let recorded = routed - .transaction()? - .warming - .context("the first step recorded no Warming evidence")?; - assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); - - // The route preflight needs the host's nginx; its receipt is recorded - // here so the phase can move on. - let current = resident(&routed.world)?; - let mut next = current.value.clone(); - next.route_preflight = Some(candidate_preflight(&next)?); - replace_transaction(&routed.world.state_store, ¤t, &next)?; - - routed.drive_until(|t| t.phase == DeploymentPhase::Fencing)?; - assert_eq!( - routed.stub.candidate_hits.load(Ordering::SeqCst), - 1, - "a recorded Warming answer is never asked for again" - ); - assert_eq!(routed.transaction()?.warming, Some(recorded)); - Ok(()) - } - - #[test] - fn a_stateful_candidate_may_not_answer_its_warming_challenge_active() { - let mut expected = fixture_generation(FIXTURE_GENERATION).unwrap().1.expected; - expected.write_lease_required = true; - assert_eq!(route_proof_warming_states(&expected), ["warming"]); - expected.write_lease_required = false; - assert_eq!(route_proof_warming_states(&expected), ["warming", "active"]); - } - - #[test] - fn a_stateful_candidate_that_answers_warming_active_is_refused() -> Result<()> { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; - routed.stub.set_state("active"); - let refusal = routed - .step() - .expect_err("a stateful candidate cannot be active before it holds a lease"); - assert!(format!("{refusal:#}").contains("not warming"), "{refusal:#}"); - assert!(routed.transaction()?.warming.is_none()); - Ok(()) - } - - // ----------------------------------------------------------------- - // What a challenged presence must prove. - // ----------------------------------------------------------------- - - #[test] - fn a_presence_holds_the_write_lease_exactly_when_it_is_not_warming() -> Result<()> { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; - let authority = routed - .world - .engine - .runtime_authority(&routed.transaction()?)?; - let lease = sha256_id(b"the granted lease"); - let other = sha256_id(b"some other lease"); - let judge = |state: &'static str, - held: Option<&str>, - current: Option<&str>, - states: &[&str]| - -> Result<()> { - let challenged = now_millis()?; - routed.stub.set_state(state); - *routed.stub.lease.lock().unwrap() = held.map(str::to_owned); - let bytes = routed.stub.answer("candidate-probe")?; - routed - .world - .engine - .authenticate_challenged_presence( - &authority, - states, - current, - "candidate-probe", - challenged, - now_millis()?, - &bytes, - ) - .map(|_| ()) - }; - let either = ["warming", "active"]; - judge("warming", None, None, &either)?; - judge("active", Some(&lease), Some(&lease), &either)?; - for (state, held, current, named) in [ - // Warming holds nothing, whatever Idunn has granted. - ("warming", Some(lease.as_str()), None, "write lease"), - ("warming", Some(lease.as_str()), Some(lease.as_str()), "write lease"), - // Anything else holds exactly the current grant. - ("active", None, Some(lease.as_str()), "exact current process write lease"), - ("active", Some(lease.as_str()), None, "exact current process write lease"), - ( - "active", - Some(other.as_str()), - Some(lease.as_str()), - "exact current process write lease", - ), - ] { - let error = judge(state, held, current, &either) - .expect_err("a presence with the wrong lease is refused"); - assert!( - format!("{error:#}").contains(named), - "{state} {held:?} {current:?}: {error:#}" - ); - } - Ok(()) - } - - #[test] - fn a_presence_minted_before_its_challenge_is_refused() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let authority = routed - .world - .engine - .runtime_authority(&routed.transaction()?)?; - let bytes = routed.stub.answer("candidate-probe")?; - let minted = now_millis()?; - // Answered before the challenge that claims it: an old presence - // replayed for a new question. - let replay = routed - .world - .engine - .authenticate_challenged_presence( - &authority, - &["warming"], - None, - "candidate-probe", - minted + 1_000, - minted + 1_000, - &bytes, - ) - .expect_err("a presence minted before its challenge"); - assert!( - format!("{replay:#}").contains("minted before its challenge"), - "{replay:#}" - ); - let early = routed - .world - .engine - .authenticate_challenged_presence( - &authority, - &["warming"], - None, - "candidate-probe", - minted + 1_000, - minted, - &bytes, - ) - .expect_err("a receipt before its challenge"); - assert!(format!("{early:#}").contains("predates its challenge"), "{early:#}"); - Ok(()) - } - - #[test] - fn direct_warming_evidence_is_bound_to_the_bytes_it_authenticated() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let id = routed.transaction()?.transaction_id; - let direct = |evidence: RuntimePresenceEvidence| WarmingEvidence::RouteProofDirect { evidence }; - - let (evidence, present) = routed.answer_now()?; - SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present)?; - - // The digest is the authenticated presence's, but the bytes are not. - let (mut evidence, present) = routed.answer_now()?; - evidence.canonical_bytes.push(0); - let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) - .expect_err("bytes that are not the authenticated presence"); - assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); - - // The bytes are the authenticated presence's, but the digest is not. - let (mut evidence, present) = routed.answer_now()?; - evidence.canonical_sha256 = sha256_id(b"another presence"); - let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) - .expect_err("a digest that is not the authenticated presence's"); - assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); - - // Evidence about another answer altogether. - let (other, _) = routed.answer_now()?; - let (_, present) = routed.answer_now()?; - SequenceAdmittedWarming::from_direct_presence(id, direct(other), present) - .expect_err("evidence about another answer"); - Ok(()) - } - - // ----------------------------------------------------------------- - // Validation: candidate-vouched evidence belongs to route-proof targets. - // ----------------------------------------------------------------- - - #[test] - fn candidate_vouched_evidence_is_refused_on_a_target_odin_reports_on() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let (evidence, _) = routed.answer_now()?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let odin_reported = transaction_at(&world, DeploymentPhase::Warming)?; - assert_eq!( - ReadinessClass::of(odin_reported.expected.as_ref().unwrap()), - Ok(ReadinessClass::OdinCorrelated) - ); - - let mut warming = odin_reported.clone(); - warming.warming = Some(WarmingEvidence::RouteProofDirect { - evidence: evidence.clone(), - }); - assert!(error_text(warming.validate()).contains("not route-proof")); - - let mut ready = odin_reported.clone(); - ready.ready = Some(ReadinessEvidence::RouteProof { - evidence: evidence.clone(), - }); - assert!(error_text(ready.validate()).contains("not route-proof")); - - // Direct Odin warming is Odin's alone, by class: a route-proof - // target may not claim it, and a target that provides the - // rendezvous may, whatever it is named. - let mut direct = routed.transaction()?; - direct.warming = Some(WarmingEvidence::FirstOdinDirect { - evidence: evidence.clone(), - }); - assert!(error_text(direct.validate()).contains("reserved for Odin")); - let odin = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; - let mut direct = odin.transaction()?; - assert_eq!(direct.target, "service"); - direct.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); - direct.validate()?; - Ok(()) - } - - #[test] - fn a_route_proof_transaction_past_awaiting_ready_needs_the_candidates_ready_proof() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let routing = routed.transaction()?; - routing.validate()?; - let mut bare = routing.clone(); - bare.ready = None; - assert!(error_text(bare.validate()).contains("lacks Ready evidence")); - routed.promote()?; - let mut bare = routed.transaction()?; - assert_eq!(bare.phase, DeploymentPhase::Committing); - bare.ready = None; - assert!(error_text(bare.validate()).contains("lacks Ready evidence")); - Ok(()) - } - - // ----------------------------------------------------------------- - // The class is the Expected's alone. Evidence collected under another - // class is held and reported, never stepped, never repaired. - // ----------------------------------------------------------------- - - /// Odin's Ready for the world's target, as pre-B3 code recorded it for - /// every routed target. - fn odin_receipt_for( - routed: &RoutedWorld, - transaction: &DeploymentTransaction, - sequence: u64, - ) -> Result { - odin_receipt_with(routed, transaction, sequence, true) - } - - fn odin_receipt_with( - routed: &RoutedWorld, - transaction: &DeploymentTransaction, - sequence: u64, - ready: bool, - ) -> Result { - let now = now_millis()?; - let bytes = signed_correlation(&routed.world, transaction, sequence, ready)?; - let authenticated = routed.world.engine.authenticate_topology_bytes( - &ControlSnapshot::read(&routed.world.state_store)?, - transaction, - &bytes, - transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256), - now, - )?; - TopologyEvidence::from_authenticated(&authenticated, now) - } - - fn stored(generation: &AdmittedGeneration) -> Stored { - Stored { - value: generation.clone(), - envelope: CultCacheEnvelope { - key: generation.target.clone(), - r#type: AdmittedGeneration::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), - }, - } - } - - #[test] - fn a_pre_b3_transaction_over_a_route_proof_target_is_held_and_reported_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - - // Ready as pre-B3 Idunn wrote it: an Odin receipt. - let current = resident(&routed.world)?; - let mut legacy = current.value.clone(); - let receipt = odin_receipt_for(&routed, &legacy, 9)?; - legacy.latest_odin_observation = Some(receipt.clone()); - legacy.odin_publisher_sequence_cursor = 9; - legacy.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: receipt }); - replace_transaction(&routed.world.state_store, ¤t, &legacy)?; - - let disagreement = ReadinessDisagreement::WrongVoucher { - target: "service".into(), - required: ReadinessClass::RouteProof, - collected: Voucher::Odin, - }; - assert_eq!(legacy.readiness_disagreement(), Some(disagreement.clone())); - // The record is readable and boot accepts it: held, not refused. - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.validate_durable_authority(&snapshot)?; - - let hits = ( - routed.stub.candidate_hits.load(Ordering::SeqCst), - routed.stub.stable_hits.load(Ordering::SeqCst), - ); - for _ in 0..4 { - assert!(!routed.world.engine.run_scheduler_tick()?); - } - let held = resident(&routed.world)?; - assert_eq!(held.value.phase, DeploymentPhase::Routing); - assert!(held.value.pre_fencing_abort.is_none()); - assert!(held.value.post_fencing_abort.is_none()); - assert_eq!(held.value.ready, legacy.ready, "nothing re-derived the evidence"); - assert_eq!( - held.value.last_error.as_deref(), - Some(disagreement.to_string().as_str()) - ); - // Reported once: later ticks leave the record alone. - routed.world.engine.run_scheduler_tick()?; - assert_eq!(resident(&routed.world)?.envelope, held.envelope); - assert_eq!( - hits, - ( - routed.stub.candidate_hits.load(Ordering::SeqCst), - routed.stub.stable_hits.load(Ordering::SeqCst) - ), - "a held transaction is asked nothing" - ); - routed.assert_odin_untouched()?; - - // An abort already under way is not held: it never reads the class. - let mut aborting = held.value.clone(); - aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); - replace_transaction(&routed.world.state_store, &held, &aborting)?; - let after = resident(&routed.world)?; - let _ = routed.world.engine.resume_candidate(&after); - assert_ne!( - resident(&routed.world)?.value.last_error.as_deref(), - Some(disagreement.to_string().as_str()), - "an abort in progress runs instead of being held" - ); - Ok(()) - } - - #[test] - fn a_pre_b3_generation_over_a_route_proof_target_boots_and_is_reported_not_refreshed() - -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let at_routing = routed.transaction()?; - routed.promote()?; - routed.step()?; - - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - let current = snapshot.admitted.first().context("nothing was admitted")?; - // Receipts that would not survive being re-proved: authentic, but - // not Ready. Boot re-proves against the current Odin authority and - // requires exact semantic Ready, never what a generation stores, so - // this record is only bootable if boot does not re-prove it. - let receipt = odin_receipt_with(&routed, &at_routing, 9, false)?; - let mut legacy = current.value.clone(); - legacy.ready = ReadinessEvidence::OdinCorrelated { - evidence: receipt.clone(), - }; - legacy.latest_odin_observation = Some(receipt); - // The stored authority is present because an Odin-tagged record - // carries one; it is not what boot reads. - let stranger = cultnet_rs::enroll_service_identity_at::( - &routed.world.root.join("identities/stranger.cc"), - )?; - legacy.odin_authority = Some(AdmittedOdinAuthority::from_anchor(&stranger.trust_anchor()?)?); - legacy.odin_publisher_sequence_cursor = 9; - assert!( - SingleFileMessagePackBackingStore::new(&routed.world.state_store) - .compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: legacy.target.clone(), - current: Some(current.envelope.clone()), - }], - &[admitted_envelope(&legacy, now_millis()?)?], - )? - ); - - assert_eq!( - legacy.readiness(), - Err(ReadinessDisagreement::WrongVoucher { - target: "service".into(), - required: ReadinessClass::RouteProof, - collected: Voucher::Odin, - }) - ); - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - let current = snapshot.admitted.first().context("the generation is gone")?; - // The class chooses the path: no Odin receipt is re-proved at boot - // and none is refreshed, though the evidence is an Odin receipt. - routed.world.engine.validate_durable_authority(&snapshot)?; - assert!(!routed.world.engine.refresh_admitted_topology(&snapshot, current)?); - routed.assert_odin_untouched()?; - - routed.world.engine.supervise_one_admitted_generation()?; - assert!( - routed - .world - .engine - .fault_reports - .lock() - .unwrap() - .contains_key("generation:service"), - "supervision reports the disagreement" - ); - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn only_a_transaction_that_still_reads_its_class_is_held_for_its_evidence() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let at_routing = routed.transaction()?; - // What pre-B3 Idunn would have collected: an Odin receipt. - let mut legacy = at_routing.clone(); - legacy.ready = Some(ReadinessEvidence::OdinCorrelated { - evidence: odin_receipt_for(&routed, &at_routing, 9)?, - }); - let disagreement = legacy.readiness_disagreement(); - assert!(disagreement.is_some()); - assert_eq!(legacy.held_disagreement(), disagreement); - - // Finished work, and an abort under way, never read the class. - let mut completed = legacy.clone(); - completed.completion = Some(TransactionCompletion::Admitted { - generation_id: "generation-1".into(), - }); - assert_eq!(completed.held_disagreement(), None); - let mut aborting = legacy.clone(); - aborting.pre_fencing_abort = Some(PreFencingAbort { - error: "operator abort".into(), - candidate_cleanup: CleanupEvidence::Pending, - topology_reconciliation: CleanupEvidence::Pending, - source_cleanup: CleanupEvidence::Pending, - }); - assert_eq!(aborting.held_disagreement(), None); - let mut aborting = legacy; - aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); - assert_eq!(aborting.held_disagreement(), None); - Ok(()) - } - - #[test] - fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { - let world = EngineFixture::new()?; - // Odin's store is unreadable: any read of it is an error. - std::fs::write(&world.engine.options.odin_correlation_store, b"odin is not a cache")?; - - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - let target = generation.expected.target.clone(); - assert_eq!(generation.readiness(), Ok(ReadinessClass::OdinCorrelated)); - generation.expected.dependencies.clear(); - generation.expected.route = None; - let undeclared = UndeclaredReadiness { target: target.clone() }; - assert_eq!( - generation.readiness(), - Err(ReadinessDisagreement::Undeclared(undeclared.clone())) - ); - assert!( - !world - .engine - .refresh_admitted_topology(&ControlSnapshot::default(), &stored(&generation))? - ); - - // The mirror: Odin declared, but the receipt is the candidate's. - let (_, mut odin_class) = fixture_generation(FIXTURE_GENERATION)?; - odin_class.ready = route_proof_evidence(); - assert_eq!( - odin_class.readiness(), - Err(ReadinessDisagreement::WrongVoucher { - target: target.clone(), - required: ReadinessClass::OdinCorrelated, - collected: Voucher::Candidate, - }) - ); - - // A transaction says the same about what it has collected. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let (evidence, _) = routed.answer_now()?; - let odin_world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&odin_world, DeploymentPhase::Warming)?; - assert_eq!(seeded.readiness_disagreement(), None); - let mut collected = seeded.clone(); - collected.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); - assert_eq!( - collected.readiness_disagreement(), - Some(ReadinessDisagreement::WrongVoucher { - target: "service".into(), - required: ReadinessClass::OdinCorrelated, - collected: Voucher::Candidate, - }) - ); - let mut bare = seeded.clone(); - bare.expected.as_mut().unwrap().dependencies.clear(); - assert_eq!( - bare.readiness_disagreement(), - Some(ReadinessDisagreement::Undeclared(UndeclaredReadiness { - target: "service".into() - })) - ); - let mut no_expected = seeded; - no_expected.expected = None; - assert_eq!(no_expected.readiness_disagreement(), None); - Ok(()) - } - - // ----------------------------------------------------------------- - // Odin is whichever target provides the rendezvous. - // ----------------------------------------------------------------- - - #[test] - fn odin_is_the_target_that_provides_the_rendezvous_not_the_target_named_odin() -> Result<()> { - let world = EngineFixture::new()?; - let (_, fixture) = fixture_generation(FIXTURE_GENERATION)?; - - // A target named "odin" that merely depends on Odin is not Odin. - let mut named = fixture.clone(); - named.target = "odin".into(); - named.expected.target = "odin".into(); - let mut snapshot = ControlSnapshot::default(); - snapshot.admitted.push(stored(&named)); - assert!(snapshot.admitted_odin().is_none()); - assert_eq!( - world.engine.current_odin_authority(&snapshot)?, - world.engine.bootstrap_odin_authority - ); - - // The target that provides the rendezvous is, whatever it is called. - let mut provider = fixture.clone(); - provider.target = "verse".into(); - provider.expected.target = "verse".into(); - provider.expected.dependencies.clear(); - provide_odin(&mut provider.expected); - let authority = AdmittedOdinAuthority::from_anchor(&world.odin_signer.trust_anchor()?)?; - provider.odin_authority = Some(authority.clone()); - snapshot.admitted.push(stored(&provider)); - assert_eq!( - snapshot.admitted_odin().map(|found| found.value.target.as_str()), - Some("verse") - ); - assert_eq!( - snapshot.admitted_odin().and_then(|found| found.value.odin_authority.clone()), - Some(authority) - ); - Ok(()) - } - - #[test] - fn a_target_that_provides_the_rendezvous_warms_as_odin_whatever_it_is_named() -> Result<()> { - // Nothing else has been admitted that could observe it, so Odin - // observes itself directly -- and only a stateful incarnation may. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; - assert_eq!(routed.transaction()?.target, "service"); - let refusal = routed - .step() - .expect_err("a stateless Odin cannot be observed directly"); - assert!( - format!("{refusal:#}").contains("direct Odin warming must be a stateful incarnation"), - "{refusal:#}" - ); - Ok(()) - } - - #[test] - fn direct_odin_warming_is_reserved_to_odin_by_class_not_by_name() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let mut named = routed.transaction()?; - named.target = "odin".into(); - let error = routed - .world - .engine - .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) - .expect_err("a target named odin that is not Odin"); - assert!( - format!("{error:#}").contains("reserved for a stateful Odin incarnation"), - "{error:#}" - ); - Ok(()) - } - - #[test] - fn a_stateful_target_that_provides_the_rendezvous_is_observed_directly_whatever_it_is_named() - -> Result<()> { - let provider = build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true, true, CommandKind::Continuity)?; - let transaction = provider.transaction()?; - assert_eq!(transaction.target, "service"); - assert!(transaction.expected.as_ref().unwrap().write_lease_required); - provider.stub.set_state("warming"); - *provider.stub.reply.lock().unwrap() = Reply::FirstOdinWarming; - let challenged_at = now_millis()?; - let answer = provider.stub.answer("warming-probe")?; - provider.world.engine.authenticate_first_odin_warming_presence( - &transaction, - "warming-probe", - challenged_at, - now_millis()?, - &answer, - )?; - - // The reverse: named odin, stateful, but nothing provides the - // rendezvous. - let stateful = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; - let mut named = stateful.transaction()?; - assert!(named.expected.as_ref().unwrap().write_lease_required); - named.target = "odin".into(); - let error = stateful - .world - .engine - .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) - .expect_err("a stateful target named odin that does not provide the rendezvous"); - assert!( - format!("{error:#}").contains("reserved for a stateful Odin incarnation"), - "{error:#}" - ); - Ok(()) - } - - // ----------------------------------------------------------------- - // A bad answer is an error. Only silence waits. - // ----------------------------------------------------------------- - - #[test] - fn a_bad_answer_to_a_challenge_is_an_error_and_only_silence_waits() -> Result<()> { - for reply in [Reply::ForeignChallengeId, Reply::HttpFailure] { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - *routed.stub.reply.lock().unwrap() = reply; - let error = match routed.world.engine.challenge_candidate( - &routed.transaction()?, - &["warming"], - None, - ) { - Err(error) => error, - Ok(_) => bail!("a bad answer was taken for silence"), - }; - assert!( - matches!( - error.downcast_ref::(), - Some(ChallengeFailure::Refused(_)) - ), - "{error:#}" - ); - // Before the fence a candidate that answers wrongly is aborted, - // not waited on. - routed.world.engine.run_scheduler_tick()?; - let after = routed.transaction()?; - assert!(after.pre_fencing_abort.is_some(), "the bad answer aborted it"); - assert!(after.warming.is_none()); - } - - let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - silent.stub.hang_up.store(true, Ordering::SeqCst); - assert!(matches!( - silent.world.engine.challenge_candidate(&silent.transaction()?, &["warming"], None)?, - CandidateAnswer::Silent(_) - )); - Ok(()) - } - - // ----------------------------------------------------------------- - // B2: the route gate, the meters and the supervision pass, through - // the Engine, over the world's stub host programs. - // ----------------------------------------------------------------- - - impl RoutedWorld { - /// Run a fresh world to an admitted generation whose transaction - /// is retired: what admitted-route supervision looks at. - fn admit(&self) -> Result<()> { - self.stub.set_state("active"); - self.run_to_routing()?; - self.promote()?; - self.drive_until(|transaction| transaction.is_terminal())?; - assert!(self.world.engine.retire_one_terminal_transaction()?); - Ok(()) - } - - fn fragment_path(&self) -> PathBuf { - self.world.root.join("service.conf") - } - - /// The target's route ledger holds `entries` recent actuations. - fn ledger(&self, entries: usize) -> Result<()> { - let now = now_millis()?; - let mut meters = TargetSupervision::new("service"); - meters.route_actuations = (0..entries as u64).map(|offset| now - 1_000 + offset).collect(); - set_meters(&self.world, &meters) - } - - #[cfg(unix)] - fn reloads(&self) -> usize { - self.world.route_stubs.count("systemctl reload") - } - - /// Something else rewrote the fragment: the admitted membership drifted. - fn drift(&self) -> Result<()> { - std::fs::write(self.fragment_path(), b"drifted bytes\n")?; - Ok(()) - } - - fn tick(&self) -> Result { - self.world.engine.supervise_one_admitted_generation() - } - } - - // A store that reads but cannot be written (ENOSPC on the state volume) - // is simulated with RLIMIT_FSIZE, which is process-wide: the test - // re-runs itself in a child process so no sibling test loses its writes. - // Returns true in the child, which runs the body; the parent returns - // false after asserting the child passed and hands back its stderr. - #[cfg(unix)] - fn in_unwritable_store_child(test: &str) -> Result> { - if std::env::var_os("IDUNN_TEST_UNWRITABLE_STORE_CHILD").is_some() { - return Ok(None); - } - let path = module_path!().split_once("::").map_or("", |(_, rest)| rest); - let output = std::process::Command::new(std::env::current_exe()?) - .args(["--exact", &format!("{path}::{test}"), "--nocapture", "--test-threads=1"]) - .env("IDUNN_TEST_UNWRITABLE_STORE_CHILD", "1") - .output()?; - let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); - assert!( - output.status.success(), - "child failed:\n{}\n{stderr}", - String::from_utf8_lossy(&output.stdout) - ); - assert!( - String::from_utf8_lossy(&output.stdout).contains("1 passed"), - "the child ran no test: {}", - String::from_utf8_lossy(&output.stdout) - ); - Ok(Some(stderr)) - } - - /// Cap file growth one byte under the store's size: reads work, every - /// write fails. Child process only. - #[cfg(unix)] - fn break_store_writes(world: &EngineFixture) { - let size = std::fs::metadata(&world.state_store).unwrap().len(); - unsafe { - libc::signal(libc::SIGXFSZ, libc::SIG_IGN); - let limit = libc::rlimit { rlim_cur: size - 1, rlim_max: libc::RLIM_INFINITY }; - assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0); - } - assert!( - set_meters(world, &TargetSupervision::new("probe")).is_err(), - "the store is still writable" - ); - assert!(ControlSnapshot::read(&world.state_store).is_ok(), "the store is unreadable"); - } - - #[cfg(unix)] - fn challenge_due(world: &EngineFixture) -> Option { - world.engine.challenge_wait.lock().unwrap().get("service").map(|wait| wait.due_unix_millis) - } - - #[cfg(unix)] - #[test] - fn an_unwritable_store_neither_spins_a_failing_challenge_nor_reports_progress() -> Result<()> { - let name = "an_unwritable_store_neither_spins_a_failing_challenge_nor_reports_progress"; - let Some(stderr) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.route_stubs.refuse_reloads()?; - routed.drift()?; - let before = routed.reloads(); - let began = now_millis()?; - break_store_writes(&routed.world); - let stored_wait = |routed: &RoutedWorld| -> Result> { - Ok(ControlSnapshot::read(&routed.world.state_store)? - .admitted - .iter() - .find_map(|stored| stored.value.route_supervision.as_ref()) - .and_then(|state| state.next_challenge_at_unix_millis)) - }; - let stored = stored_wait(&routed)?; - - // Fifty laps: one attempt, nothing reported as progress. - for _ in 0..50 { - assert!(!routed.tick()?, "a write the store refused was reported as progress"); - } - assert_eq!(routed.reloads() - before, 1, "the failing reload spun"); - assert_eq!(stored_wait(&routed)?, stored, "the record changed though writes fail"); - let due = challenge_due(&routed.world).expect("no in-memory wait after a failed record"); - let age = routed.world.engine.options.topology_maximum_age_millis; - assert!(due >= began + age, "the wait is shorter than one observation age: {due}"); - - // The wait expiring is the only thing that admits another - // attempt, and each one is again a single reload with no - // progress. The failed record is said once, not per attempt. - for attempt in 2..=4 { - routed.world.engine.challenge_wait.lock().unwrap().clear(); - assert!(!routed.tick()?); - assert!(!routed.tick()?); - assert_eq!(routed.reloads() - before, attempt); - } - return Ok(()); - }; - assert_eq!( - stderr.matches("could not record the failed route challenge of service").count(), - 1, - "{stderr}" - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_unwritable_store_challenges_a_stale_route_once_per_wait() -> Result<()> { - let name = "an_unwritable_store_challenges_a_stale_route_once_per_wait"; - let Some(stderr) = in_unwritable_store_child(name)? else { - let mut routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.engine.options.topology_maximum_age_millis = 8_000; - std::thread::sleep(Duration::from_millis(8_300)); - break_store_writes(&routed.world); - assert!(!routed.tick()?, "a write the store refused was reported as progress"); - let first = challenge_due(&routed.world).expect("the proved challenge left no wait"); - for _ in 0..5 { - std::thread::sleep(Duration::from_millis(3)); - assert!(!routed.tick()?); - } - assert_eq!(challenge_due(&routed.world), Some(first), "the route was challenged again"); - return Ok(()); - }; - assert_eq!( - stderr.matches("could not record the proved route challenge of service").count(), - 1, - "{stderr}" - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_landed_challenge_record_is_progress_and_leaves_no_in_memory_wait() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.route_stubs.refuse_reloads()?; - routed.drift()?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.challenge_wait.lock().unwrap().insert( - "service".into(), - ChallengeWait { due_unix_millis: 0, stamp: snapshot.target_stamp("service") }, - ); - assert!(routed.tick()?, "a landed record is progress"); - assert_eq!(challenge_due(&routed.world), None); - Ok(()) - } - - /// A fragment that re-drifts after every restore, with the store - /// unwritable, is reloaded once per wait, however many laps run. - #[cfg(unix)] - #[test] - fn a_re_drifting_fragment_is_reloaded_once_per_wait_when_the_store_is_unwritable() -> Result<()> { - let name = "a_re_drifting_fragment_is_reloaded_once_per_wait_when_the_store_is_unwritable"; - let Some(_) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let before = routed.reloads(); - break_store_writes(&routed.world); - for _ in 0..30 { - routed.drift()?; - assert!(!routed.tick()?, "a refused write was reported as progress"); - } - assert_eq!(routed.reloads() - before, 1, "a re-drifting fragment reloaded every lap"); - for attempt in 2..=4 { - routed.world.engine.challenge_wait.lock().unwrap().clear(); - routed.drift()?; - assert!(!routed.tick()?); - routed.drift()?; - assert!(!routed.tick()?); - assert_eq!(routed.reloads() - before, attempt); - } - return Ok(()); - }; - Ok(()) - } - - /// The wait of a proved challenge whose record failed is one - /// observation age from the attempt. - #[cfg(unix)] - #[test] - fn a_proved_challenge_that_cannot_be_recorded_waits_one_observation_age() -> Result<()> { - let name = "a_proved_challenge_that_cannot_be_recorded_waits_one_observation_age"; - let Some(_) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let before = routed.reloads(); - break_store_writes(&routed.world); - let began = now_millis()?; - routed.drift()?; - assert!(!routed.tick()?); - assert_eq!(routed.reloads() - before, 1, "the repair did not run"); - let due = challenge_due(&routed.world).expect("no wait"); - let age = routed.world.engine.options.topology_maximum_age_millis; - assert!(due >= began + age, "the wait is shorter than one age: {}", due - began); - return Ok(()); - }; - Ok(()) - } - - /// The stored schedule sets the in-memory wait: with three failures - /// recorded, the fourth failed challenge waits eight ages (capped), and - /// no repair runs inside that wait. - #[cfg(unix)] - #[test] - fn a_failed_challenge_that_cannot_be_recorded_waits_as_long_as_the_stored_schedule() -> Result<()> { - let name = "a_failed_challenge_that_cannot_be_recorded_waits_as_long_as_the_stored_schedule"; - let Some(_) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - edit_incumbent(&routed.world, |generation| { - let state = generation.route_supervision.as_mut().unwrap(); - state.consecutive_failures = 3; - state.next_challenge_at_unix_millis = Some(1); - state.degraded_since_unix_millis = Some(1); - })?; - break_store_writes(&routed.world); - let began = now_millis()?; - routed.drift()?; - assert!(!routed.tick()?); - let due = challenge_due(&routed.world).expect("no wait"); - let age = routed.world.engine.options.topology_maximum_age_millis; - let want = (age * 8).min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); - assert!(due >= began + want, "the wait ignores the stored doubling: {}", due - began); - let reloads = routed.reloads(); - for _ in 0..10 { - routed.drift()?; - assert!(!routed.tick()?); - } - assert_eq!(routed.reloads(), reloads, "the route was repaired inside its wait"); - return Ok(()); - }; - Ok(()) - } - - /// A lost race is progress, and the next pass is governed by the - /// winner's record, not by an in-memory wait of the loser. - #[cfg(unix)] - #[test] - fn a_lost_race_leaves_no_in_memory_wait() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.route_stubs.refuse_reloads()?; - let old = ControlSnapshot::read(&routed.world.state_store)?; - let stale = old.admitted_for("service").context("no generation")?; - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().last_challenge_at_unix_millis = Some(5); - })?; - routed.drift()?; - assert!(routed.world.engine.supervise_admitted_route(&old, stale)?); - assert_eq!(challenge_due(&routed.world), None, "a lost race left an in-memory wait"); - Ok(()) - } - - /// The wait left by a full disk does not outlive it: once any write for - /// the target lands, the stored schedule owns the target again and a - /// drift is repaired without waiting. - #[cfg(unix)] - #[test] - fn a_wait_left_by_a_full_disk_ends_when_any_write_for_the_target_lands() -> Result<()> { - let name = "a_wait_left_by_a_full_disk_ends_when_any_write_for_the_target_lands"; - let Some(_) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let before = routed.reloads(); - break_store_writes(&routed.world); - routed.drift()?; - assert!(!routed.tick()?); - assert!(challenge_due(&routed.world).is_some(), "no wait to outlive"); - assert_eq!(routed.reloads() - before, 1); - - // Space is freed. Nothing has landed yet, so the wait still holds. - unsafe { - let limit = libc::rlimit { rlim_cur: libc::RLIM_INFINITY, rlim_max: libc::RLIM_INFINITY }; - assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0); - } - routed.drift()?; - assert!(!routed.tick()?); - assert_eq!(routed.reloads() - before, 1, "the wait did not hold before any write landed"); - - // Something else writes for the target: the wait is over. - routed.ledger(1)?; - routed.drift()?; - assert!(routed.tick()?, "the drift was not repaired"); - assert_eq!(routed.reloads() - before, 2, "a stale wait held the repair"); - assert_eq!(challenge_due(&routed.world), None); - return Ok(()); - }; - Ok(()) - } - - /// A wait belongs to an admitted target: once the target is retired, - /// the wait is gone. - #[cfg(unix)] - #[test] - fn a_wait_ends_when_its_target_is_retired() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.challenge_wait.lock().unwrap().insert( - "service".into(), - ChallengeWait { - due_unix_millis: now_millis()? + 500_000, - stamp: snapshot.target_stamp("service"), - }, - ); - assert!(!routed.tick()?); - assert!(challenge_due(&routed.world).is_some(), "an unchanged store dropped the wait"); - assert!( - SingleFileMessagePackBackingStore::new(&routed.world.state_store) - .delete_batch_if_unchanged(&[snapshot.admitted_for("service").unwrap().envelope.clone()])? - ); - routed.tick()?; - assert_eq!(challenge_due(&routed.world), None, "a retired target kept its wait"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_full_ledger_still_restores_the_admitted_and_incumbent_route() -> Result<()> { - let full = ROUTE_ACTUATION_CEILING; - // The newest recorded Survival, or 0; the Forward ledger stays full. - let newest = |world: &EngineFixture| -> Result { - let meters = meters_of(world, "service")?; - assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); - Ok(meters.route_survivals.last().copied().unwrap_or(0)) - }; - - // (i) A post-fence abort withdraws the candidate's membership. - let aborting = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - aborting.ledger(full)?; - let before = newest(&aborting.world)?; - aborting - .world - .engine - .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; - aborting.step()?; - assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); - assert!(newest(&aborting.world)? > before, "the withdrawal was not recorded"); - - // (ii) Supervision repairs a drifted admitted fragment. - let supervised = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - supervised.admit()?; - supervised.ledger(full)?; - let (before, reloads) = (newest(&supervised.world)?, supervised.reloads()); - supervised.drift()?; - assert!(supervised.tick()?); - assert_eq!(supervised.reloads(), reloads + 1, "the repair did not run"); - assert!(newest(&supervised.world)? > before, "the repair was not recorded"); - assert!(admitted(&supervised)?.route_supervision.unwrap().degraded_since_unix_millis.is_none()); - - // (iii) A failed proof at Routing rolls the route back, though the - // install that preceded it took the last free slot. The rollback - // is recorded and leaves the Forward ledger exactly full. - let failing = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - failing.stub.set_state("active"); - failing.run_to_routing()?; - failing.ledger(full - 1)?; - failing.stub.hang_up.store(true, Ordering::SeqCst); - let before = now_millis()?; - assert!(failing.step().is_err(), "an unanswered stable route was admitted"); - assert_eq!(failing.reloads(), 2, "install then rollback"); - assert!(newest(&failing.world)? >= before, "the rollback was not recorded"); - assert!(!failing.fragment_path().exists(), "the rollback left the candidate's route"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_continuity_restart_is_never_refused_by_a_full_ledger() -> Result<()> { - // Warming: the preflight of a continuity runs and is recorded, - // and the Forward ledger is left as it was. - let warming = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - warming.step()?; - assert!(warming.transaction()?.warming.is_some()); - warming.ledger(ROUTE_ACTUATION_CEILING)?; - let ledger = meters_of(&warming.world, "service")?.route_actuations; - warming.step()?; - assert!(warming.transaction()?.route_preflight.is_some(), "the preflight was refused"); - let after = meters_of(&warming.world, "service")?; - assert_eq!(after.route_survivals.len(), 1, "the preflight was not recorded"); - assert_eq!(after.route_actuations, ledger, "a survival was counted"); - - // Routing: its install runs and is recorded, and not counted. - let routing = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routing.stub.set_state("active"); - routing.run_to_routing()?; - routing.ledger(ROUTE_ACTUATION_CEILING)?; - let ledger = meters_of(&routing.world, "service")?.route_actuations; - routing.step()?; - assert!(routing.reloads() >= 1, "the install did not run"); - assert!(routing.fragment_path().exists()); - let after = meters_of(&routing.world, "service")?; - assert_eq!(after.route_survivals.len(), 1, "the install was not recorded"); - assert_eq!(after.route_actuations, ledger, "a survival was counted"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_survival_that_cannot_be_recorded_runs_and_a_forward_is_refused() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let transaction = routed.transaction()?; - let preflight = transaction.route_preflight.clone().context("no preflight")?; - let driver = routed.world.engine.route_driver( - transaction - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1 - .route - .context("no route binding")?, - ); - // The ledger cannot be read, so nothing can be recorded. - std::fs::write(&routed.world.state_store, b"unreadable")?; - let engine = &routed.world.engine; - - let refused = engine - .route_gate("service", CommandKind::Deploy) - .admit(RouteActuation::Forward) - .unwrap_err(); - assert!(refused.downcast_ref::().is_none(), "{refused:#}"); - assert!(engine.charge_route_actuation("service", RouteActuation::Forward).is_err()); - - engine.charge_route_actuation("service", RouteActuation::Survival)?; - driver.withdraw_candidate_membership( - &preflight, - &engine.route_gate("service", CommandKind::Deploy), - )?; - assert_eq!(routed.reloads(), 1, "the unrecorded withdrawal did not run"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_second_failed_challenge_waits_twice_as_long_before_the_next_repair() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - let elapse = |routed: &RoutedWorld| { - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = - Some(1); - }) - }; - - routed.drift()?; - assert!(routed.tick()?); - elapse(&routed)?; - routed.drift()?; - assert!(routed.tick()?); - let state = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(state.consecutive_failures, 2); - assert_eq!( - state.next_challenge_at_unix_millis.unwrap() - state.last_challenge_at_unix_millis.unwrap(), - 2 * DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS - ); - - // The wait is longer than one observation age: no challenge, so no - // repair and no reload, however often the pass runs. - let reloads = routed.reloads(); - for _ in 0..20 { - routed.drift()?; - routed.tick()?; - } - assert_eq!(routed.reloads(), reloads, "the route was repaired inside its wait"); - assert_eq!( - admitted(&routed)?.route_supervision.unwrap().consecutive_failures, - 2 - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_proved_challenge_that_cannot_be_recorded_ends_the_pass_without_failing_it() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let old = ControlSnapshot::read(&routed.world.state_store)?; - let stale = old.admitted_for("service").context("no generation")?; - // Something else writes the generation after the pass read it. - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().last_challenge_at_unix_millis = Some(5); - })?; - routed.drift()?; - assert!(routed.world.engine.supervise_admitted_route(&old, stale)?); - assert_eq!( - admitted(&routed)?.route_supervision.unwrap().last_challenge_at_unix_millis, - Some(5), - "the stale write replaced the newer generation" - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_first_routed_deploy_is_metered_and_refused_at_the_ceiling() -> Result<()> { - let metered = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - metered.stub.set_state("active"); - metered.run_to_routing()?; - assert!(ControlSnapshot::read(&metered.world.state_store)?.targets.is_empty()); - metered.step()?; - assert_eq!(meters_of(&metered.world, "service")?.route_actuations.len(), 1); - - let refused = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - refused.stub.set_state("active"); - refused.run_to_routing()?; - refused.ledger(ROUTE_ACTUATION_CEILING)?; - let error = refused.step().unwrap_err(); - assert!(error.downcast_ref::().is_some(), "{error:#}"); - assert_eq!(refused.world.route_stubs.count(""), 0, "a refused install ran a program"); - assert!(!refused.fragment_path().exists()); - - // Past the fence the refusal is resumable, not an abort: the resume - // backoff spaces it and no actuation runs while it waits. - assert!(!refused.world.engine.resume_one_transaction()?); - let waiting = refused.transaction()?; - assert_eq!(waiting.phase, DeploymentPhase::Routing); - assert!(waiting.post_fencing_abort.is_none()); - assert!(waiting.last_error.as_deref().is_some_and(|text| text.contains("ceiling"))); - assert_eq!(refused.world.route_stubs.count(""), 0); - Ok(()) - } - - #[test] - fn a_stepped_back_clock_is_settled_where_the_decision_is_made() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let ahead = now_millis()? + 86_400_000; - - // A supervision pass that would decide from the restart log settles - // it first: the write ends the pass, and nothing is minted from - // entries that only look recent because the clock stepped back. - let mut restarts = TargetSupervision::new("service"); - restarts.continuity_restarts = vec![ahead; CONTINUITY_PAGING_RESTARTS]; - set_meters(&routed.world, &restarts)?; - routed.workload.kill(); - assert!(routed.tick()?); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let settled = meters_of(&routed.world, "service")?; - assert!(settled.continuity_restarts.iter().all(|&at| at < ahead)); - - // A refused forward actuation writes the clamp, so the window runs - // from now and not from the day the clock stood on. - let mut ledger = TargetSupervision::new("service"); - ledger.route_actuations = vec![ahead; ROUTE_ACTUATION_CEILING]; - set_meters(&routed.world, &ledger)?; - let refused = routed - .world - .engine - .charge_route_actuation("service", RouteActuation::Forward) - .unwrap_err(); - assert!(refused.downcast_ref::().is_some()); - let settled = meters_of(&routed.world, "service")?; - assert!(settled.route_actuations.iter().all(|&at| at < ahead)); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_refused_preflight_starts_no_private_mount_unit_and_fails_at_once_with_the_reopen_time() - -> Result<()> { - let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; - routed.step()?; - assert!(routed.transaction()?.warming.is_some()); - routed.ledger(ROUTE_ACTUATION_CEILING)?; - let reopens_at = meters_of(&routed.world, "service")? - .route_reopens_at(now_millis()?) - .context("the ledger is not full")?; - - // The refusal is an error of its own phase, before the fence: the - // deployment fails now, and its error carries the reopen time. - let id = routed.transaction()?.transaction_id; - routed.world.engine.resume_one_transaction()?; - let abort = routed - .transaction()? - .pre_fencing_abort - .context("the refusal did not abort the deployment")?; - assert!(abort.error.contains(&format!("reopens at unix ms {reopens_at}")), "{}", abort.error); - for _ in 0..8 { - if record_of(&routed.world, &id)?.completion.is_some() { - break; - } - routed.step()?; - } - let Some(TransactionCompletion::FailedBeforeFencing { error }) = - record_of(&routed.world, &id)?.completion - else { - let record = record_of(&routed.world, &id)?; - bail!( - "the deployment did not fail before the fence: phase {:?} completion {:?} abort {:?} error {:?}", - record.phase, record.completion, record.pre_fencing_abort, record.last_error - ); - }; - assert!(error.contains(&format!("reopens at unix ms {reopens_at}")), "{error}"); - assert_eq!(routed.world.route_stubs.count(""), 0, "a program ran"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.workload.kill(); - routed.stub.hang_up.store(true, Ordering::SeqCst); - routed.drift()?; - - // Tick 1: the repair charges the target's meters and the proof - // fails. Both are writes, so they end the pass: nothing else is - // decided from the snapshot they made stale. - assert!(routed.tick()?); - let state = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(state.consecutive_failures, 1); - assert!(state.degraded_since_unix_millis.is_some()); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let meters = meters_of(&routed.world, "service")?; - assert_eq!(meters.route_survivals.len(), 2); - assert!(meters.route_actuations.is_empty()); - - // Tick 2: the route waits, and the dead unit is restarted from a - // fresh read, counted in the same write that mints it. - assert!(routed.tick()?); - assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); - assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_proved_challenge_clears_the_degradation() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - routed.drift()?; - assert!(routed.tick()?); - let degraded = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(degraded.consecutive_failures, 1); - assert!(degraded.degraded_since_unix_millis.is_some()); - assert!(degraded.next_challenge_at_unix_millis.is_some()); - - routed.stub.hang_up.store(false, Ordering::SeqCst); - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); - })?; - routed.drift()?; - assert!(routed.tick()?); - let healed = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(healed.consecutive_failures, 0); - assert_eq!(healed.next_challenge_at_unix_millis, None); - assert_eq!(healed.degraded_since_unix_millis, None); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_failing_reload_that_puts_the_drift_back_backs_the_route_off() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.route_stubs.refuse_reloads()?; - let reloads = routed.reloads(); - routed.drift()?; - for _ in 0..200 { - routed.tick()?; - } - assert_eq!(routed.reloads() - reloads, 1, "the route was reloaded again and again"); - assert_eq!(std::fs::read(routed.fragment_path())?, b"drifted bytes\n"); - assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_candidate_whose_route_proof_fails_reloads_at_most_twice_the_ceiling() -> Result<()> { - let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - let mut last = None; - for _ in 0..40 { - last = Some(routed.step().unwrap_err()); - } - // Every install is a counted Forward and is followed by one - // recorded rollback, so the rollbacks are bounded by the installs: - // twelve installs spend the window, each reloads twice, and every - // later attempt is refused before it runs anything. - assert_eq!(routed.reloads(), 2 * ROUTE_ACTUATION_CEILING); - let meters = meters_of(&routed.world, "service")?; - assert_eq!(meters.route_used(now_millis()?), ROUTE_ACTUATION_CEILING); - assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); - let last = last.unwrap(); - assert!(last.downcast_ref::().is_some(), "{last:#}"); - Ok(()) - } - - /// The rescue (ruling 3d A): a target whose continuity spent every - /// restart, each charging its preflight and install as Survival, is - /// still deployed: its route changes are the ceiling's first Forward. - #[cfg(unix)] - #[test] - fn a_target_that_spent_its_restarts_can_still_be_deployed() -> Result<()> { - let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; - routed.stub.set_state("active"); - routed.step()?; - assert!(routed.transaction()?.warming.is_some()); - set_meters( - &routed.world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - for _ in 0..2 * CONTINUITY_PAGING_RESTARTS { - routed.world.engine.charge_route_actuation("service", RouteActuation::Survival)?; - } - let spent = meters_of(&routed.world, "service")?; - assert!(spent.restarts_page(now_millis()?)); - assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_PAGING_RESTARTS); - - // Warming's preflight and Routing's install are both admitted. - routed.step()?; - assert!(routed.transaction()?.route_preflight.is_some(), "the preflight was refused"); - routed.run_to_routing()?; - routed.promote()?; - assert!(routed.fragment_path().exists(), "the install was refused"); - assert!(routed.reloads() >= 1); - let after = meters_of(&routed.world, "service")?; - assert_eq!(after.route_actuations.len(), 2, "preflight and install are the Forward"); - Ok(()) - } - - // ----------------------------------------------------------------- - // B5: every post-fencing phase ends. - // ----------------------------------------------------------------- - - impl RoutedWorld { - /// One scheduler pass over the transaction, as the daemon runs it: - /// through the deadline resolver, with the resume backoff spent. - fn resume(&self) -> Result<()> { - self.world.engine.resume_backoff.lock().unwrap().clear(); - self.world.engine.resume_one_transaction()?; - Ok(()) - } - - fn resume_until_terminal(&self) -> Result { - for _ in 0..12 { - if latest(&self.world)?.completion.is_some() { - break; - } - self.resume()?; - } - let finished = latest(&self.world)?; - ensure!(finished.completion.is_some(), "never finished: {:?}", finished.last_error); - Ok(finished) - } - } - - /// The live transaction's phase deadline is already behind it. - fn pass_deadline(world: &EngineFixture) -> Result<()> { - let current = resident(world)?; - let mut next = current.value.clone(); - let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; - deadline.entered_at_unix_millis = 1; - deadline.deadline_at_unix_millis = 2; - replace_transaction(&world.state_store, ¤t, &next) - } - - /// Soul's F1 probe, committed. A continuity's route changes are - /// Survival, so no ceiling refuses its install and rollback while its - /// route proof keeps failing. The Routing deadline ends it: no attempt - /// runs after the deadline. - #[cfg(unix)] - #[test] - fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - for _ in 0..3 { - routed.resume()?; - } - // Before the deadline each attempt installs and rolls back, and no - // resume moves the deadline. - assert_eq!(routed.reloads(), 6); - let waiting = routed.transaction()?; - assert_eq!(waiting.phase, DeploymentPhase::Routing); - assert!(waiting.post_fencing_abort.is_none(), "aborted before its deadline"); - assert!(waiting.last_error.is_some()); - assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); - - pass_deadline(&routed.world)?; - routed.resume()?; - let abort = routed - .transaction()? - .post_fencing_abort - .context("the passed deadline did not end the phase")?; - assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); - let finished = routed.resume_until_terminal()?; - assert!( - matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - ), - "{:?}", - finished.completion - ); - // After the deadline no attempt runs; the withdrawal writes and - // reloads once, whatever the rolled-back disk shows. - assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); - Ok(()) - } - - /// The live shape of F1. A continuity reuses the admitted Expected, so - /// its membership is the admitted fragment already on disk: while its - /// proof fails, the install and the rollback write nothing and reload - /// nothing, and neither does the withdrawal after its deadline. - #[cfg(unix)] - #[test] - fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() - -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let current = resident(&routed.world)?; - let mut next = current.value.clone(); - let binding = next.plan.as_ref().unwrap().parsed_inputs()?.1; - let admitted = NginxRouteDriver::new(binding.route.context("no route binding")?) - .render(next.expected.as_ref().context("no Expected")?)?; - let preflight = next.route_preflight.as_mut().context("no preflight")?; - preflight.incumbent_runtime_instance_id = Some(sha256_id(b"admitted-instance")); - preflight.incumbent_membership_sha256 = Some(sha256_id(&admitted)); - preflight.incumbent_configuration = Some(admitted.clone()); - replace_transaction(&routed.world.state_store, ¤t, &next)?; - std::fs::write(routed.fragment_path(), &admitted)?; - - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - for _ in 0..5 { - routed.resume()?; - } - assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); - assert!(routed.transaction()?.last_error.is_some(), "the proof did not fail"); - assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); - - pass_deadline(&routed.world)?; - routed.resume_until_terminal()?; - assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); - assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); - Ok(()) - } - - /// The abort's withdrawal wrote the baseline back and Idunn died before - /// the reload: the disk already shows the baseline, and the retry still - /// reloads, so nginx does not keep proxying to the stopped candidate. - #[cfg(unix)] - #[test] - fn an_abort_whose_reload_a_crash_cut_off_reloads_on_retry() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - let installed = routed.reloads(); - routed - .world - .engine - .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; - std::fs::remove_file(routed.fragment_path())?; - routed.resume()?; - assert_eq!( - routed.transaction()?.post_fencing_abort.context("no abort")?.route_restoration, - CleanupEvidence::Complete - ); - assert_eq!(routed.reloads(), installed + 1, "the withdrawal trusted unloaded bytes"); - Ok(()) - } - - /// Soul's F2 probe, committed: a post-fence withdrawal whose - /// `ufw delete` keeps failing withdraws the firewall allow first, so its - /// retries reload nginx not at all. - #[cfg(unix)] - #[test] - fn a_withdrawal_whose_firewall_step_keeps_failing_never_reloads() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - let installed = routed.reloads(); - routed.world.route_stubs.refuse_ufw_deletes()?; - routed - .world - .engine - .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; - for _ in 0..20 { - routed.resume()?; - } - assert_eq!(routed.reloads(), installed, "a failing withdrawal reloaded nginx"); - assert_eq!(routed.world.route_stubs.count("ufw delete"), 20); - assert!(routed.fragment_path().exists(), "the route went before its firewall rule"); - Ok(()) - } - - /// A record lifted from before deadlines existed is timed once, from - /// the first time the resolver sees it, and never again. - #[test] - fn a_post_fence_record_without_a_deadline_is_timed_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let current = resident(&routed.world)?; - let mut lifted = current.value.clone(); - lifted.phase_deadline = None; - replace_transaction(&routed.world.state_store, ¤t, &lifted)?; - let engine = &routed.world.engine; - - let before = now_millis()?; - assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, Some(true)); - let stamped = routed.transaction()?.phase_deadline.context("the record was not timed")?; - assert_eq!(stamped.phase, DeploymentPhase::Fencing); - assert!(stamped.entered_at_unix_millis >= before); - assert_eq!( - stamped.deadline_at_unix_millis - stamped.entered_at_unix_millis, - u64::from(lifted.plan.as_ref().unwrap().phase_deadlines().fencing_seconds) * 1000 - ); - assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, None); - assert_eq!(routed.transaction()?.phase_deadline, Some(stamped)); - Ok(()) - } - - /// `idunn expire` writes a request and nothing else; the resolver ends - /// the phase from it, and the spent request is deleted. - #[test] - fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { - let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let refused = expire(&early.world.state_store, "continuity-service", "operator") - .expect_err("a transaction before the fence was expired"); - assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); - - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let untouched = resident(&routed.world)?.envelope; - expire(&routed.world.state_store, "continuity-service", "operator")?; - assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - assert_eq!(snapshot.expiry_requests.len(), 1); - assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); - - routed.resume()?; - let abort = routed - .transaction()? - .post_fencing_abort - .context("the expiry did not end the phase")?; - assert!(abort.error.contains("Fencing phase was expired by the operator"), "{}", abort.error); - routed.resume()?; - assert!( - ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), - "the spent request stayed" - ); - Ok(()) - } - - /// The `expire` verb as the CLI parses and runs it: the options land - /// where they say, and it asks only about its own command's live - /// transaction. - #[test] - fn idunn_expire_parses_its_options_and_asks_only_about_its_own_live_transaction() - -> Result<()> { - let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); - assert_eq!( - parse(args(&["expire", "up-service", "--requested-by", "alice", "--state-store", "/s/c.cc"]).into_iter())?, - Command::Expire { - state_store: PathBuf::from("/s/c.cc"), - command_id: "up-service".into(), - requested_by: "alice".into(), - } - ); - let help = parse(args(&["expire", "up-service", "--help"]).into_iter()).unwrap_err(); - assert!(format!("{help}").contains("idunn expire "), "{help}"); - assert!(parse(args(&["expire", "up-service", "--force"]).into_iter()).is_err()); - assert!(parse(args(&["expire"]).into_iter()).is_err()); - - // The verb runs: a store with nothing live refuses. - let empty = TempDir::new()?; - let store = empty.path().join("control.cc").display().to_string(); - let refused = run(args(&["expire", "up-service", "--state-store", &store]).into_iter()) - .expect_err("expired a command with no transaction"); - assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); - - // Another target's deployment in flight is not this command's. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let now = now_millis()?; - let other = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-other".into(), - kind: CommandKind::Deploy, - selector: "other".into(), - requested_by: "test".into(), - requested_at_unix_millis: now, - }; - let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; - assert!(SingleFileMessagePackBackingStore::new(&routed.world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: other.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: sealing.transaction_id.clone(), - current: None, - }, - ], - &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], - )?); - expire(&routed.world.state_store, "continuity-service", "operator")?; - let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].value.command_id, "continuity-service"); - Ok(()) - } - - /// The resolver reads a request as its own transaction's only: with two - /// live transactions, a request for the other leaves this one running. - #[test] - fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let now = now_millis()?; - let other = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-other".into(), - kind: CommandKind::Deploy, - selector: "other".into(), - requested_by: "test".into(), - requested_at_unix_millis: now, - }; - let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; - // The request names a command whose id has this transaction's - // length and first bytes, and differs only in its last. - let near = "continuity-servicx"; - assert_eq!(near.len(), "continuity-service".len()); - let request = ExpiryRequest { - schema_version: EXPIRY_REQUEST_SCHEMA.into(), - command_id: near.into(), - requested_by: "operator".into(), - requested_at_unix_millis: now, - }; - // And commands whose ids are a strict prefix of this one's, and - // extend it: a match is the whole id, in neither direction a part. - let prefix = ExpiryRequest { - command_id: "continuity-servic".into(), - ..request.clone() - }; - let extension = ExpiryRequest { - command_id: "continuity-service-2".into(), - ..request.clone() - }; - let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); - let absent = |r#type: &str, key: &str| CultCacheExpectedEnvelope { - r#type: r#type.into(), - key: key.into(), - current: None, - }; - assert!(store.compare_exchange( - &[ - absent(DeploymentCommand::TYPE, &other.command_id), - absent(DeploymentTransaction::TYPE, &sealing.transaction_id), - absent(ExpiryRequest::TYPE, &request.command_id), - absent(ExpiryRequest::TYPE, &prefix.command_id), - absent(ExpiryRequest::TYPE, &extension.command_id), - ], - &[ - command_envelope(&other, now)?, - transaction_envelope(&sealing, now)?, - expiry_request_envelope(&request, now)?, - expiry_request_envelope(&prefix, now)?, - expiry_request_envelope(&extension, now)?, - ], - )?); - assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.len(), 3); - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - assert_eq!(snapshot.transactions.iter().filter(|t| !t.value.is_terminal()).count(), 2); - let mine = snapshot - .transactions - .iter() - .find(|t| t.value.command_id == "continuity-service") - .context("no continuity")?; - assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); - let after = ControlSnapshot::read(&routed.world.state_store)?; - let unchanged = after - .transactions - .iter() - .find(|t| t.value.command_id == "continuity-service") - .context("no continuity")?; - assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); - Ok(()) - } - - /// Q3 at the deadline, through the Engine. A continuity whose candidate - /// was granted a lease and never proved it held it fails as - /// `lease-not-adopted`; one whose Ready proved it restarts the admitted - /// release. A deployment whose candidate holds a lease is left to the - /// operator: nothing is aborted and its step still runs. - #[test] - fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { - let granted = |routed: &RoutedWorld| -> Result { - routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; - Ok(routed - .transaction()? - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("no granted lease")? - .to_owned()) - }; - - let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - granted(&unadopted)?; - pass_deadline(&unadopted.world)?; - unadopted.resume()?; - let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); - assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); - let finished = unadopted.resume_until_terminal()?; - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - )); - - let adopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - let lease = granted(&adopted)?; - adopted.stub.set_state("active"); - *adopted.stub.lease.lock().unwrap() = Some(lease.clone()); - adopted.step()?; - let ready = adopted.transaction()?; - let Some(ReadinessEvidence::RouteProof { evidence }) = &ready.ready else { - bail!("the candidate was not Ready"); - }; - let adoption = ready.lease_adoption.clone().context("Ready recorded no adoption")?; - assert_eq!(adoption.write_lease_sha256, lease); - assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); - assert_eq!(adoption.source, AdoptionSource::Direct); - adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; - pass_deadline(&adopted.world)?; - adopted.resume()?; - let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); - let finished = adopted.resume_until_terminal()?; - assert!( - matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestartAdmitted, - .. - }) - ), - "{:?}", - finished.completion - ); - - let deploy = - build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; - granted(&deploy)?; - pass_deadline(&deploy.world)?; - for _ in 0..3 { - deploy.resume()?; - } - let held = deploy.transaction()?; - assert!(held.post_fencing_abort.is_none(), "a leased deployment was aborted"); - assert_eq!(held.phase, DeploymentPhase::AwaitingReady); - assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); - Ok(()) - } - - /// B5 Soul A. The physical grant lands before `Granted` is durable. A - /// continuity that dies in that window and passes its deadline is - /// aborted from `Prepared`; the abort still withdraws the lease on - /// disk, so the next fencing of the target is not refused. - #[test] - fn an_abort_from_a_prepared_lease_withdraws_a_grant_already_on_disk() -> Result<()> { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; - let current = resident(&routed.world)?; - let mut crashed = current.value.clone(); - let Some(LeasingEvidence::Granted { lease, lease_sha256 }) = crashed.leasing.clone() else { - bail!("no granted lease"); - }; - crashed.leasing = Some(LeasingEvidence::Prepared { lease: lease.clone(), lease_sha256 }); - replace_transaction(&routed.world.state_store, ¤t, &crashed)?; - let driver = CultCacheWriteLeaseDriver::new( - "service", - routed.world.root.join("lease/process-write-lease.cc"), - ); - assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); - - pass_deadline(&routed.world)?; - routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); - routed.resume_until_terminal()?; - assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); - // What the next fencing does to a candidate's lease path. - driver.revoke_exact(None)?; - Ok(()) - } - - /// B5 Soul F. A stateful candidate Odin reports on: its Ready word - /// carries the granted lease, and that Ready records the adoption, - /// sourced from Odin's topology and naming the presence Odin observed. - #[test] - fn an_odin_ready_word_records_a_stateful_candidates_lease_adoption() -> Result<()> { - let routed = build_routed_world( - Odin::Unreachable, - 1, - DeploymentPhase::Fencing, - true, - true, - CommandKind::Continuity, - )?; - for _ in 0..8 { - if routed.transaction()?.phase == DeploymentPhase::AwaitingReady { - break; - } - routed.step()?; - } - let waiting = routed.transaction()?; - assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady, "{:?}", waiting.last_error); - let lease = waiting - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("no granted lease")? - .to_owned(); - assert_eq!(waiting.lease_adoption, None); - - let ready = CultCacheEnvelope { - key: crate::drivers::incarnation_key_of( - &waiting.target, - &waiting.expected.as_ref().unwrap().canonical_sha256()?, - ), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: signed_correlation_as( - &routed.world, - &waiting, - waiting.odin_publisher_sequence_cursor + 1, - true, - "active", - Some(&lease), - b"presence-ready", - )?, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), - }; - replace_odin_entry(&routed.world.engine.options.odin_correlation_store, ready)?; - // One step admits the new sequence, the next records Ready from it. - for _ in 0..3 { - if routed.transaction()?.ready.is_some() { - break; - } - routed.step()?; - } - let adopted = routed.transaction()?; - assert!( - matches!(adopted.ready, Some(ReadinessEvidence::OdinCorrelated { .. })), - "{:?}", - adopted.last_error - ); - let adoption = adopted.lease_adoption.context("Odin's Ready recorded no adoption")?; - assert_eq!(adoption.source, AdoptionSource::OdinTopology); - assert_eq!(adoption.write_lease_sha256, lease); - assert_eq!(adoption.signed_presence_sha256, sha256_id(b"presence-ready")); - Ok(()) - } - - /// F16: Commit goes on from the topology it admitted, so Odin - /// re-stamping before the step and again during the final proof does - /// not send it round again. - #[test] - fn commit_completes_while_odin_restamps_every_tick() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - let transaction = routed.transaction()?; - let cursor = transaction.odin_publisher_sequence_cursor; - let store = routed.world.engine.options.odin_correlation_store.clone(); - replace_odin_entry(&store, odin_ready_envelope(&routed.world, &transaction, cursor + 1)?)?; - let later = odin_ready_envelope(&routed.world, &transaction, cursor + 2)?; - *routed.stub.on_stable_hit.lock().unwrap() = Some(Box::new(move || { - replace_odin_entry(&store, later.clone()).expect("Odin re-stamps"); - })); - routed.step()?; - let generation = admitted(&routed)?; - assert_eq!( - generation.latest_odin_observation.context("no Odin receipt")?.publisher_sequence, - cursor + 2, - "commit did not go on from the re-stamped topology" - ); - Ok(()) - } - - fn odin_ready_envelope( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ) -> Result { - let expected = transaction.expected.as_ref().context("no Expected")?; - Ok(CultCacheEnvelope { - key: crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: signed_correlation(world, transaction, sequence, true)?, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), - }) - } - - fn replace_odin_entry(store: &Path, envelope: CultCacheEnvelope) -> Result<()> { - let backing = SingleFileMessagePackBackingStore::new(store); - let current = backing - .pull_all_read_only_snapshot()? - .into_iter() - .find(|entry| entry.key == envelope.key && entry.r#type == envelope.r#type); - ensure!( - backing.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: envelope.r#type.clone(), - key: envelope.key.clone(), - current, - }], - &[envelope], - )?, - "the Odin store changed under the re-stamp" - ); - Ok(()) - } - } -} +use std::collections::{BTreeMap, BTreeSet}; +use std::env; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use cultnet_rs::{ + AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, + IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, + IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, + IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, + IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, + OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, + ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, + authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, + correlate_runtime_presence_claim, derive_service_identity_id, + evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, + verify_idunn_deployment_brake_authorization, verify_runtime_authority, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::deployment::{ + DependencyKind, OperatorBinding, RolloutStrategy, RouteBinding, WorkloadBinding, + capability_compatible, +}; +use crate::deployment_plan::{ + CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, +}; +use crate::drivers::{ + ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, + GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, + LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, + RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, + SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, + WriteLeasePort, +}; +use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; +use crate::host_actuator::{ + HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, + HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, +}; + +const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2"; +const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v4"; +const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3"; +const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2"; +const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v4"; +const ADMITTED_GENERATION_SCHEMA_V3: &str = "idunn.admitted_generation.v3"; +const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; +/// The capability whose declaration makes a target Odin-correlated. +pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; +const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v2"; +/// The seven-key layout written before continuity's streak, anchor and +/// re-materialization bound. Read and migrated by `read_supervision_record` +/// and `migrate_control_store_to_current_schema`; delete it, `LegacyTargetSupervisionV1` +/// and those two arms once every store has opened under a build that +/// migrates (the key-34 precedent's deletion line). +const TARGET_SUPERVISION_SCHEMA_V1: &str = "idunn.target_supervision.v1"; +const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1"; +/// The number of charged continuity attempts inside +/// `CONTINUITY_RESTART_WINDOW_MILLIS` at which the ContinuityExhausted incident +/// opens and is held. It gates no actuation: continuity never stops +/// (ruling budget-never-stop). +const CONTINUITY_PAGING_RESTARTS: usize = 6; +const CONTINUITY_RESTART_WINDOW_MILLIS: u64 = 3_600_000; +/// The base of the streak backoff: the wait after the first charged attempt; +/// it doubles with each further one in the streak. +const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; +/// The longest streak backoff between charged continuity attempts (ruling +/// continuity-backoff-defaults). +const CONTINUITY_BACKOFF_CEILING_MILLIS: u64 = 300_000; +/// How long an admitted generation must have run, with no attempt failing on +/// it, before the attempt streak resets (ruling continuity-backoff-defaults). +const CONTINUITY_SUSTAINED_RUN_MILLIS: u64 = 600_000; + +/// Forward route actuations (fragment write, firewall, `nginx -t`, reload, +/// private-mount validation) one target may perform per sliding window. A +/// healthy target performs none; a legitimate deployment performs one or two. +/// Only deployments count: a Survival actuation is recorded beside this +/// ceiling, never counted against it and never refused by it, so a target +/// that is being restarted can still be rescued by a deploy. +/// Survival is bounded elsewhere: a continuity's actuations by the streak +/// backoff (at most one attempt per `CONTINUITY_BACKOFF_CEILING_MILLIS` once +/// the streak is long) times what one transaction can do before its phase +/// deadline ends it (none, over its admitted route), and an admitted-route +/// repair by the challenge backoff. A withdrawal that keeps failing retries, +/// reloading each time, under the resume backoff alone: the abort has no +/// deadline. Nothing bounds +/// reloads across the host. +const ROUTE_ACTUATION_CEILING: usize = 12; +const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; +/// How many Survival route actuation times a target keeps, for `status` +/// only: they are a record, not a meter, and no decision reads them. +const ROUTE_SURVIVAL_RECORD: usize = 12; +/// The longest wait between route challenges while proofs keep failing. The +/// shortest is the observation max age. +const ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS: u64 = 600_000; + +/// How long continuity waits after it could not prepare a restart (the +/// projection would not demote) before trying again. +const CONTINUITY_DEFERRAL_MILLIS: u64 = 30_000; + +const DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS: u64 = 30_000; +const DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS: u64 = 2_000; + +/// An immutable request. Execution state deliberately does not fit in this +/// record; status is derived from the transactions created for it. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_command", + schema = "idunn.deployment_command.v2" +)] +struct DeploymentCommand { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + command_id: String, + #[cultcache(key = 2)] + kind: CommandKind, + #[cultcache(key = 3)] + selector: String, + #[cultcache(key = 4)] + requested_by: String, + #[cultcache(key = 5)] + requested_at_unix_millis: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum CommandKind { + Deploy, + Continuity, +} + +impl DeploymentCommand { + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == DEPLOYMENT_COMMAND_SCHEMA, + "deployment command schema is unsupported" + ); + require_id(&self.command_id, "deployment command id")?; + require_selector(&self.selector)?; + require_value(&self.requested_by, "deployment requester")?; + ensure!( + self.requested_at_unix_millis > 0, + "deployment command has no request time" + ); + match self.kind { + CommandKind::Deploy => ensure!( + !self.command_id.starts_with("continuity-"), + "operator deployment uses the continuity command namespace" + ), + CommandKind::Continuity => ensure!( + self.command_id.starts_with("continuity-") + && !self.selector.starts_with("profile:"), + "continuity command identity or selector is invalid" + ), + } + Ok(()) + } +} + +/// An operator's request that a live transaction's post-fencing phase end now +/// (`idunn expire`). The CLI only writes this record. The daemon's deadline +/// resolver reads it as a deadline that has passed, and the record is deleted +/// once its transaction no longer has a phase to end. No CLI writes a +/// transaction field. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.expiry_request", schema = "idunn.expiry_request.v1")] +struct ExpiryRequest { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + command_id: String, + #[cultcache(key = 2)] + requested_by: String, + #[cultcache(key = 3)] + requested_at_unix_millis: u64, +} + +impl ExpiryRequest { + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == EXPIRY_REQUEST_SCHEMA, + "expiry request schema is unsupported" + ); + require_id(&self.command_id, "expiry request command id")?; + require_value(&self.requested_by, "expiry requester")?; + ensure!( + self.requested_at_unix_millis > 0, + "expiry request has no request time" + ); + Ok(()) + } +} + +fn expiry_request_envelope(value: &ExpiryRequest, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.command_id, + ExpiryRequest::TYPE, + EXPIRY_REQUEST_SCHEMA, + value, + now, + ) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +#[repr(u8)] +enum DeploymentPhase { + Sealing, + Starting, + Warming, + Fencing, + Leasing, + AwaitingReady, + Routing, + Committing, + Complete, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct DeploymentAuthorization { + authorization_id: String, + brake_sha256: String, + canonical_brake_bytes: Vec, + authorized_at_unix_millis: u64, +} + +impl DeploymentAuthorization { + fn validate_shape(&self) -> Result<()> { + require_id(&self.authorization_id, "deployment authorization id")?; + ensure!( + !self.canonical_brake_bytes.is_empty() + && self.brake_sha256 == sha256_id(&self.canonical_brake_bytes) + && self.authorized_at_unix_millis > 0, + "deployment authorization receipt is incomplete" + ); + let record: IdunnDeploymentBrakeRecord = + rmp_serde::from_slice(&self.canonical_brake_bytes)?; + record.validate()?; + ensure!( + rmp_serde::to_vec(&record)? == self.canonical_brake_bytes + && record.authorization_id.as_deref() == Some(&self.authorization_id), + "deployment authorization receipt is noncanonical or mismatched" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct TopologyEvidence { + canonical_bytes: Vec, + canonical_sha256: String, + signer_identity_id: String, + publisher_sequence: u64, + admitted_at_unix_millis: u64, +} + +impl TopologyEvidence { + fn from_authenticated( + topology: &AuthenticatedOdinRuntimeTopologyCorrelation, + admitted_at_unix_millis: u64, + ) -> Result { + let record = topology.record(); + let evidence = Self { + canonical_bytes: topology.canonical_bytes().to_vec(), + canonical_sha256: sha256_id(topology.canonical_bytes()), + signer_identity_id: record.signer_identity_id.clone(), + publisher_sequence: record.publisher_sequence, + admitted_at_unix_millis, + }; + evidence.validate_shape()?; + Ok(evidence) + } + + fn validate_shape(&self) -> Result<()> { + ensure!( + !self.canonical_bytes.is_empty() + && self.canonical_sha256 == sha256_id(&self.canonical_bytes), + "topology evidence bytes or digest are invalid" + ); + require_id(&self.signer_identity_id, "Odin topology signer")?; + ensure!( + self.publisher_sequence > 0 && self.admitted_at_unix_millis > 0, + "topology evidence sequence or admission time is invalid" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RuntimePresenceEvidence { + canonical_bytes: Vec, + canonical_sha256: String, + message_id: String, + challenged_at_unix_millis: u64, + admitted_at_unix_millis: u64, +} + +impl RuntimePresenceEvidence { + fn from_present( + present: &cultnet_rs::VerifiedRuntimePresence, + message_id: String, + challenged_at_unix_millis: u64, + admitted_at_unix_millis: u64, + ) -> Result { + let evidence = Self { + canonical_bytes: present.canonical_bytes().to_vec(), + canonical_sha256: present.signed_presence_sha256().to_owned(), + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + }; + evidence.validate_shape()?; + Ok(evidence) + } + + fn validate_shape(&self) -> Result<()> { + ensure!( + !self.canonical_bytes.is_empty() + && self.canonical_sha256 == sha256_id(&self.canonical_bytes), + "runtime presence evidence bytes or digest are invalid" + ); + require_id(&self.message_id, "runtime presence challenge")?; + ensure!( + self.challenged_at_unix_millis > 0 + && self.admitted_at_unix_millis >= self.challenged_at_unix_millis, + "runtime presence evidence timeline is invalid" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "source", rename_all = "kebab-case", deny_unknown_fields)] +enum WarmingEvidence { + OdinTopology { evidence: TopologyEvidence }, + FirstOdinDirect { evidence: RuntimePresenceEvidence }, + /// A route-proof target's candidate answered Idunn's challenge itself. + RouteProofDirect { evidence: RuntimePresenceEvidence }, +} + +impl WarmingEvidence { + fn validate_shape(&self) -> Result<()> { + match self { + Self::OdinTopology { evidence } => evidence.validate_shape(), + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + evidence.validate_shape() + } + } + } + + fn voucher(&self) -> Voucher { + match self { + Self::OdinTopology { .. } | Self::FirstOdinDirect { .. } => Voucher::Odin, + Self::RouteProofDirect { .. } => Voucher::Candidate, + } + } + + /// The presence Idunn challenged for itself, when no Odin carried it. + fn direct(&self) -> Option<&RuntimePresenceEvidence> { + match self { + Self::OdinTopology { .. } => None, + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + Some(evidence) + } + } + } +} + +/// Opaque capability minted only after the transaction CAS stores exact +/// Warming evidence. Adjacent drivers receive only the candidate identity and +/// signed-presence digest needed to bind the write lease; they cannot inspect +/// or construct semantic admission. +#[derive(Clone, Debug)] +pub struct SequenceAdmittedWarming { + transaction_id: String, + signed_presence_sha256: String, + runtime_instance_id: String, +} + +impl SequenceAdmittedWarming { + fn from_topology( + transaction_id: String, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + ) -> Result { + let record = authenticated.record(); + let signed_presence_sha256 = record + .signed_presence_sha256 + .clone() + .context("Warming topology has no signed presence")?; + let runtime_instance_id = record + .runtime_instance_id + .clone() + .context("Warming topology has no runtime instance")?; + Ok(Self { + transaction_id, + signed_presence_sha256, + runtime_instance_id, + }) + } + + fn from_direct_presence( + transaction_id: String, + warming: WarmingEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + ) -> Result { + let evidence = warming + .direct() + .context("direct Warming evidence is an Odin topology receipt")?; + ensure!( + present.canonical_bytes() == evidence.canonical_bytes + && present.signed_presence_sha256() == evidence.canonical_sha256, + "direct Warming evidence differs from its authenticated presence" + ); + let runtime_instance_id = present.record().runtime_instance_id.clone(); + Ok(Self { + transaction_id, + signed_presence_sha256: evidence.canonical_sha256.clone(), + runtime_instance_id, + }) + } + + #[cfg(test)] + pub(crate) fn for_test( + transaction_id: impl Into, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + ) -> Result { + let record = authenticated.record(); + ensure!( + record.present + && !record.ready + && record.observed_presence_state.as_deref() == Some("warming") + && record.observed_write_lease_sha256.is_none() + && record.disagreements.is_empty(), + "test receipt is not exact Warming evidence" + ); + let transaction_id = transaction_id.into(); + require_id(&transaction_id, "test transaction id")?; + let signed_presence_sha256 = record + .signed_presence_sha256 + .clone() + .context("test Warming receipt has no signed presence")?; + let runtime_instance_id = record + .runtime_instance_id + .clone() + .context("test Warming receipt has no runtime instance")?; + let token = Self::from_topology(transaction_id, authenticated)?; + ensure!( + token.signed_presence_sha256 == signed_presence_sha256 + && token.runtime_instance_id == runtime_instance_id, + "test Warming token extraction changed" + ); + Ok(token) + } + + pub(crate) fn transaction_id(&self) -> &str { + &self.transaction_id + } + + pub(crate) fn signed_presence_sha256(&self) -> &str { + &self.signed_presence_sha256 + } + + pub(crate) fn runtime_instance_id(&self) -> &str { + &self.runtime_instance_id + } +} + +/// Ready proof with the same construction boundary. Dependency planning and +/// promotion receive this token, never caller-supplied digest/sequence tuples. +#[derive(Clone, Debug)] +pub(crate) struct SequenceAdmittedReady { + transaction_id: String, + evidence: TopologyEvidence, + expected: IdunnExpectedIncarnationRecord, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, +} + +impl SequenceAdmittedReady { + #[cfg(test)] + pub(crate) fn for_test( + expected: &IdunnExpectedIncarnationRecord, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + admitted_at_unix_millis: u64, + ) -> Result { + expected.validate()?; + let record = authenticated.record(); + ensure!( + record.expected_projection_sha256 == expected.canonical_sha256()? + && record.target == expected.target + && record.runtime_id == expected.runtime_id + && record.present + && record.ready + && record.observed_presence_state.as_deref() == Some("active") + && record.disagreements.is_empty(), + "test receipt is not exact Ready evidence" + ); + let evidence = + TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; + Ok(Self { + transaction_id: "test-sequence-admission".into(), + evidence, + expected: expected.clone(), + authenticated, + }) + } + + pub(crate) fn authenticated(&self) -> &AuthenticatedOdinRuntimeTopologyCorrelation { + &self.authenticated + } + + pub(crate) fn expected(&self) -> &IdunnExpectedIncarnationRecord { + &self.expected + } + + pub(crate) fn transaction_id(&self) -> &str { + &self.transaction_id + } + + pub(crate) fn evidence_sha256(&self) -> &str { + &self.evidence.canonical_sha256 + } + + pub(crate) fn publisher_sequence(&self) -> u64 { + self.evidence.publisher_sequence + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +enum FencingEvidence { + SkippedStateless, + Revoked { + incumbent_lease_sha256: Option, + candidate_lease_path_verified_empty: bool, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +enum LeasingEvidence { + SkippedStateless, + Prepared { + lease: IdunnProcessWriteLeaseRecord, + lease_sha256: String, + }, + Granted { + lease: IdunnProcessWriteLeaseRecord, + lease_sha256: String, + }, +} + +impl LeasingEvidence { + fn lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { + match self { + Self::SkippedStateless | Self::Prepared { .. } => None, + Self::Granted { lease, .. } => Some(lease), + } + } + + /// The lease Idunn issued, granted or only prepared. A prepared lease may + /// already be on disk: the physical grant runs before `Granted` is durable. + fn issued_lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { + match self { + Self::SkippedStateless => None, + Self::Prepared { lease, .. } | Self::Granted { lease, .. } => Some(lease), + } + } + + fn prepared_lease(&self) -> Option<(&IdunnProcessWriteLeaseRecord, &str)> { + match self { + Self::Prepared { + lease, + lease_sha256, + } => Some((lease, lease_sha256)), + Self::SkippedStateless | Self::Granted { .. } => None, + } + } + + fn lease_sha256(&self) -> Option<&str> { + match self { + Self::SkippedStateless | Self::Prepared { .. } => None, + Self::Granted { lease_sha256, .. } => Some(lease_sha256), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +enum RoutingEvidence { + SkippedUnrouted, + Promoted { + observation: RouteObservation, + promoted_at_unix_millis: u64, + }, +} + +impl RoutingEvidence { + fn observation(&self) -> Option<&RouteObservation> { + match self { + Self::SkippedUnrouted => None, + Self::Promoted { observation, .. } => Some(observation), + } + } + + fn promoted_at_unix_millis(&self) -> Option { + match self { + Self::SkippedUnrouted => None, + Self::Promoted { + promoted_at_unix_millis, + .. + } => Some(*promoted_at_unix_millis), + } + } +} + +/// The states a route-proof candidate may answer its Warming challenge in. A +/// stateful one is warming until it holds a lease it cannot hold yet, and the +/// lease binds that warming presence. A stateless one may have finished +/// warming before the first challenge landed. +fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'static [&'static str] { + if expected.write_lease_required { + &["warming"] + } else { + &["warming", "active"] + } +} + +/// A challenged runtime answered as something that disagrees with current +/// authority. Each disagreement is named, so a capacity below the Expected +/// minimum reads as that and not as a generic refusal. It stays the error's +/// typed value; callers read `disagreements`, and the text is only its +/// rendering. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PresenceDisagrees { + pub(crate) disagreements: Vec, +} + +impl std::fmt::Display for PresenceDisagrees { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let named = self + .disagreements + .iter() + .map(|disagreement| { + format!( + "{} (expected {}, observed {})", + disagreement.code, + disagreement.expected.as_deref().unwrap_or("none"), + disagreement.observed.as_deref().unwrap_or("none"), + ) + }) + .collect::>() + .join("; "); + write!( + formatter, + "route proof answered with a runtime that disagrees with current authority: {named}" + ) + } +} + +impl std::error::Error for PresenceDisagrees {} + +/// What a direct challenge to a candidate endpoint produced. +enum CandidateAnswer { + /// No answer: the reason it is still being waited on. + Silent(String), + Answered { + evidence: RuntimePresenceEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + }, +} + +//// Which kind of proof admits a target as Ready, derived from what the target's +/// recipe declares and from nothing else. Idunn infers no way to prove +/// readiness: a target that provides `odin.verse-rendezvous` is Odin itself, one +/// that declares a `shared-infrastructure odin.verse-rendezvous` dependency +/// advertises into the Verse and is Odin-correlated, and a routed one that +/// declares neither answers Idunn's own challenge and is route-proof. A target +/// with no route and no Odin dependency declares no proof at all, and is +/// refused. Odin collects the same receipts an Odin-correlated target does, and +/// its admitted generation is what names the Odin authority, so it is never +/// route-proof. +/// +/// `of` is the only place a class is decided, and Expected is digest-bound, so +/// the class cannot drift from the incarnation it describes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ReadinessClass { + OdinSelf, + OdinCorrelated, + RouteProof, +} + +/// A recipe that declares neither a route nor an Odin dependency, and is not +/// Odin: it has said nothing about how it proves readiness. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct UndeclaredReadiness { + pub(crate) target: String, +} + +impl std::fmt::Display for UndeclaredReadiness { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "target {} declares no way to prove readiness: its recipe must declare a route \ + (Idunn challenges it) or a shared-infrastructure {ODIN_RENDEZVOUS_CAPABILITY} \ + dependency (Odin correlates it)", + self.target + ) + } +} + +impl std::error::Error for UndeclaredReadiness {} + +impl ReadinessClass { + /// The class of a recipe's declarations. Plan compilation and `of` both + /// call it, so admission refuses exactly what `of` would refuse. + pub(crate) fn declared( + target: &str, + provides_odin: bool, + declares_odin: bool, + routed: bool, + ) -> Result { + if provides_odin { + Ok(Self::OdinSelf) + } else if declares_odin { + Ok(Self::OdinCorrelated) + } else if routed { + Ok(Self::RouteProof) + } else { + Err(UndeclaredReadiness { + target: target.to_owned(), + }) + } + } + + pub(crate) fn of( + expected: &IdunnExpectedIncarnationRecord, + ) -> Result { + Self::declared( + &expected.target, + expected + .capabilities + .iter() + .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY), + expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }), + expected.route.is_some(), + ) + } + + /// The class of `expected`, provided every receipt collected under it was + /// vouched for by that class's voucher. The one way a stored record is + /// asked what class it is. + fn confirmed( + expected: &IdunnExpectedIncarnationRecord, + collected: impl IntoIterator, + ) -> Result { + let required = Self::of(expected).map_err(ReadinessDisagreement::Undeclared)?; + match collected + .into_iter() + .find(|voucher| *voucher != required.voucher()) + { + None => Ok(required), + Some(collected) => Err(ReadinessDisagreement::WrongVoucher { + target: expected.target.clone(), + required, + collected, + }), + } + } + + /// Who vouches for a target of this class: Odin's correlation, or the + /// candidate answering Idunn. + fn voucher(self) -> Voucher { + match self { + Self::OdinSelf | Self::OdinCorrelated => Voucher::Odin, + Self::RouteProof => Voucher::Candidate, + } + } +} + +/// Who vouched for a piece of stored readiness evidence. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Voucher { + Odin, + Candidate, +} + +/// A stored record whose readiness evidence is not what its own Expected +/// requires. A record admitted before the target's recipe declared how it +/// proves readiness looks like this. It is reported and held, never repaired: +/// re-deriving the evidence around it would make the evidence tag a second +/// authority for the class. +#[derive(Clone, Debug, PartialEq, Eq)] +enum ReadinessDisagreement { + Undeclared(UndeclaredReadiness), + WrongVoucher { + target: String, + required: ReadinessClass, + collected: Voucher, + }, +} + +impl std::fmt::Display for ReadinessDisagreement { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Undeclared(undeclared) => write!(formatter, "{undeclared}"), + Self::WrongVoucher { + target, + required, + collected, + } => write!( + formatter, + "target {target} is {required:?} but its stored readiness evidence was vouched \ + for by {collected:?}: it was collected under a different class, and is held \ + until the target is redeployed" + ), + } + } +} + +impl std::error::Error for ReadinessDisagreement {} + +/// The receipt that admitted a candidate as Ready, tagged by who vouched for +/// it. The tag records what was collected; what a target must collect is +/// `ReadinessClass::of(expected)`, and a tag that differs is a +/// `ReadinessDisagreement`, never a second opinion about the class. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)] +enum ReadinessEvidence { + OdinCorrelated { evidence: TopologyEvidence }, + RouteProof { evidence: RuntimePresenceEvidence }, +} + +impl ReadinessEvidence { + fn voucher(&self) -> Voucher { + match self { + Self::OdinCorrelated { .. } => Voucher::Odin, + Self::RouteProof { .. } => Voucher::Candidate, + } + } + + fn validate_shape(&self) -> Result<()> { + match self { + Self::OdinCorrelated { evidence } => evidence.validate_shape(), + Self::RouteProof { evidence } => evidence.validate_shape(), + } + } + + fn odin(&self) -> Option<&TopologyEvidence> { + match self { + Self::OdinCorrelated { evidence } => Some(evidence), + Self::RouteProof { .. } => None, + } + } +} + +// Signed evidence that the candidate process picked up the lease Idunn +/// granted: its presence carries `write_lease_sha256` equal to the grant. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LeaseAdoptionEvidence { + write_lease_sha256: String, + signed_presence_sha256: String, + source: AdoptionSource, + observed_at_unix_millis: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum AdoptionSource { + Direct, + OdinTopology, +} + +impl LeaseAdoptionEvidence { + /// The adoption a stateful candidate's Ready proves: its signed presence + /// held exactly the granted lease. `None` for a stateless candidate. + fn with_ready( + granted_lease_sha256: Option<&str>, + signed_presence_sha256: &str, + source: AdoptionSource, + observed_at_unix_millis: u64, + ) -> Option { + granted_lease_sha256.map(|lease| Self { + write_lease_sha256: lease.to_owned(), + signed_presence_sha256: signed_presence_sha256.to_owned(), + source, + observed_at_unix_millis, + }) + } + + /// Whether this evidence is about exactly the lease that was granted. + fn names(&self, leasing: &LeasingEvidence) -> bool { + leasing.lease_sha256() == Some(self.write_lease_sha256.as_str()) + } + + fn validate_shape(&self) -> Result<()> { + require_id(&self.write_lease_sha256, "adopted lease digest")?; + require_id(&self.signed_presence_sha256, "lease adoption presence digest")?; + ensure!( + self.observed_at_unix_millis > 0, + "lease adoption has no observation time" + ); + Ok(()) + } +} + +/// When the transaction entered its current post-fencing phase and when that +/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PhaseDeadline { + phase: DeploymentPhase, + entered_at_unix_millis: u64, + deadline_at_unix_millis: u64, +} + +impl PhaseDeadline { + /// `None` for a phase that has no deadline (pre-fencing and Complete). + fn entering( + phase: DeploymentPhase, + plan: &CompiledDeploymentPlan, + now: u64, + ) -> Option { + let deadlines = plan.phase_deadlines(); + let seconds = match phase { + DeploymentPhase::Fencing => deadlines.fencing_seconds, + DeploymentPhase::Leasing => deadlines.leasing_seconds, + DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, + DeploymentPhase::Routing => deadlines.routing_seconds, + DeploymentPhase::Committing => deadlines.committing_seconds, + _ => return None, + }; + Some(Self { + phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: now.saturating_add(u64::from(seconds) * 1000), + }) + } +} + +/// How the deadline resolver ends a post-fencing phase. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PhaseEnd { + /// Abort through the post-fencing path. `lease_not_adopted` names a + /// granted lease with no adoption evidence. + Abort { lease_not_adopted: bool }, + /// A stateful deployment whose candidate holds an issued lease: the phase + /// is reported and not resolved. + OperatorRequired, +} + +/// How a post-fencing failure recovers. `RestoreIncumbent` is what a +/// post-fencing abort has always done and is the default for records that +/// predate the field. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "action", rename_all = "kebab-case", deny_unknown_fields)] +enum TerminalRecovery { + RestartAdmitted, + #[default] + RestoreIncumbent, + OperatorRequired { + reason: String, + }, +} + +/// Route supervision's durable memory for one admitted routed incarnation: +/// challenge pacing and the degradation mark. It belongs to the incarnation, so +/// a new generation starts from `default()`. What a target may actuate outlives +/// every generation and lives in `TargetSupervision`. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RouteSupervisionState { + last_challenge_at_unix_millis: Option, + consecutive_failures: u32, + next_challenge_at_unix_millis: Option, + /// Set while route proof keeps failing. Marks the route degraded so + /// dependents stop selecting it; it never authorizes a restart. + degraded_since_unix_millis: Option, +} + +impl RouteSupervisionState { + /// The next challenge is not due yet. + fn is_waiting(&self, now: u64, maximum_age_millis: u64) -> bool { + self.next_challenge_at_unix_millis.is_some_and(|next| { + is_waiting( + now, + next, + ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(maximum_age_millis), + ) + }) + } + + /// A failed proof, or a failed repair of the fragment it needs. Observation + /// state only: the route is marked degraded and the next challenge waits, + /// twice as long each time, from the observation max age up to a cap. + fn record_failed_challenge(&mut self, now: u64, maximum_age_millis: u64) { + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + let doublings = self.consecutive_failures.saturating_sub(1).min(16); + let wait = maximum_age_millis + .saturating_mul(1 << doublings) + .min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS) + .max(maximum_age_millis); + self.last_challenge_at_unix_millis = Some(now); + self.next_challenge_at_unix_millis = Some(now.saturating_add(wait)); + self.degraded_since_unix_millis.get_or_insert(now); + } + + fn record_proved_challenge(&mut self, now: u64) { + self.consecutive_failures = 0; + self.last_challenge_at_unix_millis = Some(now); + self.next_challenge_at_unix_millis = None; + self.degraded_since_unix_millis = None; + } + + fn validate(&self) -> Result<()> { + if let Some(since) = self.degraded_since_unix_millis { + ensure!(since > 0, "route degradation has no start time"); + } + Ok(()) + } +} + +/// What one target has been allowed to do, kept across every generation and +/// present before the first: Forward route actuations and continuity restarts, +/// each a sliding log of the times they happened, and a record of Survival +/// route actuations. Owned by the meter primitives below; +/// the route driver only asks, and no generation write carries it. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.target_supervision", + schema = "idunn.target_supervision.v2" +)] +struct TargetSupervision { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + /// The newest `ROUTE_ACTUATION_CEILING` Forward actuation times, + /// ascending: the only entries the ceiling counts. + #[cultcache(key = 2)] + route_actuations: Vec, + /// The newest `CONTINUITY_PAGING_RESTARTS` restart times, ascending: the + /// paging threshold's log. + #[cultcache(key = 3)] + continuity_restarts: Vec, + /// A restart whose projection could not be prepared waits until this time. + #[cultcache(key = 4)] + continuity_deferred_until: Option, + #[cultcache(key = 5)] + continuity_deferral_reason: Option, + /// The newest `ROUTE_SURVIVAL_RECORD` Survival actuation times, ascending. + /// Visibility only: never counted, never refused. + #[cultcache(key = 6)] + route_survivals: Vec, + // Keys 7-9 are why this is v2: a v1 record is lifted by + // `LegacyTargetSupervisionV1` with no streak, no anchor and no + // re-materialization. + /// Charged attempts since the last sustained run. Owned by continuity + /// supervision; it decides the next attempt's backoff step. + #[cultcache(key = 7)] + continuity_attempt_streak: u32, + /// When a dead-and-free pass began waiting out the next attempt. Owned by + /// continuity supervision; it decides when the wait is over. Older than + /// the admitted generation, it decides nothing. + #[cultcache(key = 8)] + continuity_backoff_from_unix_millis: Option, + /// When the admitted generation's one uncharged re-materialization was + /// minted. Owned by continuity supervision; it can only withhold a free + /// re-materialization, never grant one. + #[cultcache(key = 9)] + continuity_rematerialized_at_unix_millis: Option, +} + +/// What continuity supervision does about the next attempt for a dead target. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ContinuityBackoff { + /// Mint now, carrying this streak into the attempt. + Due { streak: u32 }, + /// Start the wait: write the anchor and mint nothing this pass. + Begin, + /// The wait has not run out. + Waiting, +} + +/// The entries of a sliding log still inside `window` at `now`. An entry from +/// the future means the clock stepped back; it counts as `now`, so a backwards +/// step holds a meter for at most one window however far the clock moved. +fn live_entries(log: &[u64], now: u64, window: u64) -> Vec { + log.iter() + .map(|&at| at.min(now)) + .filter(|&at| now - at < window) + .collect() +} + +/// Record `now`, keeping the newest `limit` entries: all a meter needs to +/// decide "at most `limit` in any window". +fn record_entry(log: &mut Vec, now: u64, limit: usize) { + clamp_to(log, now); + log.push(now); + if log.len() > limit { + log.drain(..log.len() - limit); + } +} + +/// Make a stepped-back clock's future entries the present. `live_entries` +/// only reads them that way, so a log that is never written would keep +/// looking recent until the clock caught up; writing the clamp is what makes +/// a backwards step cost one window and no more. +fn clamp_to(log: &mut [u64], now: u64) { + for at in log.iter_mut() { + *at = (*at).min(now); + } +} + +impl TargetSupervision { + fn new(target: &str) -> Self { + Self { + schema_version: TARGET_SUPERVISION_SCHEMA.into(), + target: target.into(), + route_actuations: Vec::new(), + continuity_restarts: Vec::new(), + continuity_deferred_until: None, + continuity_deferral_reason: None, + route_survivals: Vec::new(), + continuity_attempt_streak: 0, + continuity_backoff_from_unix_millis: None, + continuity_rematerialized_at_unix_millis: None, + } + } + + /// Clamp every log to `now`. Callers that decide from a log write the + /// result back when it changed. + fn settle(&mut self, now: u64) { + clamp_to(&mut self.route_actuations, now); + clamp_to(&mut self.continuity_restarts, now); + clamp_to(&mut self.route_survivals, now); + } + + fn route_used(&self, now: u64) -> usize { + live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() + } + + /// When the oldest counted actuation leaves the window, if the ceiling is reached. + fn route_reopens_at(&self, now: u64) -> Option { + let live = live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS); + (live.len() >= ROUTE_ACTUATION_CEILING) + .then(|| live[live.len() - ROUTE_ACTUATION_CEILING] + ROUTE_ACTUATION_WINDOW_MILLIS) + } + + /// Charge one route actuation. A `Forward` change is counted against the + /// ceiling and refused at it; a `Survival` is only recorded, never + /// counted and never refused. + fn charge_route(&mut self, now: u64, kind: RouteActuation) -> Result<(), RouteActuationRefused> { + self.settle(now); + match kind { + RouteActuation::Forward => { + if let Some(reopens_at) = self.route_reopens_at(now) { + return Err(RouteActuationRefused { + target: self.target.clone(), + used: self.route_used(now), + reopens_at_unix_millis: reopens_at, + }); + } + record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING); + } + RouteActuation::Survival => { + record_entry(&mut self.route_survivals, now, ROUTE_SURVIVAL_RECORD); + } + } + Ok(()) + } + + fn survivals_in_window(&self, now: u64) -> usize { + live_entries(&self.route_survivals, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() + } + + fn restarts_used(&self, now: u64) -> usize { + live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len() + } + + /// The paging threshold of the ContinuityExhausted incident; it gates no actuation. + fn restarts_page(&self, now: u64) -> bool { + self.restarts_used(now) >= CONTINUITY_PAGING_RESTARTS + } + + /// Whether continuity must still wait out the deferral after a projection + /// that would not demote. + fn continuity_deferred(&self, now: u64) -> bool { + self.continuity_deferred_until + .is_some_and(|due| is_waiting(now, due, CONTINUITY_DEFERRAL_MILLIS)) + } + + /// The doubling steps the next attempt waits, if it waits. A streak of `n` + /// waits `n - 1` steps. The generation's one re-materialization is an + /// attempt that failed on it though it charged no streak, so the attempt + /// after it waits the base step: every attempt leaves a gap. + fn owed_steps(&self, admitted_at: u64) -> Option { + match self.continuity_attempt_streak.checked_sub(1) { + None if self.rematerialization_spent(admitted_at) => Some(0), + steps => steps, + } + } + + /// When the wait for the next charged attempt ends, from the anchor and + /// the streak; the one spelling `continuity_backoff` and `status` share. + /// An anchor older than the admitted generation decides nothing. + fn backoff_due(&self, admitted_at: u64) -> Option { + let from = self + .continuity_backoff_from_unix_millis + .filter(|&from| from >= admitted_at)?; + let wait = self.owed_steps(admitted_at).map_or(0, |steps| { + backoff_wait( + CONTINUITY_RESTART_BACKOFF_MILLIS, + steps, + CONTINUITY_BACKOFF_CEILING_MILLIS, + ) + }); + Some(from.saturating_add(wait)) + } + + /// What to do about the next attempt for the dead admitted generation + /// admitted at `admitted_at`. The streak resets only after a sustained + /// run: the generation has lived the sustained time and no attempt, charged + /// or the one re-materialization, has been minted since it was admitted. + /// A standing anchor belongs to a death the supervision pass has seen + /// through; an observed-running pass clears it, so it never hides a reset. + fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { + let streak = self.continuity_attempt_streak; + if let Some(due) = self.backoff_due(admitted_at) { + return if self.owed_steps(admitted_at).is_some() + && is_waiting(now, due, CONTINUITY_BACKOFF_CEILING_MILLIS) + { + ContinuityBackoff::Waiting + } else { + ContinuityBackoff::Due { streak } + }; + } + let sustained = self + .continuity_restarts + .last() + .into_iter() + .chain(&self.continuity_rematerialized_at_unix_millis) + .all(|&last| admitted_at >= last) + && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; + if sustained || self.owed_steps(admitted_at).is_none() { + ContinuityBackoff::Due { streak: 0 } + } else { + ContinuityBackoff::Begin + } + } + + /// One charged attempt scheduled at `streak`. It ends the wait and any + /// deferral: the projection was prepared. + fn record_restart(&mut self, now: u64, streak: u32) { + record_entry(&mut self.continuity_restarts, now, CONTINUITY_PAGING_RESTARTS); + self.continuity_attempt_streak = streak.saturating_add(1); + self.continuity_backoff_from_unix_millis = None; + self.continuity_deferred_until = None; + self.continuity_deferral_reason = None; + } + + /// The target was found dead and free: the wait for its next attempt runs from now. + fn begin_backoff(&mut self, now: u64) { + self.continuity_backoff_from_unix_millis = Some(now); + } + + /// The admitted workload was seen running: the death the anchor waited + /// out is over, and the next death waits from its own discovery. + fn end_backoff(&mut self) { + self.continuity_backoff_from_unix_millis = None; + } + + /// The one uncharged re-materialization scheduled. The log and the streak + /// are left alone. + fn record_rematerialization(&mut self, now: u64) { + self.continuity_rematerialized_at_unix_millis = Some(now); + self.continuity_backoff_from_unix_millis = None; + self.continuity_deferred_until = None; + self.continuity_deferral_reason = None; + } + + /// Whether the generation admitted at `admitted_at` has had its free re-materialization. + fn rematerialization_spent(&self, admitted_at: u64) -> bool { + self.continuity_rematerialized_at_unix_millis + .is_some_and(|at| at >= admitted_at) + } + + fn defer_continuity(&mut self, now: u64, reason: &str) { + self.continuity_deferred_until = Some(now.saturating_add(CONTINUITY_DEFERRAL_MILLIS)); + self.continuity_deferral_reason = Some(reason.to_owned()); + } + + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == TARGET_SUPERVISION_SCHEMA, + "target supervision schema is unsupported" + ); + require_id(&self.target, "supervised target")?; + for (log, limit, what) in [ + (&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"), + (&self.continuity_restarts, CONTINUITY_PAGING_RESTARTS, "continuity restart"), + (&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"), + ] { + ensure!(log.len() <= limit, "{what} log exceeds its bound"); + ensure!( + log.iter().all(|&at| at > 0) && log.windows(2).all(|pair| pair[0] <= pair[1]), + "{what} log is not an ascending list of times" + ); + } + ensure!( + self.continuity_backoff_from_unix_millis != Some(0) + && self.continuity_rematerialized_at_unix_millis != Some(0), + "continuity backoff anchor or re-materialization has no time" + ); + ensure!( + self.continuity_deferred_until.is_some() == self.continuity_deferral_reason.is_some(), + "continuity deferral has a time or a reason but not both" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "kebab-case", deny_unknown_fields)] +enum TransactionCompletion { + Admitted { generation_id: String }, + FailedBeforeFencing { error: String }, + FailedAfterFencing { + error: String, + #[serde(default)] + recovery: TerminalRecovery, + }, +} + +use cleanup_evidence::CleanupEvidence; + +/// Where the legacy marker lives, so that only the lift can make one. +/// +/// `LegacyLift` has a private field: outside this module no expression can +/// construct it, so no other code can write `CleanupEvidence::Legacy(..)`. +/// Decoding a stored record is the one other way in, and it stays here too. +/// `transaction_envelope` refuses a record carrying the marker; the schema +/// migration, which persists what the lift produced, writes through +/// `migrate_transaction_record`, and a fault note on an existing record through +/// `last_error_envelope`; both are defined here beside the only constructor, +/// and the encoder they share is private to this module. +mod cleanup_evidence { + use super::*; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(super) struct LegacyLift(()); + + const LEGACY_MARKER: &str = "legacy-skipped-before-b1"; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(super) enum CleanupEvidence { + Pending, + Skipped, + Complete, + /// Set only by the legacy lift, on a terminal continuity abort written + /// before the single resolution rule: it issued an activation and + /// recorded `Skipped`. Boot reconciliation, not the record, owes that + /// residue's demotion. The marker is what lets validation accept the + /// old shape without reopening a finished transaction. + Legacy(LegacyLift), + } + + impl CleanupEvidence { + pub(super) fn is_complete(self) -> bool { + !matches!(self, Self::Pending) + } + + pub(super) fn is_legacy_marker(self) -> bool { + matches!(self, Self::Legacy(_)) + } + } + + impl Serialize for CleanupEvidence { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(match self { + Self::Pending => "pending", + Self::Skipped => "skipped", + Self::Complete => "complete", + Self::Legacy(_) => LEGACY_MARKER, + }) + } + } + + impl<'de> Deserialize<'de> for CleanupEvidence { + fn deserialize>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + match text.as_str() { + "pending" => Ok(Self::Pending), + "skipped" => Ok(Self::Skipped), + "complete" => Ok(Self::Complete), + LEGACY_MARKER => Ok(Self::Legacy(LegacyLift(()))), + other => Err(serde::de::Error::unknown_variant( + other, + &["pending", "skipped", "complete", LEGACY_MARKER], + )), + } + } + } + + impl DeploymentTransaction { + pub(super) fn carries_legacy_marker(&self) -> bool { + self.pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .into_iter() + .chain( + self.post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation), + ) + .any(CleanupEvidence::is_legacy_marker) + } + } + + /// The schema migration's one entry point: read a stored v2, v3 or v4 + /// transaction (lifting older ones) and return its key and current-shape + /// envelope. The record may carry the marker the lift just set. + pub(super) fn migrate_transaction_record( + envelope: &CultCacheEnvelope, + ) -> Result<(String, CultCacheEnvelope)> { + let value = read_transaction_record(envelope)?; + let next = encode_transaction(&value, value.updated_at_unix_millis)?; + Ok((value.transaction_id, next)) + } + + /// The envelope for an existing record with only its `last_error` set. The + /// record may already carry the marker; this cannot add one. + pub(super) fn last_error_envelope( + value: &DeploymentTransaction, + detail: &str, + ) -> Result { + let mut next = value.clone(); + next.last_error = Some(detail.to_string()); + encode_transaction(&next, next.updated_at_unix_millis) + } + +/// A continuity abort written before the single resolution rule recorded no +/// projection cleanup although its transaction had issued an activation, and +/// so left that activation standing. +/// +/// Nothing is reopened. A record already terminal keeps its terminal phase and +/// completion and is marked the legacy marker, the one shape validation +/// accepts for a terminal abort that owes nothing yet issued an activation; +/// boot reconciliation, the single owner of that residue, demotes it by exact +/// activation. A record still in flight lifts to `Pending`: it already holds +/// its target's authority (a live record claims it before and after the lift), +/// so the lift adds no claimant, and its own abort demotes the activation. +/// Only the control store's read applies this: history describes what happened +/// and is lifted unchanged. +pub(super) fn owe_legacy_continuity_projection(transaction: &mut DeploymentTransaction) { + if transaction.command_kind != CommandKind::Continuity || transaction.activation.is_none() { + return; + } + let terminal = transaction.completion.is_some(); + if let Some(abort) = transaction.pre_fencing_abort.as_mut() + && abort.topology_reconciliation == CleanupEvidence::Skipped + { + abort.topology_reconciliation = if terminal { + CleanupEvidence::Legacy(LegacyLift(())) + } else { + CleanupEvidence::Pending + }; + } + if let Some(abort) = transaction.post_fencing_abort.as_mut() + && abort.topology_reconciliation == CleanupEvidence::Skipped + && !terminal + { + abort.topology_reconciliation = CleanupEvidence::Pending; + } +} + +} + +/// Rollback of a transaction that has already fenced the incumbent. +/// +/// Fencing stops the admitted incarnation and revokes its write lease, so a +/// candidate that then dies for good leaves the target with nothing serving it. +/// Retrying forever cannot fix that -- a transient unit with `Restart=no` will +/// not come back -- and it holds the target, leaving every later command queued +/// behind a transaction that can never finish. +/// +/// So the candidate's own artifacts are withdrawn in the order that never +/// leaves two writers: route first, then the write lease, then the process, +/// then the projection. Reconciling the projection restores the incumbent's +/// admitted Expected where there is one, which is what lets continuity bring +/// the incumbent back; where there is none, the failed Expected is withdrawn. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PostFencingAbort { + error: String, + route_restoration: CleanupEvidence, + lease_withdrawal: CleanupEvidence, + candidate_cleanup: CleanupEvidence, + topology_reconciliation: CleanupEvidence, + source_cleanup: CleanupEvidence, +} + +impl PostFencingAbort { + fn is_complete(&self) -> bool { + self.route_restoration.is_complete() + && self.lease_withdrawal.is_complete() + && self.candidate_cleanup.is_complete() + && self.topology_reconciliation.is_complete() + && self.source_cleanup.is_complete() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PreFencingAbort { + error: String, + candidate_cleanup: CleanupEvidence, + topology_reconciliation: CleanupEvidence, + source_cleanup: CleanupEvidence, +} + +impl PreFencingAbort { + fn is_complete(&self) -> bool { + self.candidate_cleanup.is_complete() + && self.topology_reconciliation.is_complete() + && self.source_cleanup.is_complete() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +enum IncumbentCleanupEvidence { + SkippedNoIncumbent, + Pending { + generation_id: String, + workload: WorkloadObservation, + }, + Complete { + generation_id: String, + }, +} + +impl IncumbentCleanupEvidence { + fn is_complete(&self) -> bool { + matches!(self, Self::SkippedNoIncumbent | Self::Complete { .. }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum SourceCleanupEvidence { + SkippedContinuity, + Pending, + Complete, +} + +impl SourceCleanupEvidence { + fn is_complete(self) -> bool { + !matches!(self, Self::Pending) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PostCommitCleanup { + incumbent: IncumbentCleanupEvidence, + source: SourceCleanupEvidence, +} + +impl PostCommitCleanup { + fn is_complete(&self) -> bool { + self.incumbent.is_complete() && self.source.is_complete() + } +} + +/// The sole owner of an in-flight deployment decision. Every actuator result +/// is durable here before a later phase is entered. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_transaction", + schema = "idunn.deployment_transaction.v4" +)] +struct DeploymentTransaction { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + transaction_id: String, + #[cultcache(key = 2)] + command_id: String, + #[cultcache(key = 3)] + command_kind: CommandKind, + #[cultcache(key = 4)] + target: String, + #[cultcache(key = 5)] + ordinal: u32, + #[cultcache(key = 6)] + phase: DeploymentPhase, + #[cultcache(key = 7)] + created_at_unix_millis: u64, + #[cultcache(key = 8)] + updated_at_unix_millis: u64, + #[cultcache(key = 9)] + incumbent_generation_id: Option, + #[cultcache(key = 10)] + plan: Option, + #[cultcache(key = 11)] + frozen_source: Option, + #[cultcache(key = 12)] + sealed_release: Option, + #[cultcache(key = 13)] + installed_release: Option, + #[cultcache(key = 14)] + expected: Option, + #[cultcache(key = 15)] + expected_publication_sha256: Option, + #[cultcache(key = 16)] + deployment_authorization: Option, + #[cultcache(key = 17)] + lifecycle_authorized_at_unix_millis: Option, + #[cultcache(key = 18)] + activation: Option, + #[cultcache(key = 19)] + workload: Option, + #[cultcache(key = 20)] + activation_publication_sha256: Option, + #[cultcache(key = 21)] + latest_odin_observation: Option, + #[cultcache(key = 22)] + warming: Option, + #[cultcache(key = 23)] + route_preflight: Option, + #[cultcache(key = 24)] + isolation: Option, + #[cultcache(key = 25)] + fencing: Option, + #[cultcache(key = 26)] + leasing: Option, + #[cultcache(key = 27)] + ready: Option, + #[cultcache(key = 28)] + routing: Option, + #[cultcache(key = 29)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 30)] + last_error: Option, + #[cultcache(key = 31)] + completion: Option, + #[cultcache(key = 32)] + pre_fencing_abort: Option, + #[cultcache(key = 33)] + post_commit_cleanup: Option, + // because every transaction already durable predates this field: + // an absent key 34 decodes as None, which is exactly "this transaction has + // no post-fencing abort". Without it the daemon refuses to read its own + // control store and crashloops. + #[cultcache(key = 34, default)] + post_fencing_abort: Option, + /// Set by `transition` on entry to each post-fencing phase. + #[cultcache(key = 35)] + phase_deadline: Option, + #[cultcache(key = 36)] + lease_adoption: Option, +} + +impl DeploymentTransaction { + /// The one writer of `phase`. It stamps the update time and the phase + /// deadline together, so a record outside Fencing..=Committing carries no + /// deadline by construction. No other code assigns `phase`. + fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { + self.phase = phase; + self.updated_at_unix_millis = now; + self.phase_deadline = self + .plan + .as_ref() + .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); + } + + /// A live transaction past the fence that is not already being aborted: + /// the only kind whose phase a deadline, or an operator's expiry, ends. + fn resolves_by_deadline(&self) -> bool { + self.completion.is_none() + && self.pre_fencing_abort.is_none() + && self.post_fencing_abort.is_none() + && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) + } + + /// What an ended post-fencing phase resolves to (Q3). A candidate that was + /// never issued a write lease cannot have written, so the transaction + /// aborts to the incumbent through the post-fencing path, stateless or not. + /// A continuity always aborts: its candidate is the admitted release, and + /// supervision restarts that release under the continuity backoff. A + /// deployment whose candidate was issued a lease may have written under a + /// new state contract, and restoring the incumbent over that is not + /// Idunn's call: it is left for the operator. + fn phase_end(&self) -> PhaseEnd { + let leasing = self.leasing.as_ref(); + let granted = leasing.and_then(LeasingEvidence::lease).is_some(); + let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); + match self.command_kind { + CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, + _ => PhaseEnd::Abort { + lease_not_adopted: granted && self.lease_adoption.is_none(), + }, + } + } + + /// The recovery a completed post-fencing abort names. A continuity whose + /// candidate adopted its lease restarts the admitted release; everything + /// else restores the incumbent. + fn terminal_recovery(&self) -> TerminalRecovery { + if self.command_kind == CommandKind::Continuity && self.lease_adoption.is_some() { + TerminalRecovery::RestartAdmitted + } else { + TerminalRecovery::RestoreIncumbent + } + } + + /// Whether this transaction's binding declares stop-then-start. A + /// transaction without a plan yet cannot, so it reads as false. + fn rollout_stops_incumbent_first(&self) -> bool { + self.plan + .as_ref() + .and_then(|plan| plan.parsed_inputs().ok()) + .is_some_and(|(_, binding)| binding.rollout.strategy == RolloutStrategy::StopThenStart) + } + + fn new( + command: &DeploymentCommand, + target: String, + ordinal: u32, + incumbent: Option<&AdmittedGeneration>, + now: u64, + ) -> Result { + let transaction = Self { + schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), + transaction_id: format!("tx-{}", Uuid::new_v4()), + command_id: command.command_id.clone(), + command_kind: command.kind, + target, + ordinal, + phase: DeploymentPhase::Sealing, + created_at_unix_millis: now, + updated_at_unix_millis: now, + incumbent_generation_id: incumbent.map(|value| value.generation_id.clone()), + plan: None, + frozen_source: None, + sealed_release: None, + installed_release: None, + expected: None, + expected_publication_sha256: None, + deployment_authorization: None, + lifecycle_authorized_at_unix_millis: None, + activation: None, + workload: None, + activation_publication_sha256: None, + latest_odin_observation: None, + warming: None, + route_preflight: None, + isolation: None, + fencing: None, + leasing: None, + ready: None, + routing: None, + odin_publisher_sequence_cursor: incumbent + .map_or(0, |value| value.odin_publisher_sequence_cursor), + last_error: None, + completion: None, + pre_fencing_abort: None, + post_fencing_abort: None, + post_commit_cleanup: None, + phase_deadline: None, + lease_adoption: None, + }; + transaction.validate()?; + Ok(transaction) + } + + fn from_continuity( + command: &DeploymentCommand, + incumbent: &AdmittedGeneration, + now: u64, + ) -> Result { + let mut transaction = + Self::new(command, incumbent.target.clone(), 0, Some(incumbent), now)?; + transaction.plan = Some(incumbent.plan.clone()); + transaction.sealed_release = Some(incumbent.sealed_release.clone()); + transaction.installed_release = Some(incumbent.installed_release.clone()); + transaction.expected = Some(incumbent.expected.clone()); + transaction.validate()?; + Ok(transaction) + } + + /// Whether an abort owes the projection a reconciliation. A deployment + /// published its own Expected key. A continuity shares the admitted key, + /// whose Expected is never withdrawn, so it owes only the demotion of the + /// activation it issued. + fn abort_topology_reconciliation(&self) -> CleanupEvidence { + let owes = match self.command_kind { + CommandKind::Deploy => self.expected_publication_sha256.is_some(), + CommandKind::Continuity => self.activation.is_some(), + }; + if owes { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + } + } + + /// Whether an abort owes the candidate's durable cleanup: it does once an + /// activation was issued or a workload was observed, either of which leaves + /// something behind. + fn candidate_cleanup_owed(&self) -> CleanupEvidence { + if self.activation.is_some() || self.workload.is_some() { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + } + } + + /// Whether an abort's recorded candidate cleanup is what + /// `candidate_cleanup_owed` says it must be: owed cleanup is pending or + /// done, unowed cleanup is skipped. + fn candidate_cleanup_recorded(&self, recorded: CleanupEvidence) -> bool { + matches!( + (self.candidate_cleanup_owed(), recorded), + ( + CleanupEvidence::Pending, + CleanupEvidence::Pending | CleanupEvidence::Complete + ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) + ) + } + + fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { + let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; + let detail = truncate(&format!("{error:#}"), 2048); + transaction.enter_phase(DeploymentPhase::Complete, now); + transaction.last_error = Some(detail.clone()); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: detail.clone(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Skipped, + }); + transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { error: detail }); + transaction.validate()?; + Ok(transaction) + } + + fn is_terminal(&self) -> bool { + if self.phase != DeploymentPhase::Complete { + return false; + } + match &self.completion { + Some(TransactionCompletion::FailedBeforeFencing { .. }) => self + .pre_fencing_abort + .as_ref() + .is_some_and(PreFencingAbort::is_complete), + Some(TransactionCompletion::FailedAfterFencing { .. }) => self + .post_fencing_abort + .as_ref() + .is_some_and(PostFencingAbort::is_complete), + Some(TransactionCompletion::Admitted { .. }) => self + .post_commit_cleanup + .as_ref() + .is_some_and(PostCommitCleanup::is_complete), + None => false, + } + } + + /// Complete transactions retain retryable cleanup work, but admission has + /// already transferred current-incarnation authority to AdmittedGeneration. + fn owns_target_authority(&self) -> bool { + self.phase != DeploymentPhase::Complete + } + + /// Admission has moved to the new generation at Complete, but an exact + /// draining incumbent still reserves its process and candidate endpoint. + /// No later mutation for the same target may overlap that cleanup. + fn blocks_new_target_mutation(&self) -> bool { + self.owns_target_authority() + || self + .post_commit_cleanup + .as_ref() + .is_some_and(|cleanup| !cleanup.is_complete()) + } + + /// Where what this transaction has collected disagrees with what its own + /// Expected requires. There is nothing to disagree with before Expected. + fn readiness_disagreement(&self) -> Option { + let expected = self.expected.as_ref()?; + ReadinessClass::confirmed( + expected, + self.warming + .iter() + .map(WarmingEvidence::voucher) + .chain(self.ready.iter().map(ReadinessEvidence::voucher)), + ) + .err() + } + + /// The disagreement that holds this transaction: evidence collected under + /// another class is held and reported, not advanced and not repaired, since + /// no step can make it the right kind and running one anyway only fails it + /// every tick. What is finished, or being finished (a completion, or an + /// abort under way), never reads the class, so it is left to run. + fn held_disagreement(&self) -> Option { + let finishing = self.completion.is_some() + || self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some(); + self.readiness_disagreement().filter(|_| !finishing) + } + + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, + "deployment transaction schema is unsupported" + ); + require_id(&self.transaction_id, "deployment transaction id")?; + require_id(&self.command_id, "transaction command id")?; + require_id(&self.target, "transaction target")?; + ensure!( + self.created_at_unix_millis > 0 && self.updated_at_unix_millis > 0, + "deployment transaction timestamps are invalid" + ); + if let Some(generation) = &self.incumbent_generation_id { + require_id(generation, "incumbent generation id")?; + } + if let Some(plan) = &self.plan { + plan.validate()?; + ensure!( + plan.parsed_inputs()?.1.target == self.target, + "transaction plan belongs to another target" + ); + } + if let (Some(plan), Some(frozen)) = (&self.plan, &self.frozen_source) { + frozen.validate_against(plan)?; + } + if let (Some(plan), Some(release)) = (&self.plan, &self.sealed_release) { + release.validate_against(plan)?; + } + if let Some(expected) = &self.expected { + expected.validate()?; + ensure!(expected.target == self.target, "Expected target differs"); + if let Some(plan) = &self.plan { + ensure!(expected.plan_id == plan.plan_id, "Expected plan differs"); + } + if let Some(release) = &self.sealed_release { + ensure!( + expected.sealed_release_id == release.sealed_release_id, + "Expected release differs" + ); + } + if let Some(digest) = &self.expected_publication_sha256 { + ensure!( + digest == &expected.canonical_sha256()?, + "Expected publication receipt differs" + ); + } + } + if let (Some(expected), Some(activation)) = (&self.expected, &self.activation) { + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()?, + "activation belongs to another Expected projection" + ); + if let Some(workload) = &self.workload { + ensure!( + workload.runtime_instance_id() == activation.runtime_instance_id, + "workload belongs to another activation" + ); + } + if let Some(digest) = &self.activation_publication_sha256 { + ensure!( + digest == &activation.canonical_sha256()?, + "activation publication receipt differs" + ); + } + } + ensure!( + self.workload.is_none() || self.activation.is_some(), + "workload observation exists without its prepared activation" + ); + let ready_odin = self.ready.as_ref().and_then(ReadinessEvidence::odin); + for evidence in [self.latest_odin_observation.as_ref(), ready_odin] + .into_iter() + .flatten() + { + evidence.validate_shape()?; + ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, + "topology evidence exceeds the transaction replay cursor" + ); + } + if let Some(ready) = &self.ready { + ready.validate_shape()?; + // A candidate-vouched receipt is only ever written by this code, so + // one on a target that is not route-proof is forged. The other + // direction -- an Odin receipt on a route-proof target -- is what a + // pre-B3 record looks like, and is a ReadinessDisagreement to hold, + // not a record to refuse: refusing it would refuse to boot. + if let (Voucher::Candidate, Some(expected)) = (ready.voucher(), &self.expected) { + ensure!( + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" + ); + } + } + if let Some(warming) = &self.warming { + warming.validate_shape()?; + match warming { + WarmingEvidence::OdinTopology { evidence } => ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, + "Warming topology evidence exceeds the transaction replay cursor" + ), + WarmingEvidence::FirstOdinDirect { .. } => ensure!( + self.expected.as_ref().is_some_and(|expected| { + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::OdinSelf) + }), + "direct Warming evidence is reserved for Odin observing itself" + ), + WarmingEvidence::RouteProofDirect { .. } => ensure!( + self.expected.as_ref().is_none_or(|expected| { + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof) + }), + "direct route-proof Warming evidence for a target that is not route-proof" + ), + } + } + if let Some(leasing) = &self.leasing { + if let Some(expected) = &self.expected { + ensure!( + matches!( + (expected.write_lease_required, leasing), + (true, LeasingEvidence::Prepared { .. }) + | (true, LeasingEvidence::Granted { .. }) + | (false, LeasingEvidence::SkippedStateless) + ), + "lease evidence differs from the Expected state contract" + ); + } + let lease_and_sha256 = match leasing { + LeasingEvidence::SkippedStateless => None, + LeasingEvidence::Prepared { + lease, + lease_sha256, + } + | LeasingEvidence::Granted { + lease, + lease_sha256, + } => Some((lease, lease_sha256)), + }; + if let Some((lease, lease_sha256)) = lease_and_sha256 { + lease.validate()?; + ensure!( + lease.canonical_sha256()? == *lease_sha256, + "lease evidence digest differs" + ); + } + } + if let (Some(expected), Some(fencing)) = (&self.expected, &self.fencing) { + match fencing { + FencingEvidence::SkippedStateless => ensure!( + !expected.write_lease_required, + "stateful candidate claims stateless fencing" + ), + FencingEvidence::Revoked { + candidate_lease_path_verified_empty, + .. + } => ensure!( + *candidate_lease_path_verified_empty == expected.write_lease_required, + "candidate lease-path fencing differs from Expected" + ), + } + } + if let Some(error) = &self.last_error { + require_detail(error, "transaction error")?; + } + if let Some(deadline) = &self.phase_deadline { + ensure!( + deadline.phase == self.phase + && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) + .contains(&deadline.phase) + && deadline.entered_at_unix_millis > 0 + && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, + "phase deadline does not describe the current post-fencing phase" + ); + } + if let Some(adoption) = &self.lease_adoption { + adoption.validate_shape()?; + ensure!( + self.leasing + .as_ref() + .is_some_and(|leasing| adoption.names(leasing)), + "lease adoption does not name the granted lease" + ); + } + if let Some(authorization) = &self.deployment_authorization { + authorization.validate_shape()?; + } + if let Some(abort) = &self.pre_fencing_abort { + require_detail(&abort.error, "pre-fencing abort error")?; + ensure!( + self.post_commit_cleanup.is_none(), + "aborted transaction also carries post-commit cleanup" + ); + ensure!( + self.candidate_cleanup_recorded(abort.candidate_cleanup), + "abort candidate cleanup differs from its prepared activation" + ); + ensure!( + matches!( + ( + self.abort_topology_reconciliation(), + abort.topology_reconciliation + ), + ( + CleanupEvidence::Pending, + CleanupEvidence::Pending | CleanupEvidence::Complete + ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) + ) || (self.command_kind == CommandKind::Continuity + && self.phase == DeploymentPhase::Complete + && self.completion.is_some() + && abort.topology_reconciliation.is_legacy_marker()), + "abort topology cleanup differs from what the transaction projected" + ); + ensure!( + matches!( + (self.command_kind, abort.source_cleanup), + ( + CommandKind::Deploy, + CleanupEvidence::Pending | CleanupEvidence::Complete + ) | (CommandKind::Continuity, CleanupEvidence::Skipped) + ) || (self.command_kind == CommandKind::Deploy + && abort.source_cleanup == CleanupEvidence::Skipped + && self.plan.is_none() + && self.frozen_source.is_none() + && self.expected.is_none() + && self.phase == DeploymentPhase::Complete), + "abort source cleanup differs from command work" + ); + ensure!( + self.phase < DeploymentPhase::Fencing || self.phase == DeploymentPhase::Complete, + "pre-fencing abort crossed the fencing boundary" + ); + } + + if let Some(abort) = &self.post_fencing_abort { + require_detail(&abort.error, "post-fencing abort error")?; + ensure!( + self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_none(), + "post-fencing abort collides with another terminal path" + ); + ensure!( + self.phase >= DeploymentPhase::Fencing, + "post-fencing abort exists before the fence" + ); + ensure!( + self.candidate_cleanup_recorded(abort.candidate_cleanup), + "abort candidate cleanup differs from its prepared activation" + ); + } + + let failed_after_fencing = matches!( + self.completion, + Some(TransactionCompletion::FailedAfterFencing { .. }) + ); + if failed_after_fencing { + ensure!( + self.phase == DeploymentPhase::Complete, + "terminal failure is not Complete" + ); + let abort = required( + &self.post_fencing_abort, + "terminal post-fence abort evidence", + )?; + let TransactionCompletion::FailedAfterFencing { error, recovery } = + self.completion.as_ref().unwrap() + else { + unreachable!() + }; + if let TerminalRecovery::OperatorRequired { reason } = recovery { + require_detail(reason, "operator-required recovery reason")?; + } + ensure!( + abort.is_complete() && abort.error == *error, + "terminal failure lacks complete matching abort evidence" + ); + return Ok(()); + } + + let failed = matches!( + self.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + ); + if failed { + ensure!( + self.phase == DeploymentPhase::Complete, + "terminal failure is not Complete" + ); + let abort = required(&self.pre_fencing_abort, "terminal abort evidence")?; + let TransactionCompletion::FailedBeforeFencing { error } = + self.completion.as_ref().unwrap() + else { + unreachable!() + }; + ensure!( + abort.is_complete() && abort.error == *error, + "terminal failure lacks complete matching abort evidence" + ); + } else { + ensure!( + !matches!( + self.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + ), + "non-failure branch carries failure completion" + ); + if self.phase >= DeploymentPhase::Starting { + ensure!( + self.plan.is_some() + && self.sealed_release.is_some() + && self.installed_release.is_some() + && self.expected.is_some(), + "Starting transaction lacks sealed release evidence" + ); + // Expected is published as Starting's first step, after the + // brake; from the activation onward it must be there. + ensure!( + self.expected_publication_sha256.is_some() + || (self.activation.is_none() && self.phase == DeploymentPhase::Starting), + "started transaction has not published its Expected" + ); + match self.command_kind { + CommandKind::Deploy => ensure!( + self.frozen_source.is_some() && self.deployment_authorization.is_some(), + "deployment entered Starting without frozen source and brake consumption" + ), + CommandKind::Continuity => ensure!( + self.lifecycle_authorized_at_unix_millis.is_some() + && self.deployment_authorization.is_none(), + "continuity entered Starting without its lifecycle gate" + ), + } + } + if self.phase >= DeploymentPhase::Warming { + ensure!( + self.activation.is_some() + && self.workload.is_some() + && self.activation_publication_sha256.is_some(), + "Warming transaction lacks observed activation" + ); + } + if self.phase >= DeploymentPhase::Fencing { + ensure!( + self.warming.is_some() && self.isolation.is_some(), + "Fencing transaction lacks warming or isolation evidence" + ); + ensure!( + self.expected + .as_ref() + .and_then(|expected| expected.route.as_ref()) + .is_some() + == self.route_preflight.is_some(), + "route preflight does not match routed Expected" + ); + if let Some(preflight) = &self.route_preflight { + preflight.validate()?; + } + } + if self.phase >= DeploymentPhase::Leasing { + ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence"); + } + if self.phase >= DeploymentPhase::AwaitingReady { + ensure!( + matches!( + self.leasing.as_ref(), + Some(LeasingEvidence::SkippedStateless) + | Some(LeasingEvidence::Granted { .. }) + ), + "AwaitingReady lacks finalized lease evidence" + ); + } + if self.phase >= DeploymentPhase::Routing { + ensure!(self.ready.is_some(), "Routing lacks Ready evidence"); + } + if self.phase >= DeploymentPhase::Committing { + ensure!(self.routing.is_some(), "Committing lacks routing evidence"); + } + if let Some(RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + }) = &self.routing + { + observation.validate()?; + ensure!(*promoted_at_unix_millis > 0, "route promotion has no time"); + } + if self.phase == DeploymentPhase::Complete { + ensure!( + matches!( + self.completion, + Some(TransactionCompletion::Admitted { .. }) + ), + "successful Complete transaction lacks admission receipt" + ); + ensure!( + self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_some(), + "admitted transaction lacks exclusive post-commit cleanup evidence" + ); + let TransactionCompletion::Admitted { generation_id } = + self.completion.as_ref().unwrap() + else { + unreachable!() + }; + ensure!( + generation_id == &format!("generation-{}", self.transaction_id), + "admission completion names another transaction generation" + ); + } else { + ensure!( + self.completion.is_none() && self.post_commit_cleanup.is_none(), + "non-Complete transaction has completion or post-commit state" + ); + } + } + if let Some(cleanup) = &self.post_commit_cleanup { + ensure!( + self.phase == DeploymentPhase::Complete + && matches!( + self.completion, + Some(TransactionCompletion::Admitted { .. }) + ), + "post-commit cleanup exists without an admitted Complete transaction" + ); + match (&self.incumbent_generation_id, &cleanup.incumbent) { + (None, IncumbentCleanupEvidence::SkippedNoIncumbent) => {} + ( + Some(expected), + IncumbentCleanupEvidence::Pending { generation_id, .. } + | IncumbentCleanupEvidence::Complete { generation_id }, + ) if expected == generation_id => {} + _ => bail!("post-commit incumbent cleanup names another generation"), + } + ensure!( + matches!( + (self.command_kind, cleanup.source), + (CommandKind::Deploy, SourceCleanupEvidence::Pending) + | (CommandKind::Deploy, SourceCleanupEvidence::Complete) + | ( + CommandKind::Continuity, + SourceCleanupEvidence::SkippedContinuity + ) + ), + "post-commit source cleanup differs from command kind" + ); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AdmittedOdinAuthority { + signer_identity_id: String, + signer_public_key: Vec, +} + +impl AdmittedOdinAuthority { + fn from_anchor(anchor: &ServiceIdentityTrustAnchor) -> Result { + ensure!( + derive_service_identity_id::(&anchor.public_key)? + == anchor.identity_id, + "Odin topology anchor identity differs from its key" + ); + Ok(Self { + signer_identity_id: anchor.identity_id.clone(), + signer_public_key: anchor.public_key.clone(), + }) + } + + fn validate(&self) -> Result<()> { + require_id(&self.signer_identity_id, "admitted Odin signer")?; + ensure!( + derive_service_identity_id::(&self.signer_public_key)? + == self.signer_identity_id, + "admitted Odin signer identity differs from its key" + ); + Ok(()) + } +} + +/// The only current-incarnation owner. Unit state, route state, topology +/// projections, and CLI output are observations of this record, never peers. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.admitted_generation", + schema = "idunn.admitted_generation.v4" +)] +struct AdmittedGeneration { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + generation_id: String, + #[cultcache(key = 3)] + command_id: String, + #[cultcache(key = 4)] + transaction_id: String, + #[cultcache(key = 5)] + admitted_at_unix_millis: u64, + #[cultcache(key = 6)] + plan: CompiledDeploymentPlan, + #[cultcache(key = 7)] + sealed_release: SealedRelease, + #[cultcache(key = 8)] + installed_release: InstalledReleaseObservation, + #[cultcache(key = 9)] + expected: IdunnExpectedIncarnationRecord, + #[cultcache(key = 10)] + activation: IdunnRuntimeActivationRecord, + #[cultcache(key = 11)] + workload: WorkloadObservation, + #[cultcache(key = 12)] + leasing: LeasingEvidence, + /// The receipt that admitted this incarnation, tagged by proof class. + #[cultcache(key = 13)] + ready: ReadinessEvidence, + /// Present exactly when `ready` is Odin-correlated. + #[cultcache(key = 14)] + latest_odin_observation: Option, + #[cultcache(key = 15)] + routing: RoutingEvidence, + /// Present exactly when `ready` is Odin-correlated. + #[cultcache(key = 16)] + odin_authority: Option, + /// Meaningful only for an Odin-correlated generation. + #[cultcache(key = 17)] + odin_publisher_sequence_cursor: u64, + // Keys 18 (route repair start) and 20 (continuity backoff) are retired. + // The slots stay as gaps: the layout is positional, so a slot cannot be + // reused without a bump. + /// Present exactly when `routing` is Promoted. + #[cultcache(key = 19)] + route_supervision: Option, + /// Why this generation is held or down, for `status`. Cleared when the + /// generation is replaced. + #[cultcache(key = 21)] + last_error: Option, +} + +/// The Odin-side receipts of an Odin-correlated generation. Decisions that +/// consume Odin evidence read it through here, so a route-proof generation +/// cannot be mistaken for one. +struct OdinReceipts<'a> { + ready: &'a TopologyEvidence, + latest: &'a TopologyEvidence, +} + +impl AdmittedGeneration { + /// This generation's class, or where its evidence disagrees with it. The + /// class comes from its Expected, never from the tag on its Ready receipt. + fn readiness(&self) -> Result { + ReadinessClass::confirmed(&self.expected, [self.ready.voucher()]) + } + + fn odin_receipts(&self) -> Result> { + match ( + &self.ready, + &self.latest_odin_observation, + &self.odin_authority, + ) { + (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(_)) => { + Ok(OdinReceipts { + ready: evidence, + latest, + }) + } + _ => bail!("admitted generation of {} is not Odin-correlated", self.target), + } + } + + fn from_transaction( + transaction: &DeploymentTransaction, + odin_authority: Option, + now: u64, + ) -> Result { + ensure!( + transaction.phase == DeploymentPhase::Committing, + "only Committing can create an admitted generation" + ); + let ready = required(&transaction.ready, "Ready receipt")?.clone(); + let (latest_odin_observation, odin_authority) = match &ready { + ReadinessEvidence::OdinCorrelated { .. } => ( + Some( + required(&transaction.latest_odin_observation, "latest Odin receipt")? + .clone(), + ), + Some(required(&odin_authority, "Odin authority")?.clone()), + ), + ReadinessEvidence::RouteProof { .. } => (None, None), + }; + let routing = required(&transaction.routing, "route disposition")?.clone(); + let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. }) + .then(RouteSupervisionState::default); + let generation = Self { + schema_version: ADMITTED_GENERATION_SCHEMA.into(), + target: transaction.target.clone(), + generation_id: format!("generation-{}", transaction.transaction_id), + command_id: transaction.command_id.clone(), + transaction_id: transaction.transaction_id.clone(), + admitted_at_unix_millis: now, + plan: required(&transaction.plan, "transaction plan")?.clone(), + sealed_release: required(&transaction.sealed_release, "sealed release")?.clone(), + installed_release: required(&transaction.installed_release, "installed release")? + .clone(), + expected: required(&transaction.expected, "Expected projection")?.clone(), + activation: required(&transaction.activation, "activation")?.clone(), + workload: required(&transaction.workload, "workload")?.clone(), + leasing: required(&transaction.leasing, "lease disposition")?.clone(), + ready, + latest_odin_observation, + routing, + odin_authority, + odin_publisher_sequence_cursor: transaction.odin_publisher_sequence_cursor, + route_supervision, + last_error: None, + }; + generation.validate()?; + Ok(generation) + } + + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == ADMITTED_GENERATION_SCHEMA, + "admitted generation schema is unsupported" + ); + require_id(&self.target, "admitted target")?; + require_id(&self.generation_id, "admitted generation id")?; + require_id(&self.command_id, "admitted command id")?; + require_id(&self.transaction_id, "admitted transaction id")?; + ensure!(self.admitted_at_unix_millis > 0, "admission has no time"); + self.plan.validate()?; + self.sealed_release.validate_against(&self.plan)?; + self.expected.validate()?; + self.activation.validate()?; + self.ready.validate_shape()?; + match ( + &self.ready, + &self.latest_odin_observation, + &self.odin_authority, + ) { + (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(authority)) => { + latest.validate_shape()?; + authority.validate()?; + ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor + && latest.publisher_sequence == self.odin_publisher_sequence_cursor, + "admitted generation evidence does not describe one incarnation" + ); + } + (ReadinessEvidence::RouteProof { .. }, None, None) => ensure!( + ReadinessClass::of(&self.expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" + ), + _ => bail!("admitted readiness evidence and Odin receipts disagree"), + } + ensure!( + self.target == self.expected.target + && self.expected.plan_id == self.plan.plan_id + && self.expected.sealed_release_id == self.sealed_release.sealed_release_id + && self.activation.expected_projection_sha256 + == self.expected.canonical_sha256()? + && self.activation.runtime_instance_id == self.workload.runtime_instance_id(), + "admitted generation evidence does not describe one incarnation" + ); + ensure!( + self.expected.write_lease_required == self.leasing.lease().is_some(), + "admitted write-lease disposition differs from Expected" + ); + ensure!( + self.expected.route.is_some() == self.routing.observation().is_some(), + "admitted route disposition differs from Expected" + ); + if let Some(route) = self.routing.observation() { + route.validate()?; + ensure!( + route.runtime_instance_id == self.activation.runtime_instance_id, + "admitted route observation belongs to another runtime instance" + ); + } + ensure!( + self.route_supervision.is_some() + == matches!(&self.routing, RoutingEvidence::Promoted { .. }), + "route supervision state exists exactly for a promoted route" + ); + if let Some(state) = &self.route_supervision { + state.validate()?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RuntimeOptions { + state_store: PathBuf, + bindings_dir: PathBuf, + source_root: PathBuf, + staging_root: PathBuf, + topology_store: PathBuf, + /// Operator incidents, published world-readable beside the topology + /// projection. Not part of the control store: see `incident`. + incident_store: PathBuf, + odin_correlation_store: PathBuf, + odin_trust_anchor: PathBuf, + idunn_identity_store: PathBuf, + deployment_brake_operator_anchor: PathBuf, + source_identity: Option, + topology_maximum_age_millis: u64, + topology_maximum_future_skew_millis: u64, + poll_millis: u64, + /// Where host actuators dial in. `None` means no host-actuator workload + /// can be served; a binding that names one then fails at its first + /// driver call, not at startup, so an Idunn without hosts is unchanged. + host_actuator_bind: Option, + /// Where route drivers find nginx, systemctl, ufw and systemd-run, and + /// their scratch directory. Every route driver is built from this. + route_actuators: RouteActuators, +} + +impl Default for RuntimeOptions { + fn default() -> Self { + Self { + state_store: PathBuf::from("/var/lib/gamecult/idunn/control.cc"), + bindings_dir: PathBuf::from("/etc/gamecult/idunn/bindings"), + source_root: PathBuf::from("/var/lib/gamecult/idunn/sources"), + staging_root: PathBuf::from("/var/lib/gamecult/idunn/staging"), + topology_store: PathBuf::from("/var/lib/gamecult/idunn/topology.cc"), + incident_store: PathBuf::from("/var/lib/gamecult/idunn-projection/incidents.cc"), + odin_correlation_store: PathBuf::from( + "/var/lib/gamecult/odin/idunn-runtime-topology.cc", + ), + odin_trust_anchor: PathBuf::from("/etc/gamecult/idunn/odin-topology-anchor.cc"), + idunn_identity_store: PathBuf::from( + "/var/lib/gamecult/idunn/idunn-service-identity.cc", + ), + deployment_brake_operator_anchor: PathBuf::from( + "/etc/gamecult/idunn/deployment-brake-operator-anchor.cc", + ), + source_identity: None, + topology_maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, + topology_maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, + poll_millis: 500, + host_actuator_bind: None, + route_actuators: RouteActuators::default(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Command { + Serve(RuntimeOptions), + Up { + selector: String, + requested_by: String, + state_store: PathBuf, + wait: bool, + timeout_seconds: u64, + }, + Status { + state_store: PathBuf, + incident_store: PathBuf, + command_id: Option, + }, + Cancel { + state_store: PathBuf, + command_id: String, + requested_by: String, + }, + Expire { + state_store: PathBuf, + command_id: String, + requested_by: String, + }, + Validate { + recipe: PathBuf, + binding: Option, + }, +} + +pub fn run(args: impl Iterator) -> Result<()> { + match parse(args)? { + Command::Serve(options) => serve(options), + Command::Up { + selector, + requested_by, + state_store, + wait, + timeout_seconds, + } => submit( + &state_store, + &selector, + &requested_by, + wait, + timeout_seconds, + ), + Command::Status { + state_store, + incident_store, + command_id, + } => status( + &mut std::io::stdout().lock(), + &state_store, + &incident_store, + command_id.as_deref(), + ), + Command::Cancel { + state_store, + command_id, + requested_by, + } => cancel(&state_store, &command_id, &requested_by), + Command::Expire { + state_store, + command_id, + requested_by, + } => expire(&state_store, &command_id, &requested_by), + Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + } +} + +/// Offline admission check for authoring. Reads a recipe, optionally reads a +/// binding, and runs the same `admit` the daemon runs at load. It opens no +/// store, contacts no host, and actuates nothing, so it is safe to run against +/// a candidate binding before installing it. +fn validate(recipe_path: &Path, binding_path: Option<&Path>) -> Result<()> { + let recipe_text = fs::read_to_string(recipe_path) + .with_context(|| format!("reading recipe {}", recipe_path.display()))?; + let recipe = crate::deployment::TargetDeclaration::parse(&recipe_text) + .with_context(|| format!("validating recipe {}", recipe_path.display()))?; + println!( + "recipe ok: {} declares target {}", + recipe_path.display(), + recipe.target + ); + + let Some(binding_path) = binding_path else { + return Ok(()); + }; + let binding_text = fs::read_to_string(binding_path) + .with_context(|| format!("reading operator binding {}", binding_path.display()))?; + let binding = OperatorBinding::parse(&binding_text) + .with_context(|| format!("validating operator binding {}", binding_path.display()))?; + ensure!( + binding.target == recipe.target, + "operator binding targets {} but the recipe declares {}", + binding.target, + recipe.target + ); + binding + .admit(&recipe) + .with_context(|| format!("admitting {} against its recipe", binding.target))?; + println!( + "binding ok: {} admits target {}", + binding_path.display(), + binding.target + ); + Ok(()) +} + +fn parse(args: impl Iterator) -> Result { + let mut args = args.peekable(); + let command = args.next().ok_or_else(|| anyhow!(usage()))?; + match command.as_str() { + "serve" => parse_serve(args), + "up" => parse_up(args), + "status" => parse_status(args), + "cancel" => parse_cancel(args), + "expire" => parse_expire(args), + "validate" => parse_validate(args), + "--help" | "-h" | "help" => bail!(usage()), + _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), + } +} + +fn parse_serve(mut args: impl Iterator) -> Result { + let mut options = RuntimeOptions::default(); + let mut source_uid = None; + let mut source_gid = None; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, &argument)?, + "--bindings-dir" => options.bindings_dir = path_value(&mut args, &argument)?, + "--source-root" => options.source_root = path_value(&mut args, &argument)?, + "--staging-root" => options.staging_root = path_value(&mut args, &argument)?, + "--topology-store" => options.topology_store = path_value(&mut args, &argument)?, + "--incident-store" => options.incident_store = path_value(&mut args, &argument)?, + "--odin-correlation-store" => { + options.odin_correlation_store = path_value(&mut args, &argument)? + } + "--odin-trust-anchor" => options.odin_trust_anchor = path_value(&mut args, &argument)?, + "--idunn-identity-store" => { + options.idunn_identity_store = path_value(&mut args, &argument)? + } + "--deployment-brake-operator-anchor" => { + options.deployment_brake_operator_anchor = path_value(&mut args, &argument)? + } + "--topology-maximum-age-millis" => { + options.topology_maximum_age_millis = u64_value(&mut args, &argument)? + } + "--topology-maximum-future-skew-millis" => { + options.topology_maximum_future_skew_millis = u64_value(&mut args, &argument)? + } + "--source-uid" => source_uid = Some(u32_value(&mut args, &argument)?), + "--source-gid" => source_gid = Some(u32_value(&mut args, &argument)?), + "--poll-millis" => options.poll_millis = u64_value(&mut args, &argument)?, + "--host-actuator-bind" => { + let value = string_value(&mut args, &argument)?; + options.host_actuator_bind = Some( + value + .parse() + .with_context(|| format!("{argument} is not a socket address"))?, + ); + } + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn serve option {argument:?}"), + } + } + ensure!( + options.poll_millis > 0 && options.topology_maximum_age_millis > 0, + "poll and topology maximum age must be positive" + ); + options.source_identity = match (source_uid, source_gid) { + (Some(uid), Some(gid)) => { + ensure!( + uid > 0 && gid > 0, + "source UID and GID must be unprivileged" + ); + Some(ProcessIdentity { uid, gid }) + } + (None, None) => None, + _ => bail!("--source-uid and --source-gid must be supplied together"), + }; + Ok(Command::Serve(options)) +} + +fn parse_up(mut args: impl Iterator) -> Result { + let selector = args + .next() + .ok_or_else(|| anyhow!("idunn up requires a service or profile selector"))?; + require_selector(&selector)?; + let mut state_store = RuntimeOptions::default().state_store; + let mut requested_by = env::var("SUDO_USER") + .or_else(|_| env::var("USER")) + .or_else(|_| env::var("USERNAME")) + .unwrap_or_else(|_| "operator".into()); + let mut wait = true; + let mut timeout_seconds = 1800; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--requested-by" => requested_by = string_value(&mut args, &argument)?, + "--no-wait" => wait = false, + "--timeout-seconds" => timeout_seconds = u64_value(&mut args, &argument)?, + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn up option {argument:?}"), + } + } + ensure!(timeout_seconds > 0, "--timeout-seconds must be positive"); + require_value(&requested_by, "deployment requester")?; + Ok(Command::Up { + selector, + requested_by, + state_store, + wait, + timeout_seconds, + }) +} + +fn parse_status(mut args: impl Iterator) -> Result { + let defaults = RuntimeOptions::default(); + let mut state_store = defaults.state_store; + let mut incident_store = defaults.incident_store; + let mut command_id = None; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--incident-store" => incident_store = path_value(&mut args, &argument)?, + "--command" => command_id = Some(string_value(&mut args, &argument)?), + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn status option {argument:?}"), + } + } + Ok(Command::Status { + state_store, + incident_store, + command_id, + }) +} + +fn parse_expire(mut args: impl Iterator) -> Result { + let command_id = args + .next() + .ok_or_else(|| anyhow!("idunn expire requires a command id"))?; + require_id(&command_id, "command id")?; + let mut state_store = RuntimeOptions::default().state_store; + let mut requested_by = env::var("SUDO_USER") + .or_else(|_| env::var("USER")) + .or_else(|_| env::var("USERNAME")) + .unwrap_or_else(|_| "operator".into()); + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--requested-by" => requested_by = string_value(&mut args, &argument)?, + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn expire option {argument:?}"), + } + } + require_value(&requested_by, "expiry requester")?; + Ok(Command::Expire { + state_store, + command_id, + requested_by, + }) +} + +/// Ask the daemon to end a live transaction's post-fencing phase now. This +/// writes a request and nothing else: the daemon's deadline resolver ends the +/// phase exactly as it ends one whose deadline has passed. +fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { + let snapshot = ControlSnapshot::read(store_path)?; + let mut live = snapshot + .transactions + .iter() + .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); + let current = live + .next() + .context("command has no live transaction to expire")?; + ensure!( + live.next().is_none(), + "command has multiple live transactions" + ); + ensure!( + current.value.resolves_by_deadline(), + "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" + ); + let now = now_millis()?; + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: command_id.to_owned(), + requested_by: requested_by.to_owned(), + requested_at_unix_millis: now, + }; + ensure!( + SingleFileMessagePackBackingStore::new(store_path) + .insert_entry_if_absent(expiry_request_envelope(&request, now)?)?, + "an expiry request for this command is already waiting" + ); + println!("{command_id} expiry requested; Idunn ends its phase as an expired deadline"); + Ok(()) +} + +fn parse_cancel(mut args: impl Iterator) -> Result { + let command_id = args + .next() + .ok_or_else(|| anyhow!("idunn cancel requires a deployment command id"))?; + require_id(&command_id, "deployment command id")?; + let mut state_store = RuntimeOptions::default().state_store; + let mut requested_by = env::var("SUDO_USER") + .or_else(|_| env::var("USER")) + .or_else(|_| env::var("USERNAME")) + .unwrap_or_else(|_| "operator".into()); + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--requested-by" => requested_by = string_value(&mut args, &argument)?, + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn cancel option {argument:?}"), + } + } + require_value(&requested_by, "cancellation requester")?; + Ok(Command::Cancel { + state_store, + command_id, + requested_by, + }) +} + +/// Withdraw queued work, a held deployment that has not fenced, or ask Idunn to +/// clean up a live stateless candidate. Stateful candidates past fencing cannot be cancelled because their prior +/// writer may already be stopped and its state boundary needs explicit repair. +fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { + let snapshot = ControlSnapshot::read(store_path)?; + let command = snapshot + .commands + .iter() + .find(|stored| stored.value.command_id == command_id) + .context("deployment command is unknown or already retired")?; + ensure!( + command.value.kind == CommandKind::Deploy, + "only deployment commands can be cancelled; continuity is Idunn's own" + ); + let mut live = snapshot + .transactions + .iter() + .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); + if let Some(current) = live.next() { + ensure!( + live.next().is_none(), + "deployment command has multiple live transactions" + ); + ensure!( + cancel_is_safe_for_live_stateless(¤t.value) + || cancel_is_safe_for_held_prefence(¤t.value), + "live deployment can only be cancelled before fencing when Idunn holds it, or after fencing when its candidate is stateless" + ); + let error = format!("cancelled by {requested_by}"); + let mut next = current.value.clone(); + if cancel_is_safe_for_held_prefence(&next) { + next.pre_fencing_abort = Some(pre_fencing_abort_intent(&next, &error)); + } else { + next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); + } + next.last_error = Some(error); + next.updated_at_unix_millis = now_millis()?; + replace_transaction(store_path, current, &next)?; + println!("{command_id} cancellation requested; Idunn is performing exact candidate cleanup"); + return Ok(()); + } + let now = now_millis()?; + let refusal = DeploymentTransaction::rejected( + &command.value, + anyhow!("cancelled by {requested_by} before freezing"), + now, + )?; + let envelope = transaction_envelope(&refusal, now)?; + ensure!( + SingleFileMessagePackBackingStore::new(store_path).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command_id.to_owned(), + current: Some(command.envelope.clone()), + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: refusal.transaction_id.clone(), + current: None, + }, + ], + std::slice::from_ref(&envelope), + )?, + "deployment command changed before it could be cancelled" + ); + archive_terminal_transaction(store_path, &envelope)?; + println!("{command_id} cancelled"); + Ok(()) +} + +/// A deployment Idunn holds before it fenced anything has touched nothing of +/// the incumbent, so an operator may withdraw it. Idunn never does so itself. +fn cancel_is_safe_for_held_prefence(transaction: &DeploymentTransaction) -> bool { + transaction.command_kind == CommandKind::Deploy + && transaction.phase < DeploymentPhase::Fencing + && transaction.held_disagreement().is_some() +} + +fn cancel_is_safe_for_live_stateless(transaction: &DeploymentTransaction) -> bool { + transaction.command_kind == CommandKind::Deploy + && transaction.phase >= DeploymentPhase::Fencing + && matches!( + transaction.fencing.as_ref(), + Some(FencingEvidence::SkippedStateless) + ) + && transaction.completion.is_none() + && transaction.pre_fencing_abort.is_none() + && transaction.post_fencing_abort.is_none() + && transaction.post_commit_cleanup.is_none() +} + +fn parse_validate(mut args: impl Iterator) -> Result { + let mut recipe = None; + let mut binding = None; + while let Some(argument) = args.next() { + match argument.as_str() { + "--recipe" => recipe = Some(path_value(&mut args, &argument)?), + "--binding" => binding = Some(path_value(&mut args, &argument)?), + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn validate option {argument:?}"), + } + } + let recipe = recipe.ok_or_else(|| anyhow!("validate requires --recipe"))?; + Ok(Command::Validate { recipe, binding }) +} + +#[derive(Clone)] +struct Stored { + envelope: CultCacheEnvelope, + value: T, +} + +#[derive(Default)] +struct ControlSnapshot { + commands: Vec>, + transactions: Vec>, + admitted: Vec>, + targets: Vec>, + expiry_requests: Vec>, +} + +impl ControlSnapshot { + fn read(path: &Path) -> Result { + if !path.exists() { + return Ok(Self::default()); + } + let mut snapshot = Self::default(); + for envelope in SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading Idunn control snapshot")? + { + match envelope.r#type.as_str() { + DeploymentCommand::TYPE => { + ensure!( + envelope.schema_id.as_deref() == Some(DEPLOYMENT_COMMAND_SCHEMA), + "Idunn control store contains an unsupported command" + ); + let value: DeploymentCommand = decode_record(&envelope)?; + value.validate()?; + ensure!( + envelope.key == value.command_id, + "deployment command key differs from its identity" + ); + snapshot.commands.push(Stored { envelope, value }); + } + DeploymentTransaction::TYPE => { + let value = read_transaction_record(&envelope)?; + ensure!( + envelope.key == value.transaction_id, + "deployment transaction key differs from its identity" + ); + snapshot.transactions.push(Stored { envelope, value }); + } + AdmittedGeneration::TYPE => { + let value = read_generation_record(&envelope)?; + ensure!( + envelope.key == value.target, + "admitted generation key is not its target" + ); + snapshot.admitted.push(Stored { envelope, value }); + } + TargetSupervision::TYPE => { + let value = read_supervision_record(&envelope)?; + ensure!( + envelope.key == value.target, + "target supervision key is not its target" + ); + snapshot.targets.push(Stored { envelope, value }); + } + ExpiryRequest::TYPE => { + ensure!( + envelope.schema_id.as_deref() == Some(EXPIRY_REQUEST_SCHEMA), + "Idunn control store contains an unsupported expiry request" + ); + let value: ExpiryRequest = decode_record(&envelope)?; + value.validate()?; + ensure!( + envelope.key == value.command_id, + "expiry request key is not its command" + ); + snapshot.expiry_requests.push(Stored { envelope, value }); + } + _ => bail!("Idunn control store contains a foreign document"), + } + } + snapshot.validate_relations()?; + Ok(snapshot) + } + + fn validate_relations(&self) -> Result<()> { + let command_ids = self + .commands + .iter() + .map(|stored| stored.value.command_id.as_str()) + .collect::>(); + ensure!( + command_ids.len() == self.commands.len(), + "Idunn control store contains duplicate commands" + ); + let transaction_ids = self + .transactions + .iter() + .map(|stored| stored.value.transaction_id.as_str()) + .collect::>(); + ensure!( + transaction_ids.len() == self.transactions.len(), + "Idunn control store contains duplicate transactions" + ); + let admitted_targets = self + .admitted + .iter() + .map(|stored| stored.value.target.as_str()) + .collect::>(); + ensure!( + admitted_targets.len() == self.admitted.len(), + "Idunn control store contains duplicate current generations" + ); + ensure!( + self.targets + .iter() + .map(|stored| stored.value.target.as_str()) + .collect::>() + .len() + == self.targets.len(), + "Idunn control store contains duplicate target supervision records" + ); + for transaction in &self.transactions { + let command = self + .commands + .iter() + .find(|candidate| candidate.value.command_id == transaction.value.command_id) + .context("deployment transaction has no immutable command")?; + ensure!( + command.value.kind == transaction.value.command_kind, + "transaction kind differs from its immutable command" + ); + } + let mut live_targets = BTreeSet::new(); + for transaction in self + .transactions + .iter() + .filter(|stored| stored.value.owns_target_authority()) + { + ensure!( + live_targets.insert(transaction.value.target.as_str()), + "multiple transactions claim current-incarnation authority for one target" + ); + if let Some(fencing) = transaction + .value + .fencing + .as_ref() + .filter(|_| transaction.value.phase < DeploymentPhase::Complete) + { + let incumbent = self.admitted_for(&transaction.value.target); + let incumbent_matches = + match (&transaction.value.incumbent_generation_id, incumbent) { + (None, None) => true, + (Some(expected), Some(current)) => current.value.generation_id == *expected, + _ => false, + }; + ensure!( + incumbent_matches, + "fenced transaction incumbent differs from current admission" + ); + let incumbent_lease_sha256 = incumbent + .and_then(|stored| stored.value.leasing.lease()) + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose()?; + match fencing { + FencingEvidence::SkippedStateless => ensure!( + incumbent_lease_sha256.is_none(), + "stateless fencing omitted the admitted incumbent lease" + ), + FencingEvidence::Revoked { + incumbent_lease_sha256: recorded, + .. + } => { + ensure!( + recorded == &incumbent_lease_sha256, + "fencing evidence names another incumbent lease" + ); + ensure!( + incumbent_lease_sha256.is_some() + || transaction + .value + .expected + .as_ref() + .is_some_and(|expected| expected.write_lease_required), + "stateless transaction did not use its explicit fencing skip" + ); + } + } + } + } + let mut authorizations = BTreeSet::new(); + for authorization in self + .transactions + .iter() + .filter_map(|stored| stored.value.deployment_authorization.as_ref()) + { + ensure!( + authorizations.insert(authorization.authorization_id.as_str()), + "deployment authorization was consumed more than once" + ); + } + Ok(()) + } + + /// Every stored record that concerns `target`, as read: the generation, the + /// meters and its transactions. Two reads with equal stamps saw no write + /// for the target between them. + fn target_stamp(&self, target: &str) -> Vec { + let mut stamp = self + .admitted + .iter() + .filter(|stored| stored.value.target == target) + .map(|stored| stored.envelope.clone()) + .chain( + self.targets + .iter() + .filter(|stored| stored.value.target == target) + .map(|stored| stored.envelope.clone()), + ) + .chain( + self.transactions + .iter() + .filter(|stored| stored.value.target == target) + .map(|stored| stored.envelope.clone()), + ) + .collect::>(); + stamp.sort_by(|a, b| (&a.r#type, &a.key).cmp(&(&b.r#type, &b.key))); + stamp + } + + fn admitted_for(&self, target: &str) -> Option<&Stored> { + self.admitted + .iter() + .find(|stored| stored.value.target == target) + } + + fn supervision_for(&self, target: &str) -> Option<&Stored> { + self.targets + .iter() + .find(|stored| stored.value.target == target) + } + + /// The target's meters, or empty ones: a target that was never metered has + /// used nothing. + fn supervision_or_new(&self, target: &str) -> TargetSupervision { + self.supervision_for(target) + .map_or_else(|| TargetSupervision::new(target), |stored| stored.value.clone()) + } + + /// The admitted generation of the target that provides + /// `odin.verse-rendezvous`: Odin is whichever target says so, not + /// whichever is named "odin". + fn admitted_odin(&self) -> Option<&Stored> { + self.admitted.iter().find(|stored| { + ReadinessClass::of(&stored.value.expected) + .is_ok_and(|class| class == ReadinessClass::OdinSelf) + }) + } + + fn transaction_for_command(&self, command_id: &str) -> Vec<&DeploymentTransaction> { + let mut values = self + .transactions + .iter() + .filter(|stored| stored.value.command_id == command_id) + .map(|stored| &stored.value) + .collect::>(); + values.sort_by_key(|value| value.ordinal); + values + } + + fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool { + self.transactions.iter().any(|stored| { + stored.value.owns_target_authority() + && stored.value.command_id == transaction.command_id + && stored.value.ordinal < transaction.ordinal + }) + } + + /// The highest Odin publisher sequence this target has ever *acted on*. + /// + /// A transaction that failed contributes nothing. Its observations were + /// never built upon -- the deployment was abandoned and its projection + /// withdrawn -- so treating them as admitted history only raises the bar + /// for the next attempt. Odin does not reuse a sequence, so nothing is + /// weakened by forgetting the readings of an attempt that came to nothing; + /// what would be weakened is refusing every retry after a failure, which is + /// how a target ends up unable to deploy at all. + fn max_odin_sequence(&self, target: &str, signer_identity_id: &str) -> u64 { + let transaction_max = self + .transactions + .iter() + .filter_map(|stored| { + stored + .value + .latest_odin_observation + .as_ref() + .filter(|evidence| { + stored.value.target == target + && evidence.signer_identity_id == signer_identity_id + && !matches!( + stored.value.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + | Some(TransactionCompletion::FailedAfterFencing { .. }) + ) + }) + .map(|evidence| evidence.publisher_sequence) + }) + .max() + .unwrap_or(0); + let admitted_max = self + .admitted + .iter() + .filter(|stored| { + stored.value.target == target + && stored + .value + .odin_authority + .as_ref() + .is_some_and(|authority| authority.signer_identity_id == signer_identity_id) + }) + .map(|stored| stored.value.odin_publisher_sequence_cursor) + .max() + .unwrap_or(0); + transaction_max.max(admitted_max) + } +} + +pub(crate) fn decode_record(envelope: &CultCacheEnvelope) -> Result +where + T: for<'de> Deserialize<'de> + Serialize, +{ + let value: T = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&value)? == envelope.payload, + "Idunn control store contains a noncanonical record" + ); + Ok(value) +} + +fn command_envelope(value: &DeploymentCommand, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.command_id, + DeploymentCommand::TYPE, + DEPLOYMENT_COMMAND_SCHEMA, + value, + now, + ) +} + +/// The transaction layout of v2 and v3 records: `ready` is a bare Odin receipt, +/// and there are no phase deadline or lease adoption slots. It exists only to +/// decode records written before v4. `migrate_control_store_to_current_schema` +/// rewrites every one in `control.cc` at boot, but `history.cc` is never +/// migrated or pruned, so this struct dies only when history is pruned or +/// migrated. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_transaction", + schema = "idunn.deployment_transaction.v3" +)] +struct LegacyDeploymentTransaction { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + transaction_id: String, + #[cultcache(key = 2)] + command_id: String, + #[cultcache(key = 3)] + command_kind: CommandKind, + #[cultcache(key = 4)] + target: String, + #[cultcache(key = 5)] + ordinal: u32, + #[cultcache(key = 6)] + phase: DeploymentPhase, + #[cultcache(key = 7)] + created_at_unix_millis: u64, + #[cultcache(key = 8)] + updated_at_unix_millis: u64, + #[cultcache(key = 9)] + incumbent_generation_id: Option, + #[cultcache(key = 10)] + plan: Option, + #[cultcache(key = 11)] + frozen_source: Option, + #[cultcache(key = 12)] + sealed_release: Option, + #[cultcache(key = 13)] + installed_release: Option, + #[cultcache(key = 14)] + expected: Option, + #[cultcache(key = 15)] + expected_publication_sha256: Option, + #[cultcache(key = 16)] + deployment_authorization: Option, + #[cultcache(key = 17)] + lifecycle_authorized_at_unix_millis: Option, + #[cultcache(key = 18)] + activation: Option, + #[cultcache(key = 19)] + workload: Option, + #[cultcache(key = 20)] + activation_publication_sha256: Option, + #[cultcache(key = 21)] + latest_odin_observation: Option, + #[cultcache(key = 22)] + warming: Option, + #[cultcache(key = 23)] + route_preflight: Option, + #[cultcache(key = 24)] + isolation: Option, + #[cultcache(key = 25)] + fencing: Option, + #[cultcache(key = 26)] + leasing: Option, + #[cultcache(key = 27)] + ready: Option, + #[cultcache(key = 28)] + routing: Option, + #[cultcache(key = 29)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 30)] + last_error: Option, + #[cultcache(key = 31)] + completion: Option, + #[cultcache(key = 32)] + pre_fencing_abort: Option, + #[cultcache(key = 33)] + post_commit_cleanup: Option, + // A v2 record has no key 34: it predates post-fencing aborts. + #[cultcache(key = 34, default)] + post_fencing_abort: Option, +} + +impl LegacyDeploymentTransaction { + /// Every legacy Ready receipt is an Odin receipt: no route-proof class + /// existed. The new slots start empty, which is what an older transaction + /// had. + fn into_current(self) -> DeploymentTransaction { + DeploymentTransaction { + schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), + transaction_id: self.transaction_id, + command_id: self.command_id, + command_kind: self.command_kind, + target: self.target, + ordinal: self.ordinal, + phase: self.phase, + created_at_unix_millis: self.created_at_unix_millis, + updated_at_unix_millis: self.updated_at_unix_millis, + incumbent_generation_id: self.incumbent_generation_id, + plan: self.plan, + frozen_source: self.frozen_source, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + expected_publication_sha256: self.expected_publication_sha256, + deployment_authorization: self.deployment_authorization, + lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, + activation: self.activation, + workload: self.workload, + activation_publication_sha256: self.activation_publication_sha256, + latest_odin_observation: self.latest_odin_observation, + warming: self.warming, + route_preflight: self.route_preflight, + isolation: self.isolation, + fencing: self.fencing, + leasing: self.leasing, + ready: self + .ready + .map(|evidence| ReadinessEvidence::OdinCorrelated { evidence }), + routing: self.routing, + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + last_error: self.last_error, + completion: self.completion, + pre_fencing_abort: self.pre_fencing_abort, + post_commit_cleanup: self.post_commit_cleanup, + post_fencing_abort: self.post_fencing_abort, + phase_deadline: None, + lease_adoption: None, + } + } +} + +/// The v2 admitted-generation layout. Same lifetime rule as +/// `LegacyDeploymentTransaction`. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.admitted_generation", + schema = "idunn.admitted_generation.v2" +)] +struct LegacyAdmittedGeneration { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + generation_id: String, + #[cultcache(key = 3)] + command_id: String, + #[cultcache(key = 4)] + transaction_id: String, + #[cultcache(key = 5)] + admitted_at_unix_millis: u64, + #[cultcache(key = 6)] + plan: CompiledDeploymentPlan, + #[cultcache(key = 7)] + sealed_release: SealedRelease, + #[cultcache(key = 8)] + installed_release: InstalledReleaseObservation, + #[cultcache(key = 9)] + expected: IdunnExpectedIncarnationRecord, + #[cultcache(key = 10)] + activation: IdunnRuntimeActivationRecord, + #[cultcache(key = 11)] + workload: WorkloadObservation, + #[cultcache(key = 12)] + leasing: LeasingEvidence, + #[cultcache(key = 13)] + ready: TopologyEvidence, + #[cultcache(key = 14)] + latest_odin_observation: TopologyEvidence, + #[cultcache(key = 15)] + routing: RoutingEvidence, + #[cultcache(key = 16)] + odin_authority: AdmittedOdinAuthority, + #[cultcache(key = 17)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 18)] + route_repair_started_at_unix_millis: Option, +} + +impl LegacyAdmittedGeneration { + /// Every v2 generation was admitted on Odin receipts, so it migrates as + /// Odin-correlated with those receipts unchanged. Route supervision state + /// starts empty for a promoted route. The v2 repair start decided nothing + /// after S1 and is not carried. + fn into_current(self) -> AdmittedGeneration { + let route_supervision = matches!(&self.routing, RoutingEvidence::Promoted { .. }) + .then(RouteSupervisionState::default); + AdmittedGeneration { + schema_version: ADMITTED_GENERATION_SCHEMA.into(), + target: self.target, + generation_id: self.generation_id, + command_id: self.command_id, + transaction_id: self.transaction_id, + admitted_at_unix_millis: self.admitted_at_unix_millis, + plan: self.plan, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + activation: self.activation, + workload: self.workload, + leasing: self.leasing, + ready: ReadinessEvidence::OdinCorrelated { + evidence: self.ready, + }, + latest_odin_observation: Some(self.latest_odin_observation), + routing: self.routing, + odin_authority: Some(self.odin_authority), + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + route_supervision, + last_error: None, + } + } +} + +/// The v3 admitted-generation layout. Its meters (`actuations`, the continuity +/// backoff, the repair start) are retired: no released Idunn ever wrote a +/// nonzero one, and a record that shows one is refused instead of lifted. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.admitted_generation", + schema = "idunn.admitted_generation.v3" +)] +struct LegacyAdmittedGenerationV3 { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + generation_id: String, + #[cultcache(key = 3)] + command_id: String, + #[cultcache(key = 4)] + transaction_id: String, + #[cultcache(key = 5)] + admitted_at_unix_millis: u64, + #[cultcache(key = 6)] + plan: CompiledDeploymentPlan, + #[cultcache(key = 7)] + sealed_release: SealedRelease, + #[cultcache(key = 8)] + installed_release: InstalledReleaseObservation, + #[cultcache(key = 9)] + expected: IdunnExpectedIncarnationRecord, + #[cultcache(key = 10)] + activation: IdunnRuntimeActivationRecord, + #[cultcache(key = 11)] + workload: WorkloadObservation, + #[cultcache(key = 12)] + leasing: LeasingEvidence, + #[cultcache(key = 13)] + ready: ReadinessEvidence, + #[cultcache(key = 14)] + latest_odin_observation: Option, + #[cultcache(key = 15)] + routing: RoutingEvidence, + #[cultcache(key = 16)] + odin_authority: Option, + #[cultcache(key = 17)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 18)] + route_repair_started_at_unix_millis: Option, + #[cultcache(key = 19)] + route_supervision: Option, + #[cultcache(key = 20)] + continuity_backoff: LegacyContinuityBackoffV3, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyRouteSupervisionStateV3 { + last_challenge_at_unix_millis: Option, + consecutive_failures: u32, + next_challenge_at_unix_millis: Option, + degraded_since_unix_millis: Option, + actuations: LegacyActuationWindowV3, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyActuationWindowV3 { + window_started_at_unix_millis: u64, + count: u32, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyContinuityBackoffV3 { + window_started_at_unix_millis: Option, + attempts: u32, + next_restart_at_unix_millis: Option, +} + +impl LegacyAdmittedGenerationV3 { + /// Drops the retired meters. The lift creates no `TargetSupervision`: every + /// target starts unmetered, which is what no released binary ever changed. + fn into_current(self) -> Result { + ensure!( + self.continuity_backoff.attempts == 0 + && self + .route_supervision + .as_ref() + .is_none_or(|state| state.actuations.count == 0), + "admitted generation of {} records route actuations or continuity restarts that no released Idunn wrote; refusing to drop them", + self.target + ); + Ok(AdmittedGeneration { + schema_version: ADMITTED_GENERATION_SCHEMA.into(), + target: self.target, + generation_id: self.generation_id, + command_id: self.command_id, + transaction_id: self.transaction_id, + admitted_at_unix_millis: self.admitted_at_unix_millis, + plan: self.plan, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + activation: self.activation, + workload: self.workload, + leasing: self.leasing, + ready: self.ready, + latest_odin_observation: self.latest_odin_observation, + routing: self.routing, + odin_authority: self.odin_authority, + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + route_supervision: self.route_supervision.map(|state| RouteSupervisionState { + last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, + consecutive_failures: state.consecutive_failures, + next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, + degraded_since_unix_millis: state.degraded_since_unix_millis, + }), + last_error: None, + }) + } +} + +/// Decode a v2 or v3 transaction and lift it to the current shape, without +/// validating it. History reads stop here; the control store validates. +fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result { + let schema = match envelope.schema_id.as_deref() { + Some(schema @ (DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3)) => { + schema + } + _ => bail!("Idunn control store contains an unsupported transaction"), + }; + let value: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload) + .with_context(|| format!("decoding a {schema} deployment transaction"))?; + ensure!( + value.schema_version == schema, + "stored transaction schema differs from its envelope" + ); + ensure!( + schema != DEPLOYMENT_TRANSACTION_SCHEMA_V2 || value.post_fencing_abort.is_none(), + "a v2 transaction cannot carry post-fencing abort evidence" + ); + Ok(value.into_current()) +} + +/// Read one stored transaction, lifting a v2 or v3 record to the current shape. +/// +/// The control store refuses noncanonical bytes: a record must re-encode to +/// exactly what is stored. That check is what makes tampering visible, and it +/// also means adding a field is a schema change: an older record re-encodes +/// with extra keys and would be read as tampered. +/// +/// So an older record is decoded by its own layout and lifted. Byte-exactness +/// cannot apply across a version boundary -- the bytes are a different schema +/// by definition -- so the lift leans on the full semantic `validate()` +/// instead, and everything downstream sees only the current shape. +fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result { + match envelope.schema_id.as_deref() { + Some(DEPLOYMENT_TRANSACTION_SCHEMA) => { + let value: DeploymentTransaction = decode_record(envelope)?; + value.validate()?; + Ok(value) + } + _ => { + let mut value = lift_legacy_transaction(envelope)?; + cleanup_evidence::owe_legacy_continuity_projection(&mut value); + value.validate()?; + Ok(value) + } + } +} + +/// Read one stored admitted generation, lifting a v2 or v3 record to the +/// current shape by the same rule as `read_transaction_record`. +fn read_generation_record(envelope: &CultCacheEnvelope) -> Result { + match envelope.schema_id.as_deref() { + Some(ADMITTED_GENERATION_SCHEMA) => { + let value: AdmittedGeneration = decode_record(envelope)?; + value.validate()?; + Ok(value) + } + Some(ADMITTED_GENERATION_SCHEMA_V3) => { + let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload) + .context("decoding a v3 admitted generation")?; + ensure!( + legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V3, + "stored generation schema differs from its envelope" + ); + let value = legacy.into_current()?; + value.validate()?; + Ok(value) + } + Some(ADMITTED_GENERATION_SCHEMA_V2) => { + let legacy: LegacyAdmittedGeneration = rmp_serde::from_slice(&envelope.payload) + .context("decoding a v2 admitted generation")?; + ensure!( + legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V2, + "stored generation schema differs from its envelope" + ); + let value = legacy.into_current(); + value.validate()?; + Ok(value) + } + _ => bail!("Idunn control store contains an unsupported admitted generation"), + } +} + +/// The v1 target-supervision layout: the first seven keys of the current one. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.target_supervision", + schema = "idunn.target_supervision.v1" +)] +struct LegacyTargetSupervisionV1 { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + route_actuations: Vec, + #[cultcache(key = 3)] + continuity_restarts: Vec, + #[cultcache(key = 4)] + continuity_deferred_until: Option, + #[cultcache(key = 5)] + continuity_deferral_reason: Option, + #[cultcache(key = 6)] + route_survivals: Vec, +} + +impl LegacyTargetSupervisionV1 { + /// No streak, no anchor, no re-materialization: the same defaults a fresh + /// record starts with. + fn into_current(self) -> TargetSupervision { + TargetSupervision { + schema_version: TARGET_SUPERVISION_SCHEMA.into(), + target: self.target, + route_actuations: self.route_actuations, + continuity_restarts: self.continuity_restarts, + continuity_deferred_until: self.continuity_deferred_until, + continuity_deferral_reason: self.continuity_deferral_reason, + route_survivals: self.route_survivals, + continuity_attempt_streak: 0, + continuity_backoff_from_unix_millis: None, + continuity_rematerialized_at_unix_millis: None, + } + } +} + +/// Read one stored target supervision record, lifting a v1 record to the +/// current shape by the same rule as `read_generation_record`. One schema id +/// names one shape: a v1 record that carries ten keys, or a v2 record that +/// carries seven, is refused. +fn read_supervision_record(envelope: &CultCacheEnvelope) -> Result { + let value = match envelope.schema_id.as_deref() { + Some(TARGET_SUPERVISION_SCHEMA) => decode_record::(envelope)?, + Some(TARGET_SUPERVISION_SCHEMA_V1) => { + let legacy: LegacyTargetSupervisionV1 = decode_record(envelope) + .context("decoding a v1 target supervision record")?; + ensure!( + legacy.schema_version == TARGET_SUPERVISION_SCHEMA_V1, + "stored supervision schema differs from its envelope" + ); + legacy.into_current() + } + _ => bail!("Idunn control store contains an unsupported target supervision record"), + }; + value.validate()?; + ensure!( + envelope.key == value.target, + "target supervision key is not its target" + ); + Ok(value) +} + +/// Rewrite every older record as current, once, before the engine runs. +/// +/// The read path lifts older records on its own, so this is convergence rather +/// than correctness: without it the store keeps records in several shapes for +/// as long as the oldest one survives. Each rewrite is a compare-exchange +/// against the exact stored envelope, so a record that changed underneath is +/// left alone rather than clobbered. +fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { + if !store_path.exists() { + return Ok(0); + } + let store = SingleFileMessagePackBackingStore::new(store_path); + let stale = store + .pull_all_read_only_snapshot() + .context("reading Idunn control snapshot for migration")? + .into_iter() + .filter(|envelope| { + let schema = envelope.schema_id.as_deref(); + (envelope.r#type == DeploymentTransaction::TYPE + && matches!( + schema, + Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) + )) + || (envelope.r#type == AdmittedGeneration::TYPE + && matches!( + schema, + Some(ADMITTED_GENERATION_SCHEMA_V2 | ADMITTED_GENERATION_SCHEMA_V3) + )) + || (envelope.r#type == TargetSupervision::TYPE + && schema == Some(TARGET_SUPERVISION_SCHEMA_V1)) + }) + .collect::>(); + let mut migrated = 0; + for envelope in stale { + let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { + let (key, next) = cleanup_evidence::migrate_transaction_record(&envelope)?; + (DeploymentTransaction::TYPE, key, next) + } else if envelope.r#type == TargetSupervision::TYPE { + let value = read_supervision_record(&envelope)?; + let next = target_supervision_envelope(&value, now_millis()?)?; + (TargetSupervision::TYPE, value.target, next) + } else { + let value = read_generation_record(&envelope)?; + let next = admitted_envelope(&value, value.admitted_at_unix_millis)?; + (AdmittedGeneration::TYPE, value.target, next) + }; + next.stored_at = envelope.stored_at.clone(); + ensure!( + store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: record_type.into(), + key, + current: Some(envelope.clone()), + }], + &[next], + )?, + "control record changed during schema migration" + ); + migrated += 1; + } + Ok(migrated) +} + +fn transaction_envelope(value: &DeploymentTransaction, now: u64) -> Result { + ensure!( + !value.carries_legacy_marker(), + "only the legacy lift may produce a legacy-marked record; it cannot be written as a new one" + ); + encode_transaction(value, now) +} + +fn encode_transaction(value: &DeploymentTransaction, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.transaction_id, + DeploymentTransaction::TYPE, + DEPLOYMENT_TRANSACTION_SCHEMA, + value, + now, + ) +} + +fn admitted_envelope(value: &AdmittedGeneration, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.target, + AdmittedGeneration::TYPE, + ADMITTED_GENERATION_SCHEMA, + value, + now, + ) +} + +fn target_supervision_envelope(value: &TargetSupervision, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.target, + TargetSupervision::TYPE, + TARGET_SUPERVISION_SCHEMA, + value, + now, + ) +} + +pub(crate) fn typed_envelope( + key: &str, + record_type: &str, + schema: &str, + value: &T, + now: u64, +) -> Result { + Ok(CultCacheEnvelope { + key: key.into(), + r#type: record_type.into(), + payload: rmp_serde::to_vec(value)?, + stored_at: rfc3339_millis(now)?, + schema_id: Some(schema.into()), + }) +} + +/// `history.cc`, beside the control store. Terminal transactions live here so +/// the control store holds only what still gates a decision. +fn history_store_path(state_store: &Path) -> PathBuf { + state_store.with_file_name("history.cc") +} + +/// Move one finished transaction out of the live set. +/// +/// History first, then the live copy: a crash between the two leaves the +/// record in both, which reads as still-resident and is archived again on the +/// next attempt. The other order can lose it, and a completion nobody can +/// observe is the failure that actually costs an operator something. +/// +/// `insert_entry_if_absent` makes the repeat a no-op, and the delete is a +/// compare-and-swap on the exact envelope archived, so a transaction that +/// changed underneath us is left alone rather than dropped. +fn archive_terminal_transaction(state_store: &Path, envelope: &CultCacheEnvelope) -> Result<()> { + SingleFileMessagePackBackingStore::new(&history_store_path(state_store)) + .insert_entry_if_absent(envelope.clone()) + .context("archiving a finished transaction to history")?; + SingleFileMessagePackBackingStore::new(state_store) + .delete_batch_if_unchanged(std::slice::from_ref(envelope)) + .context("retiring a finished transaction from the control store")?; + // The command goes with its last transaction. A command is consumed by + // being frozen once; left resident after its transaction retired, it read + // as queued again and was frozen again on the next tick, which is how one + // `idunn up` became an unbounded series of deployment attempts. + let transaction = read_transaction_record(envelope)?; + let snapshot = ControlSnapshot::read(state_store)?; + if !snapshot + .transaction_for_command(&transaction.command_id) + .is_empty() + { + return Ok(()); + } + if let Some(command) = snapshot + .commands + .iter() + .find(|stored| stored.value.command_id == transaction.command_id) + { + SingleFileMessagePackBackingStore::new(&history_store_path(state_store)) + .insert_entry_if_absent(command.envelope.clone()) + .context("archiving a consumed command to history")?; + SingleFileMessagePackBackingStore::new(state_store) + .delete_batch_if_unchanged(std::slice::from_ref(&command.envelope)) + .context("retiring a consumed command from the control store")?; + } + Ok(()) +} + +/// What `history.cc` yielded: the transactions it decoded and the entries it +/// could not, each with its key and reason. +struct HistoryRead { + transactions: Vec, + undecodable: Vec<(String, String)>, +} + +impl HistoryRead { + /// What is missing from `transactions`, or `None` when nothing is. + fn report(&self) -> Option { + if self.undecodable.is_empty() { + return None; + } + let keys = self + .undecodable + .iter() + .map(|(key, error)| format!("{key} ({error})")) + .collect::>() + .join("; "); + Some(format!( + "Idunn history: {} archived transaction(s) could not be decoded and are not counted \ + by status or continuity backoff: {keys}", + self.undecodable.len() + )) + } +} + +/// Finished transactions. `control.cc` keeps the byte-exact canonical check +/// because its records still gate decisions. History describes what already +/// happened, so an undecodable entry is reported and skipped rather than +/// allowed to refuse the read -- that asymmetry is the whole reason the two +/// files are separate. A file that cannot be read at all is an error: the +/// caller decides whether an empty answer is safe. +fn read_history(state_store: &Path) -> Result { + let path = history_store_path(state_store); + if !path.exists() { + return Ok(HistoryRead { + transactions: Vec::new(), + undecodable: Vec::new(), + }); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .with_context(|| format!("Idunn cannot read {}", path.display()))?; + let (transactions, undecodable) = decode_history_transactions(envelopes); + Ok(HistoryRead { + transactions, + undecodable, + }) +} + +/// History for display only (status). An unreadable file shows as empty and +/// says so; decisions use `Engine::history_for_decision`, which does not. +fn read_history_transactions(state_store: &Path) -> Vec { + match read_history(state_store) { + Ok(history) => { + if let Some(report) = history.report() { + eprintln!("{report}"); + } + history.transactions + } + Err(error) => { + eprintln!("{error:#}; every archived transaction is invisible to status"); + Vec::new() + } + } +} + +/// How long to wait after `failures` consecutive failures: one poll interval +/// after the first, doubling each time, never past the ceiling. +fn backoff_wait(base_millis: u64, failures: u32, ceiling_millis: u64) -> u64 { + base_millis + .saturating_mul(1u64 << failures.min(20)) + .min(ceiling_millis) +} + +/// Whether an attempt due at `not_before` must still wait at `now`. A due time +/// further ahead than any wait its owner can set means the clock stepped back, +/// so the attempt is due: a step stalls nothing longer than the wait itself. +fn is_waiting(now: u64, not_before: u64, longest_wait: u64) -> bool { + now < not_before && not_before - now <= longest_wait +} + +/// Whether a transaction moved between two reads. An error note (`last_error`) +/// and its timestamp are commentary on a stuck step, not a step. +fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) -> bool { + let mut after = after.clone(); + after.last_error.clone_from(&before.last_error); + after.updated_at_unix_millis = before.updated_at_unix_millis; + &after != before +} + +/// Says a fault once on stderr while it lasts. A scheduler tick that hits the same +/// unreadable file every half second must not print it every half second. +/// +/// It is a trace and owns nothing else: it is in memory, keyed by free-form +/// strings, and an Idunn restart forgets it. Whether the operator is told +/// belongs to the incident record (`incident`), never to this map. +#[derive(Default)] +struct ReportOnce { + last: Mutex>, +} + +impl ReportOnce { + /// The text to print now: `report` when it differs from the last one + /// offered, nothing while the same fault stands or once it has cleared. + fn offer(&self, report: Option) -> Option { + let mut last = self.last.lock().expect("report mutex"); + if *last == report { + return None; + } + last.clone_from(&report); + report + } +} + +/// Decode every archived transaction. An entry that fails is returned with its +/// key and reason instead of vanishing, so the reader can say how many records +/// its answer is missing. +/// +/// `LegacyDeploymentTransaction` decodes the v2 and v3 layouts. `history.cc` is +/// never migrated or pruned, so it dies only when history is pruned or +/// migrated, not at boot. +fn decode_history_transactions( + envelopes: Vec, +) -> (Vec, Vec<(String, String)>) { + let mut transactions = Vec::new(); + let mut undecodable = Vec::new(); + for envelope in envelopes + .into_iter() + .filter(|envelope| envelope.r#type == DeploymentTransaction::TYPE) + { + let decoded = match envelope.schema_id.as_deref() { + Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) => { + lift_legacy_transaction(&envelope) + } + _ => rmp_serde::from_slice::(&envelope.payload) + .map_err(anyhow::Error::from), + }; + match decoded { + Ok(transaction) => transactions.push(transaction), + Err(error) => undecodable.push((envelope.key, format!("{error:#}"))), + } + } + (transactions, undecodable) +} + +/// Consumed commands, read as leniently as their transactions. +fn read_history_commands(state_store: &Path) -> Vec { + let path = history_store_path(state_store); + if !path.exists() { + return Vec::new(); + } + let Ok(envelopes) = SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot() + else { + return Vec::new(); + }; + envelopes + .into_iter() + .filter(|envelope| envelope.r#type == DeploymentCommand::TYPE) + .filter_map(|envelope| rmp_serde::from_slice::(&envelope.payload).ok()) + .collect() +} + +fn replace_transaction( + store_path: &Path, + current: &Stored, + next: &DeploymentTransaction, +) -> Result<()> { + next.validate()?; + ensure!( + current.value.transaction_id == next.transaction_id, + "transaction replacement changes identity" + ); + let envelope = transaction_envelope(next, next.updated_at_unix_millis)?; + ensure!( + SingleFileMessagePackBackingStore::new(store_path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: current.value.transaction_id.clone(), + current: Some(current.envelope.clone()), + }], + std::slice::from_ref(&envelope), + )?, + "deployment transaction changed before its compare-exchange" + ); + if next.is_terminal() { + archive_terminal_transaction(store_path, &envelope)?; + } + Ok(()) +} + +#[derive(Clone)] +struct LoadedBinding { + binding: OperatorBinding, + bytes: Vec, +} + +fn load_bindings(directory: &Path) -> Result> { + let mut paths = fs::read_dir(directory) + .with_context(|| format!("reading Idunn binding directory {}", directory.display()))? + .map(|entry| entry.map(|entry| entry.path())) + .collect::>>()?; + paths.retain(|path| path.extension() == Some(std::ffi::OsStr::new("toml"))); + paths.sort(); + let mut bindings = BTreeMap::new(); + for path in paths { + let bytes = fs::read(&path) + .with_context(|| format!("reading operator binding {}", path.display()))?; + let text = std::str::from_utf8(&bytes) + .with_context(|| format!("operator binding {} is not UTF-8", path.display()))?; + let binding = OperatorBinding::parse(text) + .with_context(|| format!("validating operator binding {}", path.display()))?; + let target = binding.target.clone(); + ensure!( + bindings + .insert(target.clone(), LoadedBinding { binding, bytes }) + .is_none(), + "operator binding target {target} is duplicated" + ); + } + ensure!(!bindings.is_empty(), "Idunn binding directory is empty"); + validate_route_bindings(&bindings)?; + Ok(bindings) +} + +fn validate_route_bindings(bindings: &BTreeMap) -> Result<()> { + let routes = bindings + .iter() + .filter_map(|(target, loaded)| { + loaded + .binding + .route + .as_ref() + .map(|route| (target.as_str(), route)) + }) + .collect::>(); + validate_route_binding_set(&routes) +} + +fn validate_route_binding_set(routes: &[(&str, &RouteBinding)]) -> Result<()> { + for (index, (target, route)) in routes.iter().enumerate() { + let (stable_host, stable_port) = route.stable_socket()?; + let private_host = route + .private_host + .parse::() + .context("validated route private host stopped being an IP address")?; + ensure!( + stable_host != private_host + || !(route.private_port_start..=route.private_port_end).contains(&stable_port), + "route {target} stable socket overlaps its candidate port range" + ); + + for (other_target, other) in &routes[index + 1..] { + ensure!( + route.route_id != other.route_id, + "route id {} is shared by targets {target} and {other_target}", + route.route_id + ); + ensure!( + route.config_path != other.config_path, + "route fragment {} is shared by targets {target} and {other_target}", + route.config_path.display() + ); + let other_stable = other.stable_socket()?; + ensure!( + route.driver != other.driver || (stable_host, stable_port) != other_stable, + "stable route socket {}:{} is shared by targets {target} and {other_target}", + stable_host, + stable_port + ); + if route.driver != other.driver { + continue; + } + let other_private_host = other + .private_host + .parse::() + .context("validated route private host stopped being an IP address")?; + let ranges_overlap = route.private_host == other.private_host + && route.private_port_start <= other.private_port_end + && other.private_port_start <= route.private_port_end; + ensure!( + !ranges_overlap, + "candidate port ranges overlap for targets {target} and {other_target}" + ); + ensure!( + stable_host != other_private_host + || !(other.private_port_start..=other.private_port_end).contains(&stable_port), + "stable route socket for {target} overlaps {other_target}'s candidate range" + ); + ensure!( + other_stable.0 != private_host + || !(route.private_port_start..=route.private_port_end) + .contains(&other_stable.1), + "stable route socket for {other_target} overlaps {target}'s candidate range" + ); + } + } + Ok(()) +} + +fn resolve_selector( + bindings: &BTreeMap, + selector: &str, +) -> Result> { + let mut targets = if let Some(profile) = selector.strip_prefix("profile:") { + bindings + .values() + .filter(|binding| binding.binding.profiles.contains(profile)) + .map(|binding| binding.binding.target.clone()) + .collect::>() + } else { + ensure!( + bindings.contains_key(selector), + "unknown deployment target {selector}" + ); + vec![selector.to_owned()] + }; + ensure!( + !targets.is_empty(), + "unknown or empty deployment profile {selector}" + ); + targets.sort_by(|left, right| { + (left != "odin") + .cmp(&(right != "odin")) + .then_with(|| left.cmp(right)) + }); + Ok(targets) +} + +fn submit( + store_path: &Path, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + if let Some(parent) = store_path.parent() { + fs::create_dir_all(parent)?; + } + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: format!("up-{}", Uuid::new_v4()), + kind: CommandKind::Deploy, + selector: selector.into(), + requested_by: requested_by.into(), + requested_at_unix_millis: now, + }; + command.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(store_path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, now)?], + )?, + "deployment command id collided" + ); + println!("{}", command.command_id); + if !wait { + return Ok(()); + } + let deadline = now.saturating_add(timeout_seconds.saturating_mul(1000)); + loop { + let snapshot = ControlSnapshot::read(store_path)?; + ensure!( + snapshot + .commands + .iter() + .any(|stored| stored.value.command_id == command.command_id), + "submitted deployment command disappeared" + ); + let transactions = snapshot.transaction_for_command(&command.command_id); + if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { + if let Some(error) = + transactions + .iter() + .find_map(|transaction| match &transaction.completion { + Some(TransactionCompletion::FailedBeforeFencing { error }) + | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => { + Some(error.as_str()) + } + _ => None, + }) + { + bail!("deployment failed: {error}") + } + println!("succeeded {} target(s)", transactions.len()); + return Ok(()); + } + if now_millis()? >= deadline { + bail!("deployment command timed out") + } + thread::sleep(Duration::from_millis(250)); + } +} + +fn status( + out: &mut impl Write, + store_path: &Path, + incident_store: &Path, + command_id: Option<&str>, +) -> Result<()> { + let snapshot = ControlSnapshot::read(store_path)?; + // Commands stay resident; their finished transactions do not. Without + // history a completed command would report as though it had never run, + // which is the operator surface R11 refused to trade away for a smaller + // live set. + let archived = read_history_transactions(store_path); + let archived_commands = read_history_commands(store_path); + let mut commands = snapshot + .commands + .iter() + .map(|stored| &stored.value) + .chain(archived_commands.iter()) + .collect::>(); + commands.sort_by_key(|command| command.requested_at_unix_millis); + if let Some(command_id) = command_id { + commands.retain(|command| command.command_id == command_id); + ensure!(!commands.is_empty(), "deployment command is unknown"); + } + for command in commands { + let mut transactions = snapshot.transaction_for_command(&command.command_id); + transactions.extend( + archived + .iter() + .filter(|value| value.command_id == command.command_id), + ); + transactions.sort_by_key(|value| value.ordinal); + let (state, detail) = derived_command_status(&transactions); + writeln!( + out, + "{} {} {} {}", + command.command_id, command.selector, state, detail + )?; + // Naming one command asks about that command, so print what a stuck + // transaction is actually waiting on. A gate reason lives in + // `last_error` and was never rendered anywhere, which left "Sealing" + // looking identical whether the brake had not authorized the + // transaction or the phase was simply slow. The release and deployment + // ids are here because they are exactly what a brake release must name. + if command_id.is_some() { + for transaction in transactions { + writeln!(out, " transaction {}", transaction.transaction_id)?; + writeln!( + out, + " target {} phase {:?}", + transaction.target, transaction.phase + )?; + if let Some(expected) = &transaction.expected { + writeln!(out, " runtime {}", expected.runtime_id)?; + writeln!(out, " release {}", expected.sealed_release_id)?; + } + writeln!( + out, + " odin publisher cursor {}", + transaction.odin_publisher_sequence_cursor + )?; + if let Some(evidence) = &transaction.latest_odin_observation { + writeln!( + out, + " latest odin observation sequence {}", + evidence.publisher_sequence + )?; + } + if let Some(reason) = &transaction.last_error { + writeln!(out, " waiting on {reason}")?; + } + // A Complete transaction that is not terminal still owns its + // target through unfinished cleanup. Without this an operator + // sees "Complete" and a stale reason and cannot tell what is + // holding the target. + if let Some(completion) = &transaction.completion { + writeln!(out, " completion {completion:?}")?; + } + if let Some(abort) = &transaction.pre_fencing_abort { + writeln!(out, " pre-fencing abort {abort:?}")?; + } + if let Some(abort) = &transaction.post_fencing_abort { + writeln!(out, " post-fencing abort {abort:?}")?; + } + if let Some(cleanup) = &transaction.post_commit_cleanup { + writeln!(out, " post-commit cleanup {cleanup:?}")?; + } + writeln!( + out, + " terminal {} owns-target {}", + transaction.is_terminal(), + transaction.blocks_new_target_mutation() + )?; + } + } + } + // What supervision holds for each target: the union of the admitted + // generations and the metered targets, so a target that has only been + // deployed to (no generation yet) still shows what it has used. + for line in render_targets(&snapshot, incident_store, now_millis()?) { + writeln!(out, "{line}")?; + } + Ok(()) +} + +/// Every supervised target with its meters and its incidents. An incident +/// store that cannot be read costs the incident lines and nothing else. +fn render_targets(snapshot: &ControlSnapshot, incident_store: &Path, now: u64) -> Vec { + let incidents = IncidentStore::new(incident_store).read(); + let mut lines = Vec::new(); + for target in supervised_targets(snapshot) { + lines.extend(render_supervision( + target, + snapshot.admitted_for(target).map(|stored| &stored.value), + snapshot.supervision_for(target).map(|stored| &stored.value), + now, + )); + if let Ok(records) = &incidents { + lines.extend(render_incidents(records, target, now)); + } + } + if let Err(error) = &incidents { + lines.push(format!("incidents unreadable: {}", error.root_cause())); + } + lines +} + +/// Every target supervision holds something for: the admitted generations +/// and the metered targets. +fn supervised_targets(snapshot: &ControlSnapshot) -> BTreeSet<&str> { + snapshot + .admitted + .iter() + .map(|stored| stored.value.target.as_str()) + .chain(snapshot.targets.iter().map(|stored| stored.value.target.as_str())) + .collect() +} + +/// One target's meters and route pacing, one line each, times in unix +/// milliseconds. The counts are what is still inside the window at `now`. +fn render_supervision( + target: &str, + generation: Option<&AdmittedGeneration>, + supervision: Option<&TargetSupervision>, + now: u64, +) -> Vec { + let at = |time: Option| time.map_or("none".to_owned(), |millis| millis.to_string()); + let meters = supervision.cloned().unwrap_or_else(|| TargetSupervision::new(target)); + let mut lines = vec![format!( + "target {target} {}", + generation.map_or("no-admitted-generation", |value| value.generation_id.as_str()) + )]; + if let Some(reason) = generation.and_then(|value| value.last_error.as_deref()) { + lines.push(format!(" held: {reason}")); + } + lines.push(format!( + " continuity attempts-in-hour {} pages-at {} streak {} next-attempt-at {}", + meters.restarts_used(now), + CONTINUITY_PAGING_RESTARTS, + meters.continuity_attempt_streak, + at(generation.and_then(|value| meters.backoff_due(value.admitted_at_unix_millis))) + )); + if let Some(rematerialized) = meters.continuity_rematerialized_at_unix_millis { + lines.push(format!(" continuity rematerialized-at {rematerialized}")); + } + if let (Some(until), Some(reason)) = ( + meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + ) { + lines.push(format!(" continuity deferred until {until}: {reason}")); + } + if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { + lines.push(format!( + " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", + match route.degraded_since_unix_millis { + Some(since) => format!("degraded-since {since}"), + None => "healthy".to_owned(), + }, + route.consecutive_failures, + at(route.last_challenge_at_unix_millis), + at(route.next_challenge_at_unix_millis) + )); + } + if generation.is_none_or(|value| value.route_supervision.is_some()) + || !meters.route_actuations.is_empty() + || !meters.route_survivals.is_empty() + { + lines.push(format!( + " route actuations {}/{} in window, reopens-at {}", + meters.route_used(now), + ROUTE_ACTUATION_CEILING, + at(meters.route_reopens_at(now)) + )); + // A record, not a meter: only the newest are kept, so a saturated + // count says "at least". + let survivals = meters.survivals_in_window(now); + lines.push(format!( + " route survival actuations {}{} in window (uncounted), last-at {}", + if survivals == ROUTE_SURVIVAL_RECORD { "at least " } else { "" }, + survivals, + at(meters.route_survivals.last().copied()) + )); + } + lines +} + +fn derived_command_status(transactions: &[&DeploymentTransaction]) -> (&'static str, String) { + if transactions.is_empty() { + return ("queued", String::new()); + } + // A live transaction is the command's present tense. Reporting an older + // attempt's failure while a new one is sealing hid a running deployment + // behind the word "failed". + let current = transactions + .iter() + .filter(|transaction| !transaction.is_terminal()) + .map(|transaction| format!("{}:{:?}", transaction.target, transaction.phase)) + .collect::>(); + if !current.is_empty() { + return ("running", current.join(",")); + } + if let Some(error) = transactions + .iter() + .find_map(|transaction| match &transaction.completion { + Some(TransactionCompletion::FailedBeforeFencing { error }) + | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => Some(error.clone()), + _ => None, + }) + { + return ("failed", error); + } + if transactions.iter().all(|transaction| { + transaction.is_terminal() + && matches!( + transaction.completion, + Some(TransactionCompletion::Admitted { .. }) + ) + }) { + return ( + "succeeded", + transactions + .iter() + .map(|transaction| transaction.target.as_str()) + .collect::>() + .join(","), + ); + } + ("complete", String::new()) +} + +struct ProcessLock { + file: File, + #[cfg(not(unix))] + path: PathBuf, +} + +impl ProcessLock { + fn acquire(store_path: &Path) -> Result { + let path = sibling_path(store_path, ".daemon.lock"); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + #[cfg(unix)] + { + use std::os::fd::AsRawFd; + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(&path) + .with_context(|| format!("opening Idunn daemon lock {}", path.display()))?; + let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + ensure!(result == 0, "another Idunn daemon owns this control store"); + Ok(Self { file }) + } + #[cfg(not(unix))] + { + let file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .with_context(|| format!("another Idunn daemon owns {}", path.display()))?; + Ok(Self { file, path }) + } + } +} + +impl Drop for ProcessLock { + fn drop(&mut self) { + #[cfg(unix)] + { + use std::os::fd::AsRawFd; + let _ = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; + } + #[cfg(not(unix))] + { + let _ = fs::remove_file(&self.path); + } + } +} + +/// One target's in-memory challenge wait: when it is due, and the stored +/// records of the target as they were when the record failed to land. +struct ChallengeWait { + due_unix_millis: u64, + stamp: Vec, +} + +struct Engine { + options: RuntimeOptions, + idunn_signer: ServiceIdentitySigner, + idunn_anchor: ServiceIdentityTrustAnchor, + bootstrap_odin_authority: AdmittedOdinAuthority, + source: Arc, + docker_runner: DockerRunnerDriver, + systemd_workload: Arc, + host_actuators: Option, + history_report: ReportOnce, + /// One `ReportOnce` per faulting record, so one wedged transaction says + /// its fault once and never drowns another's. + fault_reports: Mutex>, + /// Retry pacing for a transaction whose step keeps failing, by + /// transaction id: consecutive failures and the earliest next attempt. + /// Process-local on purpose: a restart retries at once, and no schema + /// carries a clock that only pacing reads. + resume_backoff: Mutex>, + /// When each routed target's next challenge is due, held only when the + /// stored `RouteSupervisionState` could not record the last one. The stored + /// value owns the schedule and survives restarts; this spaces attempts + /// while the record did not land. An entry ends the moment any write for + /// its target lands (its stamp no longer matches the store) or the target + /// is retired or withdrawn: it never outlives the episode that made it. + challenge_wait: Mutex>, +} + +/// The route driver's door to the actuation ceiling: every group of host +/// mutations a driver makes for `target` is charged through here. The gate +/// owns which charge a request is: a driver asks for the change it is making, +/// and the command that owns the transaction decides whether that change is +/// deployment (`Forward`, refusable) or the continuity of the admitted +/// generation (`Survival`, never refused). +struct EngineRouteGate<'a> { + engine: &'a Engine, + target: &'a str, + command: CommandKind, +} + +impl RouteActuationGate for EngineRouteGate<'_> { + fn admit(&self, requested: RouteActuation) -> Result<()> { + let kind = match self.command { + CommandKind::Deploy => requested, + CommandKind::Continuity => RouteActuation::Survival, + }; + self.engine.charge_route_actuation(self.target, kind) + } +} + +#[derive(Clone, Copy)] +struct ResumeBackoff { + failures: u32, + not_before_unix_millis: u64, +} + +/// Longest wait between attempts at a failing transaction step. +const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; + +impl Engine { + fn open(options: RuntimeOptions) -> Result { + Self::open_with_systemd_workload( + options, + Arc::new(SystemdTransientWorkloadDriver::default()), + ) + } + + /// `open` with the systemd workload port supplied. It is the seam that + /// lets the phase engine run past Fencing without a systemd to talk to. + fn open_with_systemd_workload( + options: RuntimeOptions, + systemd_workload: Arc, + ) -> Result { + let idunn_signer = + open_service_identity_at::(&options.idunn_identity_store) + .context("opening Idunn activation identity")?; + let idunn_anchor = idunn_signer.trust_anchor()?; + let odin_anchor = read_trust_anchor::(&options.odin_trust_anchor) + .context("reading bootstrap Odin topology anchor")?; + let bootstrap_odin_authority = AdmittedOdinAuthority::from_anchor(&odin_anchor)?; + let source: Arc = Arc::new(GitSourceDriver::new( + &options.source_root, + options.staging_root.join("frozen-sources"), + options.source_identity, + )); + let host_actuators = options + .host_actuator_bind + .map(|bind| HostActuatorHub::bind(bind).map(|hub| Arc::new(Mutex::new(hub)))) + .transpose()?; + if let Some(hub) = &host_actuators { + eprintln!( + "Idunn host actuator hub listening on {}", + hub.lock().expect("hub mutex").local_addr()? + ); + let service_signer = + open_service_identity_at::(&options.idunn_identity_store) + .context("opening Idunn identity for the host actuator hub thread")?; + let bindings_dir = options.bindings_dir.clone(); + spawn_hub_service(Arc::clone(hub), service_signer, move || { + host_anchors_from(&bindings_dir) + }); + } + Ok(Self { + options, + idunn_signer, + idunn_anchor, + bootstrap_odin_authority, + source, + docker_runner: DockerRunnerDriver::default(), + systemd_workload, + host_actuators, + history_report: ReportOnce::default(), + fault_reports: Mutex::default(), + resume_backoff: Mutex::default(), + challenge_wait: Mutex::default(), + }) + } + + fn host_anchors(&self) -> BTreeMap { + host_anchors_from(&self.options.bindings_dir) + } + + /// Every route driver comes from here, so the actuator paths are the + /// options' and nothing else's. + fn route_driver(&self, binding: RouteBinding) -> NginxRouteDriver { + NginxRouteDriver::with_actuators(binding, &self.options.route_actuators) + } + + fn route_gate<'a>(&'a self, target: &'a str, command: CommandKind) -> EngineRouteGate<'a> { + EngineRouteGate { + engine: self, + target, + command, + } + } + + /// Charge one route actuation to the target's `TargetSupervision` before + /// the driver acts. The record is created by the first charge, so a first + /// deployment is metered like any other. A `Forward` is counted against + /// the sliding ceiling; a refused one is the typed `RouteActuationRefused`, + /// and one that cannot be recorded is refused too: no charge, no + /// actuation. A `Survival` is never counted or refused, only recorded when + /// it can be, and runs regardless when it cannot: an unwritable store must + /// not stop a rollback, a withdrawal or a repair. + fn charge_route_actuation(&self, target: &str, kind: RouteActuation) -> Result<()> { + match self.record_route_actuation(target, kind) { + Ok(charged) => charged.map_err(anyhow::Error::new), + Err(error) if kind == RouteActuation::Survival => { + eprintln!( + "Idunn could not record a survival route actuation of {target}; it proceeds unrecorded: {error:#}" + ); + Ok(()) + } + Err(error) => Err(error), + } + } + + /// The outer error is the ledger failing to read or write; the inner one + /// is the ceiling refusing. + fn record_route_actuation( + &self, + target: &str, + kind: RouteActuation, + ) -> Result> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let seen = snapshot.supervision_for(target); + let mut next = snapshot.supervision_or_new(target); + let now = now_millis()?; + let charged = next.charge_route(now, kind); + // A refusal writes nothing unless the clock had stepped back and the + // log needed settling. + if charged.is_ok() || seen.is_some_and(|stored| stored.value != next) { + self.write_target_supervision(seen, &next, now)?; + } + Ok(charged) + } + + /// Replace a target's meters, or create them when `seen` is `None`, by + /// compare-exchange against exactly what was read. + fn write_target_supervision( + &self, + seen: Option<&Stored>, + next: &TargetSupervision, + now: u64, + ) -> Result<()> { + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: TargetSupervision::TYPE.into(), + key: next.target.clone(), + current: seen.map(|stored| stored.envelope.clone()), + }], + &[target_supervision_envelope(next, now)?], + )?, + "target supervision changed before its meters were written" + ); + Ok(()) + } + + /// Replace an admitted generation by compare-exchange against the envelope + /// that was read. `Ok(false)` is a lost race: another writer moved the + /// record, so the caller's snapshot is stale. `Err` is a write the store + /// refused, which changed nothing. + fn replace_generation( + &self, + seen: &Stored, + next: &AdmittedGeneration, + now: u64, + ) -> Result { + Ok(SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: seen.value.target.clone(), + current: Some(seen.envelope.clone()), + }], + &[admitted_envelope(next, now)?], + )?) + } + + /// Say a fault once while it lasts, under `key`. + fn report_once(&self, key: &str, message: String) { + let offered = self + .fault_reports + .lock() + .expect("fault report mutex") + .entry(key.to_string()) + .or_default() + .offer(Some(message)); + if let Some(message) = offered { + eprintln!("{message}"); + } + } + + fn incidents(&self) -> IncidentStore { + IncidentStore::new(&self.options.incident_store) + } + + /// Settle the outcome of one incident operation. A failure never leaves + /// this function: it is traced once per episode, under the operation and + /// the subject it was decided for (`*` for a pass over the whole store), + /// and continuity decides exactly as it would have with a healthy store. + /// Each decision site has its own key, so one site's success never ends + /// another site's episode. The message names the operation and the root + /// cause and never a path or a value. Incident writes are never progress. + fn record_incident(&self, operation: &str, subject: &str, outcome: Result) { + let key = format!("incident-store:{operation}:{subject}"); + match outcome { + Ok(_) => self.clear_fault(&key), + Err(error) => self.report_once( + &key, + format!( + "Idunn could not {operation} an operator incident: {}", + error.root_cause() + ), + ), + } + } + + /// Close every open incident whose subject is a target that is no longer + /// admitted, then retire what has been closed past retention. + fn reconcile_incidents(&self, snapshot: &ControlSnapshot) { + let store = self.incidents(); + let close_unadmitted = || -> Result<()> { + let now = now_millis()?; + for record in store.read()? { + if record.is_open() + && record.condition.ends_with_admission() + && snapshot.admitted_for(&record.subject).is_none() + { + store.close( + record.condition, + &record.subject, + CloseReason::NoLongerAdmitted, + now, + )?; + } + } + Ok(()) + }; + self.record_incident("close", "*", close_unadmitted()); + self.record_incident("retire", "*", now_millis().and_then(|now| store.retire_closed(now))); + } + + fn host_access(&self) -> Result> { + let hub = self + .host_actuators + .as_ref() + .context("this Idunn serves no host actuators (no --host-actuator-bind)")?; + Ok(HostActuatorAccess { + hub: Arc::clone(hub), + anchors: self.host_anchors(), + signer: &self.idunn_signer, + }) + } +} + +/// The trust anchor of every host a binding names. A binding whose anchor +/// cannot be read leaves its host unattachable and is logged; the other +/// hosts are unaffected. +fn host_anchors_from(bindings_dir: &Path) -> BTreeMap { + { + let mut anchors = BTreeMap::new(); + let bindings = match load_bindings(bindings_dir) { + Ok(bindings) => bindings, + Err(error) => { + eprintln!("Idunn cannot read bindings for host anchors: {error:#}"); + return anchors; + } + }; + for loaded in bindings.values() { + let WorkloadBinding::HostActuator(workload) = &loaded.binding.workload else { + continue; + }; + match read_trust_anchor::(&workload.host_trust_anchor_store) + { + Ok(anchor) => { + anchors.insert(workload.host.clone(), anchor); + } + Err(error) => eprintln!( + "Idunn cannot read the actuator anchor for host {}: {error:#}", + workload.host + ), + } + } + anchors + } +} + +impl Engine { + /// The runner the plan's binding declares. Every runner in one binding + /// is of one kind; the binding validated that. + fn runner_for(&self, plan: &CompiledDeploymentPlan) -> Result> { + let (_, binding) = plan.parsed_inputs()?; + Ok(match &binding.workload { + WorkloadBinding::SystemdTransient(_) => Box::new(self.docker_runner.clone()), + WorkloadBinding::HostActuator(_) => Box::new(HostActuatorRunnerDriver { + access: self.host_access()?, + }), + }) + } + + /// The workload driver the plan's binding declares. + fn workload_for(&self, plan: &CompiledDeploymentPlan) -> Result> { + let (_, binding) = plan.parsed_inputs()?; + Ok(match &binding.workload { + WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), + WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { + access: self.host_access()?, + }), + }) + } + + /// History for a decision that would be wrong on an empty answer: the + /// continuity restart ceiling and the check that a command is consumed. + /// `None` means `history.cc` cannot be read, so the decision is not made; + /// a history that merely has no entries is `Some`. The fault is printed + /// once while it lasts, not on every tick. + fn history_for_decision(&self) -> Option> { + let (report, transactions) = match read_history(&self.options.state_store) { + Ok(history) => (history.report(), Some(history.transactions)), + Err(error) => ( + Some(format!( + "{error:#}; continuity and command retirement stop until it is readable" + )), + None, + ), + }; + if let Some(text) = self.history_report.offer(report) { + eprintln!("{text}"); + } + transactions + } + + fn topology(&self) -> CultCacheTopologyDriver { + CultCacheTopologyDriver { + projection_store: self.options.topology_store.clone(), + correlation_store: self.options.odin_correlation_store.clone(), + } + } + + fn current_odin_authority(&self, snapshot: &ControlSnapshot) -> Result { + // The bootstrap key stands only until Odin itself is admitted. Once it + // is, its own generation names the authority; an admitted Odin that + // carries none has no authority to name, and falling back to the + // bootstrap key would let a stale key vouch for the Verse. + let authority = match snapshot.admitted_odin() { + None => self.bootstrap_odin_authority.clone(), + Some(stored) => stored.value.odin_authority.clone().with_context(|| { + "admitted Odin generation carries no Odin authority (route-proof readiness)" + })?, + }; + authority.validate()?; + Ok(authority) + } + + fn trusted_topology_context(&self, now: u64) -> OdinTopologyAuthenticationContext { + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + } + } + + /// Re-prove the evidence that still gates a decision. + /// + /// Terminal transactions are skipped: they describe what already happened + /// and authorize nothing further, so re-proving them only creates ways for + /// history to refuse a boot. On yggdrasil that is 259 of 260 records. + /// + /// The operator anchor is read on demand rather than up front. Reading it + /// unconditionally meant an absent brake artifact refused startup against + /// an *empty* store -- a deployment brake gating Idunn itself, which + /// `F:\Projects\CLAUDE.md` forbids outright: a target brake "may never gate + /// Idunn itself or unrelated service lifecycle". + fn validate_durable_authority(&self, snapshot: &ControlSnapshot) -> Result<()> { + let mut operator_anchor = None; + for stored in &snapshot.transactions { + let transaction = &stored.value; + if transaction.is_terminal() { + continue; + } + if let Some(authorization) = &transaction.deployment_authorization { + authorization.validate_shape()?; + let record: IdunnDeploymentBrakeRecord = + rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + if operator_anchor.is_none() { + operator_anchor = + Some(read_trust_anchor::( + &self.options.deployment_brake_operator_anchor, + )?); + } + verify_idunn_deployment_brake_authorization( + &record, + operator_anchor.as_ref().expect("read directly above"), + )?; + let expected = required(&transaction.expected, "authorized Expected projection")?; + ensure!( + record.authorized_release_id.as_deref() + == Some(expected.sealed_release_id.as_str()) + && record.authorized_deployment_id.as_deref() + == Some(transaction.transaction_id.as_str()) + && record.runtime_id == expected.runtime_id, + "durable deployment authorization names another release or transaction" + ); + } + let lease = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + if let Some(evidence) = &transaction.latest_odin_observation { + let authenticated = self.authenticate_topology_bytes( + snapshot, + transaction, + &evidence.canonical_bytes, + lease, + evidence.admitted_at_unix_millis, + )?; + validate_authenticated_evidence(evidence, &authenticated)?; + } + if let Some(evidence) = &transaction.warming { + match evidence { + WarmingEvidence::OdinTopology { evidence } => { + let authenticated = self.authenticate_topology_bytes( + snapshot, + transaction, + &evidence.canonical_bytes, + None, + evidence.admitted_at_unix_millis, + )?; + validate_authenticated_evidence(evidence, &authenticated)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(snapshot, transaction)?; + ensure!( + is_semantic_warming( + required(&transaction.expected, "Warming Expected projection",)?, + required(&transaction.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?, + "durable Warming gate is not supported by current runtime evidence" + ); + } + WarmingEvidence::FirstOdinDirect { evidence } => { + self.authenticate_first_odin_warming_presence( + transaction, + &evidence.message_id, + evidence.challenged_at_unix_millis, + evidence.admitted_at_unix_millis, + &evidence.canonical_bytes, + )?; + } + WarmingEvidence::RouteProofDirect { evidence } => { + self.reauthenticate_route_proof( + transaction, + evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + evidence.admitted_at_unix_millis, + )?; + } + } + } + if let Some(ReadinessEvidence::RouteProof { evidence }) = &transaction.ready { + self.reauthenticate_route_proof( + transaction, + evidence, + &["active"], + lease, + evidence.admitted_at_unix_millis, + )?; + } + if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) { + let authenticated = self.authenticate_topology_bytes( + snapshot, + transaction, + &evidence.canonical_bytes, + lease, + evidence.admitted_at_unix_millis, + )?; + validate_authenticated_evidence(evidence, &authenticated)?; + ensure!( + is_semantic_ready(&authenticated), + "durable Ready label is not exact semantic Ready" + ); + } + } + let odin_authority = self.current_odin_authority(snapshot)?; + for stored in &snapshot.admitted { + let generation = &stored.value; + // Only a generation whose own Expected says Odin vouches for it has + // Odin receipts to re-prove. One whose evidence disagrees with its + // class is held and reported by supervision, not re-proved here. + if !matches!(generation.readiness(), Ok(class) if class != ReadinessClass::RouteProof) + { + continue; + } + let odin = generation.odin_receipts()?; + let authority = self.runtime_authority_parts( + &generation.plan, + &generation.expected, + &generation.activation, + )?; + let latest = authenticate_odin_runtime_topology_correlation( + &odin.latest.canonical_bytes, + &authority, + generation.leasing.lease_sha256(), + &odin_authority.signer_public_key, + self.trusted_topology_context(odin.latest.admitted_at_unix_millis), + )?; + validate_authenticated_evidence(odin.latest, &latest)?; + let ready = authenticate_odin_runtime_topology_correlation( + &odin.ready.canonical_bytes, + &authority, + generation.leasing.lease_sha256(), + &odin_authority.signer_public_key, + self.trusted_topology_context(odin.ready.admitted_at_unix_millis), + )?; + validate_authenticated_evidence(odin.ready, &ready)?; + ensure!( + is_semantic_ready(&ready), + "admitted generation Ready label is not exact semantic Ready" + ); + } + Ok(()) + } +} + +/// What the boot reconciliation of failed continuity projections found. +#[derive(Debug, Default, PartialEq, Eq)] +struct ProjectionReconciliation { + /// Failed continuity transactions whose exact activation was demoted. + demoted: Vec, + /// Targets whose projection names an activation that neither the admitted + /// generation, a live transaction, nor a failed issuer accounts for. They + /// are reported and left alone. + unexplained: Vec, +} + +impl Engine { + /// Once at boot: demote the activation a failed continuity left standing, + /// where the projection still names exactly the activation that failed + /// transaction issued. A continuity abort before the single resolution + /// rule left such residue (and the lift owes the same demotion to any + /// abort still resident). The issuer is identified by exact activation, + /// from history and from resident terminal records not yet retired to it. + /// + /// Nothing is adopted: an activation with no failed issuer is never + /// demoted or kept on anyone's behalf, only reported. + fn reconcile_failed_continuity_projections(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut outcome = ProjectionReconciliation::default(); + // Resident issuers need no history; only the history issuers wait on it. + let history = self.history_for_decision(); + if history.is_none() { + eprintln!( + "Idunn cannot read history at boot: failed continuities archived there are not \ + reconciled, resident ones still are" + ); + } + let topology = self.topology(); + let issuers = history + .iter() + .flatten() + .chain(snapshot.transactions.iter().map(|stored| &stored.value)) + .filter(|transaction| { + transaction.command_kind == CommandKind::Continuity + && matches!( + transaction.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + | Some(TransactionCompletion::FailedAfterFencing { .. }) + ) + }); + for issuer in issuers { + let (Some(expected), Some(activation), Some(plan)) = + (&issuer.expected, &issuer.activation, &issuer.plan) + else { + continue; + }; + let demoted = (|| -> Result { + if topology.projected_activation(expected)?.as_ref() != Some(activation) { + return Ok(false); + } + let provider_anchor = self.provider_anchor_for_plan(plan)?; + topology.demote_to_expected_only(expected, &provider_anchor, activation, None)?; + Ok(true) + })(); + match demoted { + Ok(true) => outcome.demoted.push(issuer.transaction_id.clone()), + Ok(false) => {} + Err(error) => eprintln!( + "Idunn could not reconcile the projection of failed continuity {}: {error:#}", + issuer.transaction_id + ), + } + } + for generation in &snapshot.admitted { + let projected = match topology.projected_activation(&generation.value.expected) { + Ok(projected) => projected, + Err(error) => { + eprintln!( + "Idunn cannot read the projected activation of {}: {error:#}", + generation.value.target + ); + continue; + } + }; + let Some(projected) = projected else { + continue; + }; + let accounted = projected == generation.value.activation + || snapshot.transactions.iter().any(|stored| { + stored.value.target == generation.value.target + && !stored.value.is_terminal() + && stored.value.activation.as_ref() == Some(&projected) + }); + if !accounted { + outcome.unexplained.push(generation.value.target.clone()); + } + } + Ok(outcome) + } +} + +/// Everything the daemon does once, before its loop: lock, migrate, validate, +/// reconcile the projection. The lock is returned so the caller holds it for as +/// long as the engine runs. +fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { + for path in [ + &options.state_store, + &options.topology_store, + &options.staging_root, + ] { + let directory = if path.extension().is_some() { + path.parent() + .context("configured Idunn path has no parent")? + } else { + path.as_path() + }; + fs::create_dir_all(directory) + .with_context(|| format!("creating Idunn directory {}", directory.display()))?; + } + let lock = ProcessLock::acquire(&options.state_store)?; + let migrated = migrate_control_store_to_current_schema(&options.state_store) + .context("migrating Idunn control records to the current schema")?; + if migrated > 0 { + println!( + "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA}" + ); + } + ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; + let engine = Engine::open(options)?; + engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; + match engine.reconcile_failed_continuity_projections() { + Ok(outcome) => { + for transaction_id in &outcome.demoted { + println!("demoted the activation failed continuity {transaction_id} left projected"); + } + for target in &outcome.unexplained { + eprintln!( + "Idunn found an activation projected for {target} that no admitted generation, \ + live transaction or failed continuity accounts for; left as it is" + ); + } + } + Err(error) => eprintln!("Idunn boot projection reconciliation failed: {error:#}"), + } + Ok((lock, engine)) +} + +fn serve(options: RuntimeOptions) -> Result<()> { + let (_lock, engine) = boot(options)?; + serve_while(&engine, || true) +} + +/// The daemon's loop: tick, sleep, and go again while `again` says so. `serve` +/// says so forever; a test says so a few times. +fn serve_while(engine: &Engine, mut again: impl FnMut() -> bool) -> Result<()> { + while again() { + match engine.run_scheduler_tick() { + Ok(true) => continue, + Ok(false) => {} + // A fault in one transaction is that transaction's problem. Idunn + // runs under Restart=always, so propagating it here turns a single + // unreadable record into a crashloop that takes every unrelated + // target's continuity down with it -- the daemon-survival organ + // killed by the thing it exists to survive. Log against the tick + // and keep going; the fault recurs every poll until it is fixed, + // which is louder than a restart loop and cheaper than an outage. + Err(error) => eprintln!("Idunn scheduler tick failed: {error:#}"), + } + thread::sleep(Duration::from_millis(engine.options.poll_millis)); + } + Ok(()) +} + +impl Engine { + /// One resident terminal transaction goes to history per tick. + /// + /// Transactions that finished before finished transactions travelled to + /// history are still resident with their commands, gating nothing and + /// consuming their commands correctly, but growing the store every decision + /// reads. They leave the same way a transaction finishing today does. + fn retire_one_terminal_transaction(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + // A record that cannot be archived (history unreadable, a torn write) + // is that record's fault: the next terminal record is still tried, and + // the tick goes on to supervise and freeze. + for stored in snapshot + .transactions + .iter() + .filter(|stored| stored.value.is_terminal()) + { + match archive_terminal_transaction(&self.options.state_store, &stored.envelope) { + Ok(()) => { + eprintln!( + "Idunn retired resident terminal transaction {} to history", + stored.value.transaction_id + ); + return Ok(true); + } + Err(error) => self.note_fault( + "could not retire terminal transaction", + &stored.value.transaction_id, + &error, + ), + } + } + Ok(false) + } + + /// One pass of the scheduler. An unreadable control store is the tick's + /// fault and is returned; anything one transaction or one admitted + /// generation does wrong is logged against it and never stops the others. + fn run_scheduler_tick(&self) -> Result { + if self.retire_one_terminal_transaction()? { + return Ok(true); + } + let transaction_progress = self.resume_one_transaction()?; + let continuity_progress = self.supervise_one_admitted_generation()?; + // Freezing is not gated on the others' progress: a queued command + // whose target is free must not wait behind an unrelated target's + // wedge. `freeze_command` already refuses a busy target, so this is + // still at most one freeze per tick and never a second claimant. + let froze = self.freeze_one_queued_command()?; + Ok(transaction_progress || continuity_progress || froze) + } + + /// Say a per-record fault once while it lasts and leave it in the + /// record's `last_error`, which status renders. Recording is best effort: + /// a store too broken to take the note is already in stderr. + fn note_fault(&self, what: &str, transaction_id: &str, error: &anyhow::Error) { + let detail = truncate(&format!("{error:#}"), 2048); + let offered = { + let mut reports = self.fault_reports.lock().expect("fault report mutex"); + reports + .entry(transaction_id.to_string()) + .or_default() + .offer(Some(detail.clone())) + }; + let recorded = self.record_last_error(transaction_id, &detail); + if offered.is_some() { + eprintln!("Idunn {what} {transaction_id}: {detail}"); + if let Err(error) = recorded { + eprintln!("Idunn could not leave that in the record of {transaction_id}: {error:#}"); + } + } + } + + fn clear_fault(&self, transaction_id: &str) { + self.fault_reports + .lock() + .expect("fault report mutex") + .remove(transaction_id); + } + + /// Write `detail` as the record's `last_error`, touching nothing else and + /// not archiving: a terminal record whose archive is the fault stays put. + fn record_last_error(&self, transaction_id: &str, detail: &str) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let Some(stored) = snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == transaction_id) + else { + return Ok(()); + }; + if stored.value.last_error.as_deref() == Some(detail) { + return Ok(()); + } + let envelope = cleanup_evidence::last_error_envelope(&stored.value, detail)?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction_id.to_string(), + current: Some(stored.envelope.clone()), + }], + &[envelope], + )?, + "transaction changed before its fault note" + ); + Ok(()) + } + + /// Startup and every later loop use the same order: unfinished ownership + /// work first, admitted-body continuity second, new commands last. A + /// waiting transaction yields without relaxing ordinal order inside its + /// own command, so one target brake cannot suspend unrelated continuity. + fn resume_one_transaction(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; + let mut candidates = snapshot + .transactions + .iter() + .filter(|stored| !stored.value.is_terminal()) + .collect::>(); + candidates.sort_by_key(|stored| { + ( + stored.value.created_at_unix_millis, + stored.value.ordinal, + stored.value.transaction_id.as_str(), + ) + }); + for current in candidates { + if snapshot.has_earlier_authority_sibling(¤t.value) { + continue; + } + match self.resume_candidate(current) { + Ok(moved) => progressed |= moved, + Err(error) => { + self.note_fault( + "could not resume transaction", + ¤t.value.transaction_id, + &error, + ); + } + } + } + Ok(progressed) + } + + /// Advance one transaction, turning its failure into the durable record + /// its phase calls for. Returns whether its record changed. + /// + /// "Moved" means the transaction's state advanced: a phase change, a + /// completion, or a durable abort step. Rewriting `last_error` to say the + /// same step failed again in new words is not movement, or a wedged + /// transaction would report progress every tick. + fn resume_candidate(&self, current: &Stored) -> Result { + let id = current.value.transaction_id.as_str(); + if let Some(moved) = self.resolve_phase_deadline(current)? { + return Ok(moved); + } + if self + .resume_backoff + .lock() + .expect("resume backoff mutex") + .get(id) + .is_some_and(|backoff| { + now_millis().is_ok_and(|now| { + is_waiting(now, backoff.not_before_unix_millis, RESUME_BACKOFF_CEILING_MILLIS) + }) + }) + { + return Ok(false); + } + if let Some(disagreement) = current.value.held_disagreement() { + self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); + return Ok(false); + } + let advanced = self.advance_transaction(current); + if advanced.is_ok() { + // The step ran and succeeded: the fault, if any, has recovered, so + // a later failure is news. A skipped or failed attempt clears nothing. + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + self.clear_fault(id); + } + if let Err(error) = advanced { + let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; + let latest = latest_snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared while recording an execution error")?; + if latest.value.is_terminal() { + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + return Ok(true); + } + self.back_off(id)?; + if latest.value.phase < DeploymentPhase::Fencing { + // Before the fence an abort that cannot finish a step is + // resumable, never a post-fence abort: that path refuses a + // pre-fence phase, so choosing it only wedged the tick. + // + // A publish that lost to a held lock is not a failed step: the + // holder lets go, so the transaction stays in its phase with + // the error recorded and the backoff above retries it. The + // phase deadline, not an abort, bounds the wait. + if latest.value.pre_fencing_abort.is_none() + && error.downcast_ref::().is_none() + { + self.begin_pre_fencing_abort(latest, error)?; + } else { + self.record_resumable_error(latest, &error)?; + } + } else if latest.value.post_fencing_abort.is_none() + && self.candidate_is_permanently_stopped(&latest.value)? + { + // Past the fence an error is resumable while the candidate + // can still recover. This one cannot: its transient unit + // has failed and carries Restart=no, so retrying would hold + // the target forever behind a transaction that can never + // finish. + self.begin_post_fencing_abort(latest, error)?; + } else { + // Resumable only until the phase deadline: the resolver ends + // the phase before the next step once it has passed. + self.record_resumable_error(latest, &error)?; + } + } + let after = ControlSnapshot::read(&self.options.state_store)?; + let live = after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared while checking scheduler progress")?; + Ok(state_advanced(¤t.value, &live.value)) + } + + /// Wait twice as long after each consecutive failure of a transaction's + /// step, from one poll interval up to a ceiling. + fn back_off(&self, transaction_id: &str) -> Result<()> { + let now = now_millis()?; + let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); + let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); + let wait = backoff_wait(self.options.poll_millis, failures, RESUME_BACKOFF_CEILING_MILLIS); + all.insert( + transaction_id.to_string(), + ResumeBackoff { + failures: failures.saturating_add(1), + not_before_unix_millis: now.saturating_add(wait), + }, + ); + Ok(()) + } + + fn freeze_one_queued_command(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let live_commands = snapshot + .transactions + .iter() + .map(|stored| stored.value.command_id.clone()) + .collect::>(); + let mut candidates = snapshot + .commands + .iter() + .filter(|stored| { + stored.value.kind == CommandKind::Deploy + && !live_commands.contains(&stored.value.command_id) + }) + .collect::>(); + if candidates.is_empty() { + return Ok(false); + } + // A command is consumed by any transaction, live or retired. Commands + // whose transactions were retired to history before commands travelled + // with them are still resident; they are not queued, they are + // history that never moved. Retire them here and never freeze them. + // With history unreadable, whether a command was consumed is unknown: + // freezing it again could run a finished deployment twice. + let Some(history) = self.history_for_decision() else { + return Ok(false); + }; + let historical_commands = history + .into_iter() + .map(|transaction| transaction.command_id) + .collect::>(); + let mut retired = false; + candidates.retain(|stored| { + if !historical_commands.contains(&stored.value.command_id) { + return true; + } + match SingleFileMessagePackBackingStore::new(&history_store_path( + &self.options.state_store, + )) + .insert_entry_if_absent(stored.envelope.clone()) + .and_then(|_| { + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .delete_batch_if_unchanged(std::slice::from_ref(&stored.envelope)) + }) { + Ok(_) => { + eprintln!( + "Idunn retired consumed command {}: its transactions are history", + stored.value.command_id + ); + retired = true; + } + Err(error) => eprintln!( + "Idunn left consumed command {} resident: {error:#}", + stored.value.command_id + ), + } + false + }); + if retired { + return Ok(true); + } + candidates.sort_by_key(|stored| stored.value.requested_at_unix_millis); + // Oldest first, but a command whose target is busy does not block the + // ones behind it for other targets. + for command in candidates { + if self.freeze_command(&snapshot, command)? { + return Ok(true); + } + } + Ok(false) + } + + /// Freeze one queued command into its transactions. `Ok(false)` means its + /// target is busy and nothing was written. + fn freeze_command( + &self, + snapshot: &ControlSnapshot, + command: &Stored, + ) -> Result { + let bindings = match load_bindings(&self.options.bindings_dir) { + Ok(bindings) => bindings, + Err(error) => { + eprintln!( + "Idunn left command {} queued because operator bindings are invalid: {error:#}", + command.value.command_id + ); + return Ok(false); + } + }; + let targets = match resolve_selector(&bindings, &command.value.selector) { + Ok(targets) => targets, + Err(error) => { + let now = now_millis()?; + let rejected = DeploymentTransaction::rejected(&command.value, error, now)?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: rejected.transaction_id.clone(), + current: None, + }], + &[transaction_envelope(&rejected, now)?], + )?, + "bad selector changed before refusal was recorded" + ); + return Ok(true); + } + }; + let busy = snapshot + .transactions + .iter() + .filter(|stored| stored.value.blocks_new_target_mutation()) + .map(|stored| stored.value.target.as_str()) + .collect::>(); + if targets.iter().any(|target| busy.contains(target.as_str())) { + return Ok(false); + } + let now = now_millis()?; + let transactions = targets + .into_iter() + .enumerate() + .map(|(ordinal, target)| { + DeploymentTransaction::new( + &command.value, + target.clone(), + u32::try_from(ordinal)?, + snapshot.admitted_for(&target).map(|stored| &stored.value), + now, + ) + }) + .collect::>>()?; + let expected = transactions + .iter() + .map(|transaction| CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }) + .collect::>(); + let next = transactions + .iter() + .map(|transaction| transaction_envelope(transaction, now)) + .collect::>>()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange(&expected, &next)?, + "queued command lost its ordered transaction creation CAS" + ); + Ok(true) + } + + fn supervise_one_admitted_generation(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + // A wait belongs to the failed record it spaces. Once any write for its + // target has landed, or the target is no longer admitted, the stored + // schedule owns the target again. + self.challenge_wait + .lock() + .expect("challenge wait mutex") + .retain(|target, wait| snapshot.target_stamp(target) == wait.stamp); + self.reconcile_incidents(&snapshot); + let mut progressed = false; + let mut admitted = snapshot.admitted.iter().collect::>(); + admitted.sort_by_key(|stored| stored.value.target.as_str()); + for current in admitted { + if let Err(disagreement) = current.value.readiness() { + self.note_fault( + "holds admitted generation", + &format!("generation:{}", current.value.target), + &anyhow::Error::new(disagreement), + ); + } + let blocker = snapshot.transactions.iter().find(|stored| { + stored.value.target == current.value.target + && stored.value.blocks_new_target_mutation() + }); + if blocker.is_some_and(|stored| { + let owns_from = if stored.value.rollout_stops_incumbent_first() { + // The incumbent is stopped at Starting by design; a + // continuity restart in that window would fight the + // candidate for the same host resources. + DeploymentPhase::Starting + } else { + DeploymentPhase::Fencing + }; + stored.value.phase >= owns_from && stored.value.phase < DeploymentPhase::Complete + }) { + continue; + } + match self.supervise_admitted_route(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + } + match self.refresh_admitted_topology(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn preserved admitted {} after rejecting topology observation: {error:#}", + current.value.target + ), + } + match self.retire_stale_incarnations(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn preserved a stale {} incarnation projection: {error:#}", + current.value.target + ), + } + // Restore the admitted Expected before anything is asked of the + // workload, and only the Expected. + // + // A target that cannot see its own Expected cannot start, and an + // aborted deployment leaves exactly that gap, so this cannot wait + // for an observation the dead process will never provide. The + // activation is deliberately not restored: it names one running + // incarnation, every restart issues a fresh one, and republishing + // the admitted one leaves the projection naming an incarnation that + // no longer exists -- which the workload rightly refuses to adopt. + // That record is published from observation once the process is up. + if blocker.is_none() + || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) + { + let repair = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.admitted_expected_projection_is_exact( + ¤t.value.expected, + &provider_anchor, + )? { + ensure!( + topology.publish_expected(¤t.value.expected, &provider_anchor)? + == current.value.expected.canonical_sha256()?, + "admitted Expected projection repair differs" + ); + } + Ok(()) + })(); + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); + } + } + let mut operational_error = None; + let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { + workload.observe( + ¤t.value.expected, + ¤t.value.activation, + ¤t.value.workload, + ) + }) { + Ok(observation) => { + let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { + let lease_health = (|| -> Result { + let lease_path = current + .value + .plan + .parsed_inputs()? + .1 + .process_write_lease + .context("admitted lease has no operator binding")? + .record_path; + let driver = + CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + if driver.observe_exact(lease)? { + return Ok(true); + } + if driver.observe_empty()? { + return Ok(false); + } + bail!("write-lease store contains unexpected authority") + })(); + match lease_health { + Ok(true) => false, + Ok(false) => { + operational_error = + Some(anyhow!("admitted physical write lease is missing")); + true + } + Err(error) => { + eprintln!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target + ); + continue; + } + } + } else { + false + }; + if lease_is_missing { + None + } else { + Some(observation) + } + } + Err(error) if error.downcast_ref::().is_some() => { + // The host cannot be asked right now (its actuator is + // between sessions, or Idunn just restarted). That is + // silence, not a death; counting it as one burned every + // continuity attempt in the reattach window on + // 2026-09-11 and left a healthy Muninn demoted. + eprintln!( + "Idunn cannot observe admitted {} this tick: {error:#}", + current.value.target + ); + continue; + } + Err(error) => { + operational_error = Some(error); + None + } + }; + if observation.is_some() { + let seen = snapshot.supervision_for(¤t.value.target); + if let Some(stored) = seen.filter(|stored| { + stored.value.continuity_backoff_from_unix_millis.is_some() + }) { + let mut next = stored.value.clone(); + next.end_backoff(); + self.write_target_supervision(Some(stored), &next, now_millis()?)?; + } + // The admitted workload was seen running: the only evidence + // that ends an exhaustion incident, once the window no longer holds it. + // The window sliding alone is not health and never closes one. + self.record_incident( + "close", + ¤t.value.target, + now_millis().and_then(|now| { + // Running is not enough while the window still holds + // the restarts that exhausted it: a release that + // survives one poll after each restart is still the + // same incident. + if snapshot + .supervision_or_new(¤t.value.target) + .restarts_page(now) + { + return Ok(false); + } + self.incidents().close( + IncidentCondition::ContinuityExhausted, + ¤t.value.target, + CloseReason::Recovered, + now, + ) + }), + ); + continue; + } + let workload_error = operational_error + .context("admitted operational state has neither observation nor error")?; + // A held generation is a record the recipe never declared + // readiness for. Idunn reports it and does not decide for it: a + // continuity minted over it copies the undeclared Expected, is + // held at once, and owns the target forever, so the declaring + // redeploy that would clear the hold could never freeze. The dead + // held target stays down and free. + if current.value.readiness().is_err() { + self.note_fault( + "holds a dead admitted generation down; declare readiness in its recipe and redeploy", + &format!("generation-down:{}", current.value.target), + &anyhow!("admitted generation {} is not running", current.value.generation_id), + ); + continue; + } + // Continuity restarts a release; it cannot repair one, and it + // never stops (budget-never-stop). Every charged attempt is spaced + // by a doubling, capped wait, so a queued deployment always gets + // a free gap (map B10). The log outlives every generation and only + // pages. Nothing here reads history: an unreadable history file + // must not stop crash recovery. + let now = now_millis()?; + let supervision = snapshot.supervision_or_new(¤t.value.target); + // A stepped-back clock leaves entries in the future. Settle them + // before deciding from them; the write ends this target's pass. + let mut settled = supervision.clone(); + settled.settle(now); + if settled != supervision { + self.write_target_supervision( + snapshot.supervision_for(¤t.value.target), + &settled, + now, + )?; + progressed = true; + continue; + } + let continuity_key = format!("continuity:{}", current.value.target); + + // Continuity defers to any live transaction for its target and + // never aborts one (handover-phase-split). + if blocker.is_some() { + continue; + } + if supervision.restarts_page(now) { + // The record comes first and the trace after it: the trace is + // this record's shadow, never its substitute. + self.record_incident( + "open", + ¤t.value.target, + self.incidents().open( + IncidentCondition::ContinuityExhausted, + ¤t.value.target, + now, + ), + ); + self.report_once( + &continuity_key, + format!( + "Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff (streak {}): {workload_error:#}", + current.value.target, + supervision.restarts_used(now), + supervision.continuity_attempt_streak, + ), + ); + } else { + self.clear_fault(&continuity_key); + } + if supervision.continuity_deferred(now) { + continue; + } + // A unit the host lost to a reboot is re-materialized once per + // generation, uncharged and with no wait; the driver owns the + // evidence and supervision owns the budget. An unanswerable + // driver is not evidence of a reboot. + let admitted_at = current.value.admitted_at_unix_millis; + let rematerialize = !supervision.rematerialization_spent(admitted_at) + && match self.workload_for(¤t.value.plan).and_then(|workload| { + workload.lost_to_host_reboot(¤t.value.workload, admitted_at) + }) { + Ok(lost) => lost, + Err(error) => { + self.note_fault( + "could not tell whether the host rebooted under", + &format!("reboot:{}", current.value.target), + &error, + ); + false + } + }; + let mut streak = 0; + if !rematerialize { + match supervision.continuity_backoff(now, admitted_at) { + ContinuityBackoff::Waiting => continue, + ContinuityBackoff::Begin => { + let mut began = supervision.clone(); + began.begin_backoff(now); + self.write_target_supervision( + snapshot.supervision_for(¤t.value.target), + &began, + now, + )?; + progressed = true; + continue; + } + ContinuityBackoff::Due { streak: due } => streak = due, + } + } + + // The projected activation names an incarnation that is gone. Every + // restart is issued a fresh one, so leaving the old record standing + // makes the projection describe a process that no longer exists -- + // and a workload that reads it refuses to adopt an activation that + // is not its own, which is a restart loop rather than a recovery. + // Demote to Expected-only, the same shape an aborted deployment + // leaves, and let the restart publish its own activation once it is + // observed. + let demotion = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.projected_activation_is_present(¤t.value.expected)? { + return Ok(()); + } + topology.demote_to_expected_only( + ¤t.value.expected, + &provider_anchor, + ¤t.value.activation, + current.value.leasing.lease(), + )?; + Ok(()) + })(); + // Expected-only is the precondition of a restart: a continuity + // minted over a projection that still names an activation would + // fail between preparing and publishing its own, and its abort + // could not resolve. So a failed demotion mints nothing. The + // failure is written to the generation, and the next attempt is + // not before the deferral ends. + if let Err(error) = demotion { + eprintln!( + "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", + current.value.target + ); + let mut deferred = supervision.clone(); + deferred.defer_continuity(now, &truncate(&format!("{error:#}"), 2048)); + self.write_target_supervision( + snapshot.supervision_for(¤t.value.target), + &deferred, + now, + )?; + progressed = true; + continue; + } + // An engaged lifecycle brake means no continuity transaction is + // minted at all. A transaction that exists and waits on the brake + // owns the target, and a target owned by a parked restart accepts + // no deployment -- which is the one thing a lifecycle brake must + // never gate. The Sealing-phase check remains as the guard for a + // transaction minted just before the brake was engaged. + if !self.lifecycle_allows_generation(current, now)? { + continue; + } + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: format!("continuity-{}", Uuid::new_v4()), + kind: CommandKind::Continuity, + selector: current.value.target.clone(), + requested_by: "idunn-continuity".into(), + requested_at_unix_millis: now, + }; + command.validate()?; + let transaction = + DeploymentTransaction::from_continuity(&command, ¤t.value, now)?; + // The attempt is charged, or recorded as the one re-materialization, + // in the same CAS that mints it, so a crash cannot schedule one the + // meters never saw. + let mut counted = supervision; + if rematerialize { + counted.record_rematerialization(now); + } else { + counted.record_restart(now, streak); + } + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: TargetSupervision::TYPE.into(), + key: counted.target.clone(), + current: snapshot + .supervision_for(&counted.target) + .map(|stored| stored.envelope.clone()), + }, + ], + &[ + command_envelope(&command, now)?, + transaction_envelope(&transaction, now)?, + target_supervision_envelope(&counted, now)?, + ], + )?, + "continuity scheduling lost its command/transaction/meter CAS" + ); + progressed = true; + } + Ok(progressed) + } + + fn supervise_admitted_route( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + ) -> Result { + let Some(expected_route) = current.value.expected.route.as_ref() else { + ensure!( + matches!(¤t.value.routing, RoutingEvidence::SkippedUnrouted), + "unrouted admitted generation carries route authority" + ); + return Ok(false); + }; + let RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + } = ¤t.value.routing + else { + bail!("routed admitted generation has no promoted route receipt") + }; + let now = now_millis()?; + // A route whose proofs keep failing is challenged on a widening + // schedule, not every tick. + if current + .value + .route_supervision + .as_ref() + .is_some_and(|state| state.is_waiting(now, self.options.topology_maximum_age_millis)) + { + return Ok(false); + } + if self + .challenge_wait + .lock() + .expect("challenge wait mutex") + .get(¤t.value.target) + .is_some_and(|wait| { + is_waiting( + now, + wait.due_unix_millis, + ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(self.options.topology_maximum_age_millis), + ) + }) + { + return Ok(false); + } + ensure!( + observation.route_id == expected_route.route_id + && observation.runtime_instance_id == current.value.activation.runtime_instance_id, + "admitted route receipt names another incarnation" + ); + let binding = current.value.plan.parsed_inputs()?.1; + let lease_driver = if let Some(lease) = current.value.leasing.lease() { + let lease_path = &binding + .process_write_lease + .as_ref() + .context("stateful admitted generation has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + ensure!( + driver.observe_exact(lease)?, + "admitted process write lease is no longer exact" + ); + Some(driver) + } else { + None + }; + let route_binding = binding + .route + .context("routed admitted generation has no operator route binding")?; + let driver = self.route_driver(route_binding); + let gate = self.route_gate(¤t.value.target, CommandKind::Continuity); + + if driver.observe_membership(¤t.value.expected, &observation.membership_sha256)? + && route_observation_is_current( + observation.observed_at_unix_millis, + now, + self.options.topology_maximum_age_millis, + self.options.topology_maximum_future_skew_millis, + ) + { + return Ok(false); + } + let route_key = format!("route:{}", current.value.target); + // A failed repair or a failed proof changes observation state only: it + // marks the route degraded and widens the wait. That record is written + // like a proved one: when it lands the pass ends (`Ok(true)`), because + // the caller's snapshot no longer matches the generation; when the store + // refuses it the pass reports no progress and the in-memory wait below + // spaces the next attempt. + let challenge = (|| -> Result { + // Actuates only when the fragment on disk differs from the + // admitted membership; an exact fragment makes this a no-op. It is + // survival: restoring the admitted route is never refused. + driver.restore_admitted_membership( + ¤t.value.expected, + &observation.membership_sha256, + &gate, + )?; + let authority = self.runtime_authority_parts( + ¤t.value.plan, + ¤t.value.expected, + ¤t.value.activation, + )?; + let refreshed = self.prove_stable_route_against( + ¤t.value.expected, + ¤t.value.activation, + &authority, + current.value.leasing.lease_sha256(), + &driver, + observation.membership_sha256.clone(), + )?; + ensure!( + driver.observe_membership(¤t.value.expected, &refreshed.membership_sha256)?, + "admitted route membership changed during its continuity challenge" + ); + if let (Some(lease_driver), Some(lease)) = (&lease_driver, current.value.leasing.lease()) + { + ensure!( + lease_driver.observe_exact(lease)?, + "admitted process write lease changed during its continuity challenge" + ); + } + Ok(refreshed) + })(); + let mut next = current.value.clone(); + // The wait the record carries, also held in memory until a record lands. + let (recorded, wait_until) = match challenge { + Ok(refreshed) => { + next.routing = RoutingEvidence::Promoted { + observation: refreshed, + promoted_at_unix_millis: *promoted_at_unix_millis, + }; + if let Some(state) = next.route_supervision.as_mut() { + state.record_proved_challenge(now); + } + self.clear_fault(&route_key); + ( + "proved", + now.saturating_add(self.options.topology_maximum_age_millis), + ) + } + Err(error) => { + if let Some(state) = next.route_supervision.as_mut() { + state.record_failed_challenge(now, self.options.topology_maximum_age_millis); + } + self.report_once( + &route_key, + format!( + "Idunn could not prove admitted {} on its stable route: {error:#}", + current.value.target + ), + ); + let due = next + .route_supervision + .as_ref() + .and_then(|state| state.next_challenge_at_unix_millis) + .unwrap_or(now); + ("failed", due) + } + }; + // Progress is a store that changed: our record landed, or another + // writer beat it (the snapshot is stale either way, so the pass ends). + // A write the store refused changed nothing: no progress, the scheduler + // sleeps if nothing else moved, and the in-memory wait spaces the next + // challenge. + let record_key = format!("route-record:{}", current.value.target); + match self.replace_generation(current, &next, now) { + Ok(_) => { + self.challenge_wait + .lock() + .expect("challenge wait mutex") + .remove(¤t.value.target); + self.clear_fault(&record_key); + Ok(true) + } + Err(record) => { + self.challenge_wait + .lock() + .expect("challenge wait mutex") + .insert( + current.value.target.clone(), + ChallengeWait { + due_unix_millis: wait_until, + stamp: snapshot.target_stamp(¤t.value.target), + }, + ); + self.report_once( + &record_key, + format!( + "Idunn could not record the {recorded} route challenge of {}: {}", + current.value.target, + // The root cause only: the error chain names a fresh temp file each + // attempt, which would defeat once-per-fault. + record.root_cause() + ), + ); + Ok(false) + } + } + } + + fn refresh_admitted_topology( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + ) -> Result { + if !matches!(current.value.readiness(), Ok(class) if class != ReadinessClass::RouteProof) { + return Ok(false); + } + let odin = current.value.odin_receipts()?; + let Some(received) = self.topology().receive( + ¤t.value.target, + ¤t.value.expected.canonical_sha256()?, + )? + else { + return Ok(false); + }; + ensure!( + received.target == current.value.target, + "topology transport substituted admitted target" + ); + let now = now_millis()?; + let authority = self.runtime_authority_parts( + ¤t.value.plan, + ¤t.value.expected, + ¤t.value.activation, + )?; + let odin_authority = self.current_odin_authority(snapshot)?; + let authenticated = match authenticate_odin_runtime_topology_correlation( + &received.canonical_bytes, + &authority, + current.value.leasing.lease_sha256(), + &odin_authority.signer_public_key, + self.trusted_topology_context(now), + ) { + Ok(authenticated) => authenticated, + // An aged-out correlation is nothing new about the admitted + // generation, not a rejected observation. + Err(error) if is_stale_observation(&error) => return Ok(false), + Err(error) => return Err(error), + }; + let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; + if !sequence_requires_admission( + Some(odin.latest), + snapshot.max_odin_sequence(¤t.value.target, &evidence.signer_identity_id), + &evidence, + )? { + return Ok(false); + } + let mut next = current.value.clone(); + next.latest_odin_observation = Some(evidence.clone()); + next.odin_publisher_sequence_cursor = evidence.publisher_sequence; + if is_semantic_ready(&authenticated) { + next.ready = ReadinessEvidence::OdinCorrelated { evidence }; + } + next.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: current.value.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&next, now)?], + )?, + "admitted generation changed before topology sequence CAS" + ); + Ok(true) + } + + fn advance_transaction(&self, current: &Stored) -> Result<()> { + if current.value.post_fencing_abort.is_some() && current.value.completion.is_none() { + return self.advance_post_fencing_abort(current); + } + if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() { + return self.advance_pre_fencing_abort(current); + } + match current.value.phase { + DeploymentPhase::Sealing => self.advance_sealing(current), + DeploymentPhase::Starting => self.advance_starting(current), + DeploymentPhase::Warming => self.advance_warming(current), + DeploymentPhase::Fencing => self.advance_fencing(current), + DeploymentPhase::Leasing => self.advance_leasing(current), + DeploymentPhase::AwaitingReady => self.advance_awaiting_ready(current), + DeploymentPhase::Routing => self.advance_routing(current), + DeploymentPhase::Committing => self.advance_committing(current), + DeploymentPhase::Complete => self.advance_post_commit_cleanup(current), + } + } + + fn advance_sealing(&self, current: &Stored) -> Result<()> { + if current.value.plan.is_none() { + ensure!( + current.value.command_kind == CommandKind::Deploy, + "continuity transaction lost its admitted plan" + ); + let bindings = load_bindings(&self.options.bindings_dir)?; + let loaded = bindings + .get(¤t.value.target) + .context("operator binding disappeared before sealing")?; + let now = now_millis()?; + let resolved = + self.source + .resolve(&loaded.binding, ¤t.value.transaction_id, now)?; + let provider_snapshot = ControlSnapshot::read(&self.options.state_store)?; + let providers = self.current_ready_provider_tokens(&provider_snapshot)?; + let candidate_port = self.select_candidate_port(&loaded.binding)?; + let plan = compile_deployment_plan( + &resolved.recipe_bytes, + &loaded.bytes, + resolved.facts, + format!("incarnation-{}", current.value.transaction_id), + candidate_port, + now, + &providers, + )?; + // Idunn infers no way to prove readiness. A recipe that declares + // none is refused here, before anything is frozen, built or installed. + plan.readiness_class()?; + return self.persist_same_phase(current, |next| { + next.plan = Some(plan); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + + if current.value.command_kind == CommandKind::Deploy + && current.value.frozen_source.is_none() + { + let now = now_millis()?; + let plan = required(¤t.value.plan, "transaction plan")?; + let receipt = self.source.freeze(¤t.value.transaction_id, plan)?; + return self.persist_same_phase(current, |next| { + next.frozen_source = Some(receipt); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + + if current.value.sealed_release.is_none() + || current.value.installed_release.is_none() + || current.value.expected.is_none() + { + ensure!( + current.value.command_kind == CommandKind::Deploy, + "continuity transaction lost its admitted release evidence" + ); + let now = now_millis()?; + let plan = required(¤t.value.plan, "transaction plan")?; + let frozen_receipt = required(¤t.value.frozen_source, "frozen source")?; + let frozen = self.source.observe_frozen(plan, frozen_receipt)?; + let materialized = self.runner_for(plan)?.materialize( + &frozen, + plan, + &self.options.staging_root, + now, + )?; + let installed = self.workload_for(plan)?.install(plan, &materialized)?; + let expected = materialized.release.expected_projection(plan)?; + return self.persist_same_phase(current, |next| { + next.sealed_release = Some(materialized.release); + next.installed_release = Some(installed); + next.expected = Some(expected); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + + // The candidate's Expected is published in Starting, after the brake + // has admitted this exact transaction. Publishing it is the first + // Verse-visible change a deployment makes, and the brake gates changes. + let now = now_millis()?; + let mut next = current.value.clone(); + match current.value.command_kind { + CommandKind::Deploy => { + let Some(authorization) = self.deployment_authorization(current, now)? else { + return self.record_gate_wait( + current, + "deployment brake has not released this exact transaction", + ); + }; + next.deployment_authorization = Some(authorization); + } + CommandKind::Continuity => { + if !self.lifecycle_allows(current, now)? { + return self + .record_gate_wait(current, "lifecycle brake denies continuity restart"); + } + next.lifecycle_authorized_at_unix_millis = Some(now); + } + } + validate_live_providers_for_deploy(current.value.command_kind, || { + self.validate_selected_providers_current(required( + ¤t.value.plan, + "transaction plan", + )?) + })?; + next.enter_phase(DeploymentPhase::Starting, now); + next.last_error = None; + replace_transaction(&self.options.state_store, current, &next) + } + + fn advance_starting(&self, current: &Stored) -> Result<()> { + let expected = required(¤t.value.expected, "Expected projection")?; + let plan = required(¤t.value.plan, "transaction plan")?; + let release = required(¤t.value.sealed_release, "sealed release")?; + let installed = required(¤t.value.installed_release, "installed release")?; + if current.value.expected_publication_sha256.is_none() { + // Published under the candidate's own incarnation key, beside + // whatever the target's admitted incarnation currently projects. + // Nothing the incumbent reads about itself changes here. + let now = now_millis()?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; + let digest = self + .topology() + .publish_expected(expected, &provider_anchor)?; + return self.persist_same_phase(current, |next| { + next.expected_publication_sha256 = Some(digest); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + if current.value.activation.is_none() { + validate_live_providers_for_deploy(current.value.command_kind, || { + self.validate_selected_providers_current(plan) + })?; + let now = now_millis()?; + let runtime_instance_id = runtime_instance_id(¤t.value.transaction_id)?; + let launch = IdunnRuntimeActivationLaunch::issue( + expected, + runtime_instance_id, + now, + &self.idunn_signer, + )?; + let activation = self + .workload_for(plan)? + .prepare_activation(plan, expected, launch)?; + return self.persist_same_phase(current, |next| { + next.activation = Some(activation); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + if current.value.workload.is_none() { + let now = now_millis()?; + let activation = required(¤t.value.activation, "activation")?; + if current.value.rollout_stops_incumbent_first() { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? { + self.workload_for(&incumbent.value.plan)? + .stop(&incumbent.value.workload) + .context("stopping the incumbent before the candidate starts")?; + } + } + let observation = self + .workload_for(plan)? + .start_prepared(plan, release, installed, expected, activation)?; + return self.persist_same_phase(current, |next| { + next.workload = Some(observation); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + if current.value.activation_publication_sha256.is_none() { + let now = now_millis()?; + let activation = required(¤t.value.activation, "activation")?; + let workload = required(¤t.value.workload, "workload")?; + self.workload_for(required(¤t.value.plan, "transaction plan")?)? + .observe(expected, activation, workload)?; + let digest = self + .topology() + .publish_observed_activation(expected, activation, workload)?; + return self.persist_same_phase(current, |next| { + next.activation_publication_sha256 = Some(digest); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + let mut next = current.value.clone(); + next.enter_phase(DeploymentPhase::Warming, now_millis()?); + next.last_error = None; + replace_transaction(&self.options.state_store, current, &next) + } + + fn advance_warming(&self, current: &Stored) -> Result<()> { + let expected = required(¤t.value.expected, "Expected projection")?; + let activation = required(¤t.value.activation, "activation")?; + let workload = required(¤t.value.workload, "workload")?; + self.workload_for(required(¤t.value.plan, "transaction plan")?)? + .observe(expected, activation, workload)?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; + let class = ReadinessClass::of(expected)?; + if current.value.warming.is_none() && class == ReadinessClass::RouteProof { + // A route-proof target's warming is Idunn's own challenge to the + // candidate endpoint. No Odin is read: the target declared no + // reason to be aware of one. + return match self.challenge_candidate( + ¤t.value, + route_proof_warming_states(expected), + None, + )? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { evidence, present } => { + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + self.persist_same_phase(current, |next| { + next.warming = Some(warming); + Ok(()) + }) + } + }; + } + if current.value.warming.is_none() { + // Odin's warming is observed directly when nothing else can observe + // it. That is the first bootstrap, and it is also every continuity + // restart of Odin: the incumbent that would do the observing is the + // very process being restarted, so it cannot report on its own + // return. A deployment of Odin is not included -- there the healthy + // incumbent observes the candidate, which is the point. + // Odin's warming is observed directly exactly when no Odin can + // observe it: the first bootstrap, a continuity restart of Odin + // itself, or any moment the admitted Odin has no usable correlation + // to report through. The rule is the same one that justified the + // bootstrap case -- "observed through Odin" is not available -- and + // it stays scoped to this one target. Every other target keeps + // waiting for Odin, which is what makes Odin the root of the chain. + // + // Asked once. Admitting the latest topology persists the sequence + // cursor, so asking twice with the same record fails the second + // time as "transaction changed before topology admission". + let is_odin = class == ReadinessClass::OdinSelf; + let odin_observation = if is_odin + && snapshot.admitted_odin().is_some() + && current.value.command_kind != CommandKind::Continuity + { + self.admit_latest_topology(current, None)? + } else { + None + }; + let odin_observes_itself = is_odin && odin_observation.is_none(); + if odin_observes_itself { + ensure!( + expected.write_lease_required, + "direct Odin warming must be a stateful incarnation" + ); + let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + return self.persist_same_phase(current, |next| { + next.warming = Some(warming); + Ok(()) + }); + } + let observation = match odin_observation { + Some(observation) => Some(observation), + None => self.admit_latest_topology(current, None)?, + }; + let Some((admitted, authenticated)) = observation else { + // Only the first-Odin bootstrap above observes presence + // directly. Every other target's warming presence arrives as + // Odin's authenticated runtime topology correlation, so until + // Odin is admitted and publishing, a candidate warms forever + // with nothing to say why. + return self.record_gate_wait( + current, + "no authenticated Odin topology correlation yet: warming presence for a \ + non-Odin target is observed through Odin, which must be admitted first", + ); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + let semantic_warming = is_semantic_warming( + expected, + activation, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?; + let latest = required( + &admitted.value.latest_odin_observation, + "sequence-admitted warming evidence", + )?; + if semantic_warming { + let evidence = latest.clone(); + let _token = SequenceAdmittedWarming::from_topology( + admitted.value.transaction_id.clone(), + authenticated, + )?; + return self.persist_same_phase(&admitted, |next| { + next.warming = Some(WarmingEvidence::OdinTopology { evidence }); + Ok(()) + }); + } else { + return Ok(()); + } + } + let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; + ensure!( + warming.transaction_id() == current.value.transaction_id + && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + "durable Warming evidence belongs to another candidate" + ); + + if expected.route.is_some() && current.value.route_preflight.is_none() { + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; + let incumbent_route = + incumbent.and_then(|generation| generation.value.routing.observation().cloned()); + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let route_binding = binding + .route + .context("routed Expected lost route binding")?; + let driver = self.route_driver(route_binding); + let receipt = driver.preflight( + expected, + &activation.runtime_instance_id, + incumbent_route.as_ref(), + &self.route_gate(¤t.value.target, current.value.command_kind), + )?; + return self.persist_same_phase(current, |next| { + next.route_preflight = Some(receipt); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + + if current.value.isolation.is_none() { + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; + // Isolation is a property of two processes running at once. A + // stopped incumbent has already released its DynamicUser UID, and + // systemd is free to hand that same UID to the candidate -- so + // comparing the candidate against the incumbent's *recorded* + // identity turns ordinary UID reuse into a permanent refusal to + // fence. There is nothing left to be isolated from. + let incumbent_workload = match incumbent { + Some(value) + if self + .workload_for(&value.value.plan)? + .is_permanently_stopped(&value.value.workload)? => + { + None + } + other => other.map(|value| &value.value.workload), + }; + let isolation = WorkloadObservation::prove_isolation(workload, incumbent_workload)?; + return self.persist_same_phase(current, |next| { + next.isolation = Some(isolation); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + + self.transition(current, DeploymentPhase::Fencing) + } + + fn advance_fencing(&self, current: &Stored) -> Result<()> { + if current.value.fencing.is_none() { + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; + let incumbent_lease = incumbent.and_then(|generation| generation.value.leasing.lease()); + let expected = required(¤t.value.expected, "Expected projection")?; + let evidence = if expected.write_lease_required || incumbent_lease.is_some() { + let incumbent_lease_path = incumbent_lease + .map(|_| -> Result { + Ok(incumbent + .context("incumbent lease has no admitted generation")? + .value + .plan + .parsed_inputs()? + .1 + .process_write_lease + .context("incumbent lease has no admitted binding")? + .record_path) + }) + .transpose()?; + if let (Some(lease), Some(path)) = (incumbent_lease, &incumbent_lease_path) { + let incumbent = incumbent.context("incumbent lease lost its generation")?; + self.workload_for(&incumbent.value.plan)? + .stop(&incumbent.value.workload) + .context("stopping the exact incumbent before revoking its lifetime-held write lease")?; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); + driver.revoke_exact(Some(lease))?; + ensure!( + driver.observe_empty()?, + "incumbent write lease remained after exact fencing" + ); + self.topology().withdraw_process_write_lease( + &incumbent.value.expected, + &incumbent.value.activation, + Some(lease), + )?; + } + let candidate_lease_path = if expected.write_lease_required { + Some( + current + .value + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .process_write_lease + .context("stateful candidate has no write-lease binding")? + .record_path, + ) + } else { + None + }; + if let Some(path) = &candidate_lease_path { + if incumbent_lease_path.as_ref() != Some(path) { + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); + driver.revoke_exact(None)?; + ensure!( + driver.observe_empty()?, + "candidate write-lease path was not empty before grant" + ); + } + } + self.topology().withdraw_process_write_lease( + expected, + required(¤t.value.activation, "candidate activation")?, + None, + )?; + FencingEvidence::Revoked { + incumbent_lease_sha256: incumbent_lease + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose()?, + candidate_lease_path_verified_empty: candidate_lease_path.is_some(), + } + } else { + FencingEvidence::SkippedStateless + }; + return self.persist_same_phase(current, |next| { + next.fencing = Some(evidence); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + self.transition(current, DeploymentPhase::Leasing) + } + + fn advance_leasing(&self, current: &Stored) -> Result<()> { + let expected = required(¤t.value.expected, "Expected projection")?; + if !expected.write_lease_required { + if current.value.leasing.is_none() { + let now = now_millis()?; + return self.persist_same_phase(current, |next| { + next.leasing = Some(LeasingEvidence::SkippedStateless); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + return self.transition(current, DeploymentPhase::AwaitingReady); + } + + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let lease_path = binding + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + + if matches!( + current.value.leasing.as_ref(), + Some(LeasingEvidence::Granted { .. }) + ) { + let activation = required(¤t.value.activation, "activation")?; + let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease) + .context("granted Leasing evidence has no write lease")?; + let recorded_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("granted Leasing evidence has no lease digest")?; + ensure!( + driver.observe_exact(lease)?, + "physical write lease disappeared after Granted became durable" + ); + self.workload_for(required(¤t.value.plan, "transaction plan")?)? + .observe( + expected, + activation, + required(¤t.value.workload, "candidate workload")?, + )?; + ensure!( + driver.grant(expected, activation, &warming, lease)? == recorded_sha256, + "replayed physical write lease differs from Granted evidence" + ); + ensure!( + self.topology() + .publish_process_write_lease(expected, activation, lease)? + == recorded_sha256, + "replayed write-lease projection differs from Granted evidence" + ); + return self.transition(current, DeploymentPhase::AwaitingReady); + } + + if let Some((lease, prepared_sha256)) = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::prepared_lease) + { + let now = now_millis()?; + let activation = required(¤t.value.activation, "activation")?; + let historical_warming = self.rehydrate_warming_token(¤t.value, now, false)?; + let physical_is_exact = driver.observe_exact(lease)?; + let warming = if physical_is_exact { + historical_warming + } else { + ensure!( + driver.observe_empty()?, + "candidate write-lease store contains authority other than its durable prepared lease" + ); + match self.rehydrate_warming_token(¤t.value, now, true) { + Ok(warming) => warming, + Err(_) => { + let Some((admitted, fresh_evidence, fresh_warming)) = + self.fresh_warming_for_lease(current, now)? + else { + return Ok(()); + }; + let replacement = self.prepare_candidate_write_lease( + &admitted.value, + &fresh_warming, + now, + )?; + let replacement_sha256 = replacement.canonical_sha256()?; + return self.persist_same_phase(&admitted, |next| { + next.warming = Some(fresh_evidence); + next.leasing = Some(LeasingEvidence::Prepared { + lease: replacement, + lease_sha256: replacement_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + } + }; + self.workload_for(required(¤t.value.plan, "transaction plan")?)? + .observe( + expected, + activation, + required(¤t.value.workload, "candidate workload")?, + )?; + if !physical_is_exact { + ensure!( + driver.observe_empty()?, + "candidate write-lease store changed before physical grant" + ); + } + let granted_sha256 = driver.grant(expected, activation, &warming, lease)?; + ensure!( + granted_sha256 == prepared_sha256, + "granted write lease differs from the durable prepared lease" + ); + let projected_sha256 = self + .topology() + .publish_process_write_lease(expected, activation, lease)?; + ensure!( + projected_sha256 == granted_sha256, + "projected write lease differs from the granted process authority" + ); + let lease = lease.clone(); + return self.persist_same_phase(current, |next| { + next.leasing = Some(LeasingEvidence::Granted { + lease, + lease_sha256: projected_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + + ensure!( + current.value.leasing.is_none(), + "stateful Leasing phase carries invalid lease evidence" + ); + ensure!( + driver.observe_empty()?, + "candidate write-lease store contains authority before lease preparation" + ); + let now = now_millis()?; + let Some((admitted, fresh_evidence, fresh_warming)) = + self.fresh_warming_for_lease(current, now)? + else { + return Ok(()); + }; + let lease = self.prepare_candidate_write_lease(&admitted.value, &fresh_warming, now)?; + let lease_sha256 = lease.canonical_sha256()?; + self.persist_same_phase(&admitted, |next| { + next.warming = Some(fresh_evidence); + next.leasing = Some(LeasingEvidence::Prepared { + lease, + lease_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }) + } + + fn advance_awaiting_ready(&self, current: &Stored) -> Result<()> { + let now = now_millis()?; + let expected = required(¤t.value.expected, "Expected projection")?; + self.observe_candidate_before_waiting(¤t.value)?; + if expected.write_lease_required { + let activation = required(¤t.value.activation, "activation")?; + let warming = self.rehydrate_warming_token(¤t.value, now, false)?; + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease) + .context("stateful transaction has no process write lease")?; + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let path = binding + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + ensure!( + CultCacheWriteLeaseDriver::new(¤t.value.target, path) + .observe(expected, activation, &warming, lease)?, + "candidate process write lease is no longer exact" + ); + } + if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { + // Ready is the candidate answering Idunn's own challenge Active, + // holding the exact lease Idunn granted. Once recorded it is + // history; the stable route's challenge is what keeps proving it. + if matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ) { + return self.transition(current, DeploymentPhase::Routing); + } + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + return match self.challenge_candidate(¤t.value, &["warming", "active"], lease)? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { present, .. } if present.record().state == "warming" => { + self.record_gate_wait(current, "candidate is still warming") + } + CandidateAnswer::Answered { evidence, .. } => { + let adoption = LeaseAdoptionEvidence::with_ready( + lease, + &evidence.canonical_sha256, + AdoptionSource::Direct, + evidence.admitted_at_unix_millis, + ); + self.persist_same_phase(current, |next| { + next.ready = Some(ReadinessEvidence::RouteProof { evidence }); + next.lease_adoption = adoption; + Ok(()) + }) + } + }; + } + { + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + let semantic_ready = is_semantic_ready(&authenticated); + let latest = required( + &admitted.value.latest_odin_observation, + "sequence-admitted Ready evidence", + )?; + if current.value.ready.as_ref().and_then(ReadinessEvidence::odin) == Some(latest) { + ensure!(semantic_ready, "stored Ready receipt changed meaning"); + } else if semantic_ready { + let evidence = latest.clone(); + // A Ready stateful correlation carries exactly the current + // lease (the authenticator refuses one that does not). + let adoption = authenticated + .record() + .signed_presence_sha256 + .as_deref() + .and_then(|presence| { + LeaseAdoptionEvidence::with_ready( + current_lease, + presence, + AdoptionSource::OdinTopology, + evidence.admitted_at_unix_millis, + ) + }); + let _token = SequenceAdmittedReady { + transaction_id: admitted.value.transaction_id.clone(), + evidence: evidence.clone(), + expected: expected.clone(), + authenticated, + }; + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence }); + next.lease_adoption = adoption; + Ok(()) + }); + } else { + return Ok(()); + } + } + self.transition(current, DeploymentPhase::Routing) + } + + fn advance_routing(&self, current: &Stored) -> Result<()> { + let expected = required(¤t.value.expected, "Expected projection")?; + let activation = required(¤t.value.activation, "activation")?; + self.observe_candidate_before_waiting(¤t.value)?; + if current.value.routing.is_none() { + let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { + // Ready is the candidate's own proof, recorded in AwaitingReady. + // Validation refuses Routing without a Ready receipt, and a + // receipt the other party vouched for is held before any step + // runs. No Odin correlation is read to admit a route. + current.clone() + } else { + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at route admission" + ); + let latest = required( + &admitted.value.latest_odin_observation, + "current route-admission topology evidence", + )?; + if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) + != Some(latest) + { + let latest = latest.clone(); + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); + Ok(()) + }); + } + let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; + ensure!( + ready.transaction_id() == admitted.value.transaction_id, + "Ready token belongs to another transaction" + ); + admitted + }; + validate_live_providers_for_deploy(admitted.value.command_kind, || { + self.validate_selected_providers_current(required( + &admitted.value.plan, + "transaction plan", + )?) + })?; + self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; + let evidence = if expected.route.is_some() { + let binding = admitted.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let route_binding = binding.route.context("routed plan has no route binding")?; + let preflight = required(&admitted.value.route_preflight, "route preflight")?; + let driver = self.route_driver(route_binding); + let gate = self.route_gate(&admitted.value.target, admitted.value.command_kind); + ensure!( + preflight.candidate_runtime_instance_id == activation.runtime_instance_id, + "route preflight belongs to another runtime instance" + ); + let rollback_allowed = may_rollback_route_after_failed_proof(required( + &admitted.value.fencing, + "route fencing evidence", + )?); + let membership_sha256 = driver.install( + expected, + &activation.runtime_instance_id, + preflight, + rollback_allowed, + &gate, + )?; + let observation = match self.prove_stable_route( + &admitted.value, + &driver, + membership_sha256, + ) { + Ok(observation) => observation, + Err(proof_error) if !rollback_allowed => { + return Err(proof_error).context( + "incumbent authority was fenced; candidate route remains installed for fail-closed retry", + ); + } + Err(proof_error) => { + return match driver.rollback( + expected, + &activation.runtime_instance_id, + preflight, + &gate, + ) { + Ok(()) => Err(proof_error) + .context("candidate did not answer its stable route challenge"), + Err(rollback_error) => Err(proof_error).context(format!( + "candidate did not answer its stable route challenge; exact route rollback also failed: {rollback_error:#}" + )), + }; + } + }; + ensure!( + driver.observe_membership(expected, &observation.membership_sha256)?, + "route membership changed during its signed stable-listener observation" + ); + self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; + let promoted_at_unix_millis = observation.observed_at_unix_millis; + RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + } + } else { + RoutingEvidence::SkippedUnrouted + }; + let now = now_millis()?; + return self.persist_same_phase(&admitted, |next| { + next.routing = Some(evidence); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + self.transition(current, DeploymentPhase::Committing) + } + + fn advance_committing(&self, current: &Stored) -> Result<()> { + self.observe_candidate_before_waiting(¤t.value)?; + let route_proof = ReadinessClass::of(required( + ¤t.value.expected, + "Expected projection", + )?)? == ReadinessClass::RouteProof; + let ready_current = if route_proof { + // The candidate's Ready proof is durable, and the stable route's + // challenge below is its currency. No Odin is read to commit. + current.clone() + } else { + let current_lease_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + // A newer sequence admitted here changed the record, and commit + // goes on from that record rather than returning: returning let + // Odin's next re-stamp abort every attempt (F16). + let Some((ready_current, authenticated)) = + self.admit_latest_topology(current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at admission commit" + ); + // Not require_current: the Ready receipt is durable evidence, so + // authenticating it against `now` asks a stored record to be live and + // refuses it once it ages past the 30s observation window. Route + // admission (:3906) refreshes the receipt to the latest observation and + // can therefore demand currency; by Committing the receipt is history. + // Currency here is answered by the latest observation, authenticated + // and checked for semantic readiness immediately above. + self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; + ready_current + }; + validate_live_providers_for_deploy(ready_current.value.command_kind, || { + self.validate_selected_providers_current(required( + &ready_current.value.plan, + "transaction plan", + )?) + })?; + + let expected = required(&ready_current.value.expected, "Expected projection")?; + let activation = required(&ready_current.value.activation, "activation")?; + let workload = required(&ready_current.value.workload, "workload")?; + self.workload_for(required(&ready_current.value.plan, "transaction plan")?)? + .observe(expected, activation, workload)?; + self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; + if let Some(route) = ready_current + .value + .routing + .as_ref() + .and_then(RoutingEvidence::observation) + { + let binding = ready_current + .value + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1; + let driver = + self.route_driver(binding.route.context("routed plan has no route binding")?); + ensure!( + driver.observe_membership(expected, &route.membership_sha256)?, + "route membership changed before admission commit" + ); + let current_route = self.prove_stable_route( + &ready_current.value, + &driver, + route.membership_sha256.clone(), + )?; + ensure!( + current_route.route_id == route.route_id + && current_route.runtime_instance_id == route.runtime_instance_id, + "stable route changed incarnation before admission commit" + ); + ensure!( + driver.observe_membership(expected, &route.membership_sha256)?, + "route membership changed during the final signed admission challenge" + ); + self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; + } + + let commit_current = if route_proof { + ready_current + } else { + let current_lease_sha256 = ready_current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + // The final proof's own presence makes Odin re-stamp, so this + // admission usually writes. Commit is from the record it wrote. + let Some((commit_current, authenticated)) = + self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready after the final admission challenge" + ); + self.rehydrate_ready_token(&commit_current.value, now_millis()?, false)?; + commit_current + }; + let now = now_millis()?; + validate_live_providers_for_deploy(commit_current.value.command_kind, || { + self.validate_selected_providers_current(required( + &commit_current.value.plan, + "transaction plan", + )?) + })?; + self.workload_for(required(&commit_current.value.plan, "transaction plan")?)? + .observe( + required(&commit_current.value.expected, "Expected projection")?, + required(&commit_current.value.activation, "activation")?, + required(&commit_current.value.workload, "workload")?, + )?; + self.ensure_transaction_write_lease_current(&commit_current.value, now)?; + + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; + let odin_authority = (!route_proof) + .then(|| self.current_odin_authority(&snapshot)) + .transpose()?; + let generation = AdmittedGeneration::from_transaction( + &commit_current.value, + odin_authority, + now, + )?; + let post_commit_cleanup = PostCommitCleanup { + incumbent: match incumbent { + Some(incumbent) + if incumbent_was_stopped_during_fencing(required( + &commit_current.value.fencing, + "commit fencing evidence", + )?) => + { + IncumbentCleanupEvidence::Complete { + generation_id: incumbent.value.generation_id.clone(), + } + } + Some(incumbent) => IncumbentCleanupEvidence::Pending { + generation_id: incumbent.value.generation_id.clone(), + workload: incumbent.value.workload.clone(), + }, + None => IncumbentCleanupEvidence::SkippedNoIncumbent, + }, + source: match commit_current.value.command_kind { + CommandKind::Deploy => SourceCleanupEvidence::Pending, + CommandKind::Continuity => SourceCleanupEvidence::SkippedContinuity, + }, + }; + let mut complete = commit_current.value.clone(); + complete.enter_phase(DeploymentPhase::Complete, now); + complete.last_error = None; + complete.completion = Some(TransactionCompletion::Admitted { + generation_id: generation.generation_id.clone(), + }); + complete.post_commit_cleanup = Some(post_commit_cleanup); + complete.validate()?; + + let admitted_expected = CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: generation.target.clone(), + current: incumbent.map(|stored| stored.envelope.clone()), + }; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: commit_current.value.transaction_id.clone(), + current: Some(commit_current.envelope.clone()), + }, + admitted_expected, + ], + &[ + transaction_envelope(&complete, now)?, + admitted_envelope(&generation, now)?, + ], + )?, + "incumbent or transaction changed before atomic admission commit" + ); + Ok(()) + } + + fn exact_incumbent<'a>( + &self, + snapshot: &'a ControlSnapshot, + transaction: &DeploymentTransaction, + ) -> Result>> { + let current = snapshot.admitted_for(&transaction.target); + match (&transaction.incumbent_generation_id, current) { + (None, None) => Ok(None), + (Some(expected), Some(current)) if current.value.generation_id == *expected => { + Ok(Some(current)) + } + _ => bail!("target incumbent changed after transaction creation"), + } + } + + fn incumbent_lease_sha256_for_warming( + &self, + snapshot: &ControlSnapshot, + transaction: &DeploymentTransaction, + ) -> Result> { + if let Some(fencing) = &transaction.fencing { + return match fencing { + FencingEvidence::SkippedStateless => Ok(None), + FencingEvidence::Revoked { + incumbent_lease_sha256, + .. + } => Ok(incumbent_lease_sha256.clone()), + }; + } + self.exact_incumbent(snapshot, transaction)? + .and_then(|incumbent| incumbent.value.leasing.lease()) + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose() + } + + fn fresh_warming_for_lease( + &self, + current: &Stored, + now: u64, + ) -> Result< + Option<( + Stored, + WarmingEvidence, + SequenceAdmittedWarming, + )>, + > { + let prior = self.rehydrate_warming_token(¤t.value, now, false)?; + match required(¤t.value.warming, "pre-fence Warming evidence")?.clone() { + WarmingEvidence::FirstOdinDirect { .. } => { + // Whether this transaction may observe Odin directly was + // settled when the Warming evidence was recorded, and that + // decision is durable in the transaction. Re-deciding it here + // with the narrower "no admitted Odin" test contradicted the + // caller and failed the refresh. What the refresh owes is + // freshness, and the replay check below is what provides it. + // That the target is Odin was already required by the + // rehydration above, which authenticates the recorded answer. + let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "first Odin replayed its pre-fence Warming presence" + ); + Ok(Some((current.clone(), warming, token))) + } + WarmingEvidence::RouteProofDirect { .. } => { + // A stateful route-proof candidate is warming until it holds + // the lease this refresh is about to bind, so the fresh + // presence must be warming again, and must be a new one. + let CandidateAnswer::Answered { evidence, present } = + self.challenge_candidate(¤t.value, &["warming"], None)? + else { + return Ok(None); + }; + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "route-proof candidate replayed its pre-fence Warming presence" + ); + Ok(Some((current.clone(), warming, token))) + } + WarmingEvidence::OdinTopology { + evidence: prior_evidence, + } => { + let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)? + else { + return Ok(None); + }; + let fresh_evidence = required( + &admitted.value.latest_odin_observation, + "post-fence Odin Warming observation", + )? + .clone(); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, &admitted.value)?; + if !is_semantic_warming( + required(&admitted.value.expected, "Warming Expected projection")?, + required(&admitted.value.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )? { + return Ok(None); + } + let token = SequenceAdmittedWarming::from_topology( + admitted.value.transaction_id.clone(), + authenticated, + )?; + if !provider_warming_advanced( + prior_evidence.publisher_sequence, + prior.signed_presence_sha256(), + fresh_evidence.publisher_sequence, + token.signed_presence_sha256(), + ) { + return Ok(None); + } + Ok(Some(( + admitted, + WarmingEvidence::OdinTopology { + evidence: fresh_evidence, + }, + token, + ))) + } + } + } + + fn prepare_candidate_write_lease( + &self, + transaction: &DeploymentTransaction, + warming: &SequenceAdmittedWarming, + now: u64, + ) -> Result { + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + ensure!( + expected.write_lease_required + && warming.transaction_id() == transaction.transaction_id + && warming.runtime_instance_id() == activation.runtime_instance_id, + "fresh Warming token does not own this stateful candidate" + ); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let epoch = self + .exact_incumbent(&snapshot, transaction)? + .and_then(|generation| generation.value.leasing.lease()) + .map_or(1, |lease| lease.lease_epoch.saturating_add(1)); + let lease = IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: expected + .state_schema_generation + .clone() + .context("stateful Expected has no schema generation")?, + state_contract_sha256: expected + .state_contract_sha256 + .clone() + .context("stateful Expected has no state contract")?, + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + warming_presence_sha256: warming.signed_presence_sha256().to_owned(), + lease_epoch: epoch, + issued_at_unix_millis: now, + }; + lease.validate()?; + Ok(lease) + } + + fn ensure_transaction_write_lease_current( + &self, + transaction: &DeploymentTransaction, + now: u64, + ) -> Result<()> { + let expected = required(&transaction.expected, "Expected projection")?; + let lease = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease); + ensure!( + expected.write_lease_required == lease.is_some(), + "transaction write-lease disposition differs from Expected" + ); + let Some(lease) = lease else { + return Ok(()); + }; + let activation = required(&transaction.activation, "activation")?; + let warming = self.rehydrate_warming_token(transaction, now, false)?; + let lease_path = required(&transaction.plan, "transaction plan")? + .parsed_inputs()? + .1 + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + ensure!( + CultCacheWriteLeaseDriver::new(&transaction.target, lease_path) + .observe(expected, activation, &warming, lease,)?, + "process write lease changed across the authority boundary" + ); + Ok(()) + } + + fn select_candidate_port(&self, binding: &OperatorBinding) -> Result> { + let Some(route) = &binding.route else { + return Ok(None); + }; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut used = BTreeSet::new(); + for plan in snapshot + .admitted + .iter() + .map(|stored| &stored.value.plan) + .chain( + snapshot + .transactions + .iter() + .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter_map(|stored| stored.value.plan.as_ref()), + ) + { + if let Some(port) = plan.candidate_port { + used.insert(port); + } + } + (route.private_port_start..=route.private_port_end) + .find(|port| !used.contains(port)) + .map(Some) + .context("no private candidate port remains in the operator range") + } + + fn deployment_authorization( + &self, + current: &Stored, + now: u64, + ) -> Result> { + let expected = required(¤t.value.expected, "Expected projection")?; + let plan = required(¤t.value.plan, "transaction plan")?; + let binding = plan.parsed_inputs()?.1; + let Some((record, canonical_bytes)) = + read_deployment_brake(&binding.brakes.deployment_store)? + else { + return Ok(None); + }; + let anchor = read_trust_anchor::( + &self.options.deployment_brake_operator_anchor, + )?; + if evaluate_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&record), + &anchor, + &expected.runtime_id, + &expected.sealed_release_id, + ¤t.value.transaction_id, + now, + ) + .is_err() + { + return Ok(None); + } + let authorization_id = record + .authorization_id + .clone() + .context("released brake has no authorization id")?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + ensure!( + !snapshot.transactions.iter().any(|stored| { + stored + .value + .deployment_authorization + .as_ref() + .is_some_and(|used| used.authorization_id == authorization_id) + }), + "deployment authorization was already consumed" + ); + Ok(Some(DeploymentAuthorization { + authorization_id, + brake_sha256: sha256_id(&canonical_bytes), + canonical_brake_bytes: canonical_bytes, + authorized_at_unix_millis: now, + })) + } + + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { + let plan = required(¤t.value.plan, "continuity plan")?; + let binding = plan.parsed_inputs()?.1; + let expected = required(¤t.value.expected, "continuity Expected")?; + match read_lifecycle_brake(&binding.brakes.lifecycle_store) { + Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Present(&record), + &expected.runtime_id, + ¤t.value.target, + now, + ) + .is_ok()), + Ok(None) => Ok(evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Missing, + &expected.runtime_id, + ¤t.value.target, + now, + ) + .is_ok()), + Err(_) => Ok(false), + } + } + + /// A phase that waits on Odin's evidence about the candidate must first + /// confirm the candidate still exists. Otherwise a candidate that died + /// while its correlation was pending is never observed again: the phase + /// returns early on "no evidence yet" every tick, the error that would have + /// triggered the post-fencing abort never happens, and the transaction + /// owns the target forever behind a process that is gone. + fn observe_candidate_before_waiting(&self, transaction: &DeploymentTransaction) -> Result<()> { + self.workload_for(required(&transaction.plan, "transaction plan")?)? + .observe( + required(&transaction.expected, "Expected projection")?, + required(&transaction.activation, "activation")?, + required(&transaction.workload, "candidate workload")?, + ) + .map(|_| ()) + } + + /// Withdraw one projected incarnation of this target that nothing owns: + /// not the admitted generation, not any live transaction. A replaced + /// incumbent, an aborted candidate whose reconciliation was skipped, or a + /// record left by the projection migration all end here. One per tick; + /// the projection is a shared file and each withdrawal is one CAS. + fn retire_stale_incarnations( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + ) -> Result { + let topology = self.topology(); + let target = current.value.target.as_str(); + let owned = snapshot + .transactions + .iter() + .filter(|stored| stored.value.target == target && !stored.value.is_terminal()) + .filter_map(|stored| stored.value.expected.as_ref()) + .map(IdunnExpectedIncarnationRecord::canonical_sha256) + .collect::>>()?; + let admitted_sha256 = current.value.expected.canonical_sha256()?; + let Some(stale) = topology + .projected_incarnations(target)? + .into_iter() + .find(|expected| { + expected + .canonical_sha256() + .is_ok_and(|sha256| sha256 != admitted_sha256 && !owned.contains(&sha256)) + }) + else { + return Ok(false); + }; + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + topology.withdraw_stale_incarnation(&stale, &provider_anchor)?; + eprintln!( + "Idunn withdrew stale {} incarnation {} from the projection", + target, stale.incarnation_id + ); + Ok(true) + } + + /// The lifecycle brake as it applies to restarting one admitted + /// generation: the same evaluation `lifecycle_allows` makes for a + /// continuity transaction, asked before one is minted. + fn lifecycle_allows_generation( + &self, + current: &Stored, + now: u64, + ) -> Result { + let binding = current.value.plan.parsed_inputs()?.1; + let expected = ¤t.value.expected; + match read_lifecycle_brake(&binding.brakes.lifecycle_store) { + Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Present(&record), + &expected.runtime_id, + ¤t.value.target, + now, + ) + .is_ok()), + Ok(None) => Ok(evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Missing, + &expected.runtime_id, + ¤t.value.target, + now, + ) + .is_ok()), + Err(_) => Ok(false), + } + } + + fn record_gate_wait( + &self, + current: &Stored, + detail: &str, + ) -> Result<()> { + if current.value.last_error.as_deref() == Some(detail) { + return Ok(()); + } + let mut next = current.value.clone(); + next.last_error = Some(detail.into()); + next.updated_at_unix_millis = now_millis()?; + replace_transaction(&self.options.state_store, current, &next) + } + + fn runtime_authority( + &self, + transaction: &DeploymentTransaction, + ) -> Result { + self.runtime_authority_parts( + required(&transaction.plan, "transaction plan")?, + required(&transaction.expected, "Expected projection")?, + required(&transaction.activation, "activation")?, + ) + } + + /// The stable route's proof: a challenged presence that is Active. + fn authenticate_routed_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result<(String, u64)> { + let present = self.authenticate_challenged_presence( + authority, + &["active"], + current_write_lease_sha256, + message_id, + challenged_at_unix_millis, + received_at_unix_millis, + canonical_presence, + )?; + Ok(( + present.signed_presence_sha256().to_owned(), + received_at_unix_millis, + )) + } + + /// A presence Idunn challenged for itself, from the stable endpoint or the + /// candidate's: signed by the provider and the launch's activation key, + /// bound to current authority, minted after the challenge, answering that + /// exact challenge, in one of `states`. A warming presence holds no write + /// lease; a presence in any other state holds exactly the current one. + /// Every disagreement with authority is named, so a capacity below the + /// Expected minimum reads as that and not as a generic refusal. + fn authenticate_challenged_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + states: &[&str], + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { + ensure!( + received_at_unix_millis >= challenged_at_unix_millis, + "route proof predates its challenge" + ); + let authenticated = authenticate_runtime_presence_claim( + canonical_presence, + authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: received_at_unix_millis, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + }, + )?; + let correlation = correlate_runtime_presence_claim(authenticated, authority)?; + if !correlation.disagreements().is_empty() { + return Err(PresenceDisagrees { + disagreements: correlation.disagreements().to_vec(), + } + .into()); + } + let present = correlation.into_undisputed_present()?; + let presence = present.record(); + ensure!( + presence.observed_at_unix_millis >= challenged_at_unix_millis, + "route proof returned a presence minted before its challenge" + ); + ensure!( + states.contains(&presence.state.as_str()), + "route proof runtime is {}, not {}", + presence.state, + states.join(" or ") + ); + ensure!( + presence.detail == format!("route-observation:{message_id}"), + "route proof response is not bound to the exact challenge" + ); + // A warming presence is exempt from the lease comparison: it is + // Idunn's own lease that a warming candidate cannot hold yet, and the + // authenticator above already refuses one that claims a lease. + let lease_is_current = presence.state == "warming" + || presence.write_lease_sha256.as_deref() == current_write_lease_sha256; + ensure!( + lease_is_current, + "route proof runtime does not hold the exact current process write lease" + ); + Ok(present) + } + + fn authenticate_first_odin_warming_presence( + &self, + transaction: &DeploymentTransaction, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { + // Whether direct observation is warranted at all is decided in + // advance_warming, which asks the question that matters: can any Odin + // report on this one. Re-deciding it here with the narrower "has an + // Odin ever been admitted" test only contradicted that caller. What is + // still checked is what this function itself depends on -- the target + // is Odin, and the incarnation is stateful. + let expected = required(&transaction.expected, "Odin Expected")?; + ensure!( + ReadinessClass::of(expected)? == ReadinessClass::OdinSelf + && expected.write_lease_required, + "direct Warming presence is reserved for a stateful Odin incarnation" + ); + ensure!( + received_at_unix_millis >= challenged_at_unix_millis, + "first Odin Warming observation predates its challenge" + ); + let authority = self.runtime_authority(transaction)?; + let authenticated = authenticate_runtime_presence_claim( + canonical_presence, + &authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: received_at_unix_millis, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + }, + )?; + let correlation = correlate_runtime_presence_claim(authenticated, &authority)?; + ensure!( + correlation.disagreements().is_empty(), + "first Odin Warming presence disagrees with its Expected activation" + ); + let present = correlation.into_undisputed_present()?; + let presence = present.record(); + ensure!( + presence.observed_at_unix_millis >= challenged_at_unix_millis + && presence.state == "warming" + && presence.write_lease_sha256.is_none() + && presence.detail == format!("idunn-warming:{message_id}"), + "first Odin candidate did not return exact fresh pre-lease Warming evidence" + ); + Ok(present) + } + + fn observe_first_odin_warming( + &self, + transaction: &DeploymentTransaction, + ) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + let expected = required(&transaction.expected, "first Odin Expected")?; + let binding = required(&transaction.plan, "first Odin plan")? + .parsed_inputs()? + .1; + let driver = self.route_driver( + binding + .route + .context("first Odin bootstrap has no candidate route binding")?, + ); + let message_id = format!("warming-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = driver.request_candidate_runtime_presence(expected, &message_id)?; + ensure!( + response.message_id == message_id, + "first Odin candidate transport substituted its challenge identity" + ); + let admitted_at_unix_millis = now_millis()?; + let present = self.authenticate_first_odin_warming_presence( + transaction, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok((evidence, present)) + } + + fn prove_stable_route( + &self, + transaction: &DeploymentTransaction, + driver: &NginxRouteDriver, + membership_sha256: String, + ) -> Result { + let expected = required(&transaction.expected, "route Expected projection")?; + let activation = required(&transaction.activation, "route activation")?; + let authority = self.runtime_authority(transaction)?; + let current_write_lease_sha256 = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + self.prove_stable_route_against( + expected, + activation, + &authority, + current_write_lease_sha256, + driver, + membership_sha256, + ) + } + + fn prove_stable_route_against( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + driver: &NginxRouteDriver, + membership_sha256: String, + ) -> Result { + let message_id = format!("route-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = driver.request_runtime_presence(expected, &message_id)?; + ensure!( + response.message_id == message_id, + "route transport substituted its challenge identity" + ); + let received_at_unix_millis = now_millis()?; + let (signed_presence_sha256, observed_at_unix_millis) = self.authenticate_routed_presence( + authority, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + received_at_unix_millis, + &response.canonical_presence, + )?; + let observation = RouteObservation { + route_id: driver.binding.route_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + membership_sha256, + signed_presence_sha256, + observed_at_unix_millis, + }; + observation.validate()?; + Ok(observation) + } + + fn runtime_authority_parts( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + let provider_anchor = self.provider_anchor_for_plan(plan)?; + verify_runtime_authority( + expected, + activation, + &self.idunn_anchor, + &provider_anchor.public_key, + ) + } + + fn provider_anchor_for_plan( + &self, + plan: &CompiledDeploymentPlan, + ) -> Result { + let binding = plan.parsed_inputs()?.1; + let provider_anchor = read_trust_anchor::( + &binding.runtime_identity.trust_anchor_store, + )?; + ensure!( + provider_anchor.schema_version + == ::TRUST_ANCHOR_SCHEMA, + "provider runtime presence trust anchor schema is unsupported" + ); + Ok(provider_anchor) + } + + fn authenticate_topology_bytes( + &self, + snapshot: &ControlSnapshot, + transaction: &DeploymentTransaction, + canonical_bytes: &[u8], + current_write_lease_sha256: Option<&str>, + trusted_received_at: u64, + ) -> Result { + let authority = self.runtime_authority(transaction)?; + let odin_authority = self.current_odin_authority(snapshot)?; + authenticate_odin_runtime_topology_correlation( + canonical_bytes, + &authority, + current_write_lease_sha256, + &odin_authority.signer_public_key, + self.trusted_topology_context(trusted_received_at), + ) + } + + /// Persist every newly authenticated publisher sequence before deciding + /// whether it means Warming, Ready, degraded, or disagreement. + fn admit_latest_topology( + &self, + current: &Stored, + current_write_lease_sha256: Option<&str>, + ) -> Result< + Option<( + Stored, + AuthenticatedOdinRuntimeTopologyCorrelation, + )>, + > { + // Odin keeps one correlation per incarnation, so the transaction's own + // Expected selects exactly the evidence about it. A correlation about + // the incarnation this one is replacing lives under another key and is + // never handed back here. + let expected_sha256 = + required(¤t.value.expected, "Expected projection")?.canonical_sha256()?; + let Some(received) = self + .topology() + .receive(¤t.value.target, &expected_sha256)? + else { + return Ok(None); + }; + ensure!( + received.target == current.value.target, + "topology transport substituted target" + ); + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let live = snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared before topology admission")?; + ensure!( + live.envelope == current.envelope, + "transaction changed before topology admission" + ); + let authenticated = match self.authenticate_topology_bytes( + &snapshot, + &live.value, + &received.canonical_bytes, + current_write_lease_sha256, + now, + ) { + Ok(authenticated) => authenticated, + // Odin re-stamps a correlation only when its facts change, and a + // live presence changes them every heartbeat. A correlation that + // has aged out therefore says one thing: nothing about this + // incarnation has arrived lately. That is absence to wait on, not + // evidence to refuse; it failed every Heimdall deployment and the + // first observed Odin deployment as a fault. + Err(error) if is_stale_observation(&error) => return Ok(None), + Err(error) => return Err(error), + }; + let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; + if !sequence_requires_admission( + live.value.latest_odin_observation.as_ref(), + snapshot.max_odin_sequence(&live.value.target, &evidence.signer_identity_id), + &evidence, + )? { + return Ok(Some((live.clone(), authenticated))); + } + let mut next = live.value.clone(); + next.latest_odin_observation = Some(evidence.clone()); + next.odin_publisher_sequence_cursor = evidence.publisher_sequence; + next.updated_at_unix_millis = now; + next.last_error = None; + replace_transaction(&self.options.state_store, live, &next)?; + let admitted = ControlSnapshot::read(&self.options.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == next.transaction_id) + .context("sequence-admitted transaction disappeared")?; + ensure!( + admitted.value.latest_odin_observation.as_ref() == Some(&evidence) + && admitted.value.odin_publisher_sequence_cursor == evidence.publisher_sequence, + "topology sequence was not durably admitted" + ); + Ok(Some((admitted, authenticated))) + } + + fn rehydrate_warming_token( + &self, + transaction: &DeploymentTransaction, + now: u64, + require_current: bool, + ) -> Result { + let evidence = required(&transaction.warming, "durable warming evidence")?.clone(); + match evidence { + WarmingEvidence::OdinTopology { evidence } => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let authenticated = self.authenticate_topology_bytes( + &snapshot, + transaction, + &evidence.canonical_bytes, + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + validate_authenticated_evidence(&evidence, &authenticated)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, transaction)?; + ensure!( + is_semantic_warming( + required(&transaction.expected, "Warming Expected projection")?, + required(&transaction.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?, + "durable warming evidence no longer satisfies the Warming gate" + ); + SequenceAdmittedWarming::from_topology( + transaction.transaction_id.clone(), + authenticated, + ) + } + WarmingEvidence::FirstOdinDirect { evidence } => { + let present = self.authenticate_first_odin_warming_presence( + transaction, + &evidence.message_id, + evidence.challenged_at_unix_millis, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + &evidence.canonical_bytes, + )?; + SequenceAdmittedWarming::from_direct_presence( + transaction.transaction_id.clone(), + WarmingEvidence::FirstOdinDirect { evidence }, + present, + ) + } + WarmingEvidence::RouteProofDirect { evidence } => { + let present = self.reauthenticate_route_proof( + transaction, + &evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + SequenceAdmittedWarming::from_direct_presence( + transaction.transaction_id.clone(), + WarmingEvidence::RouteProofDirect { evidence }, + present, + ) + } + } + } + + /// A route-proof presence Idunn recorded, authenticated again as of `at`. + fn reauthenticate_route_proof( + &self, + transaction: &DeploymentTransaction, + evidence: &RuntimePresenceEvidence, + states: &[&str], + current_write_lease_sha256: Option<&str>, + at_unix_millis: u64, + ) -> Result { + let authority = self.runtime_authority(transaction)?; + self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &evidence.message_id, + evidence.challenged_at_unix_millis, + at_unix_millis, + &evidence.canonical_bytes, + ) + } + + /// One direct challenge to a route-proof target's candidate endpoint. + /// Failing to reach or hear the candidate is only silence: a process that + /// is still starting does not answer, and that is waiting, not a fault. + /// What does answer is authenticated, and a bad answer is an error, as is + /// a challenge Idunn could not make: those stay a `ChallengeFailure` in the + /// error's chain. + fn challenge_candidate( + &self, + transaction: &DeploymentTransaction, + states: &[&str], + current_write_lease_sha256: Option<&str>, + ) -> Result { + let expected = required(&transaction.expected, "candidate Expected")?; + let binding = required(&transaction.plan, "candidate plan")? + .parsed_inputs()? + .1; + let driver = self.route_driver( + binding + .route + .context("route-proof candidate has no route binding")?, + ); + let message_id = format!("candidate-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = match driver.request_candidate_runtime_presence(expected, &message_id) { + Ok(response) => response, + Err(ChallengeFailure::Silent(error)) => { + return Ok(CandidateAnswer::Silent(format!( + "candidate endpoint did not answer its challenge: {error:#}" + ))); + } + Err(bad @ ChallengeFailure::Refused(_)) => return Err(bad.into()), + }; + let admitted_at_unix_millis = now_millis()?; + let authority = self.runtime_authority(transaction)?; + let present = self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok(CandidateAnswer::Answered { evidence, present }) + } + + fn rehydrate_ready_token( + &self, + transaction: &DeploymentTransaction, + now: u64, + require_current: bool, + ) -> Result { + let evidence = required(&transaction.ready, "durable Ready evidence")? + .odin() + .context("durable Ready evidence is not an Odin receipt")? + .clone(); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let current_lease = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let authenticated = self.authenticate_topology_bytes( + &snapshot, + transaction, + &evidence.canonical_bytes, + current_lease, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + validate_authenticated_evidence(&evidence, &authenticated)?; + ensure!( + is_semantic_ready(&authenticated), + "durable Ready evidence no longer authenticates as Ready" + ); + Ok(SequenceAdmittedReady { + transaction_id: transaction.transaction_id.clone(), + evidence, + expected: required(&transaction.expected, "Expected projection")?.clone(), + authenticated, + }) + } + + fn current_ready_provider_tokens( + &self, + snapshot: &ControlSnapshot, + ) -> Result> { + let mut providers = Vec::new(); + for stored in &snapshot.admitted { + // No pre-filter on ready == latest: a provider that published again + // after going ready is still ready. rehydrate_admitted_ready owns + // that judgment now, and reports why when it refuses. + match self.rehydrate_admitted_ready(snapshot, &stored.value) { + Ok(provider) => providers.push(provider), + Err(error) => eprintln!( + "Idunn excluded non-current provider {}: {error:#}", + stored.value.target + ), + } + } + Ok(providers) + } + + fn rehydrate_admitted_ready( + &self, + snapshot: &ControlSnapshot, + generation: &AdmittedGeneration, + ) -> Result { + let odin = generation.odin_receipts()?; + let authority = self.runtime_authority_parts( + &generation.plan, + &generation.expected, + &generation.activation, + )?; + let odin_authority = self.current_odin_authority(snapshot)?; + let current_lease = generation.leasing.lease_sha256(); + let authenticated = authenticate_odin_runtime_topology_correlation( + &odin.ready.canonical_bytes, + &authority, + current_lease, + &odin_authority.signer_public_key, + self.trusted_topology_context(odin.ready.admitted_at_unix_millis), + )?; + validate_authenticated_evidence(odin.ready, &authenticated)?; + ensure!( + is_semantic_ready(&authenticated), + "admitted provider's Ready receipt is not Ready at its admission time" + ); + // A provider that published after going ready is the ordinary case, not + // a stale one, so the cursor is authenticated on its own terms rather + // than compared to the receipt. Authentication is what binds it to this + // incarnation -- it requires the record's current_activation_sha256 and + // runtime_instance_id to match this generation's activation -- so a + // separate identity check here would only restate it more weakly. + // Note this is the cursor as frozen at admission, not a live read of + // Odin. Both receipts are historical proofs; current workload health + // is owned by continuity and route observation, not receipt age. + let authenticated_latest = authenticate_odin_runtime_topology_correlation( + &odin.latest.canonical_bytes, + &authority, + current_lease, + &odin_authority.signer_public_key, + self.trusted_topology_context(odin.latest.admitted_at_unix_millis), + )?; + validate_authenticated_evidence(odin.latest, &authenticated_latest)?; + ensure!( + is_semantic_ready(&authenticated_latest), + "admitted provider's latest receipt is not Ready at its admission time" + ); + Ok(SequenceAdmittedReady { + transaction_id: generation.transaction_id.clone(), + evidence: odin.ready.clone(), + expected: generation.expected.clone(), + authenticated, + }) + } + + fn validate_selected_providers_current( + &self, + plan: &CompiledDeploymentPlan, + ) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + for selection in &plan.dependencies { + let Some(provider) = &selection.provider else { + ensure!( + selection.requirement.kind == DependencyKind::Optional, + "required dependency has no provider" + ); + continue; + }; + let DependencyProviderAuthority::ManagedReady { + target, + incarnation_id, + plan_id, + sealed_release_id, + expected_projection_sha256, + odin_topology_correlation_sha256, + odin_topology_publisher_sequence, + } = &provider.authority + else { + continue; + }; + let admitted = snapshot + .admitted_for(target) + .context("selected managed dependency is no longer admitted")?; + let token = self.rehydrate_admitted_ready(&snapshot, &admitted.value)?; + ensure!( + admitted.value.expected.incarnation_id == *incarnation_id + && admitted.value.plan.plan_id == *plan_id + && admitted.value.sealed_release.sealed_release_id == *sealed_release_id + && admitted.value.expected.canonical_sha256()? == *expected_projection_sha256 + && token.publisher_sequence() >= *odin_topology_publisher_sequence, + "selected dependency authority changed before actuation" + ); + ensure!( + token.publisher_sequence() != *odin_topology_publisher_sequence + || token.evidence_sha256() == odin_topology_correlation_sha256, + "selected dependency Odin sequence changed evidence" + ); + ensure!( + token + .authenticated() + .record() + .observed_capabilities + .iter() + .any(|capability| { + capability_compatible( + &selection.requirement.capability, + &selection.requirement.schema, + &selection.requirement.compatibility, + &capability.capability, + &capability.schema, + &capability.compatibility, + ) && capability.capacity >= selection.requirement.minimum_capacity + }), + "selected dependency no longer provides its required capability" + ); + } + Ok(()) + } + + fn persist_same_phase( + &self, + current: &Stored, + mutation: F, + ) -> Result<()> + where + F: FnOnce(&mut DeploymentTransaction) -> Result<()>, + { + let mut next = current.value.clone(); + mutation(&mut next)?; + ensure!( + next.phase == current.value.phase, + "same-phase evidence update changed phase" + ); + next.updated_at_unix_millis = now_millis()?; + next.last_error = None; + replace_transaction(&self.options.state_store, current, &next) + } + + fn transition( + &self, + current: &Stored, + next_phase: DeploymentPhase, + ) -> Result<()> { + ensure!( + next_phase as u8 == current.value.phase as u8 + 1, + "deployment phase transition is not adjacent" + ); + let mut next = current.value.clone(); + next.enter_phase(next_phase, now_millis()?); + next.last_error = None; + replace_transaction(&self.options.state_store, current, &next) + } + + /// Every post-fencing phase ends. Before a live transaction past the + /// fence runs its step, a phase whose deadline has passed, or whose + /// command an operator asked to expire, is resolved as `phase_end` says. + /// Returns `Some(moved)` when it decided the tick, `None` to run the step. + /// + /// The deadline is written only by `enter_phase`, so nothing here or in a + /// resume path extends it. A record lifted from before deadlines existed + /// has none; its phase is timed once, from the first time this sees it. + fn resolve_phase_deadline( + &self, + current: &Stored, + ) -> Result> { + let transaction = ¤t.value; + if !transaction.resolves_by_deadline() { + return Ok(None); + } + let now = now_millis()?; + let Some(deadline) = transaction.phase_deadline else { + let mut next = transaction.clone(); + next.enter_phase(transaction.phase, now); + if next.phase_deadline.is_none() { + return Ok(None); + } + replace_transaction(&self.options.state_store, current, &next)?; + return Ok(Some(true)); + }; + let requested = ControlSnapshot::read(&self.options.state_store)? + .expiry_requests + .iter() + .any(|request| request.value.command_id == transaction.command_id); + let cause = if now >= deadline.deadline_at_unix_millis { + format!( + "the {:?} phase passed its deadline at unix ms {}", + transaction.phase, deadline.deadline_at_unix_millis + ) + } else if requested { + format!("the {:?} phase was expired by the operator", transaction.phase) + } else { + return Ok(None); + }; + match transaction.phase_end() { + PhaseEnd::Abort { lease_not_adopted } => { + let error = if lease_not_adopted { + anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") + } else { + anyhow!("{cause}") + }; + self.begin_post_fencing_abort(current, error)?; + Ok(Some(true)) + } + PhaseEnd::OperatorRequired => { + self.report_once( + &format!("deadline:{}", transaction.transaction_id), + format!( + "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", + transaction.transaction_id + ), + ); + Ok(None) + } + } + } + + /// Delete expiry requests whose command has no phase left to end: it was + /// resolved, it finished, or its command is gone. + fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { + let spent = snapshot + .expiry_requests + .iter() + .filter(|request| { + !snapshot.transactions.iter().any(|stored| { + stored.value.command_id == request.value.command_id + && stored.value.resolves_by_deadline() + }) + }) + .map(|request| request.envelope.clone()) + .collect::>(); + if spent.is_empty() { + return Ok(false); + } + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .delete_batch_if_unchanged(&spent) + } + + fn record_resumable_error( + &self, + current: &Stored, + error: &anyhow::Error, + ) -> Result<()> { + let detail = truncate(&format!("{error:#}"), 2048); + if current.value.last_error.as_deref() == Some(detail.as_str()) { + return Ok(()); + } + let mut next = current.value.clone(); + next.last_error = Some(detail); + next.updated_at_unix_millis = now_millis()?; + replace_transaction(&self.options.state_store, current, &next) + } + + fn begin_pre_fencing_abort( + &self, + current: &Stored, + error: anyhow::Error, + ) -> Result<()> { + ensure!( + current.value.phase < DeploymentPhase::Fencing, + "post-fence transaction cannot terminal-fail" + ); + ensure!( + current.value.pre_fencing_abort.is_none(), + "pre-fencing abort intent is already durable" + ); + let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); + self.persist_same_phase(current, |next| { + next.pre_fencing_abort = Some(abort); + Ok(()) + }) + } + + /// Whether this transaction's candidate can no longer run at all. + /// + /// A transaction that has never started one has no candidate to be dead, so + /// it is not permanently stopped -- it is simply not there yet. + fn candidate_is_permanently_stopped( + &self, + transaction: &DeploymentTransaction, + ) -> Result { + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(required(&transaction.plan, "transaction plan")?)? + .is_permanently_stopped(workload) + } + + fn begin_post_fencing_abort( + &self, + current: &Stored, + error: anyhow::Error, + ) -> Result<()> { + ensure!( + current.value.phase >= DeploymentPhase::Fencing, + "pre-fence transaction must abort through the pre-fencing path" + ); + ensure!( + current.value.post_fencing_abort.is_none(), + "post-fencing abort intent is already durable" + ); + let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); + self.persist_same_phase(current, |next| { + next.post_fencing_abort = Some(abort); + Ok(()) + }) + } + + /// Withdraw the candidate in the one order that never leaves two writers: + /// take away its route, then its write lease, then the process itself, and + /// only then reconcile the projection. + fn advance_post_fencing_abort(&self, current: &Stored) -> Result<()> { + let abort = required( + ¤t.value.post_fencing_abort, + "post-fencing abort intent", + )?; + if abort.route_restoration == CleanupEvidence::Pending { + let preflight = required(¤t.value.route_preflight, "route preflight receipt")?; + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let driver = + self.route_driver(binding.route.context("routed plan has no route binding")?); + driver + .withdraw_candidate_membership( + preflight, + &self.route_gate(¤t.value.target, current.value.command_kind), + ) + .context("restoring the route the candidate found")?; + return self.persist_same_phase(current, |next| { + next.post_fencing_abort.as_mut().unwrap().route_restoration = + CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.lease_withdrawal == CleanupEvidence::Pending { + let expected = required(¤t.value.expected, "Expected projection")?; + let activation = required(¤t.value.activation, "activation")?; + // A prepared lease is withdrawn too: its physical grant may have + // landed before `Granted` was durable. Revoking an absent lease, + // and withdrawing an absent projection, are no-ops. + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + .context("post-fencing abort lost the lease it must withdraw")?; + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let lease_path = binding + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + driver + .revoke_exact(Some(lease)) + .context("revoking the abandoned candidate write lease")?; + ensure!( + driver.observe_empty()?, + "candidate write lease remained after post-fencing withdrawal" + ); + self.topology() + .withdraw_process_write_lease(expected, activation, Some(lease)) + .context("withdrawing the abandoned candidate write-lease projection")?; + return self.persist_same_phase(current, |next| { + next.post_fencing_abort.as_mut().unwrap().lease_withdrawal = + CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.candidate_cleanup == CleanupEvidence::Pending { + let plan = required(¤t.value.plan, "abandoned candidate plan")?; + if let Some(workload) = ¤t.value.workload { + self.workload_for(plan)? + .stop(workload) + .context("stopping the abandoned candidate")?; + } + self.workload_for(plan)? + .discard_prepared( + required(¤t.value.plan, "abandoned candidate plan")?, + required(¤t.value.expected, "abandoned Expected projection")?, + required(¤t.value.activation, "abandoned activation")?, + ) + .context("discarding the abandoned activation material")?; + return self.persist_same_phase(current, |next| { + next.post_fencing_abort.as_mut().unwrap().candidate_cleanup = + CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.topology_reconciliation == CleanupEvidence::Pending { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + // The failed candidate's projection is resolved first. For a + // deployment, fencing also stopped the incumbent and revoked its + // lease, so the incumbent is demoted to Expected-only under its + // key with its own exact activation, which is what lets + // continuity bring it back. A continuity candidate is the + // incumbent's key: it has nothing further to demote. + let topology = self.topology(); + let failed_provider_anchor = self.provider_anchor_for_plan(required( + ¤t.value.plan, + "failed transaction plan", + )?)?; + reconcile_failed_candidate_projection( + &topology, + ¤t.value, + &failed_provider_anchor, + )?; + if let Some(incumbent) = self + .exact_incumbent(&snapshot, ¤t.value)? + .filter(|_| current.value.command_kind == CommandKind::Deploy) + { + let admitted_provider_anchor = + self.provider_anchor_for_plan(&incumbent.value.plan)?; + let expected_sha256 = topology.demote_to_expected_only( + &incumbent.value.expected, + &admitted_provider_anchor, + &incumbent.value.activation, + incumbent.value.leasing.lease(), + )?; + ensure!( + expected_sha256 == incumbent.value.expected.canonical_sha256()?, + "restored incumbent Expected receipt differs" + ); + } + return self.persist_same_phase(current, |next| { + next.post_fencing_abort + .as_mut() + .unwrap() + .topology_reconciliation = CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.source_cleanup == CleanupEvidence::Pending { + self.source + .cleanup( + ¤t.value.transaction_id, + current.value.frozen_source.as_ref(), + ) + .context("cleaning the abandoned source")?; + return self.persist_same_phase(current, |next| { + next.post_fencing_abort.as_mut().unwrap().source_cleanup = + CleanupEvidence::Complete; + Ok(()) + }); + } + ensure!( + abort.is_complete(), + "post-fencing abort cleanup is incomplete" + ); + let mut next = current.value.clone(); + next.enter_phase(DeploymentPhase::Complete, now_millis()?); + next.last_error = Some(abort.error.clone()); + next.completion = Some(TransactionCompletion::FailedAfterFencing { + error: abort.error.clone(), + recovery: current.value.terminal_recovery(), + }); + replace_transaction(&self.options.state_store, current, &next) + } + + fn advance_pre_fencing_abort(&self, current: &Stored) -> Result<()> { + ensure!( + current.value.phase < DeploymentPhase::Fencing, + "pre-fencing abort crossed the fencing boundary" + ); + let abort = required(¤t.value.pre_fencing_abort, "pre-fencing abort intent")?; + if abort.candidate_cleanup == CleanupEvidence::Pending { + let plan = required(¤t.value.plan, "transaction plan")?; + if let Some(workload) = ¤t.value.workload { + self.workload_for(plan)? + .stop(workload) + .context("stopping exact pre-fence candidate")?; + } + self.workload_for(plan)? + .discard_prepared( + required(¤t.value.plan, "pre-fencing candidate plan")?, + required(¤t.value.expected, "pre-fencing Expected projection")?, + required(¤t.value.activation, "pre-fencing activation")?, + ) + .context("discarding exact pre-fence activation material")?; + return self.persist_same_phase(current, |next| { + next.pre_fencing_abort.as_mut().unwrap().candidate_cleanup = + CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.topology_reconciliation == CleanupEvidence::Pending { + // Before fencing a deployment never touched its incumbent: it is + // still running under its own key with its activation and lease, + // so only the candidate's records are withdrawn. A continuity + // candidate shares the admitted key and demotes what it issued. + let plan = required(¤t.value.plan, "failed transaction plan")?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; + reconcile_failed_candidate_projection( + &self.topology(), + ¤t.value, + &provider_anchor, + )?; + return self.persist_same_phase(current, |next| { + next.pre_fencing_abort + .as_mut() + .unwrap() + .topology_reconciliation = CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.source_cleanup == CleanupEvidence::Pending { + self.source + .cleanup( + ¤t.value.transaction_id, + current.value.frozen_source.as_ref(), + ) + .context("cleaning failed pre-fence source")?; + return self.persist_same_phase(current, |next| { + next.pre_fencing_abort.as_mut().unwrap().source_cleanup = CleanupEvidence::Complete; + Ok(()) + }); + } + ensure!( + abort.is_complete(), + "pre-fencing abort cleanup is incomplete" + ); + let mut next = current.value.clone(); + next.enter_phase(DeploymentPhase::Complete, now_millis()?); + next.last_error = Some(abort.error.clone()); + next.completion = Some(TransactionCompletion::FailedBeforeFencing { + error: abort.error.clone(), + }); + replace_transaction(&self.options.state_store, current, &next) + } + + fn advance_post_commit_cleanup(&self, current: &Stored) -> Result<()> { + let cleanup = required( + ¤t.value.post_commit_cleanup, + "post-commit cleanup evidence", + )?; + if let IncumbentCleanupEvidence::Pending { + generation_id, + workload, + } = &cleanup.incumbent + { + let binding = required(¤t.value.plan, "committed transaction plan")? + .parsed_inputs()? + .1; + if let Some(promoted_at_unix_millis) = required( + ¤t.value.routing, + "committed transaction routing evidence", + )? + .promoted_at_unix_millis() + { + let retire_not_before = + route_drain_deadline(promoted_at_unix_millis, binding.rollout.drain_seconds)?; + if now_millis()? < retire_not_before { + return Ok(()); + } + } + // The incumbent is the same target under the same binding kind; + // an incumbent of another kind is refused by the driver, not + // guessed at. + self.workload_for(required(¤t.value.plan, "committed transaction plan")?)? + .stop(workload) + .with_context(|| format!("retiring admitted incumbent {generation_id}"))?; + let generation_id = generation_id.clone(); + return self.persist_same_phase(current, |next| { + next.post_commit_cleanup.as_mut().unwrap().incumbent = + IncumbentCleanupEvidence::Complete { generation_id }; + Ok(()) + }); + } + if cleanup.source == SourceCleanupEvidence::Pending { + self.source + .cleanup( + ¤t.value.transaction_id, + current.value.frozen_source.as_ref(), + ) + .context("cleaning committed deployment source")?; + return self.persist_same_phase(current, |next| { + next.post_commit_cleanup.as_mut().unwrap().source = SourceCleanupEvidence::Complete; + Ok(()) + }); + } + ensure!(cleanup.is_complete(), "post-commit cleanup is incomplete"); + Ok(()) + } +} + +fn route_drain_deadline(promoted_at_unix_millis: u64, drain_seconds: u32) -> Result { + promoted_at_unix_millis + .checked_add( + u64::from(drain_seconds) + .checked_mul(1_000) + .context("route drain duration overflows milliseconds")?, + ) + .context("route drain deadline overflows Unix milliseconds") +} + +/// CultLib refuses a correlation older than the trusted window with this +/// text. Idunn reads that one refusal as "nothing new", everywhere it waits +/// on Odin's evidence; every other refusal stays a refusal. +fn is_stale_observation(error: &anyhow::Error) -> bool { + format!("{error:#}").contains("outside the trusted observation window") +} + +fn route_observation_is_current( + observed_at_unix_millis: u64, + now_unix_millis: u64, + maximum_age_millis: u64, + maximum_future_skew_millis: u64, +) -> bool { + observed_at_unix_millis <= now_unix_millis.saturating_add(maximum_future_skew_millis) + && now_unix_millis.saturating_sub(observed_at_unix_millis) <= maximum_age_millis +} + +fn provider_warming_advanced( + prior_odin_sequence: u64, + prior_signed_presence_sha256: &str, + candidate_odin_sequence: u64, + candidate_signed_presence_sha256: &str, +) -> bool { + candidate_odin_sequence > prior_odin_sequence + && candidate_signed_presence_sha256 != prior_signed_presence_sha256 +} + +/// Resolve the projection a failed candidate left. A deployment candidate owns +/// its own incarnation key and withdraws it whole. A continuity candidate +/// shares the admitted key: the admitted Expected is never withdrawn, and the +/// only thing demoted is the activation this transaction issued, exactly. +fn reconcile_failed_candidate_projection( + topology: &CultCacheTopologyDriver, + transaction: &DeploymentTransaction, + provider_anchor: &ServiceIdentityTrustAnchor, +) -> Result<()> { + let expected = required(&transaction.expected, "failed Expected projection")?; + match transaction.command_kind { + CommandKind::Deploy => topology + .withdraw_incarnation( + expected, + provider_anchor, + transaction.activation.as_ref(), + None, + ) + .context("withdrawing the failed candidate projection"), + CommandKind::Continuity => topology + .demote_to_expected_only( + expected, + provider_anchor, + required(&transaction.activation, "failed continuity activation")?, + None, + ) + .map(drop) + .context("demoting the failed continuity activation"), + } +} + +fn may_rollback_route_after_failed_proof(fencing: &FencingEvidence) -> bool { + matches!(fencing, FencingEvidence::SkippedStateless) +} + +fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { + matches!( + fencing, + FencingEvidence::Revoked { + incumbent_lease_sha256: Some(_), + .. + } + ) +} + +fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { + PreFencingAbort { + error: truncate(error, 2048), + candidate_cleanup: transaction.candidate_cleanup_owed(), + topology_reconciliation: transaction.abort_topology_reconciliation(), + source_cleanup: if transaction.command_kind == CommandKind::Deploy { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + } +} + +fn post_fencing_abort_intent( + transaction: &DeploymentTransaction, + error: &str, +) -> PostFencingAbort { + PostFencingAbort { + error: truncate(error, 2048), + route_restoration: if transaction.route_preflight.is_some() { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + lease_withdrawal: if transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + .is_some() + { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + candidate_cleanup: transaction.candidate_cleanup_owed(), + topology_reconciliation: transaction.abort_topology_reconciliation(), + source_cleanup: if transaction.command_kind == CommandKind::Deploy { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + } +} + +fn sequence_requires_admission( + latest: Option<&TopologyEvidence>, + maximum_admitted_sequence: u64, + candidate: &TopologyEvidence, +) -> Result { + candidate.validate_shape()?; + if latest.is_some_and(|existing| { + existing.canonical_bytes == candidate.canonical_bytes + && existing.canonical_sha256 == candidate.canonical_sha256 + && existing.signer_identity_id == candidate.signer_identity_id + && existing.publisher_sequence == candidate.publisher_sequence + }) { + return Ok(false); + } + ensure!( + candidate.publisher_sequence > maximum_admitted_sequence, + "Odin topology publisher sequence was replayed or reordered" + ); + Ok(true) +} + +fn validate_live_providers_for_deploy(command_kind: CommandKind, validation: F) -> Result<()> +where + F: FnOnce() -> Result<()>, +{ + match command_kind { + CommandKind::Deploy => validation(), + CommandKind::Continuity => Ok(()), + } +} + +fn validate_authenticated_evidence( + evidence: &TopologyEvidence, + authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, +) -> Result<()> { + let record = authenticated.record(); + ensure!( + authenticated.canonical_bytes() == evidence.canonical_bytes.as_slice() + && sha256_id(authenticated.canonical_bytes()) == evidence.canonical_sha256 + && record.signer_identity_id == evidence.signer_identity_id + && record.publisher_sequence == evidence.publisher_sequence, + "durable topology evidence differs from its authenticated receipt" + ); + Ok(()) +} + +fn is_semantic_warming( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + incumbent_lease_sha256: Option<&str>, + authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, +) -> Result { + let record = authenticated.record(); + if !record.present || record.observed_write_lease_sha256.is_some() { + return Ok(false); + } + let expected_projection_detail = format!( + "expected:{};activation:{}", + expected.canonical_sha256()?, + activation.canonical_sha256()? + ); + if !warming_disagreements_match_incumbent( + &expected_projection_detail, + incumbent_lease_sha256, + &record.disagreements, + ) { + return Ok(false); + } + Ok(if expected.write_lease_required { + !record.ready && record.observed_presence_state.as_deref() == Some("warming") + } else { + matches!( + record.observed_presence_state.as_deref(), + Some("warming" | "active") + ) + }) +} + +fn warming_disagreements_match_incumbent( + expected_projection_detail: &str, + incumbent_lease_sha256: Option<&str>, + disagreements: &[OdinTopologyDisagreement], +) -> bool { + if disagreements.is_empty() { + true + } else if let (Some(incumbent_lease_sha256), [disagreement]) = + (incumbent_lease_sha256, disagreements) + { + disagreement.code == "projected-write-lease" + && disagreement.expected.as_deref() == Some(expected_projection_detail) + && disagreement.observed.as_deref() == Some(incumbent_lease_sha256) + } else { + false + } +} + +fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool { + let record = authenticated.record(); + record.present + && record.ready + && record.observed_presence_state.as_deref() == Some("active") + && record.disagreements.is_empty() +} + +fn read_deployment_brake(path: &Path) -> Result)>> { + let Some(envelope) = read_single_envelope(path)? else { + return Ok(None); + }; + ensure!( + envelope.r#type == IdunnDeploymentBrakeRecord::TYPE + && envelope.schema_id.as_deref() == Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA), + "deployment brake store contains a foreign record" + ); + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&envelope.payload)?; + record.validate()?; + ensure!( + rmp_serde::to_vec(&record)? == envelope.payload && envelope.key == record.brake_id, + "deployment brake is noncanonical or keyed by another authority" + ); + Ok(Some((record, envelope.payload))) +} + +fn read_lifecycle_brake(path: &Path) -> Result> { + let Some(envelope) = read_single_envelope(path)? else { + return Ok(None); + }; + ensure!( + envelope.r#type == IdunnLifecycleBrakeRecord::TYPE + && envelope.schema_id.as_deref() == Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA), + "lifecycle brake store contains a foreign record" + ); + IdunnLifecycleBrakeRecord::decode_canonical(&envelope.payload).map(Some) +} + +fn read_single_envelope(path: &Path) -> Result> { + if !path.exists() { + return Ok(None); + } + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + match entries.as_slice() { + [] => Ok(None), + [entry] => Ok(Some(entry.clone())), + _ => bail!("single-record authority store is ambiguous"), + } +} + +pub(crate) fn read_trust_anchor( + path: &Path, +) -> Result { + let envelope = read_single_envelope(path)? + .with_context(|| format!("service identity trust anchor {} is absent", path.display()))?; + ensure!( + envelope.r#type == P::TRUST_ANCHOR_TYPE + && envelope.key == P::TRUST_ANCHOR_KEY + && envelope.schema_id.as_deref() == Some(P::TRUST_ANCHOR_SCHEMA), + "service identity trust anchor belongs to another profile" + ); + let anchor: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&anchor)? == envelope.payload + && derive_service_identity_id::

(&anchor.public_key)? == anchor.identity_id, + "service identity trust anchor is noncanonical or self-inconsistent" + ); + Ok(anchor) +} + +fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { + value.as_ref().with_context(|| format!("{label} is absent")) +} + +fn now_millis() -> Result { + Ok(SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock predates Unix epoch")? + .as_millis() + .try_into()?) +} + +fn rfc3339_millis(millis: u64) -> Result { + chrono::DateTime::from_timestamp_millis(i64::try_from(millis)?) + .context("timestamp is out of range") + .map(|value| value.to_rfc3339()) +} + +fn sibling_path(path: &Path, suffix: &str) -> PathBuf { + let mut value = path.as_os_str().to_os_string(); + value.push(suffix); + PathBuf::from(value) +} + +fn string_value(args: &mut impl Iterator, name: &str) -> Result { + args.next() + .ok_or_else(|| anyhow!("{name} requires a value")) +} + +fn path_value(args: &mut impl Iterator, name: &str) -> Result { + Ok(PathBuf::from(string_value(args, name)?)) +} + +fn u32_value(args: &mut impl Iterator, name: &str) -> Result { + string_value(args, name)? + .parse() + .with_context(|| format!("{name} requires a u32")) +} + +fn u64_value(args: &mut impl Iterator, name: &str) -> Result { + string_value(args, name)? + .parse() + .with_context(|| format!("{name} requires a u64")) +} + +fn require_selector(value: &str) -> Result<()> { + let value = value.strip_prefix("profile:").unwrap_or(value); + require_id(value, "deployment selector") +} + +pub(crate) fn require_id(value: &str, label: &str) -> Result<()> { + ensure!( + !value.is_empty() + && value.len() <= 256 + && value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/') + }), + "{label} is invalid" + ); + Ok(()) +} + +fn require_value(value: &str, label: &str) -> Result<()> { + ensure!( + !value.trim().is_empty() + && value == value.trim() + && value.len() <= 1024 + && !value.contains('\0'), + "{label} is invalid" + ); + Ok(()) +} + +fn require_detail(value: &str, label: &str) -> Result<()> { + ensure!( + !value.is_empty() && value.len() <= 2048 && !value.contains('\0'), + "{label} is invalid" + ); + Ok(()) +} + +fn truncate(value: &str, length: usize) -> String { + value.chars().take(length).collect() +} + +fn sha256_id(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut encoded = String::with_capacity(71); + encoded.push_str("sha256-"); + for byte in digest { + use std::fmt::Write as _; + write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); + } + encoded +} + +fn runtime_instance_id(transaction_id: &str) -> Result { + require_id(transaction_id, "runtime transaction id")?; + Ok(sha256_id( + format!("gamecult.idunn.runtime-instance.v1:{transaction_id}").as_bytes(), + )) +} + +fn usage() -> &'static str { + "Idunn deployment, admission, and continuity control plane\n\n\ + idunn serve [runtime options] [--host-actuator-bind ADDR]\n\ + idunn up [--state-store PATH] [--no-wait]\n\ + idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ + idunn cancel [--state-store PATH]\n\ + idunn expire [--state-store PATH]\n\ + idunn validate --recipe PATH [--binding PATH]\n\n\ + Recipes describe capability and process requirements. Idunn seals exact\n\ + source and artifacts, admits one incarnation, and delegates execution to\n\ + systemd and routing mechanics to the configured proxy." +} + +#[cfg(test)] +mod tests { + use cultnet_rs::{ + GameCultProviderHealthIdentity, OdinRuntimeTopologyCorrelationPurpose, + OdinRuntimeTopologyCorrelationRecord, enroll_service_identity_at, + export_service_identity_trust_anchor, + }; + use tempfile::TempDir; + + use super::*; + use crate::drivers::{SystemdWorkloadObservation, incarnation_key}; + use crate::incident::INCIDENT_RETENTION_MILLIS; + + /// Fixed clock for the signing fixture; correlations must land inside + /// DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS of it to authenticate. + const NOW: u64 = 1_700_000_000_000; + + fn digest(byte: char) -> String { + byte.to_string().repeat(64) + } + + fn route_binding( + route_id: &str, + stable_port: u16, + private_port_start: u16, + private_port_end: u16, + ) -> RouteBinding { + RouteBinding { + driver: crate::deployment::RouteDriver::NginxStreamTcp, + route_id: route_id.into(), + stable_endpoint: format!("tcp://127.0.0.1:{stable_port}"), + private_host: "127.0.0.1".into(), + private_port_start, + private_port_end, + config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), + reload_unit: "nginx.service".into(), + } + } + + fn command(kind: CommandKind) -> DeploymentCommand { + DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: match kind { + CommandKind::Deploy => "up-test".into(), + CommandKind::Continuity => "continuity-test".into(), + }, + kind, + selector: "ghostlight".into(), + requested_by: "operator".into(), + requested_at_unix_millis: 100, + } + } + + /// Drives the replacement gate against the real `control.cc`. The point is + /// not that the unit tests pass but that the record actually stuck on the + /// host would now commit. Run with: + /// IDUNN_LIVE_CONTROL=/path/to/control.cc cargo test live_committing -- --ignored --nocapture + #[test] + #[ignore] + fn live_committing_record_is_unstuck_by_the_replacement_gate() -> Result<()> { + let Ok(path) = std::env::var("IDUNN_LIVE_CONTROL") else { + eprintln!("IDUNN_LIVE_CONTROL unset"); + return Ok(()); + }; + let snapshot = ControlSnapshot::read(Path::new(&path))?; + for stored in &snapshot.transactions { + let value = &stored.value; + if value.completion.is_some() { + continue; + } + let (Some(ready), Some(latest)) = ( + value.ready.as_ref().and_then(ReadinessEvidence::odin), + &value.latest_odin_observation, + ) else { + continue; + }; + println!( + "LIVE {} target={} phase={:?} ready_seq={} latest_seq={}", + value.transaction_id, + value.target, + value.phase, + ready.publisher_sequence, + latest.publisher_sequence + ); + println!( + " old gate (receipt == cursor): {}", + if ready == latest { "PASS" } else { "REFUSE" } + ); + let (latest_record, _) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + &latest.canonical_bytes, + )?; + println!( + " cursor says: projection={} instance={:?} ready={} present={} disagreements={}", + latest_record.expected_projection_sha256, + latest_record.runtime_instance_id, + latest_record.ready, + latest_record.present, + latest_record.disagreements.len() + ); + let (Some(expected), Some(activation)) = (&value.expected, &value.activation) else { + println!(" transaction has not declared an incarnation yet"); + continue; + }; + println!( + " declared: projection={} instance={}", + expected.canonical_sha256()?, + activation.runtime_instance_id + ); + let binds = latest_record.expected_projection_sha256 == expected.canonical_sha256()? + && latest_record.runtime_instance_id.as_deref() + == Some(activation.runtime_instance_id.as_str()); + println!(" binds declared incarnation (what authentication enforces): {binds}"); + } + Ok(()) + } + + /// The smallest real `Engine` the scheduler needs. Idunn had no + /// Engine-level test at all, which is why "does one bad record kill the + /// daemon" was arguable rather than measurable. + struct EngineFixture { + _temp: TempDir, + engine: Engine, + state_store: PathBuf, + odin_signer: ServiceIdentitySigner, + root: PathBuf, + /// The host programs every route driver of this Engine actuates + /// through: logged stubs, so nothing touches the workstation's nginx. + #[cfg(unix)] + route_stubs: RouteStubs, + } + + /// Stub nginx, systemd-run, systemctl and ufw that append each call to one + /// log. `reload-fails` makes `systemctl reload` exit non-zero. + #[cfg(unix)] + struct RouteStubs { + calls: PathBuf, + reload_fails: PathBuf, + ufw_delete_fails: PathBuf, + } + + #[cfg(unix)] + impl RouteStubs { + fn new(root: &Path) -> Result<(RouteActuators, Self)> { + use std::os::unix::fs::PermissionsExt; + + let dir = root.join("route-stubs"); + std::fs::create_dir_all(&dir)?; + let calls = dir.join("calls"); + let reload_fails = dir.join("reload-fails"); + let ufw_delete_fails = dir.join("ufw-delete-fails"); + let program = |name: &str, refuse: &str| -> Result { + let path = dir.join(name); + std::fs::write( + &path, + format!( + "#!/bin/sh\necho \"{name} $*\" >> '{}'\n{refuse}exit 0\n", + calls.display() + ), + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))?; + Ok(path) + }; + let actuators = RouteActuators { + nginx: program("nginx", "")?, + systemd_run: program("systemd-run", "")?, + systemctl: program( + "systemctl", + &format!( + "if [ \"$1\" = reload ] && [ -e '{}' ]; then exit 1; fi\n", + reload_fails.display() + ), + )?, + ufw: program( + "ufw", + &format!( + "if [ \"$1\" = delete ] && [ -e '{}' ]; then exit 1; fi\n", + ufw_delete_fails.display() + ), + )?, + preflight_root: root.join("route-preflight"), + }; + Ok(( + actuators, + Self { + calls, + reload_fails, + ufw_delete_fails, + }, + )) + } + + fn refuse_reloads(&self) -> Result<()> { + std::fs::write(&self.reload_fails, b"x")?; + Ok(()) + } + + /// `ufw delete` exits non-zero with an error that is not "non-existent". + fn refuse_ufw_deletes(&self) -> Result<()> { + std::fs::write(&self.ufw_delete_fails, b"x")?; + Ok(()) + } + + /// How many logged calls begin with `prefix`, e.g. `systemctl reload`. + fn count(&self, prefix: &str) -> usize { + std::fs::read_to_string(&self.calls) + .unwrap_or_default() + .lines() + .filter(|line| line.starts_with(prefix)) + .count() + } + } + + impl EngineFixture { + fn new() -> Result { + Self::build(None) + } + + /// An Engine whose systemd workload port is `workload`, so the phase + /// machine can run without systemd. This is the whole seam: nginx, + /// systemctl and the network are never reached on an unrouted, + /// stateless target, and the Odin evidence is signed by the key this + /// fixture enrolled. + fn with_workload(workload: Arc) -> Result { + Self::build(Some(workload)) + } + + fn build(workload: Option>) -> Result { + let temp = TempDir::new()?; + let root = temp.path(); + std::fs::create_dir_all(root.join("identities"))?; + let idunn = root.join("identities/idunn.cc"); + enroll_service_identity_at::(&idunn)?; + let odin_private = root.join("identities/odin.cc"); + let odin_signer = enroll_service_identity_at::(&odin_private)?; + let odin_anchor = root.join("identities/odin-anchor.cc"); + export_service_identity_trust_anchor(&odin_signer, &odin_anchor)?; + + let state_store = root.join("control.cc"); + #[cfg(unix)] + let (route_actuators, route_stubs) = RouteStubs::new(root)?; + let options = RuntimeOptions { + state_store: state_store.clone(), + bindings_dir: root.join("bindings"), + source_root: root.join("sources"), + staging_root: root.join("staging"), + topology_store: root.join("topology.cc"), + incident_store: root.join("incidents.cc"), + odin_correlation_store: root.join("odin-correlation.cc"), + odin_trust_anchor: odin_anchor, + idunn_identity_store: idunn, + deployment_brake_operator_anchor: root.join("brake-anchor.cc"), + #[cfg(unix)] + route_actuators, + ..RuntimeOptions::default() + }; + let engine = match workload { + Some(workload) => Engine::open_with_systemd_workload(options, workload)?, + None => Engine::open(options)?, + }; + Ok(Self { + root: root.to_path_buf(), + _temp: temp, + engine, + state_store, + odin_signer, + #[cfg(unix)] + route_stubs, + }) + } + } + + fn terminal_transaction(target: &str) -> Result { + Ok(terminal_transaction_with_command(target)?.0) + } + + fn terminal_transaction_with_command( + target: &str, + ) -> Result<(DeploymentTransaction, DeploymentCommand)> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: format!("up-{target}"), + kind: CommandKind::Deploy, + selector: target.into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, target.into(), 0, None, 100)?; + transaction.phase = DeploymentPhase::Complete; + transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { + error: "sealed source vanished".into(), + }); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "sealed source vanished".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + assert!(transaction.is_terminal(), "fixture must be terminal"); + Ok((transaction, command)) + } + + #[test] + fn a_finished_transaction_leaves_the_live_set_and_stays_observable() -> Result<()> { + let world = EngineFixture::new()?; + let history = history_store_path(&world.state_store); + + let (live, command) = terminal_transaction_with_command("ghostlight")?; + // A transaction is only readable beside its immutable command. + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope( + &command, + command.requested_at_unix_millis + )?], + )? + ); + let mut opening = live.clone(); + opening.phase = DeploymentPhase::Sealing; + opening.completion = None; + opening.pre_fencing_abort = None; + let opening_envelope = transaction_envelope(&opening, opening.updated_at_unix_millis)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: opening.transaction_id.clone(), + current: None, + }], + std::slice::from_ref(&opening_envelope), + )? + ); + let stored = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("seeded transaction")?; + + replace_transaction(&world.state_store, &stored, &live)?; + + // Gone from the set that still gates decisions... + assert!( + ControlSnapshot::read(&world.state_store)? + .transactions + .is_empty(), + "a finished transaction stayed resident" + ); + // ...and still answerable, which is what R11 refused to trade away. + let archived = read_history_transactions(&world.state_store); + assert_eq!(archived.len(), 1); + assert_eq!(archived[0].transaction_id, live.transaction_id); + assert!(history.exists()); + // The command went with it. Left resident it would read as queued and + // be frozen again next tick; consumed, it is history beside its + // transaction and `status` still answers for it. + assert!( + ControlSnapshot::read(&world.state_store)? + .commands + .is_empty(), + "a consumed command stayed resident" + ); + let archived_commands = read_history_commands(&world.state_store); + assert_eq!(archived_commands.len(), 1); + assert_eq!(archived_commands[0].command_id, command.command_id); + assert!(!world.engine.freeze_one_queued_command()?); + Ok(()) + } + + #[test] + fn a_resident_command_whose_transactions_are_history_is_retired_not_frozen() -> Result<()> { + // Before commands travelled to history with their transactions, a + // command outlived its retired transaction in the control store and + // read as queued again. Freeze must retire it, never re-freeze it. + let world = EngineFixture::new()?; + let (finished, command) = terminal_transaction_with_command("ghostlight")?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + command_envelope(&command, command.requested_at_unix_millis)?, + )?; + SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) + .insert_entry_if_absent(transaction_envelope( + &finished, + finished.updated_at_unix_millis, + )?)?; + assert_eq!(ControlSnapshot::read(&world.state_store)?.commands.len(), 1); + + assert!(world.engine.freeze_one_queued_command()?); + + assert!( + ControlSnapshot::read(&world.state_store)? + .commands + .is_empty(), + "consumed command stayed resident" + ); + assert_eq!(read_history_commands(&world.state_store).len(), 1); + assert!( + ControlSnapshot::read(&world.state_store)? + .transactions + .is_empty(), + "a consumed command was frozen again" + ); + assert!(!world.engine.freeze_one_queued_command()?); + Ok(()) + } + + #[test] + fn a_resident_terminal_transaction_is_retired_by_the_tick_with_its_command() -> Result<()> { + let world = EngineFixture::new()?; + let (finished, command) = terminal_transaction_with_command("ghostlight")?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + command_envelope(&command, command.requested_at_unix_millis)?, + )?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + transaction_envelope(&finished, finished.updated_at_unix_millis)?, + )?; + assert!(world.engine.retire_one_terminal_transaction()?); + let live = ControlSnapshot::read(&world.state_store)?; + assert!(live.transactions.is_empty() && live.commands.is_empty()); + assert_eq!(read_history_transactions(&world.state_store).len(), 1); + assert_eq!(read_history_commands(&world.state_store).len(), 1); + assert!(!world.engine.retire_one_terminal_transaction()?); + Ok(()) + } + + #[test] + fn re_archiving_after_a_crash_window_is_a_no_op() -> Result<()> { + // History is written before the live copy is removed, so a crash + // between the two leaves the record in both. The repeat must not + // duplicate it or fail. + let world = EngineFixture::new()?; + let live = terminal_transaction("odin")?; + let envelope = transaction_envelope(&live, live.updated_at_unix_millis)?; + + archive_terminal_transaction(&world.state_store, &envelope)?; + archive_terminal_transaction(&world.state_store, &envelope)?; + + let archived = read_history_transactions(&world.state_store); + assert_eq!(archived.len(), 1, "the repeat duplicated the record"); + Ok(()) + } + + #[test] + fn an_unreadable_history_entry_is_skipped_rather_than_fatal() -> Result<()> { + // The asymmetry that justifies two files: control.cc refuses a record + // it cannot re-encode byte for byte, because those records still gate + // decisions. History describes what already happened and must never be + // able to refuse a read. + let world = EngineFixture::new()?; + let live = terminal_transaction("voidbot")?; + let envelope = transaction_envelope(&live, live.updated_at_unix_millis)?; + archive_terminal_transaction(&world.state_store, &envelope)?; + + let history = history_store_path(&world.state_store); + assert!( + SingleFileMessagePackBackingStore::new(&history).insert_entry_if_absent( + CultCacheEnvelope { + key: "tx-corrupt".into(), + r#type: DeploymentTransaction::TYPE.into(), + payload: vec![0xc1], + stored_at: rfc3339_millis(1)?, + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), + } + )? + ); + + let archived = read_history_transactions(&world.state_store); + assert_eq!(archived.len(), 1, "the readable record must survive"); + assert_eq!(archived[0].transaction_id, live.transaction_id); + Ok(()) + } + + #[test] + fn an_absent_brake_anchor_does_not_gate_an_empty_store() -> Result<()> { + // `F:\Projects\CLAUDE.md`: a target brake "may never gate Idunn itself + // or unrelated service lifecycle". Reading the operator anchor up front + // meant a missing brake artifact refused startup against a store with + // nothing in it -- the deployment authority held hostage by an artifact + // describing one target's consent. + let world = EngineFixture::new()?; + assert!( + !world + .engine + .options + .deployment_brake_operator_anchor + .exists(), + "fixture must not have written a brake anchor" + ); + let snapshot = ControlSnapshot::read(&world.state_store)?; + world.engine.validate_durable_authority(&snapshot)?; + Ok(()) + } + + #[test] + fn one_unreadable_record_faults_its_tick_without_touching_the_store() -> Result<()> { + let world = EngineFixture::new()?; + + // An empty store schedules nothing and must not fault. + assert!(!world.engine.run_scheduler_tick()?); + + // A record the control store cannot read. Idunn runs under + // Restart=always, so before run_scheduler_tick existed this + // propagated out of the loop and crashlooped the daemon -- taking + // every unrelated target's continuity with it. + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: "tx-unreadable".into(), + current: None, + }], + &[CultCacheEnvelope { + key: "tx-unreadable".into(), + r#type: DeploymentTransaction::TYPE.into(), + // 0xc1 is msgpack's never-used byte: decodable by nothing. + payload: vec![0xc1], + stored_at: rfc3339_millis(1)?, + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), + }], + )? + ); + let poisoned = std::fs::read(&world.state_store)?; + + for _ in 0..3 { + assert!( + world.engine.run_scheduler_tick().is_err(), + "the tick must report the fault rather than swallow it" + ); + } + + // Faulting is not the same as flailing: a tick that cannot read must + // not rewrite, truncate or repair the store behind the operator. + assert_eq!( + std::fs::read(&world.state_store)?, + poisoned, + "a faulting tick rewrote the control store" + ); + Ok(()) + } + + /// A real signing chain for topology gates. Until this existed no test in + /// this file could construct a signed Odin correlation -- every topology + /// fixture was `vec![byte]` -- so every gate between Idunn and Odin was + /// asserted against bytes that could not be authenticated. That is how a + /// comparison against Idunn's own wall clock survived in the admission + /// path. Modelled on `odin-daemon`'s TestWorld, which already had to build + /// this chain to test the other side of the same contract. + struct TopologyFixture { + _temp: TempDir, + idunn_anchor: ServiceIdentityTrustAnchor, + odin_signer: ServiceIdentitySigner, + provider_public_key: Vec, + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + } + + impl TopologyFixture { + fn new(target: &str) -> Result { + let temp = TempDir::new()?; + let root = temp.path().join("identities"); + std::fs::create_dir_all(&root)?; + let idunn_signer = + enroll_service_identity_at::(&root.join("idunn.cc"))?; + let idunn_anchor = idunn_signer.trust_anchor()?; + let odin_signer = + enroll_service_identity_at::(&root.join("odin.cc"))?; + let provider_signer = enroll_service_identity_at::( + &root.join("provider.cc"), + )?; + + let expected = IdunnExpectedIncarnationRecord { + schema_version: cultnet_rs::IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: target.into(), + plan_id: sha256_id(b"plan"), + incarnation_id: format!("{target}/generation-1"), + sealed_release_id: sha256_id(b"release"), + source_repository: format!("github.com/GameCult/{target}"), + source_revision: "3".repeat(40), + recipe_sha256: sha256_id(b"recipe"), + runtime_id: format!("{target}-runtime"), + expected_signer_identity_id: provider_signer.entry().identity_id.clone(), + health_contract: format!("{target}.runtime-health.v1"), + artifact_sha256: sha256_id(b"artifact"), + state_schema_generation: None, + state_contract_sha256: None, + write_lease_required: false, + route: None, + capabilities: Vec::new(), + dependencies: Vec::new(), + }; + expected.validate()?; + + // Idunn-signed, exactly as the real activation path issues it: the + // authority chain is what makes a correlation authenticable at all. + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + sha256_id(b"artifact-witness"), + NOW - 20, + &idunn_signer, + )?; + let activation = launch.activation().clone(); + + Ok(Self { + _temp: temp, + idunn_anchor, + odin_signer, + provider_public_key: provider_signer.entry().public_key.clone(), + expected, + activation, + }) + } + + fn authority(&self) -> Result { + verify_runtime_authority( + &self.expected, + &self.activation, + &self.idunn_anchor, + &self.provider_public_key, + ) + } + + fn correlation( + &self, + sequence: u64, + ready: bool, + ) -> Result { + Ok(OdinRuntimeTopologyCorrelationRecord { + schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: self.expected.target.clone(), + expected_projection_sha256: self.expected.canonical_sha256()?, + expected: true, + current_activation_sha256: Some(self.activation.canonical_sha256()?), + signed_presence_sha256: Some(sha256_id(b"presence")), + observed_presence_state: Some("active".into()), + observed_presence_publisher_sequence: Some(sequence), + observed_write_lease_sha256: None, + observed_capabilities: Vec::new(), + runtime_id: self.expected.runtime_id.clone(), + runtime_instance_id: Some(self.activation.runtime_instance_id.clone()), + present: true, + ready, + dependencies: Vec::new(), + disagreements: Vec::new(), + signer_identity_id: self.odin_signer.entry().identity_id.clone(), + publisher_sequence: sequence, + observed_at_unix_millis: NOW, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }) + } + + fn sign(&self, record: &mut OdinRuntimeTopologyCorrelationRecord) -> Result> { + record.signature = self + .odin_signer + .sign::( + &record.unsigned_signature_payload()?, + ) + .signature; + record.canonical_bytes() + } + + fn authenticate( + &self, + canonical: &[u8], + ) -> Result { + authenticate_odin_runtime_topology_correlation( + canonical, + &self.authority()?, + None, + &self.odin_signer.entry().public_key, + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: NOW, + maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, + maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, + }, + ) + } + } + + #[test] + fn the_fixture_actually_authenticates() -> Result<()> { + // First: prove the chain can REFUSE. Without this the tests below + // establish nothing -- a fixture that authenticates anything would + // make every gate look satisfied. + let world = TopologyFixture::new("odin")?; + let mut record = world.correlation(297_765, true)?; + let canonical = world.sign(&mut record)?; + world.authenticate(&canonical)?; + + // Re-encoded with the old signature over changed content. + let mut forged = record.clone(); + forged.publisher_sequence = 297_766; + assert!(world.authenticate(&forged.canonical_bytes()?).is_err()); + + let mut unsigned = world.correlation(297_765, true)?; + unsigned.signature = vec![0; 64]; + assert!(world.authenticate(&unsigned.canonical_bytes()?).is_err()); + + // Stale beyond the trusted observation window. + let mut old = world.correlation(297_765, true)?; + old.observed_at_unix_millis = NOW - DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS - 1; + let old_canonical = world.sign(&mut old)?; + assert!(world.authenticate(&old_canonical).is_err()); + Ok(()) + } + + #[test] + fn a_signed_observation_far_past_the_receipt_admits_this_incarnation() -> Result<()> { + // The yggdrasil wedge, with a real signature rather than vec![byte]: + // the cursor has run hundreds of thousands of sequences past the Ready + // receipt and still describes the incarnation the transaction declared. + let world = TopologyFixture::new("odin")?; + let mut record = world.correlation(501_713, true)?; + let canonical = world.sign(&mut record)?; + let authenticated = world.authenticate(&canonical)?; + + // Authenticating at all is the incarnation check: the authority is + // built from the declared expected and activation, and the correlation + // had to bind both to get here. Nothing further to compare. + assert!(is_semantic_ready(&authenticated)); + assert_eq!(authenticated.record().publisher_sequence, 501_713); + Ok(()) + } + + #[test] + fn authentication_binds_a_correlation_to_the_declared_incarnation() -> Result<()> { + // The failure that matters: Odin's signature is valid and the record is + // fresh, but the process it describes is not the one we activated. + // Authentication already refuses this -- the authority is built from + // this transaction's own expected and activation, and the correlation + // must bind both. Pinned here because a redundant identity gate sat on + // top of this for want of a test that could reach it. + let world = TopologyFixture::new("odin")?; + let mut restarted = world.correlation(501_713, true)?; + restarted.runtime_instance_id = Some(sha256_id(b"another-instance")); + let canonical = world.sign(&mut restarted)?; + let error = world + .authenticate(&canonical) + .expect_err("correlation must bind the current activation"); + assert!( + format!("{error:#}").contains("does not bind the current activation"), + "{error:#}" + ); + + // Same for a correlation about another Expected projection: the + // authority carries this transaction's declaration, so a correlation + // that is not about it cannot authenticate against it. + let mut reprojected = world.correlation(501_714, true)?; + reprojected.expected_projection_sha256 = sha256_id(b"another-projection"); + let canonical = world.sign(&mut reprojected)?; + assert!(world.authenticate(&canonical).is_err()); + Ok(()) + } + + #[test] + fn a_signed_observation_that_is_not_ready_does_not_admit() -> Result<()> { + let world = TopologyFixture::new("odin")?; + let mut degraded = world.correlation(501_713, false)?; + let canonical = world.sign(&mut degraded)?; + assert!(!is_semantic_ready(&world.authenticate(&canonical)?)); + + // Ready alongside a disagreement is not merely "not ready" -- the + // correlation contract refuses to encode it at all, so a publisher + // cannot assert readiness over its own dissent. Worth pinning: it is + // why is_semantic_ready's disagreement clause is belt to the schema's + // braces rather than the only thing standing there. + let mut disputed = world.correlation(501_714, true)?; + disputed.disagreements.push(OdinTopologyDisagreement { + code: "route-membership-differs".into(), + expected: Some("a".into()), + observed: Some("b".into()), + }); + assert!(world.sign(&mut disputed).is_err()); + Ok(()) + } + + // --------------------------------------------------------------------- + // Admission audit (soul/admission-audit). Each test below is a negative + // check on one claim made by c09993c..8448f83; none asserts that the + // happy path works, the tests above already do that. + // --------------------------------------------------------------------- + + impl TopologyFixture { + /// Authenticate with an explicit trusted time, against this fixture's + /// own authority and Odin key. Mirrors `rehydrate_ready_token`'s two + /// callers: `now` when require_current, `admitted_at` otherwise. + fn authenticate_at( + &self, + canonical: &[u8], + trusted_received_at_unix_millis: u64, + ) -> Result { + authenticate_odin_runtime_topology_correlation( + canonical, + &self.authority()?, + None, + &self.odin_signer.entry().public_key, + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis, + maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, + maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, + }, + ) + } + } + + /// Claim 1: deleting `observation_describes_declared_incarnation` opened + /// nothing because authentication binds the same identity. The strongest + /// attack is a correlation that is internally perfect for ANOTHER + /// incarnation of the same target -- its own projection, its own activation + /// digest, its own instance id -- signed by OUR admitted Odin, so the only + /// thing left to refuse it is the incarnation binding itself. + #[test] + fn a_perfect_correlation_about_another_incarnation_of_this_target_is_refused() -> Result<()> { + let ours = TopologyFixture::new("odin")?; + let theirs = TopologyFixture::new("odin")?; + assert_ne!( + ours.expected.canonical_sha256()?, + theirs.expected.canonical_sha256()?, + "fixtures must declare distinct incarnations for this test to mean anything" + ); + // Sanity: it is a real, Ready correlation for the other incarnation. + let mut genuine = theirs.correlation(501_713, true)?; + let genuine = theirs.sign(&mut genuine)?; + assert!(is_semantic_ready(&theirs.authenticate(&genuine)?)); + + let mut record = theirs.correlation(501_713, true)?; + let canonical = ours.sign(&mut record)?; + let error = ours + .authenticate(&canonical) + .expect_err("another incarnation's correlation must not authenticate against ours"); + assert!( + format!("{error:#}").contains("substitutes or omits Expected authority"), + "{error:#}" + ); + Ok(()) + } + + /// Claim 1, second face: same Expected, different activation. This is the + /// case the deleted gate could not see at all -- it compared instance ids + /// only -- and the one that matters when a process is re-activated under + /// the same projection. Authentication compares the activation digest. + #[test] + fn a_correlation_bound_to_another_activation_of_the_same_expected_is_refused() -> Result<()> { + let world = TopologyFixture::new("odin")?; + let other = TopologyFixture::new("odin")?; + + let mut reactivated = world.correlation(501_713, true)?; + reactivated.current_activation_sha256 = Some(other.activation.canonical_sha256()?); + let canonical = world.sign(&mut reactivated)?; + let error = world + .authenticate(&canonical) + .expect_err("activation digest must bind"); + assert!( + format!("{error:#}").contains("does not bind the current activation"), + "{error:#}" + ); + + // A Present record with no activation cannot even be encoded: the + // schema refuses it upstream of authentication ("Present topology + // state lacks an authenticated runtime session"). Either refusal is + // the right answer; what must not happen is an absent field reading as + // a match for ours. + let mut anonymous = world.correlation(501_714, true)?; + anonymous.current_activation_sha256 = None; + let error = world + .sign(&mut anonymous) + .and_then(|canonical| world.authenticate(&canonical)) + .expect_err("absent activation must not read as ours"); + assert!( + format!("{error:#}").contains("lacks an authenticated runtime session"), + "{error:#}" + ); + + let mut nameless = world.correlation(501_715, true)?; + nameless.runtime_instance_id = None; + let error = world + .sign(&mut nameless) + .and_then(|canonical| world.authenticate(&canonical)) + .expect_err("absent instance id must not read as ours"); + // Likewise refused at encoding: a Present record must carry its + // instance id ("runtime instance identity and observation evidence are + // partial"). + assert!(format!("{error:#}").contains("are partial"), "{error:#}"); + Ok(()) + } + + /// Claim 1, third face: the deleted gate also compared `target`. The + /// projection digest already covers the target, but a record can carry a + /// foreign `target` field beside our projection digest; prove + /// `validate_against_expected` refuses that on its own. + #[test] + fn a_correlation_naming_another_target_beside_our_projection_is_refused() -> Result<()> { + let world = TopologyFixture::new("odin")?; + let mut stranger = world.correlation(501_713, true)?; + stranger.target = "ghostlight".into(); + let canonical = world.sign(&mut stranger)?; + let error = world + .authenticate(&canonical) + .expect_err("target must bind"); + assert!( + format!("{error:#}").contains("substitutes or omits Expected authority"), + "{error:#}" + ); + Ok(()) + } + + /// Claim 4: the fixture's chain can refuse at every link, not only at the + /// correlation signature. If any of these passed, the tests above would be + /// asserting against a rubber stamp. + #[test] + fn the_fixture_chain_refuses_at_every_link() -> Result<()> { + let ours = TopologyFixture::new("odin")?; + let theirs = TopologyFixture::new("odin")?; + + // Authority link: activation must bind this Expected ... + assert!( + verify_runtime_authority( + &ours.expected, + &theirs.activation, + &ours.idunn_anchor, + &ours.provider_public_key + ) + .is_err(), + "another incarnation's activation verified against our Expected" + ); + // ... be signed by the Idunn we trust ... + assert!( + verify_runtime_authority( + &ours.expected, + &ours.activation, + &theirs.idunn_anchor, + &ours.provider_public_key + ) + .is_err(), + "activation verified against a foreign Idunn anchor" + ); + // ... and name the provider key the Expected selected. + assert!( + verify_runtime_authority( + &ours.expected, + &ours.activation, + &ours.idunn_anchor, + &theirs.provider_public_key + ) + .is_err(), + "a foreign provider key was accepted as the Expected signer" + ); + + // Odin link: a correlation honestly signed by another Odin identity + // (its own id, its own signature) is refused against our admitted key. + let mut record = ours.correlation(501_713, true)?; + record.signer_identity_id = theirs.odin_signer.entry().identity_id.clone(); + let canonical = theirs.sign(&mut record)?; + let error = ours + .authenticate(&canonical) + .expect_err("foreign Odin signer accepted"); + assert!( + format!("{error:#}").contains("names a different identity"), + "{error:#}" + ); + + // Odin link: our identity id, their signature bytes. + let mut record = ours.correlation(501_713, true)?; + record.signature = theirs + .odin_signer + .sign::(&record.unsigned_signature_payload()?) + .signature; + let error = ours + .authenticate(&record.canonical_bytes()?) + .expect_err("forged signature accepted"); + assert!( + format!("{error:#}").contains("signature verification failed"), + "{error:#}" + ); + Ok(()) + } + + /// Claim 2: `rehydrate_ready_token(.., require_current = false)` replays the + /// receipt's authentication at its own `admitted_at`, so the window it + /// widens is exactly the one the receipt already passed at admission. Pin + /// both halves: at `admitted_at` the receipt authenticates however old it + /// is now; at `now` past the window it does not. An `admitted_at` earlier + /// than the observation is refused by the future-skew bound, so the field + /// cannot be pulled backwards to launder a later observation either. + #[test] + fn a_durable_receipt_replays_at_its_admission_time_and_not_at_now() -> Result<()> { + let world = TopologyFixture::new("odin")?; + let mut record = world.correlation(3_841, true)?; + let canonical = world.sign(&mut record)?; + let admitted_at = NOW + 5; + + world.authenticate_at(&canonical, admitted_at)?; + let much_later = admitted_at + DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS + 1; + assert!( + world.authenticate_at(&canonical, much_later).is_err(), + "require_current would refuse" + ); + world.authenticate_at(&canonical, admitted_at)?; + + let before_observation = NOW - DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS - 1; + assert!( + world + .authenticate_at(&canonical, before_observation) + .is_err(), + "an admitted_at earlier than the observation must not authenticate" + ); + Ok(()) + } + + /// Claim 5 companion: what the live store holds, not only whether the + /// stuck record would commit. Run with: + /// IDUNN_LIVE_CONTROL=/path/to/control.cc cargo test live_store_inventory -- --ignored --nocapture + #[test] + #[ignore] + fn live_store_inventory() -> Result<()> { + let Ok(path) = std::env::var("IDUNN_LIVE_CONTROL") else { + eprintln!("IDUNN_LIVE_CONTROL unset"); + return Ok(()); + }; + let snapshot = ControlSnapshot::read(Path::new(&path))?; + println!( + "INVENTORY commands={} transactions={} admitted={}", + snapshot.commands.len(), + snapshot.transactions.len(), + snapshot.admitted.len() + ); + for stored in &snapshot.transactions { + let value = &stored.value; + println!( + "TX {} target={} kind={:?} phase={:?} complete={} ready_seq={:?} latest_seq={:?} incumbent={:?}", + value.transaction_id, + value.target, + value.command_kind, + value.phase, + value.completion.is_some(), + value + .ready + .as_ref() + .and_then(ReadinessEvidence::odin) + .map(|evidence| evidence.publisher_sequence), + value + .latest_odin_observation + .as_ref() + .map(|evidence| evidence.publisher_sequence), + value.incumbent_generation_id, + ); + } + for stored in &snapshot.admitted { + let value = &stored.value; + println!( + "ADMITTED target={} generation={} tx={} admitted_at={} ready_seq={:?} ready_admitted_at={:?} latest_seq={:?} latest_admitted_at={:?} instance={} {}", + value.target, + value.generation_id, + value.transaction_id, + value.admitted_at_unix_millis, + value.ready.odin().map(|evidence| evidence.publisher_sequence), + value.ready.odin().map(|evidence| evidence.admitted_at_unix_millis), + value + .latest_odin_observation + .as_ref() + .map(|evidence| evidence.publisher_sequence), + value + .latest_odin_observation + .as_ref() + .map(|evidence| evidence.admitted_at_unix_millis), + value.activation.runtime_instance_id, + value.workload.describe(), + ); + } + Ok(()) + } + + fn topology(sequence: u64, byte: u8) -> TopologyEvidence { + let canonical_bytes = vec![byte]; + TopologyEvidence { + canonical_sha256: sha256_id(&canonical_bytes), + canonical_bytes, + signer_identity_id: "odin-signer".into(), + publisher_sequence: sequence, + admitted_at_unix_millis: 100, + } + } + + fn workload(uid: u32, pid_namespace_id: u64, mount_namespace_id: u64) -> WorkloadObservation { + WorkloadObservation::Systemd(SystemdWorkloadObservation { + unit: format!("idunn-{uid}.service"), + unit_description: format!("Idunn test {uid}"), + invocation_id: format!("invocation-{uid}"), + exec_main_start_timestamp_monotonic: 1, + service_type: "exec".into(), + restart_policy: "always".into(), + kill_mode: "control-group".into(), + dynamic_user: true, + systemd_user: format!("u{uid}"), + systemd_group: format!("u{uid}"), + supplementary_groups: String::new(), + capability_bounding_set: String::new(), + ambient_capabilities: String::new(), + private_mounts: true, + private_pids: true, + protect_proc: "invisible".into(), + proc_subset: "pid".into(), + no_new_privileges: true, + umask: "0077".into(), + inaccessible_paths: String::new(), + load_credential: String::new(), + main_pid: uid, + process_start_time: 1, + process_uids: [uid; 4], + process_gids: [uid; 4], + process_groups: vec![uid], + process_cap_inheritable: 0, + process_cap_permitted: 0, + process_cap_effective: 0, + process_cap_bounding: 0, + process_cap_ambient: 0, + process_no_new_privileges: true, + process_namespace_pids: vec![1], + mount_namespace_id, + pid_namespace_id, + executable: PathBuf::from("/opt/test/bin/service"), + executable_device: 1, + executable_inode: 1, + executable_sha256: sha256_id(&[1]), + runtime_instance_id: sha256_id(&uid.to_be_bytes()), + working_directory: PathBuf::from("/opt/test"), + runtime_bundle: PathBuf::from("/run/test"), + command_line_sha256: sha256_id(&[2]), + environment_names: Vec::new(), + environment_contract_sha256: sha256_id(&[3]), + control_group: format!("/system.slice/idunn-{uid}.service"), + credentials_directory: None, + parent_only_file_descriptors: Vec::new(), + activation_signer_identity_id: "activation".into(), + activation_signer_public_key: vec![1; 32], + service_credentials: Vec::new(), + }) + } + + fn write_lease() -> IdunnProcessWriteLeaseRecord { + IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: "ghostlight".into(), + expected_projection_sha256: sha256_id(&[1]), + plan_id: sha256_id(&[2]), + incarnation_id: "incarnation-test".into(), + sealed_release_id: sha256_id(&[3]), + activation_witness_sha256: sha256_id(&[4]), + state_schema_generation: "ghostlight-state-v1".into(), + state_contract_sha256: sha256_id(&[5]), + runtime_id: "ghostlight".into(), + runtime_instance_id: sha256_id(&[6]), + warming_presence_sha256: sha256_id(&[7]), + lease_epoch: 1, + issued_at_unix_millis: 100, + } + } + + #[test] + fn validate_is_offline_and_declarative() { + // It must accept a recipe alone, accept a recipe plus a binding, and + // refuse to be handed anything executable. It opens no store and + // actuates nothing, which is what makes it safe to run on a candidate + // binding before that binding is installed. + let parsed = parse( + ["validate", "--recipe", "/tmp/recipe.toml"] + .into_iter() + .map(str::to_owned), + ) + .unwrap(); + assert!(matches!(parsed, Command::Validate { binding: None, .. })); + + assert!( + parse( + ["validate", "--recipe", "/r.toml", "--binding", "/b.toml"] + .into_iter() + .map(str::to_owned) + ) + .is_ok() + ); + assert!(parse(["validate"].into_iter().map(str::to_owned)).is_err()); + for rejected in [ + vec![ + "validate", + "--recipe", + "/r.toml", + "--deploy-command", + "sh -c bad", + ], + vec!["validate", "--recipe", "/r.toml", "--state-store", "/s.cc"], + ] { + assert!(parse(rejected.into_iter().map(str::to_owned)).is_err()); + } + } + + #[test] + fn cli_exposes_only_declarative_commands() { + for arguments in [ + vec!["up", "ghostlight", "--deploy-command", "sh -c bad"], + vec!["serve", "--swarm-profile", "yggdrasil-local"], + vec!["serve", "--restart-command", "bad"], + vec!["validate-runtime-admission"], + ] { + assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); + } + } + + #[test] + fn garden_path_accepts_service_and_profile_selectors() { + for selector in ["ghostlight", "profile:aetheria", "profile:full-gamecult"] { + let parsed = + parse(["up", selector, "--no-wait"].into_iter().map(str::to_owned)).unwrap(); + let Command::Up { + selector: actual, .. + } = parsed + else { + panic!("expected up command") + }; + assert_eq!(actual, selector); + } + } + + #[test] + fn route_bindings_are_one_global_socket_and_candidate_authority_map() { + let first = route_binding("first", 4103, 14103, 14111); + let second = route_binding("second", 8831, 18831, 18839); + validate_route_binding_set(&[("first", &first), ("second", &second)]).unwrap(); + + let mut duplicate_id = second.clone(); + duplicate_id.route_id = first.route_id.clone(); + assert!( + validate_route_binding_set(&[("first", &first), ("second", &duplicate_id)]).is_err() + ); + + let mut overlapping_candidates = second.clone(); + overlapping_candidates.private_port_start = 14111; + overlapping_candidates.private_port_end = 14120; + assert!( + validate_route_binding_set(&[("first", &first), ("second", &overlapping_candidates),]) + .is_err() + ); + + let mut stable_inside_other_range = second; + stable_inside_other_range.stable_endpoint = "tcp://127.0.0.1:14105".into(); + assert!( + validate_route_binding_set(&[ + ("first", &first), + ("second", &stable_inside_other_range), + ]) + .is_err() + ); + } + + #[test] + fn deployment_command_is_immutable_positional_fact() -> Result<()> { + let value = command(CommandKind::Deploy); + value.validate()?; + let encoded = rmp_serde::to_vec(&value)?; + assert_eq!(encoded[0], 0x96); + assert!(!encoded.windows(7).any(|window| window == b"running")); + assert!(!encoded.windows(5).any(|window| window == b"owner")); + Ok(()) + } + + #[test] + fn phases_are_one_exact_forward_chain() { + let phases = [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Warming, + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + DeploymentPhase::Complete, + ]; + assert!( + phases + .windows(2) + .all(|pair| pair[1] as u8 == pair[0] as u8 + 1) + ); + } + + #[test] + fn only_live_stateless_deployments_can_be_cancelled_after_fencing() { + let command = command(CommandKind::Deploy); + let mut stateless = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100).unwrap(); + stateless.phase = DeploymentPhase::Routing; + stateless.fencing = Some(FencingEvidence::SkippedStateless); + assert!(cancel_is_safe_for_live_stateless(&stateless)); + + let mut stateful = stateless.clone(); + stateful.fencing = Some(FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + }); + assert!(!cancel_is_safe_for_live_stateless(&stateful)); + + let mut unfenced = stateless.clone(); + unfenced.phase = DeploymentPhase::Sealing; + assert!(!cancel_is_safe_for_live_stateless(&unfenced)); + + let mut committed = stateless; + committed.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-test".into(), + }); + assert!(!cancel_is_safe_for_live_stateless(&committed)); + } + + #[test] + fn every_new_odin_sequence_must_advance_and_exact_retry_is_idempotent() -> Result<()> { + let first = topology(7, 1); + assert!(sequence_requires_admission(None, 6, &first)?); + assert!(!sequence_requires_admission( + Some(&first), + 7, + &topology(7, 1) + )?); + assert!(sequence_requires_admission( + Some(&first), + 7, + &topology(8, 2) + )?); + assert!(sequence_requires_admission(Some(&first), 7, &topology(7, 2)).is_err()); + assert!(sequence_requires_admission(Some(&first), 7, &topology(6, 3)).is_err()); + Ok(()) + } + + #[test] + fn odin_sequence_cursors_are_scoped_by_target_and_signer() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut ghostlight = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + ghostlight.latest_odin_observation = Some(topology(7, 1)); + ghostlight.odin_publisher_sequence_cursor = 7; + let mut odin = DeploymentTransaction::new(&command, "odin".into(), 1, None, 100)?; + odin.latest_odin_observation = Some(topology(41, 2)); + odin.odin_publisher_sequence_cursor = 41; + let envelope = |key: &str| CultCacheEnvelope { + key: key.into(), + r#type: DeploymentTransaction::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), + }; + let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), + commands: Vec::new(), + transactions: vec![ + Stored { + envelope: envelope("ghostlight"), + value: ghostlight, + }, + Stored { + envelope: envelope("odin"), + value: odin, + }, + ], + admitted: Vec::new(), + targets: Vec::new(), + }; + + assert_eq!(snapshot.max_odin_sequence("ghostlight", "odin-signer"), 7); + assert_eq!(snapshot.max_odin_sequence("odin", "odin-signer"), 41); + assert_eq!(snapshot.max_odin_sequence("ghostlight", "other-signer"), 0); + + // A transaction that failed was never built upon, so its readings must + // not raise the bar for the next attempt. Counting them is how a target + // that has failed once becomes a target that can never deploy. + let mut abandoned = snapshot; + abandoned.transactions[0].value.completion = + Some(TransactionCompletion::FailedBeforeFencing { + error: "sealed source was rejected".into(), + }); + assert_eq!(abandoned.max_odin_sequence("ghostlight", "odin-signer"), 0); + abandoned.transactions[1].value.completion = + Some(TransactionCompletion::FailedAfterFencing { + error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + assert_eq!(abandoned.max_odin_sequence("odin", "odin-signer"), 0); + Ok(()) + } + + #[test] + fn stale_odin_provider_receipt_cannot_gate_continuity_restart() -> Result<()> { + let touched = std::cell::Cell::new(false); + validate_live_providers_for_deploy(CommandKind::Continuity, || { + touched.set(true); + bail!("Odin provider receipt is stale") + })?; + assert!(!touched.get()); + assert!( + validate_live_providers_for_deploy(CommandKind::Deploy, || { + bail!("Odin provider receipt is stale") + }) + .is_err() + ); + Ok(()) + } + + #[test] + fn prepared_write_lease_is_durable_but_not_admitted_authority() -> Result<()> { + let lease = write_lease(); + lease.validate()?; + let prepared = LeasingEvidence::Prepared { + lease: lease.clone(), + lease_sha256: lease.canonical_sha256()?, + }; + let encoded = rmp_serde::to_vec(&prepared)?; + let decoded: LeasingEvidence = rmp_serde::from_slice(&encoded)?; + assert_eq!(decoded, prepared); + assert!(decoded.lease().is_none()); + assert!(decoded.lease_sha256().is_none()); + assert_eq!(decoded.prepared_lease().unwrap().0, &lease); + Ok(()) + } + + #[test] + fn runtime_instance_identity_is_stable_for_activation_prepare_replay() -> Result<()> { + let first = runtime_instance_id("tx-one")?; + assert_eq!(runtime_instance_id("tx-one")?, first); + assert_ne!(runtime_instance_id("tx-two")?, first); + Ok(()) + } + + #[test] + fn pre_fencing_abort_is_durable_before_cleanup_and_cannot_resume_as_success() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut transaction = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "sealed source was rejected".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Pending, + }); + transaction.validate()?; + assert!(!transaction.is_terminal()); + + transaction + .pre_fencing_abort + .as_mut() + .unwrap() + .source_cleanup = CleanupEvidence::Complete; + transaction.phase = DeploymentPhase::Complete; + transaction.last_error = Some("sealed source was rejected".into()); + transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { + error: "sealed source was rejected".into(), + }); + transaction.validate()?; + assert!(transaction.is_terminal()); + Ok(()) + } + + #[test] + fn post_fencing_abort_is_terminal_only_when_every_cleanup_is_complete() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut transaction = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + transaction.phase = DeploymentPhase::Complete; + transaction.post_fencing_abort = Some(PostFencingAbort { + error: "candidate died after the fence".into(), + route_restoration: CleanupEvidence::Skipped, + lease_withdrawal: CleanupEvidence::Pending, + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Pending, + }); + transaction.last_error = Some("candidate died after the fence".into()); + transaction.completion = Some(TransactionCompletion::FailedAfterFencing { + error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + // The evidence exists so that a target is released only once the + // candidate's route, lease, process and projection are actually gone. + assert!(transaction.validate().is_err()); + assert!(!transaction.is_terminal()); + + let abort = transaction.post_fencing_abort.as_mut().unwrap(); + abort.lease_withdrawal = CleanupEvidence::Complete; + abort.source_cleanup = CleanupEvidence::Complete; + transaction.validate()?; + assert!(transaction.is_terminal()); + + // The recorded error and the completion must name the same failure. + transaction.completion = Some(TransactionCompletion::FailedAfterFencing { + error: "a different story".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + assert!(transaction.validate().is_err()); + Ok(()) + } + + #[test] + fn a_pre_fence_transaction_cannot_carry_a_post_fencing_abort() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut transaction = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + assert!(transaction.phase < DeploymentPhase::Fencing); + transaction.post_fencing_abort = Some(PostFencingAbort { + error: "candidate died after the fence".into(), + route_restoration: CleanupEvidence::Skipped, + lease_withdrawal: CleanupEvidence::Skipped, + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Skipped, + }); + assert!(transaction.validate().is_err()); + Ok(()) + } + #[test] + fn candidate_cleanup_is_owed_when_an_activation_or_a_workload_exists() -> Result<()> { + // (activation issued, workload observed, expected cleanup) + let rows = [ + (false, false, CleanupEvidence::Skipped), + (true, false, CleanupEvidence::Pending), + (false, true, CleanupEvidence::Pending), + (true, true, CleanupEvidence::Pending), + ]; + for (has_activation, has_workload, expected) in rows { + let world = ContinuityProjection::after_candidate_observed()?; + let mut transaction = world.continuity_transaction()?; + if !has_activation { + transaction.activation = None; + } + transaction.workload = has_workload.then(|| workload(1000, 1, 2)); + assert_eq!( + transaction.candidate_cleanup_owed(), + expected, + "activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + pre_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "pre-fencing intent, activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "post-fencing intent, activation {has_activation}, workload {has_workload}" + ); + } + Ok(()) + } + + #[test] + fn a_stored_abort_whose_candidate_cleanup_disagrees_with_the_owed_rule_is_refused() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + // (activation issued, recorded cleanup, accepted). + // Rows with a workload are omitted: a transaction holding one is refused on + // other grounds, so they could not speak for this rule. The workload operand + // of the owed rule is pinned by the method-level test above. + let rows = [ + (false, CleanupEvidence::Skipped, true), + (false, CleanupEvidence::Pending, false), + (false, CleanupEvidence::Complete, false), + (true, CleanupEvidence::Skipped, false), + (true, CleanupEvidence::Pending, true), + (true, CleanupEvidence::Complete, true), + ]; + for (has_activation, recorded, accepted) in rows { + let label = format!("activation {has_activation}, {recorded:?}"); + let shape = |mut transaction: DeploymentTransaction| -> DeploymentTransaction { + transaction.activation = has_activation.then(|| world.candidate_activation.clone()); + transaction + }; + + let mut pre = shape(world.continuity_transaction()?); + pre.pre_fencing_abort = Some(PreFencingAbort { + error: "candidate refused".into(), + candidate_cleanup: recorded, + topology_reconciliation: pre.abort_topology_reconciliation(), + source_cleanup: CleanupEvidence::Skipped, + }); + assert_eq!(pre.validate().is_ok(), accepted, "pre-fencing, {label}"); + + let mut post = shape(DeploymentTransaction::new( + &command(CommandKind::Deploy), + "ghostlight".into(), + 0, + None, + 100, + )?); + post.phase = DeploymentPhase::Complete; + post.post_fencing_abort = Some(PostFencingAbort { + error: "candidate died after the fence".into(), + route_restoration: CleanupEvidence::Skipped, + lease_withdrawal: CleanupEvidence::Skipped, + candidate_cleanup: recorded, + topology_reconciliation: post.abort_topology_reconciliation(), + source_cleanup: CleanupEvidence::Complete, + }); + post.last_error = Some("candidate died after the fence".into()); + post.completion = Some(TransactionCompletion::FailedAfterFencing { + error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + // A Pending cleanup is not terminal, so this completed shape is + // only judged for the recorded values a terminal abort can carry. + if recorded != CleanupEvidence::Pending { + assert_eq!(post.validate().is_ok(), accepted, "post-fencing, {label}"); + } + } + Ok(()) + } + + #[test] + fn bad_selector_becomes_one_terminal_refusal_record() -> Result<()> { + let command = command(CommandKind::Deploy); + let transaction = DeploymentTransaction::rejected( + &command, + anyhow!("deployment selector is unknown"), + 100, + )?; + assert!(transaction.is_terminal()); + assert_eq!(transaction.target, command.selector); + assert!(matches!( + transaction.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + Ok(()) + } + + #[test] + fn post_commit_cleanup_keeps_exact_incumbent_work_owned_until_retired() -> Result<()> { + let incumbent = workload(1001, 40, 50); + let cleanup = PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Pending { + generation_id: "generation-old".into(), + workload: incumbent.clone(), + }, + source: SourceCleanupEvidence::Pending, + }; + let encoded = rmp_serde::to_vec(&cleanup)?; + let mut replayed: PostCommitCleanup = rmp_serde::from_slice(&encoded)?; + assert_eq!(replayed, cleanup); + assert!(!replayed.is_complete()); + replayed.incumbent = IncumbentCleanupEvidence::Complete { + generation_id: "generation-old".into(), + }; + assert!(!replayed.is_complete()); + replayed.source = SourceCleanupEvidence::Complete; + assert!(replayed.is_complete()); + Ok(()) + } + + #[test] + fn routed_incumbent_retirement_waits_for_the_declared_drain_deadline() -> Result<()> { + assert_eq!(route_drain_deadline(1_000, 30)?, 31_000); + assert!(route_drain_deadline(u64::MAX, 1).is_err()); + Ok(()) + } + + #[test] + fn admitted_route_receipt_refreshes_when_stale_or_implausibly_future_dated() { + assert!(route_observation_is_current(900, 1_000, 100, 10)); + assert!(route_observation_is_current(1_010, 1_000, 100, 10)); + assert!(!route_observation_is_current(899, 1_000, 100, 10)); + assert!(!route_observation_is_current(1_011, 1_000, 100, 10)); + } + + #[test] + fn lease_warming_requires_both_new_odin_and_new_provider_evidence() { + let old = digest('1'); + let fresh = digest('2'); + assert!(!provider_warming_advanced(7, &old, 7, &fresh)); + assert!(!provider_warming_advanced(7, &old, 8, &old)); + assert!(provider_warming_advanced(7, &old, 8, &fresh)); + } + + #[test] + fn failed_route_proof_never_restores_any_fenced_incumbent() { + assert!(!may_rollback_route_after_failed_proof( + &FencingEvidence::Revoked { + incumbent_lease_sha256: Some(digest('1')), + candidate_lease_path_verified_empty: false, + } + )); + assert!(!may_rollback_route_after_failed_proof( + &FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + } + )); + assert!(may_rollback_route_after_failed_proof( + &FencingEvidence::SkippedStateless + )); + } + + #[test] + fn fenced_writer_is_already_retired_before_post_commit_cleanup() { + assert!(incumbent_was_stopped_during_fencing( + &FencingEvidence::Revoked { + incumbent_lease_sha256: Some(digest('1')), + candidate_lease_path_verified_empty: false, + } + )); + assert!(!incumbent_was_stopped_during_fencing( + &FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + } + )); + assert!(!incumbent_was_stopped_during_fencing( + &FencingEvidence::SkippedStateless + )); + } + + #[test] + fn warming_accepts_only_the_exact_projected_incumbent_lease_disagreement() { + let expected_detail = format!("expected:{};activation:{}", digest('1'), digest('2')); + let incumbent = digest('3'); + let exact = OdinTopologyDisagreement { + code: "projected-write-lease".into(), + expected: Some(expected_detail.clone()), + observed: Some(incumbent.clone()), + }; + + assert!(warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[exact.clone()], + )); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + None, + &[exact.clone()], + )); + + let mut substituted = exact.clone(); + substituted.observed = Some(digest('4')); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[substituted], + )); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[exact.clone(), exact], + )); + } + + #[test] + fn candidate_and_incumbent_must_differ_in_all_three_native_boundaries() { + let incumbent = workload(1001, 40, 50); + let prove = |candidate: &WorkloadObservation| { + WorkloadObservation::prove_isolation(candidate, Some(&incumbent)) + }; + assert!(prove(&workload(1002, 41, 51)).is_ok()); + assert!(prove(&workload(1001, 41, 51)).is_err()); + assert!(prove(&workload(1002, 40, 51)).is_err()); + assert!(prove(&workload(1002, 41, 50)).is_err()); + } + + #[test] + fn exactly_one_inflight_transaction_may_own_a_target() -> Result<()> { + let command = command(CommandKind::Deploy); + let one = DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + let two = DeploymentTransaction::new(&command, "ghostlight".into(), 1, None, 100)?; + let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), + commands: vec![Stored { + envelope: command_envelope(&command, 100)?, + value: command, + }], + transactions: vec![ + Stored { + envelope: transaction_envelope(&one, 100)?, + value: one, + }, + Stored { + envelope: transaction_envelope(&two, 100)?, + value: two, + }, + ], + admitted: Vec::new(), + targets: Vec::new(), + }; + assert!(snapshot.validate_relations().is_err()); + Ok(()) + } + + #[test] + fn command_ordinals_do_not_turn_one_target_wait_into_a_global_brake() -> Result<()> { + let first_command = command(CommandKind::Deploy); + let first = DeploymentTransaction::new(&first_command, "odin".into(), 0, None, 100)?; + let second = DeploymentTransaction::new(&first_command, "ghostlight".into(), 1, None, 100)?; + let mut unrelated_command = command(CommandKind::Deploy); + unrelated_command.command_id = "up-unrelated".into(); + let unrelated = + DeploymentTransaction::new(&unrelated_command, "huginn".into(), 0, None, 101)?; + let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), + commands: Vec::new(), + transactions: [&first, &second, &unrelated] + .into_iter() + .map(|value| Stored { + envelope: transaction_envelope(value, 100).unwrap(), + value: value.clone(), + }) + .collect(), + admitted: Vec::new(), + targets: Vec::new(), + }; + assert!(!snapshot.has_earlier_authority_sibling(&first)); + assert!(snapshot.has_earlier_authority_sibling(&second)); + assert!(!snapshot.has_earlier_authority_sibling(&unrelated)); + Ok(()) + } + + #[test] + fn complete_cleanup_remains_retryable_without_owning_the_current_incarnation() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut transaction = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + transaction.phase = DeploymentPhase::Complete; + transaction.completion = Some(TransactionCompletion::Admitted { + generation_id: format!("generation-{}", transaction.transaction_id), + }); + transaction.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Pending, + }); + + assert!(!transaction.is_terminal()); + assert!(!transaction.owns_target_authority()); + assert!(transaction.blocks_new_target_mutation()); + Ok(()) + } + + #[test] + fn legacy_mutable_command_schema_is_rejected() -> Result<()> { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let legacy = CultCacheEnvelope { + key: "up-legacy".into(), + r#type: DeploymentCommand::TYPE.into(), + payload: rmp_serde::to_vec(&(DEPLOYMENT_COMMAND_SCHEMA, "up-legacy", "running"))?, + stored_at: rfc3339_millis(100)?, + schema_id: Some("idunn.deployment_command.v1".into()), + }; + assert!( + SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: "up-legacy".into(), + current: None, + }], + &[legacy], + )? + ); + assert!(ControlSnapshot::read(&path).is_err()); + Ok(()) + } + + #[test] + fn loadcredential_era_control_schemas_are_rejected_before_rehydration() -> Result<()> { + for (record_type, key, schema) in [ + ( + DeploymentTransaction::TYPE, + "tx-legacy-workload-observation", + "idunn.deployment_transaction.v1", + ), + ( + AdmittedGeneration::TYPE, + "legacy-admitted-target", + "idunn.admitted_generation.v1", + ), + ] { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let legacy = CultCacheEnvelope { + key: key.into(), + r#type: record_type.into(), + payload: Vec::new(), + stored_at: rfc3339_millis(100)?, + schema_id: Some(schema.into()), + }; + assert!( + SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: record_type.into(), + key: key.into(), + current: None, + }], + &[legacy], + )? + ); + assert!(ControlSnapshot::read(&path).is_err()); + } + Ok(()) + } + + #[test] + fn source_identity_is_explicit_and_atomic() { + let partial = parse( + ["serve", "--source-uid", "1001"] + .into_iter() + .map(str::to_owned), + ) + .unwrap_err() + .to_string(); + assert!(partial.contains("supplied together")); + let Command::Serve(options) = parse( + ["serve", "--source-uid", "1001", "--source-gid", "1002"] + .into_iter() + .map(str::to_owned), + ) + .unwrap() else { + panic!("expected serve") + }; + assert_eq!( + options.source_identity, + Some(ProcessIdentity { + uid: 1001, + gid: 1002 + }) + ); + } + + // ---- v2/v3 control records lifted to the v3 generation / v4 transaction ---- + // + // The bytes under tests/fixtures/idunn-control-legacy were written by the + // encoder at Idunn 9001b58 (see the README there). Nothing below encodes a + // legacy record: the legacy layout is decoded from those bytes, and the + // lifted record is compared with what the decoded legacy record says. + + const FIXTURE_GENERATION: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-with-receipts.hex"); + const FIXTURE_GENERATION_REPAIRING: &str = include_str!( + "../tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex" + ); + const FIXTURE_GENERATION_V3_ODIN: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-odin.hex"); + const FIXTURE_GENERATION_V3_ROUTE_PROOF: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex"); + const FIXTURE_TRANSACTIONS: [(&str, &str); 6] = [ + ( + "fencing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-fencing.hex"), + ), + ( + "leasing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-leasing.hex"), + ), + ( + "awaiting-ready", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex"), + ), + ( + "routing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-routing.hex"), + ), + ( + "committing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing.hex"), + ), + ( + "failed-after-fencing", + include_str!( + "../tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex" + ), + ), + ]; + const FIXTURE_TRANSACTION_V2: &str = + include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex"); + + fn fixture_envelope(text: &str, record_type: &str) -> Result { + let mut lines = text.lines(); + let schema = lines.next().context("fixture has no schema line")?; + let hex = lines.next().context("fixture has no payload line")?.trim(); + let payload = (0..hex.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&hex[index..index + 2], 16)) + .collect::, _>>()?; + let key = if record_type == AdmittedGeneration::TYPE && schema == ADMITTED_GENERATION_SCHEMA_V3 { + rmp_serde::from_slice::(&payload)?.target + } else if record_type == AdmittedGeneration::TYPE { + rmp_serde::from_slice::(&payload)?.target + } else { + rmp_serde::from_slice::(&payload)?.transaction_id + }; + Ok(CultCacheEnvelope { + key, + r#type: record_type.into(), + payload, + stored_at: rfc3339_millis(1_700_000_123_456)?, + schema_id: Some(schema.into()), + }) + } + + fn fixture_generation(text: &str) -> Result<(LegacyAdmittedGeneration, AdmittedGeneration)> { + let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; + let legacy = rmp_serde::from_slice(&envelope.payload)?; + Ok((legacy, read_generation_record(&envelope)?)) + } + + fn fixture_transaction( + text: &str, + ) -> Result<(LegacyDeploymentTransaction, DeploymentTransaction)> { + let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; + let legacy = rmp_serde::from_slice(&envelope.payload)?; + Ok((legacy, read_transaction_record(&envelope)?)) + } + + #[test] + fn a_v2_generation_lifts_to_the_decisions_it_encoded() -> Result<()> { + for text in [FIXTURE_GENERATION, FIXTURE_GENERATION_REPAIRING] { + let (legacy, lifted) = fixture_generation(text)?; + // Every receipt the old decisions read is the same receipt, now + // tagged Odin-correlated. + let odin = lifted.odin_receipts()?; + assert_eq!(odin.ready, &legacy.ready); + assert_eq!(odin.latest, &legacy.latest_odin_observation); + assert_eq!(lifted.odin_authority.as_ref(), Some(&legacy.odin_authority)); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.activation, legacy.activation); + assert_eq!(lifted.workload, legacy.workload); + assert_eq!(lifted.generation_id, legacy.generation_id); + // The new state starts empty and the plan is still a v2 plan. + assert_eq!(lifted.route_supervision, Some(RouteSupervisionState::default())); + assert_eq!(lifted.last_error, None); + assert_eq!(lifted.plan.schema, crate::deployment_plan::COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2); + assert_eq!( + lifted.plan.phase_deadlines(), + crate::deployment_plan::PhaseDeadlines::IDUNN_DEFAULTS + ); + } + Ok(()) + } + + #[test] + fn v2_and_v3_transactions_lift_to_the_decisions_they_encoded() -> Result<()> { + let mut phases = Vec::new(); + for (name, text) in FIXTURE_TRANSACTIONS { + let (legacy, lifted) = fixture_transaction(text)?; + phases.push((name, lifted.phase)); + assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + assert_eq!(lifted.phase, legacy.phase); + assert_eq!(lifted.completion, legacy.completion); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.fencing, legacy.fencing); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.warming, legacy.warming); + assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!( + lifted.ready.as_ref().and_then(ReadinessEvidence::odin), + legacy.ready.as_ref() + ); + assert_eq!(lifted.phase_deadline, None); + assert_eq!(lifted.lease_adoption, None); + assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); + } + assert_eq!( + phases.iter().map(|(_, phase)| *phase).collect::>(), + [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + DeploymentPhase::Complete, + ] + ); + + // The genuine v2 layout lifts to the same record as the v3 one. + let (_, from_v2) = fixture_transaction(FIXTURE_TRANSACTION_V2)?; + let (_, from_v3) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + assert_eq!(from_v2, from_v3); + + // A terminal failure predating recovery names the recovery it always did. + let (_, failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; + assert!(matches!( + failed.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + assert!(failed.is_terminal()); + Ok(()) + } + + #[test] + fn the_boot_migration_rewrites_every_legacy_record_once() -> Result<()> { + // The fixtures were cut from one build, so their records share keys + // and each migrates in a store of its own. + let mut fixtures = vec![ + fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?, + fixture_envelope(FIXTURE_GENERATION_REPAIRING, AdmittedGeneration::TYPE)?, + fixture_envelope(FIXTURE_TRANSACTION_V2, DeploymentTransaction::TYPE)?, + ]; + for (_, text) in FIXTURE_TRANSACTIONS { + fixtures.push(fixture_envelope(text, DeploymentTransaction::TYPE)?); + } + for envelope in fixtures { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + assert!(store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: envelope.r#type.clone(), + key: envelope.key.clone(), + current: None, + }], + std::slice::from_ref(&envelope), + )?); + let debug_of = |envelope: &CultCacheEnvelope| -> Result { + Ok(if envelope.r#type == AdmittedGeneration::TYPE { + format!("{:?}", read_generation_record(envelope)?) + } else { + format!("{:?}", read_transaction_record(envelope)?) + }) + }; + let before = debug_of(&envelope)?; + + assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); + + let after = store.pull_all_read_only_snapshot()?; + assert_eq!(after.len(), 1); + let expected_schema = if envelope.r#type == AdmittedGeneration::TYPE { + ADMITTED_GENERATION_SCHEMA + } else { + DEPLOYMENT_TRANSACTION_SCHEMA + }; + assert_eq!(after[0].schema_id.as_deref(), Some(expected_schema)); + assert_eq!(after[0].stored_at, envelope.stored_at); + // The rewritten record is canonical and means what the legacy one meant. + assert_eq!(debug_of(&after[0])?, before); + } + Ok(()) + } + + #[test] + fn history_still_reads_legacy_transactions() -> Result<()> { + let temporary = tempfile::tempdir()?; + let state_store = temporary.path().join("control.cc"); + let envelope = fixture_envelope(FIXTURE_TRANSACTIONS[5].1, DeploymentTransaction::TYPE)?; + SingleFileMessagePackBackingStore::new(&history_store_path(&state_store)) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: envelope.key.clone(), + current: None, + }], + std::slice::from_ref(&envelope), + )?; + let archived = read_history_transactions(&state_store); + assert_eq!(archived.len(), 1); + assert_eq!(archived[0].transaction_id, envelope.key); + assert_eq!(archived[0].schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + Ok(()) + } + + #[test] + fn readiness_class_follows_the_expected_odin_dependency() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut expected = generation.expected.clone(); + assert!(expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + })); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + for dependency in &mut expected.dependencies { + // Same capability, other kind: only the shared-infrastructure + // declaration makes a target Odin-correlated. + dependency.kind = "required".into(); + } + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + expected.dependencies.clear(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + // Nothing to challenge and no Odin dependency: the recipe has declared + // no way to prove readiness, and Idunn does not pick one. + expected.route = None; + assert_eq!( + ReadinessClass::of(&expected), + Err(UndeclaredReadiness { + target: expected.target.clone() + }) + ); + // Declaring the Odin dependency is the declaration, routed or not: a + // routed target that only publishes to Odin is Odin-correlated. + let odin_dependency = generation.expected.dependencies.clone(); + expected.dependencies = odin_dependency.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + expected.route = generation.expected.route.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + // Providing the rendezvous is being Odin, whatever else is declared. + expected.dependencies.clear(); + expected.route = None; + provide_odin(&mut expected); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); + Ok(()) + } + + #[test] + fn warming_evidence_is_shape_checked_whatever_its_source() { + let good = |evidence: &ReadinessEvidence| match evidence { + ReadinessEvidence::RouteProof { evidence } => evidence.clone(), + ReadinessEvidence::OdinCorrelated { .. } => unreachable!("a route-proof fixture"), + }; + let presence = good(&route_proof_evidence()); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: presence.clone() }, + WarmingEvidence::RouteProofDirect { evidence: presence.clone() }, + WarmingEvidence::OdinTopology { evidence: topology(3, 1) }, + ] { + warming.validate_shape().expect("well-formed evidence"); + } + let mut damaged = presence; + damaged.canonical_bytes.push(1); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: damaged.clone() }, + WarmingEvidence::RouteProofDirect { evidence: damaged }, + WarmingEvidence::OdinTopology { + evidence: TopologyEvidence { + canonical_sha256: "sha256:not-the-digest".into(), + ..topology(3, 1) + }, + }, + ] { + assert!(warming.validate_shape().is_err(), "{warming:?}"); + } + } + + #[test] + fn a_presence_disagreement_names_each_disagreement() { + let shortfall = PresenceDisagrees { + disagreements: vec![ + OdinTopologyDisagreement { + code: "capacity-below-minimum".into(), + expected: Some("2".into()), + observed: Some("1".into()), + }, + OdinTopologyDisagreement { + code: "state".into(), + expected: None, + observed: None, + }, + ], + }; + assert_eq!( + shortfall.to_string(), + "route proof answered with a runtime that disagrees with current authority: \ + capacity-below-minimum (expected 2, observed 1); state (expected none, observed none)" + ); + } + + fn route_proof_evidence() -> ReadinessEvidence { + let canonical_bytes = vec![7, 7, 7]; + ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + canonical_sha256: sha256_id(&canonical_bytes), + canonical_bytes, + message_id: "challenge-1".into(), + challenged_at_unix_millis: 10, + admitted_at_unix_millis: 11, + }, + } + } + + #[test] + fn route_proof_readiness_is_refused_for_a_target_that_declares_odin() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut proof = generation.clone(); + proof.ready = route_proof_evidence(); + proof.latest_odin_observation = None; + proof.odin_authority = None; + // The fixture's Expected declares Odin, so route proof is not its class. + assert!(proof.validate().is_err()); + + // Mixed shapes are refused whatever the class. + let mut mixed = generation.clone(); + mixed.latest_odin_observation = None; + assert!(mixed.validate().is_err()); + let mut mixed = generation; + mixed.odin_authority = None; + assert!(mixed.validate().is_err()); + + let mut transaction = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?.1; + transaction.validate()?; + transaction.ready = Some(route_proof_evidence()); + assert!(transaction.validate().is_err()); + Ok(()) + } + + #[test] + fn route_supervision_state_exists_exactly_for_a_promoted_route() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut missing = generation.clone(); + missing.route_supervision = None; + assert!(missing.validate().is_err()); + + let mut unrouted = generation; + unrouted.route_supervision = None; + unrouted.routing = RoutingEvidence::SkippedUnrouted; + // Unrouted while Expected still names a route: refused for that reason + // too, so the test above pins the supervision rule on its own. + assert!(unrouted.validate().is_err()); + Ok(()) + } + + #[test] + fn a_new_generation_starts_unobserved_and_undegraded() -> Result<()> { + let committing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1; + let (_, incumbent) = fixture_generation(FIXTURE_GENERATION)?; + let authority = incumbent + .odin_authority + .clone() + .context("fixture generation has no Odin authority")?; + let next = AdmittedGeneration::from_transaction( + &committing, + Some(authority), + 1_700_000_200_000, + )?; + assert_eq!(next.route_supervision, Some(RouteSupervisionState::default())); + assert_eq!(next.last_error, None); + Ok(()) + } + + #[test] + fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let plan = &generation.plan; + let defaults = plan.phase_deadlines(); + for (phase, seconds) in [ + (DeploymentPhase::Fencing, defaults.fencing_seconds), + (DeploymentPhase::Leasing, defaults.leasing_seconds), + (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), + (DeploymentPhase::Routing, defaults.routing_seconds), + (DeploymentPhase::Committing, defaults.committing_seconds), + ] { + let deadline = PhaseDeadline::entering(phase, plan, 1_000).context("no deadline")?; + assert_eq!(deadline.phase, phase); + assert_eq!(deadline.entered_at_unix_millis, 1_000); + assert_eq!(deadline.deadline_at_unix_millis, 1_000 + u64::from(seconds) * 1000); + } + for phase in [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Warming, + DeploymentPhase::Complete, + ] { + assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); + } + Ok(()) + } + + #[test] + fn transition_stamps_the_deadline_of_the_phase_it_enters() -> Result<()> { + let world = EngineFixture::new()?; + // Routing, already holding the route receipt Committing requires. + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?; + assert_eq!(transaction.phase, DeploymentPhase::Routing); + transaction.routing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1.routing; + let envelope = transaction_envelope(&transaction, transaction.updated_at_unix_millis)?; + let command = command_envelope(&command(CommandKind::Continuity), 100)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.key.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[command, envelope], + )? + ); + let stored = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("stored transaction vanished")?; + world.engine.transition(&stored, DeploymentPhase::Committing)?; + let advanced = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("advanced transaction vanished")?; + let deadline = advanced.value.phase_deadline.context("no deadline stamped")?; + assert_eq!(deadline.phase, DeploymentPhase::Committing); + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + u64::from(advanced.value.plan.as_ref().unwrap().phase_deadlines().committing_seconds) + * 1000 + ); + assert_eq!(deadline.entered_at_unix_millis, advanced.value.updated_at_unix_millis); + Ok(()) + } + + #[test] + fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let plan = transaction.plan.clone().unwrap(); + let now = transaction.updated_at_unix_millis; + transaction.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now); + transaction.validate()?; + let mut wrong_phase = transaction.clone(); + wrong_phase.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::Routing, &plan, now); + assert!(wrong_phase.validate().is_err()); + let mut backwards = transaction.clone(); + backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; + assert!(backwards.validate().is_err()); + // Entry is not ordered against the wall-clock stamp of later writes. + let mut entered_after_the_stamp = transaction; + entered_after_the_stamp.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); + entered_after_the_stamp.validate()?; + Ok(()) + } + + // --------------------------------------------------------------------- + // The phase machine, driven by the real Engine past Fencing. + // --------------------------------------------------------------------- + + /// A workload port with no systemd behind it: observing a workload + /// returns what was observed, stopping and discarding succeed, and + /// anything that would launch a process refuses. + struct StillWorkload; + + impl WorkloadPort for StillWorkload { + fn install( + &self, + _: &CompiledDeploymentPlan, + _: &crate::drivers::MaterializedRelease, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn prepare_activation( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: IdunnRuntimeActivationLaunch, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn start_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &SealedRelease, + _: &crate::drivers::InstalledReleaseObservation, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn discard_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + Ok(()) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + Ok(prior.clone()) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(false) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } + } + + /// A Continuity transaction for an unrouted, stateless target with no + /// incumbent: the shape most Engine tests need. + fn transaction_at( + world: &EngineFixture, + phase: DeploymentPhase, + ) -> Result { + seeded_transaction(world, phase, CommandKind::Continuity, None) + } + + /// The Odin publisher sequence a transaction seeded over `incumbent` + /// warms at; its Ready is the next one. Above anything an earlier + /// generation of the world admitted. + fn seeded_sequence(incumbent: Option<&AdmittedGeneration>) -> u64 { + if incumbent.is_some() { 14 } else { 4 } + } + + /// A released deployment brake receipt, signed by an operator identity + /// enrolled in the world: what a Deploy's `deployment_authorization` holds. + fn deployment_authorization( + world: &EngineFixture, + expected: &IdunnExpectedIncarnationRecord, + transaction_id: &str, + now: u64, + ) -> Result { + use cultnet_rs::{ + IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, + IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, + IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, + }; + let signer = enroll_service_identity_at::( + &world.root.join("identities/brake-operator.cc"), + )?; + export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + let mut record = IdunnDeploymentBrakeRecord { + schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), + brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), + runtime_id: expected.runtime_id.clone(), + status: "released".into(), + scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), + reason: "test rollout".into(), + observed_at_unix_millis: now, + expires_at_unix_millis: Some(now + 600_000), + authorization_id: Some("authorization-test".into()), + authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), + authorized_release_id: Some(expected.sealed_release_id.clone()), + authorized_deployment_id: Some(transaction_id.to_owned()), + authorized_by: Some(signer.trust_anchor()?.identity_id), + authorization_issued_at_unix_millis: Some(now), + authorization_expires_at_unix_millis: Some(now + 600_000), + signature_algorithm: Some("ed25519".into()), + signature: None, + private_state_exposed: false, + updated_by: "operator/test".into(), + }; + record.signature = Some( + signer + .sign::(&rmp_serde::to_vec(&record)?) + .signature, + ); + record.validate()?; + let canonical_brake_bytes = rmp_serde::to_vec(&record)?; + Ok(DeploymentAuthorization { + authorization_id: "authorization-test".into(), + brake_sha256: sha256_id(&canonical_brake_bytes), + canonical_brake_bytes, + authorized_at_unix_millis: now, + }) + } + + /// What makes a target Odin: it provides the rendezvous. + fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + }); + } + + /// The declaration that makes an unrouted fixture target Odin-correlated. + /// Idunn infers no way to prove readiness, and a target with no route has + /// nothing for Idunn to challenge, so the world says outright that Odin + /// reports on it. The plan selects no provider: these worlds never plan + /// against Odin, they only read its correlation, which names the provider + /// Expected does. + fn declare_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.dependencies.push(cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: Some("odin".into()), + provider_authority: Some("managed-incarnation".into()), + provider_expected_projection_sha256: Some(sha256_id(b"odin expected projection")), + provider_endpoint: None, + }); + } + + /// A transaction for an unrouted, stateless target, sitting at `phase` + /// (Warming or later) with every earlier phase's evidence in place. The + /// plan, sealed release, Expected, activation and Warming evidence are the + /// real constructions; only the workload and isolation observations are + /// borrowed from a recorded transaction. + /// + /// `incumbent` is an admitted generation already in the world's store. A + /// Continuity over it restarts that generation's own release, so it shares + /// the incumbent's Expected key. A Deploy over it builds a second + /// incarnation. A Deploy carries what phases past Fencing never read but + /// validation requires: a three-field frozen-source receipt and a + /// well-formed release receipt from the brake. + fn seeded_transaction( + world: &EngineFixture, + phase: DeploymentPhase, + kind: CommandKind, + incumbent: Option<&AdmittedGeneration>, + ) -> Result { + use crate::deployment_plan::tests::{ + BINDING, RECIPE, artifact_receipt, external_input_receipt, source, + }; + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = if provider_path.exists() { + open_service_identity_at::(&provider_path)? + } else { + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + provider + }; + + let (binding_head, binding_tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ) + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id); + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ); + let (plan, release) = match (kind, incumbent) { + (CommandKind::Continuity, Some(incumbent)) => ( + incumbent.plan.clone(), + incumbent.sealed_release.clone(), + ), + _ => { + let incarnation = if incumbent.is_some() { + "service-incarnation-2" + } else { + "service-incarnation-1" + }; + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + incarnation, + None, + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + (plan, release) + } + }; + let mut expected = release.expected_projection(&plan)?; + assert!(!expected.write_lease_required && expected.route.is_none()); + declare_odin(&mut expected); + + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: format!( + "{}-service{}", + match kind { + CommandKind::Deploy => "up", + CommandKind::Continuity => "continuity", + }, + if incumbent.is_some() { "-again" } else { "" } + ), + kind, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = match (kind, incumbent) { + (CommandKind::Continuity, Some(incumbent)) => { + DeploymentTransaction::from_continuity(&command, incumbent, now)? + } + _ => DeploymentTransaction::new(&command, "service".into(), 0, incumbent, now)?, + }; + let activation = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&transaction.transaction_id)?, + now, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("recorded transaction has no workload")?; + match kind { + CommandKind::Continuity => { + transaction.lifecycle_authorized_at_unix_millis = Some(now); + } + CommandKind::Deploy => { + transaction.frozen_source = Some(FrozenSourceReceipt { + transaction_id: transaction.transaction_id.clone(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + transaction.deployment_authorization = Some(deployment_authorization( + world, + &expected, + &transaction.transaction_id, + now, + )?); + } + } + transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); + transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; + transaction.isolation = recorded.isolation; + transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + transaction.sealed_release = Some(release); + transaction.expected = Some(expected); + transaction.activation = Some(activation); + transaction.workload = Some(workload); + transaction.plan = Some(plan); + // Odin's first word about the candidate, before it was Ready. + let sequence = seeded_sequence(incumbent); + let warming = signed_correlation(world, &transaction, sequence, false)?; + let authenticated = world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&world.state_store)?, + &transaction, + &warming, + None, + now, + )?; + transaction.warming = Some(WarmingEvidence::OdinTopology { + evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, + }); + transaction.odin_publisher_sequence_cursor = sequence; + transaction.enter_phase(phase, now); + transaction.validate()?; + + let store = SingleFileMessagePackBackingStore::new(&world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )?); + Ok(transaction) + } + + /// Odin's correlation for the transaction's incarnation, signed by the key + /// the Engine trusts and stamped now. + fn signed_correlation( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + ) -> Result> { + signed_correlation_as(world, transaction, sequence, ready, "active", None, b"presence") + } + + /// Odin's correlation about the transaction's incarnation, observing a + /// presence in `state` that holds `lease`, identified by `presence`. + fn signed_correlation_as( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + state: &str, + lease: Option<&str>, + presence: &[u8], + ) -> Result> { + let expected = transaction.expected.as_ref().context("no Expected")?; + let activation = transaction.activation.as_ref().context("no activation")?; + let mut record = OdinRuntimeTopologyCorrelationRecord { + schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + expected: true, + current_activation_sha256: Some(activation.canonical_sha256()?), + signed_presence_sha256: Some(sha256_id(presence)), + observed_presence_state: Some(state.into()), + observed_presence_publisher_sequence: Some(sequence), + observed_write_lease_sha256: lease.map(str::to_owned), + observed_capabilities: expected + .capabilities + .iter() + .map(|capability| cultnet_rs::GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: 1, + }) + .collect(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: Some(activation.runtime_instance_id.clone()), + present: true, + ready, + dependencies: expected + .dependencies + .iter() + .map(|dependency| cultnet_rs::OdinRuntimeDependencyEvidence { + kind: dependency.kind.clone(), + capability: dependency.capability.clone(), + schema: dependency.schema.clone(), + compatibility: dependency.compatibility.clone(), + provider_id: dependency.provider_id.clone(), + provider_authority: dependency.provider_authority.clone(), + provider_expected_projection_sha256: dependency + .provider_expected_projection_sha256 + .clone(), + provider_endpoint: dependency.provider_endpoint.clone(), + observed_capacity: Some(dependency.minimum_capacity), + provider_evidence_sha256: Some(sha256_id(b"odin provider evidence")), + ready: true, + }) + .collect(), + disagreements: Vec::new(), + signer_identity_id: world.odin_signer.entry().identity_id.clone(), + publisher_sequence: sequence, + observed_at_unix_millis: now_millis()?, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }; + record.signature = world + .odin_signer + .sign::(&record.unsigned_signature_payload()?) + .signature; + record.canonical_bytes() + } + + /// Odin publishes its Ready correlation for the transaction incarnation. + fn odin_reports_ready( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let expected = transaction.expected.as_ref().context("no Expected")?; + SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store) + .insert_entry_if_absent(CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &expected.target, + &expected.canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(world, transaction, sequence, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + })?; + Ok(()) + } + + fn resident(world: &EngineFixture) -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("the transaction left the live set") + } + + /// The transaction wherever it now lives: live, or archived once terminal. + fn latest(world: &EngineFixture) -> Result { + match resident(world) { + Ok(stored) => Ok(stored.value), + Err(_) => read_history_transactions(&world.state_store) + .into_iter() + .next() + .context("the transaction is in neither the live set nor history"), + } + } + + /// One engine step at a time until `stop` holds, recording each phase + /// entered and checking after every step that the deadline on the record + /// is exactly the current post-fencing phase's, and absent outside one. + fn drive( + world: &EngineFixture, + stop: impl Fn(&DeploymentTransaction) -> bool, + ) -> Result> { + let mut phases = Vec::new(); + for _ in 0..40 { + let Ok(current) = resident(world) else { + return Ok(phases); + }; + if stop(¤t.value) { + return Ok(phases); + } + world.engine.advance_transaction(¤t)?; + let after = latest(world)?; + match &after.phase_deadline { + Some(deadline) => assert_eq!(deadline.phase, after.phase), + None => assert!( + !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) + .contains(&after.phase), + "a post-fencing record lost its deadline" + ), + } + if phases.last() != Some(&after.phase) { + phases.push(after.phase); + } + } + bail!("the transaction did not reach the stop condition in 40 steps") + } + + #[test] + fn an_admission_runs_from_fencing_to_complete_and_commits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + + let phases = drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + assert_eq!( + phases, + [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + DeploymentPhase::Complete, + ] + ); + let finished = latest(&world)?; + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::Admitted { .. }) + )); + let snapshot = ControlSnapshot::read(&world.state_store)?; + let admitted = snapshot + .admitted_for("service") + .context("commit wrote no admitted generation")?; + assert_eq!(admitted.value.transaction_id, finished.transaction_id); + assert_eq!(Some(&admitted.value.expected), seeded.expected.as_ref()); + Ok(()) + } + + #[test] + fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Warming)?; + let warming = resident(&world)?; + assert_eq!(warming.value.phase_deadline, None); + + world + .engine + .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + let finished = latest(&world)?; + assert_eq!(finished.phase, DeploymentPhase::Complete); + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + Ok(()) + } + + #[test] + fn a_post_fence_abort_runs_to_complete() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Routing + })?; + let routing = resident(&world)?; + assert!(routing.value.phase_deadline.is_some()); + + world + .engine + .begin_post_fencing_abort(&routing, anyhow!("candidate refused"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + let finished = latest(&world)?; + assert_eq!(finished.phase, DeploymentPhase::Complete); + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { .. }) + )); + assert!( + ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .is_none(), + "an aborted candidate was admitted" + ); + Ok(()) + } + + #[test] + fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + let plan = transaction.plan.clone().unwrap(); + let base = transaction.created_at_unix_millis; + transaction.enter_phase(DeploymentPhase::Routing, base + 1_000); + let deadline = transaction.phase_deadline.context("no deadline on entry")?; + assert_eq!(deadline.phase, DeploymentPhase::Routing); + assert_eq!( + deadline.deadline_at_unix_millis, + base + 1_000 + u64::from(plan.phase_deadlines().routing_seconds) * 1000 + ); + // A same-phase write after the wall clock stepped backwards is still a + // valid record: the deadline is fixed at entry, not ordered against + // later stamps, and neither is the creation time. + transaction.updated_at_unix_millis = deadline.entered_at_unix_millis - 1; + transaction.validate()?; + transaction.updated_at_unix_millis = base - 1; + transaction.validate()?; + transaction.enter_phase(DeploymentPhase::Committing, base + 2_000); + assert_eq!( + transaction.phase_deadline.unwrap().phase, + DeploymentPhase::Committing + ); + transaction.enter_phase(DeploymentPhase::Complete, base + 3_000); + assert_eq!(transaction.phase_deadline, None); + Ok(()) + } + + #[test] + fn an_undecodable_history_record_is_counted_and_never_dropped_silently() -> Result<()> { + let (finished, _) = terminal_transaction_with_command("ghostlight")?; + let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; + let mut torn = good.clone(); + torn.key = "tx-torn".into(); + // 0xc1 is msgpack's never-used byte: decodable by nothing. + torn.payload = vec![0xc1]; + let mut foreign = good.clone(); + foreign.key = "tx-foreign".into(); + foreign.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA_V3.into()); + foreign.payload = vec![0xc1]; + + let (decoded, undecodable) = + decode_history_transactions(vec![good, torn, foreign]); + assert_eq!(decoded, [finished]); + assert_eq!( + undecodable.iter().map(|(key, _)| key.as_str()).collect::>(), + ["tx-torn", "tx-foreign"] + ); + assert!(undecodable.iter().all(|(_, error)| !error.is_empty())); + Ok(()) + } + + /// The layout `IsolationEvidence` had before c848459 turned it into an + /// untagged Linux/Host enum. That commit claimed old records decode + /// unchanged; this is the check with the old shape itself. + #[test] + fn isolation_evidence_from_before_the_host_variant_still_decodes() -> Result<()> { + #[derive(Serialize)] + struct BeforeTheHostVariant { + candidate_uid: u32, + candidate_pid_namespace_id: u64, + candidate_mount_namespace_id: u64, + incumbent_uid: Option, + incumbent_pid_namespace_id: Option, + incumbent_mount_namespace_id: Option, + } + for incumbent in [None, Some(7)] { + let old = BeforeTheHostVariant { + candidate_uid: 61_000, + candidate_pid_namespace_id: 4_026_531_836, + candidate_mount_namespace_id: 4_026_531_841, + incumbent_uid: incumbent, + incumbent_pid_namespace_id: incumbent.map(|_| 8), + incumbent_mount_namespace_id: incumbent.map(|_| 9), + }; + for bytes in [rmp_serde::to_vec(&old)?, rmp_serde::to_vec_named(&old)?] { + let decoded: IsolationEvidence = rmp_serde::from_slice(&bytes)?; + let IsolationEvidence::Linux(linux) = decoded else { + bail!("an old Linux record decoded as a host record"); + }; + assert_eq!(linux.candidate_uid, 61_000); + assert_eq!(linux.incumbent_uid, incumbent); + } + } + Ok(()) + } + + #[test] + fn an_admitted_odin_without_odin_authority_is_refused_not_replaced_by_the_bootstrap_key() -> Result<()> { + let world = EngineFixture::new()?; + let empty = ControlSnapshot::read(&world.state_store)?; + assert_eq!( + world.engine.current_odin_authority(&empty)?, + world.engine.bootstrap_odin_authority + ); + + let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; + odin.target = "odin".into(); + // Odin is the target that provides the rendezvous, whatever it is named. + odin.expected.target = "odin".into(); + odin.expected.dependencies.clear(); + provide_odin(&mut odin.expected); + odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( + &world.odin_signer.trust_anchor()?, + )?); + let mut with_authority = ControlSnapshot::default(); + with_authority.admitted.push(Stored { + value: odin.clone(), + envelope: CultCacheEnvelope { + key: "odin".into(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + }); + assert_eq!( + world.engine.current_odin_authority(&with_authority)?, + odin.odin_authority.clone().unwrap() + ); + + odin.odin_authority = None; + let mut without = ControlSnapshot::default(); + without.admitted.push(Stored { + value: odin.clone(), + envelope: CultCacheEnvelope { + key: "odin".into(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + }); + let error = world + .engine + .current_odin_authority(&without) + .expect_err("a route-proof Odin must not fall back to the bootstrap key"); + assert!(format!("{error:#}").contains("no Odin authority"), "{error:#}"); + Ok(()) + } + + fn error_text(result: Result<()>) -> String { + format!("{:#}", result.expect_err("the record must be refused")) + } + + /// A route-proof generation from a real admission: the world's Expected + /// declares no Odin dependency, so route proof is its class. + fn route_proof_generation() -> Result { + route_proof::committed_generation() + } + + #[test] + fn a_route_proof_generation_is_admitted_only_in_its_exact_shape() -> Result<()> { + let generation = route_proof_generation()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.odin_authority.is_none()); + assert!(generation.latest_odin_observation.is_none()); + assert!(generation.odin_receipts().is_err()); + + // Route proof carries no Odin receipt and no Odin authority. + let mut with_receipt = generation.clone(); + with_receipt.latest_odin_observation = Some(topology(1, 1)); + assert!(with_receipt.validate().is_err()); + let mut with_authority = generation.clone(); + with_authority.odin_authority = Some(AdmittedOdinAuthority { + signer_identity_id: "odin-signer".into(), + signer_public_key: vec![1; 32], + }); + assert!(with_authority.validate().is_err()); + + // Its readiness evidence is checked by shape. + let ReadinessEvidence::RouteProof { evidence } = &generation.ready else { + bail!("not route proof"); + }; + let mut tampered = generation.clone(); + tampered.ready = ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + canonical_sha256: sha256_id(b"another"), + ..evidence.clone() + }, + }; + assert!(error_text(tampered.validate()).contains("runtime presence evidence")); + let mut untimed = generation.clone(); + untimed.ready = ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + admitted_at_unix_millis: evidence.challenged_at_unix_millis - 1, + ..evidence.clone() + }, + }; + assert!(error_text(untimed.validate()).contains("timeline")); + Ok(()) + } + + #[test] + fn odin_correlated_readiness_evidence_is_checked_by_shape() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.validate()?; + let ReadinessEvidence::OdinCorrelated { evidence } = &generation.ready else { + bail!("fixture is not Odin-correlated"); + }; + for broken in [ + TopologyEvidence { + canonical_sha256: sha256_id(b"another"), + ..evidence.clone() + }, + TopologyEvidence { + publisher_sequence: 0, + ..evidence.clone() + }, + TopologyEvidence { + admitted_at_unix_millis: 0, + ..evidence.clone() + }, + ] { + let mut tampered = generation.clone(); + tampered.ready = ReadinessEvidence::OdinCorrelated { evidence: broken }; + assert!(tampered.validate().is_err()); + } + Ok(()) + } + + #[test] + fn lease_adoption_evidence_is_checked_by_shape_on_the_transaction() -> Result<()> { + let (_, transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let adoption = LeaseAdoptionEvidence { + write_lease_sha256: sha256_id(b"lease"), + signed_presence_sha256: sha256_id(b"presence"), + source: AdoptionSource::Direct, + observed_at_unix_millis: 100, + }; + let refused = |change: fn(&mut LeaseAdoptionEvidence)| { + let mut adopting = transaction.clone(); + let mut evidence = adoption.clone(); + change(&mut evidence); + adopting.lease_adoption = Some(evidence); + error_text(adopting.validate()) + }; + assert!(refused(|evidence| evidence.write_lease_sha256.clear()) + .contains("adopted lease digest")); + assert!(refused(|evidence| evidence.signed_presence_sha256 = "no spaces allowed".into()) + .contains("presence digest")); + assert!(refused(|evidence| evidence.observed_at_unix_millis = 0) + .contains("no observation time")); + // Well-formed but naming no granted lease: the record is refused for + // that reason and no other. + assert!(refused(|_| {}).contains("does not name the granted lease")); + Ok(()) + } + + #[test] + fn route_supervision_is_checked_on_the_generation_and_the_meters_on_their_own_record() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.validate()?; + let supervised = |change: fn(&mut RouteSupervisionState)| { + let mut tampered = generation.clone(); + change(tampered.route_supervision.as_mut().unwrap()); + error_text(tampered.validate()) + }; + assert!(supervised(|state| state.degraded_since_unix_millis = Some(0)) + .contains("no start time")); + + let meters = |change: fn(&mut TargetSupervision)| { + let mut meters = TargetSupervision::new("service"); + change(&mut meters); + error_text(meters.validate()) + }; + TargetSupervision::new("service").validate()?; + assert!(meters(|m| m.route_actuations = vec![0]).contains("ascending")); + assert!(meters(|m| m.route_actuations = vec![9, 5]).contains("ascending")); + assert!( + meters(|m| m.route_actuations = (1..=13).collect()).contains("exceeds its bound") + ); + assert!(meters(|m| m.continuity_restarts = (1..=7).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.route_survivals = (1..=13).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.route_survivals = vec![9, 5]).contains("ascending")); + assert!(meters(|m| m.continuity_deferred_until = Some(5)).contains("time or a reason")); + assert!(meters(|m| m.continuity_deferral_reason = Some("x".into())).contains("time or a reason")); + assert!(meters(|m| m.schema_version = "idunn.target_supervision.v0".into()) + .contains("unsupported")); + Ok(()) + } + + #[test] + fn max_odin_sequence_counts_admitted_generations_by_target_and_signer() -> Result<()> { + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + let authority = generation.odin_authority.clone().context("no authority")?; + generation.odin_publisher_sequence_cursor = 88; + let target = generation.target.clone(); + let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), + commands: Vec::new(), + transactions: Vec::new(), + admitted: vec![Stored { + envelope: CultCacheEnvelope { + key: target.clone(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + value: generation, + }], + targets: Vec::new(), + }; + assert_eq!(snapshot.max_odin_sequence(&target, &authority.signer_identity_id), 88); + assert_eq!(snapshot.max_odin_sequence(&target, "another-signer"), 0); + assert_eq!( + snapshot.max_odin_sequence("another-target", &authority.signer_identity_id), + 0 + ); + Ok(()) + } + + #[test] + fn a_store_holding_a_lifted_legacy_generation_reads_and_keeps_its_key_honest() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let envelope = fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?; + let backing = SingleFileMessagePackBackingStore::new(&store); + backing.insert_entry_if_absent(envelope.clone())?; + + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.admitted.len(), 1); + let lifted = &snapshot.admitted[0].value; + assert_eq!(lifted.target, envelope.key); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); + assert!(snapshot.admitted_for(&envelope.key).is_some()); + + // The lifted record is held to the same key rule as a current one. + let other = temp.path().join("misfiled.cc"); + let mut misfiled = envelope; + misfiled.key = "not-its-target".into(); + SingleFileMessagePackBackingStore::new(&other).insert_entry_if_absent(misfiled)?; + assert!(error_text(ControlSnapshot::read(&other).map(|_| ())) + .contains("admitted generation key is not its target")); + Ok(()) + } + + /// Replace Odin's one correlation for the incarnation with a newer one. + fn odin_republishes( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let path = &world.engine.options.odin_correlation_store; + let store = SingleFileMessagePackBackingStore::new(path); + let existing = store.pull_all_read_only_snapshot()?; + assert!(store.delete_batch_if_unchanged(&existing)?); + odin_reports_ready(world, transaction, sequence) + } + + fn ready_sequence(transaction: &DeploymentTransaction) -> Option { + transaction + .ready + .as_ref() + .and_then(ReadinessEvidence::odin) + .map(|evidence| evidence.publisher_sequence) + } + + #[test] + fn a_newer_odin_reading_refreshes_the_receipt_before_awaiting_ready_or_routing_advance() + -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + + // AwaitingReady: the receipt is written, then confirmed, then the + // phase advances only on a receipt equal to the latest reading. + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::AwaitingReady + && ready_sequence(transaction) == Some(5) + })?; + odin_republishes(&world, &seeded, 6)?; + world.engine.advance_transaction(&resident(&world)?)?; // admits 6 + let admitted = resident(&world)?.value; + assert_eq!(admitted.phase, DeploymentPhase::AwaitingReady); + assert_eq!(admitted.latest_odin_observation.as_ref().unwrap().publisher_sequence, 6); + assert_eq!(ready_sequence(&admitted), Some(5)); + world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 6 + let refreshed = resident(&world)?.value; + assert_eq!(refreshed.phase, DeploymentPhase::AwaitingReady); + assert_eq!(ready_sequence(&refreshed), Some(6)); + world.engine.advance_transaction(&resident(&world)?)?; + assert_eq!(resident(&world)?.value.phase, DeploymentPhase::Routing); + + // Routing: route admission demands a receipt equal to the latest + // reading, so a newer one is adopted before anything is routed. + odin_republishes(&world, &seeded, 7)?; + world.engine.advance_transaction(&resident(&world)?)?; // admits 7 + assert_eq!(ready_sequence(&resident(&world)?.value), Some(6)); + world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 7 + let routing = resident(&world)?.value; + assert_eq!(routing.phase, DeploymentPhase::Routing); + assert_eq!(ready_sequence(&routing), Some(7)); + assert!(routing.routing.is_none()); + world.engine.advance_transaction(&resident(&world)?)?; + assert!(resident(&world)?.value.routing.is_some()); + Ok(()) + } + + #[test] + fn lease_adoption_names_only_the_granted_lease() -> Result<()> { + let lease = write_lease(); + let lease_sha256 = lease.canonical_sha256()?; + let adoption = LeaseAdoptionEvidence { + write_lease_sha256: lease_sha256.clone(), + signed_presence_sha256: sha256_id(b"adopting-presence"), + source: AdoptionSource::Direct, + observed_at_unix_millis: 100, + }; + adoption.validate_shape()?; + let granted = LeasingEvidence::Granted { + lease: lease.clone(), + lease_sha256: lease_sha256.clone(), + }; + assert!(adoption.names(&granted)); + // Prepared is not granted, stateless has nothing to adopt, and another + // lease's digest is not this lease. + assert!(!adoption.names(&LeasingEvidence::Prepared { + lease, + lease_sha256, + })); + assert!(!adoption.names(&LeasingEvidence::SkippedStateless)); + let mut other = adoption.clone(); + other.write_lease_sha256 = sha256_id(b"another-lease"); + assert!(!other.names(&granted)); + + // On a transaction, adoption without a granted lease is refused. + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + transaction.lease_adoption = Some(adoption); + assert!(transaction.validate().is_err()); + Ok(()) + } + + /// Q3 as a table: which ended phases abort, which name the unadopted + /// lease, which are left to the operator, and which recovery the terminal + /// record names. + #[test] + fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let lease = write_lease(); + let sha256 = lease.canonical_sha256()?; + let granted = LeasingEvidence::Granted { + lease: lease.clone(), + lease_sha256: sha256.clone(), + }; + let prepared = LeasingEvidence::Prepared { + lease, + lease_sha256: sha256.clone(), + }; + let adoption = LeaseAdoptionEvidence::with_ready( + Some(&sha256), + &sha256_id(b"adopting-presence"), + AdoptionSource::Direct, + 1, + ); + assert!(adoption.is_some()); + assert_eq!( + LeaseAdoptionEvidence::with_ready(None, &sha256_id(b"p"), AdoptionSource::Direct, 1), + None + ); + let abort = |lease_not_adopted| PhaseEnd::Abort { lease_not_adopted }; + use CommandKind::{Continuity, Deploy}; + let cases = [ + (Deploy, None, None, abort(false)), + (Deploy, Some(LeasingEvidence::SkippedStateless), None, abort(false)), + (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), + (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), + (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), + (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), + (Continuity, Some(prepared), None, abort(false)), + (Continuity, Some(granted.clone()), None, abort(true)), + (Continuity, Some(granted), adoption.clone(), abort(false)), + ]; + for (kind, leasing, lease_adoption, expected) in cases { + transaction.command_kind = kind; + transaction.leasing = leasing.clone(); + transaction.lease_adoption = lease_adoption.clone(); + assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); + let recovery = if kind == Continuity && lease_adoption.is_some() { + TerminalRecovery::RestartAdmitted + } else { + TerminalRecovery::RestoreIncumbent + }; + assert_eq!(transaction.terminal_recovery(), recovery, "{kind:?} {lease_adoption:?}"); + } + Ok(()) + } + + #[test] + fn an_operator_required_recovery_must_say_why() -> Result<()> { + let (_, mut failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; + let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = + failed.completion.as_mut() + else { + panic!("fixture is not a post-fencing failure") + }; + *recovery = TerminalRecovery::OperatorRequired { + reason: "adopted-lease".into(), + }; + failed.validate()?; + assert!(failed.is_terminal()); + let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = + failed.completion.as_mut() + else { + unreachable!() + }; + *recovery = TerminalRecovery::OperatorRequired { + reason: String::new(), + }; + assert!(failed.validate().is_err()); + Ok(()) + } + /// The persisted projection a continuity restart leaves behind, read at + /// the file the daemon writes. + struct ContinuityProjection { + _temp: TempDir, + topology: CultCacheTopologyDriver, + provider_anchor: ServiceIdentityTrustAnchor, + expected: IdunnExpectedIncarnationRecord, + admitted_activation: IdunnRuntimeActivationRecord, + candidate_activation: IdunnRuntimeActivationRecord, + } + + impl ContinuityProjection { + /// The production sequence: the admitted incarnation dies and is + /// demoted to Expected-only, then the continuity candidate publishes + /// its own activation under the same key once it is observed. + fn after_candidate_observed() -> Result { + let temp = TempDir::new()?; + let root = temp.path(); + let idunn = enroll_service_identity_at::(&root.join("idunn.cc"))?; + let provider = enroll_service_identity_at::( + &root.join("provider.cc"), + )?; + let provider_anchor = provider.trust_anchor()?; + let mut expected = TopologyFixture::new("ghostlight")?.expected; + expected.expected_signer_identity_id = provider.entry().identity_id.clone(); + expected.artifact_sha256 = sha256_id(&[1]); + expected.validate()?; + let topology = CultCacheTopologyDriver { + projection_store: root.join("topology.cc"), + correlation_store: root.join("correlation.cc"), + }; + let admitted = workload(1, 40, 50); + let candidate = workload(2, 41, 51); + let issue = |observation: &WorkloadObservation| -> Result<_> { + Ok(IdunnRuntimeActivationLaunch::issue( + &expected, + observation.runtime_instance_id().to_owned(), + NOW, + &idunn, + )? + .activation() + .clone()) + }; + let admitted_activation = issue(&admitted)?; + let candidate_activation = issue(&candidate)?; + assert_ne!(admitted_activation, candidate_activation); + + topology.publish_expected(&expected, &provider_anchor)?; + topology.publish_observed_activation(&expected, &admitted_activation, &admitted)?; + topology.demote_to_expected_only( + &expected, + &provider_anchor, + &admitted_activation, + None, + )?; + topology.publish_expected(&expected, &provider_anchor)?; + topology.publish_observed_activation(&expected, &candidate_activation, &candidate)?; + Ok(Self { + _temp: temp, + topology, + provider_anchor, + expected, + admitted_activation, + candidate_activation, + }) + } + + fn continuity_transaction(&self) -> Result { + let mut transaction = DeploymentTransaction::new( + &command(CommandKind::Continuity), + "ghostlight".into(), + 0, + None, + 100, + )?; + transaction.expected = Some(self.expected.clone()); + transaction.activation = Some(self.candidate_activation.clone()); + Ok(transaction) + } + + /// Types of the records projected under the shared incarnation key. + fn projected_types(&self) -> Result> { + let key = incarnation_key(&self.expected)?; + Ok( + SingleFileMessagePackBackingStore::new(&self.topology.projection_store) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.key == key) + .map(|entry| entry.r#type) + .collect(), + ) + } + } + + #[test] + fn a_failed_continuity_demotes_its_own_activation_and_keeps_the_expected() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + assert!( + world + .projected_types()? + .contains(&IdunnRuntimeActivationRecord::TYPE.to_owned()) + ); + let transaction = world.continuity_transaction()?; + assert_eq!( + transaction.abort_topology_reconciliation(), + CleanupEvidence::Pending + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "candidate died").topology_reconciliation, + CleanupEvidence::Pending + ); + + reconcile_failed_candidate_projection( + &world.topology, + &transaction, + &world.provider_anchor, + )?; + + assert_eq!( + world.projected_types()?, + vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()], + "the shared key must keep its Expected and nothing else" + ); + Ok(()) + } + + #[test] + fn a_continuity_that_issued_no_activation_owes_no_projection_change() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + let mut transaction = world.continuity_transaction()?; + transaction.activation = None; + assert_eq!( + transaction.abort_topology_reconciliation(), + CleanupEvidence::Skipped + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "candidate never started") + .topology_reconciliation, + CleanupEvidence::Skipped + ); + Ok(()) + } + + #[test] + fn a_deploy_fencing_after_a_failed_continuity_resolves_with_the_admitted_activation() + -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + reconcile_failed_candidate_projection( + &world.topology, + &world.continuity_transaction()?, + &world.provider_anchor, + )?; + let settled = std::fs::read(&world.topology.projection_store)?; + + // The deploy's post-fence rollback demotes the admitted generation's + // exact activation. The failed continuity left nothing standing, so it + // resolves instead of refusing as substituted. + world.topology.demote_to_expected_only( + &world.expected, + &world.provider_anchor, + &world.admitted_activation, + None, + )?; + assert_eq!(std::fs::read(&world.topology.projection_store)?, settled); + Ok(()) + } + + #[test] + fn no_path_adopts_an_activation_the_transaction_did_not_issue() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + let before = std::fs::read(&world.topology.projection_store)?; + + // The candidate's activation is still projected. The admitted + // generation's exact activation is not what stands there, so its + // demotion refuses rather than taking the newer one. + assert!( + world + .topology + .demote_to_expected_only( + &world.expected, + &world.provider_anchor, + &world.admitted_activation, + None, + ) + .is_err() + ); + + // Nor may a continuity transaction demote an activation it did not + // issue. + let mut foreign = world.continuity_transaction()?; + foreign.activation = Some(world.admitted_activation.clone()); + assert!( + reconcile_failed_candidate_projection( + &world.topology, + &foreign, + &world.provider_anchor + ) + .is_err() + ); + assert_eq!(std::fs::read(&world.topology.projection_store)?, before); + Ok(()) + } + + // --------------------------------------------------------------------- + // B1 fix batch: the transition, the precondition, the boot reconciliation, + // history that fails closed, and the abort paths through the Engine. + // --------------------------------------------------------------------- + + /// A workload port whose process can be killed: observing it succeeds + /// until `kill`, then fails the way a dead unit does. Everything else is + /// `StillWorkload`'s. + struct SwitchWorkload { + alive: std::sync::atomic::AtomicBool, + rebooted: std::sync::atomic::AtomicBool, + } + + impl SwitchWorkload { + fn new() -> Arc { + Arc::new(Self { + alive: std::sync::atomic::AtomicBool::new(true), + rebooted: std::sync::atomic::AtomicBool::new(false), + }) + } + + fn revive(&self) { + self.alive + .store(true, std::sync::atomic::Ordering::SeqCst); + self.rebooted + .store(false, std::sync::atomic::Ordering::SeqCst); + } + + /// The host rebooted under the unit: it is gone and the driver can tell. + fn reboot(&self) { + self.rebooted + .store(true, std::sync::atomic::Ordering::SeqCst); + self.kill(); + } + + fn kill(&self) { + self.alive + .store(false, std::sync::atomic::Ordering::SeqCst); + } + } + + impl WorkloadPort for SwitchWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + Ok(()) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + ensure!( + self.alive.load(std::sync::atomic::Ordering::SeqCst), + "the workload is gone" + ); + Ok(prior.clone()) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(false) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) + } + } + + /// Admit a generation for "service" through the real phase machine and + /// retire its transaction, leaving the world with an incumbent. + fn admit_incumbent(world: &EngineFixture) -> Result { + let seeded = transaction_at(world, DeploymentPhase::Fencing)?; + odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; + drive(world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + assert!(world.engine.retire_one_terminal_transaction()?); + Ok(ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone()) + } + + #[test] + fn an_admitted_odin_correlated_generation_takes_a_newer_odin_reading_once() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let first = seeded_sequence(None) + 1; + odin_reports_ready(&world, &seeded, first)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + assert!(world.engine.retire_one_terminal_transaction()?); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + assert_eq!(admitted.readiness(), Ok(ReadinessClass::OdinCorrelated)); + assert_eq!(admitted.odin_publisher_sequence_cursor, first); + + // Nothing newer than what is held: nothing is written. + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + let envelope = current.envelope.clone(); + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + assert_eq!(incumbent_envelope(&world)?, envelope); + + // A newer reading is adopted, once. + odin_republishes(&world, &seeded, first + 4)?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(world.engine.refresh_admitted_topology(&snapshot, current)?); + let refreshed = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no generation")? + .value + .clone(); + assert_eq!(refreshed.odin_publisher_sequence_cursor, first + 4); + assert_eq!( + refreshed.latest_odin_observation.as_ref().map(|latest| latest.publisher_sequence), + Some(first + 4) + ); + assert_eq!(refreshed.ready.odin().map(|ready| ready.publisher_sequence), Some(first + 4)); + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + Ok(()) + } + + /// Change the incumbent in the store, compare-and-swap. + fn edit_incumbent( + world: &EngineFixture, + edit: impl FnOnce(&mut AdmittedGeneration), + ) -> Result { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let stored = snapshot + .admitted_for("service") + .context("no admitted generation")?; + let mut next = stored.value.clone(); + edit(&mut next); + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: "service".into(), + current: Some(stored.envelope.clone()), + }], + &[admitted_envelope(&next, now_millis()?)?], + )? + ); + Ok(next) + } + + /// Replace or create a target's meters in the store. + fn set_meters(world: &EngineFixture, meters: &TargetSupervision) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + world.engine.write_target_supervision( + snapshot.supervision_for(&meters.target), + meters, + now_millis()?, + ) + } + + fn meters_of(world: &EngineFixture, target: &str) -> Result { + Ok(ControlSnapshot::read(&world.state_store)?.supervision_or_new(target)) + } + + /// Meters for "service" whose restarts happened these many milliseconds ago. + fn restarted_ago(ages: &[u64]) -> Result { + let now = now_millis()?; + let mut meters = TargetSupervision::new("service"); + meters.continuity_restarts = ages.iter().map(|age| now - age).collect(); + meters.continuity_restarts.sort_unstable(); + // Every logged attempt belongs to one streak. + meters.continuity_attempt_streak = ages.len() as u32; + Ok(meters) + } + + /// Drop every live transaction and command: what a finished restart leaves. + fn clear_live_transactions(world: &EngineFixture) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let envelopes = snapshot + .transactions + .iter() + .map(|stored| stored.envelope.clone()) + .chain(snapshot.commands.iter().map(|stored| stored.envelope.clone())) + .collect::>(); + if envelopes.is_empty() { + return Ok(()); + } + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&envelopes)? + ); + Ok(()) + } + + fn incumbent_envelope(world: &EngineFixture) -> Result { + Ok(ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone()) + } + + /// Publish what a transaction's candidate projects: its Expected and, + /// when asked, its observed activation. + fn project_candidate( + world: &EngineFixture, + transaction: &DeploymentTransaction, + with_activation: bool, + ) -> Result<()> { + let expected = transaction.expected.as_ref().context("no Expected")?; + let anchor = world + .engine + .provider_anchor_for_plan(transaction.plan.as_ref().context("no plan")?)?; + let topology = world.engine.topology(); + topology.publish_expected(expected, &anchor)?; + if with_activation { + // The observation must name the release the Expected declares; + // the borrowed workload is patched to. + let mut workload = transaction.workload.clone().context("no workload")?; + if let WorkloadObservation::Systemd(observed) = &mut workload { + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + topology.publish_observed_activation( + expected, + transaction.activation.as_ref().context("no activation")?, + &workload, + )?; + } + Ok(()) + } + + /// Publish what an admitted generation projects while it runs. + fn project_admitted(world: &EngineFixture, generation: &AdmittedGeneration) -> Result<()> { + let anchor = world.engine.provider_anchor_for_plan(&generation.plan)?; + let topology = world.engine.topology(); + topology.publish_expected(&generation.expected, &anchor)?; + topology.publish_observed_activation( + &generation.expected, + &generation.activation, + &generation.workload, + )?; + Ok(()) + } + + /// The types of the records projected under one incarnation key, sorted. + fn projected_under( + world: &EngineFixture, + expected: &IdunnExpectedIncarnationRecord, + ) -> Result> { + let path = &world.engine.options.topology_store; + if !path.exists() { + return Ok(Vec::new()); + } + let key = incarnation_key(expected)?; + let mut types = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.key == key) + .map(|entry| entry.r#type) + .collect::>(); + types.sort(); + Ok(types) + } + + /// One transaction by id, wherever it now lives. A failed transaction is + /// archived the moment it finishes, so it is in history, not resident. + fn record_of(world: &EngineFixture, transaction_id: &str) -> Result { + if let Some(stored) = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == transaction_id) + { + return Ok(stored.value); + } + read_history_transactions(&world.state_store) + .into_iter() + .find(|transaction| transaction.transaction_id == transaction_id) + .context("the transaction is in neither the live set nor history") + } + + fn expected_only() -> Vec { + vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()] + } + + // ---- the Engine's abort paths, Deploy and incumbent ---- + + #[test] + fn a_deploy_that_fences_commits_and_retires_the_incumbent_it_replaced() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + admit_incumbent(&world)?; + let meters = restarted_ago(&[600_000, 60_000])?; + set_meters(&world, &meters)?; + let incumbent = edit_incumbent(&world, |_| {})?; + + let candidate = seeded_transaction( + &world, + DeploymentPhase::Fencing, + CommandKind::Deploy, + Some(&incumbent), + )?; + odin_reports_ready(&world, &candidate, seeded_sequence(Some(&incumbent)) + 1)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the deploy admitted nothing")? + .value + .clone(); + assert_eq!(admitted.transaction_id, candidate.transaction_id); + assert_ne!(admitted.generation_id, incumbent.generation_id); + // The restart log belongs to the target: a commit must not reset it. + assert_eq!(meters_of(&world, "service")?, meters); + assert_eq!(admitted.route_supervision, incumbent.route_supervision); + + // The commit did not stop the incumbent: it is retired after. + let committed = record_of(&world, &candidate.transaction_id)?; + let cleanup = committed.post_commit_cleanup.clone().context("no cleanup")?; + assert_eq!( + cleanup.incumbent, + IncumbentCleanupEvidence::Pending { + generation_id: incumbent.generation_id.clone(), + workload: incumbent.workload.clone(), + } + ); + assert_eq!(cleanup.source, SourceCleanupEvidence::Pending); + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + + drive(&world, |transaction| transaction.is_terminal())?; + let retired = record_of(&world, &candidate.transaction_id)? + .post_commit_cleanup + .context("no cleanup")?; + assert_eq!( + retired.incumbent, + IncumbentCleanupEvidence::Complete { + generation_id: incumbent.generation_id + } + ); + assert_eq!(retired.source, SourceCleanupEvidence::Complete); + assert!(!frozen.exists(), "the frozen source outlived its deploy"); + Ok(()) + } + + #[test] + fn a_deploy_post_fence_abort_cleans_up_and_restores_its_incumbent_exactly() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let candidate = seeded_transaction( + &world, + DeploymentPhase::Fencing, + CommandKind::Deploy, + Some(&incumbent), + )?; + project_candidate(&world, &candidate, true)?; + let candidate_expected = candidate.expected.clone().context("no Expected")?; + assert_eq!( + projected_under(&world, &candidate_expected)?.len(), + 2, + "the candidate did not project" + ); + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + let before = incumbent_envelope(&world)?; + + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + assert!(resident(&world)?.value.post_fencing_abort.is_some()); + drive(&world, |transaction| transaction.completion.is_some())?; + + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + let abort = finished.post_fencing_abort.context("no abort evidence")?; + assert_eq!(abort.topology_reconciliation, CleanupEvidence::Complete); + assert_eq!(abort.source_cleanup, CleanupEvidence::Complete); + assert!(!frozen.exists(), "the abandoned source was not cleaned"); + assert!( + projected_under(&world, &candidate_expected)?.is_empty(), + "the abandoned candidate is still projected" + ); + // Fencing stopped the incumbent: it stands as Expected-only, with the + // exact activation it ran under withdrawn, ready for continuity. + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + assert_eq!(incumbent_envelope(&world)?, before); + Ok(()) + } + + #[test] + fn a_deploy_pre_fence_abort_cleans_up_and_leaves_its_incumbent_running() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let candidate = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Deploy, + Some(&incumbent), + )?; + project_candidate(&world, &candidate, true)?; + let candidate_expected = candidate.expected.clone().context("no Expected")?; + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + let before = incumbent_envelope(&world)?; + + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let intent = resident(&world)?.value.pre_fencing_abort.context("no intent")?; + assert_eq!(intent.topology_reconciliation, CleanupEvidence::Pending); + assert_eq!(intent.source_cleanup, CleanupEvidence::Pending); + drive(&world, |transaction| transaction.completion.is_some())?; + + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + assert!(!frozen.exists(), "the abandoned source was not cleaned"); + assert!(projected_under(&world, &candidate_expected)?.is_empty()); + // Before fencing a deployment never touched its incumbent. + assert_eq!( + projected_under(&world, &incumbent.expected)?, + { + let mut types = vec![ + IdunnExpectedIncarnationRecord::TYPE.to_owned(), + IdunnRuntimeActivationRecord::TYPE.to_owned(), + ]; + types.sort(); + types + } + ); + assert_eq!(incumbent_envelope(&world)?, before); + Ok(()) + } + + #[test] + fn a_continuity_abort_over_its_incumbent_demotes_only_the_activation_it_issued() -> Result<()> { + for post_fence in [false, true] { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let candidate = seeded_transaction( + &world, + if post_fence { + DeploymentPhase::Fencing + } else { + DeploymentPhase::Warming + }, + CommandKind::Continuity, + Some(&incumbent), + )?; + // The candidate shares the incumbent's key: same Expected, its own + // activation. + assert_eq!(candidate.expected.as_ref(), Some(&incumbent.expected)); + assert_ne!(candidate.activation.as_ref(), Some(&incumbent.activation)); + project_candidate(&world, &candidate, true)?; + let before = incumbent_envelope(&world)?; + + let stored = resident(&world)?; + if post_fence { + world + .engine + .begin_post_fencing_abort(&stored, anyhow!("candidate died"))?; + assert_eq!( + resident(&world)? + .value + .post_fencing_abort + .context("no intent")? + .topology_reconciliation, + CleanupEvidence::Pending + ); + } else { + world + .engine + .begin_pre_fencing_abort(&stored, anyhow!("candidate died"))?; + assert_eq!( + resident(&world)? + .value + .pre_fencing_abort + .context("no intent")? + .topology_reconciliation, + CleanupEvidence::Pending + ); + } + drive(&world, |transaction| transaction.completion.is_some())?; + + assert_eq!( + projected_under(&world, &incumbent.expected)?, + expected_only(), + "post_fence={post_fence}: the shared key must keep its Expected and nothing else" + ); + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(finished.is_terminal()); + assert!(matches!( + finished.completion, + Some( + TransactionCompletion::FailedBeforeFencing { .. } + | TransactionCompletion::FailedAfterFencing { .. } + ) + )); + assert_eq!(incumbent_envelope(&world)?, before); + } + Ok(()) + } + + // ---- fix 1: the legacy lift owns the transition ---- + + const FIXTURE_PRE_B1_ABORTS: [(&str, &str); 3] = [ + ( + "pre-fence abort in flight", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex"), + ), + ( + "pre-fence abort already terminal", + include_str!( + "../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex" + ), + ), + ( + "post-fence abort in flight", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex"), + ), + ]; + const FIXTURE_PROVIDER_ANCHOR: &str = + include_str!("../tests/fixtures/idunn-control-legacy/provider-anchor.hex"); + /// Where the fixtures' plan binding names the provider trust anchor. + const FIXTURE_ANCHOR_PATH: &str = "/tmp/idunn-b1-legacy/provider-anchor.cc"; + + fn hex_bytes(text: &str) -> Vec { + let hex = text.trim(); + (0..hex.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("hex fixture")) + .collect() + } + + fn write_fixture_anchor() -> Result<()> { + let path = Path::new(FIXTURE_ANCHOR_PATH); + std::fs::create_dir_all(path.parent().context("anchor path has no parent")?)?; + std::fs::write(path, hex_bytes(FIXTURE_PROVIDER_ANCHOR))?; + Ok(()) + } + + fn abort_owed(transaction: &DeploymentTransaction) -> Option { + transaction + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .or_else(|| { + transaction + .post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }) + } + + /// Put a continuity's record and its command in the control store as they + /// stood: `envelope` is written as given, in whatever schema it is in. + fn make_resident( + world: &EngineFixture, + transaction: &DeploymentTransaction, + envelope: CultCacheEnvelope, + ) -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: transaction.command_id.clone(), + kind: CommandKind::Continuity, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: envelope.key.clone(), + current: None, + }, + ], + &[command_envelope(&command, 100)?, envelope], + )? + ); + Ok(()) + } + + #[test] + fn a_pre_b1_continuity_abort_lifts_without_reopening_and_boot_resolves_its_residue() + -> Result<()> { + write_fixture_anchor()?; + for (name, text) in FIXTURE_PRE_B1_ABORTS { + let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; + + // The record is what the old rule wrote: an activation was issued + // and the abort says it owes the projection nothing. + let legacy: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload)?; + assert_eq!(legacy.command_kind, CommandKind::Continuity, "{name}"); + assert!(legacy.activation.is_some(), "{name}"); + + // History describes what happened, so it lifts the record as written. + let archived = lift_legacy_transaction(&envelope)?; + assert_eq!(abort_owed(&archived), Some(CleanupEvidence::Skipped), "{name}"); + + // The control store lifts it without reopening anything. + let lifted = read_transaction_record(&envelope)?; + let terminal = archived.completion.is_some(); + if terminal { + assert!( + abort_owed(&lifted).is_some_and(CleanupEvidence::is_legacy_marker), + "{name}" + ); + assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); + assert_eq!(lifted.phase, archived.phase, "{name}"); + assert_eq!(lifted.completion, archived.completion, "{name}"); + assert!(!lifted.owns_target_authority(), "{name}"); + } else { + assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); + assert!(!lifted.is_terminal(), "{name}"); + assert_eq!(lifted.completion, None, "{name}"); + } + + // Boot: the store reads. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + make_resident(&world, &lifted, envelope.clone())?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 1, "{name}"); + project_candidate(&world, &lifted, true)?; + let expected = lifted.expected.clone().context("no Expected")?; + assert_eq!(projected_under(&world, &expected)?.len(), 2, "{name}"); + + if terminal { + // A finished record owes nothing and is not touched: boot + // reconciliation demotes the residue by its exact activation. + let before = resident(&world)?.envelope; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted, vec![lifted.transaction_id.clone()], "{name}"); + assert_eq!(resident(&world)?.envelope, before, "{name}"); + } else { + // A record in flight resolves through its own abort. + for _ in 0..12 { + let Ok(current) = resident(&world) else { break }; + if current.value.completion.is_some() { + break; + } + world.engine.advance_transaction(¤t)?; + } + assert!( + record_of(&world, &envelope.key)?.completion.is_some(), + "{name}: the abort never resolved" + ); + } + assert_eq!( + projected_under(&world, &expected)?, + expected_only(), + "{name}: the projection was not demoted to Expected-only" + ); + } + Ok(()) + } + + #[test] + fn resident_terminal_pre_b1_aborts_never_claim_target_authority() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + let first = read_transaction_record(&envelope)?; + assert_eq!(first.target, live.target); + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + + // A second resident abort of the same shape for the same target, + // already written as the marker the migration persists. + let mut second = first.clone(); + second.transaction_id = "tx-second-legacy".into(); + second.command_id = "continuity-second-legacy".into(); + make_resident( + &world, + &second, + cleanup_evidence::last_error_envelope(&second, "second")?, + )?; + + // The store reads: three records, one of them live, none of the + // finished ones claiming the target. + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 3); + let live_before = snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope + .clone(); + + // Reconciliation cleans the residue once, by exact activation. + let expected = first.expected.clone().context("no Expected")?; + project_candidate(&world, &first, true)?; + assert_eq!(projected_under(&world, &expected)?.len(), 2); + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted.len(), 1); + assert_eq!(projected_under(&world, &expected)?, expected_only()); + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!( + after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope, + live_before + ); + Ok(()) + } + + #[test] + fn boot_never_republishes_a_withdrawn_incarnation_and_history_keeps_one_copy() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let lifted = read_transaction_record(&envelope)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + make_resident(&world, &lifted, envelope.clone())?; + // A crash between the two archive steps: history already holds it. + assert!( + SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) + .insert_entry_if_absent(envelope)? + ); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + + // The incarnation was withdrawn: nothing is projected for it. + let expected = lifted.expected.clone().context("no Expected")?; + assert!(projected_under(&world, &expected)?.is_empty()); + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + assert!( + world + .engine + .reconcile_failed_continuity_projections()? + .demoted + .is_empty() + ); + assert!( + projected_under(&world, &expected)?.is_empty(), + "a withdrawn incarnation's Expected was published again" + ); + + // One copy in history, finished, and nothing resident. + let history = read_history(&world.state_store)?; + assert_eq!(history.transactions.len(), 1); + assert!(history.transactions[0].is_terminal()); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + Ok(()) + } + + #[test] + fn boot_runs_the_projection_reconciliation_serve_starts_from() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let failed = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + project_candidate(&world, &failed, true)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + // A pre-B1 abort's residue: the failed candidate's activation stands. + project_candidate(&world, &failed, true)?; + assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); + + let (_lock, _engine) = boot(world.engine.options.clone())?; + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + /// A workload whose candidate can neither be stopped nor recover: an abort + /// that cannot finish its first step. + struct WedgedWorkload; + + impl WorkloadPort for WedgedWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + StillWorkload.observe(expected, activation, prior) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + bail!("the unit will not stop") + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } + } + + #[test] + fn one_wedged_pre_fence_abort_does_not_stop_the_scheduler() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; + let wedged = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + + // Recording the error is not progress, and neither is repeating it. + assert!(!world.engine.resume_one_transaction()?); + assert!(!world.engine.resume_one_transaction()?); + // Every tick answers: the wedged abort records its error and waits, + // and the rest of the scheduler goes on to freeze the queued command. + write_service_binding(&world)?; + for _ in 0..6 { + world.engine.run_scheduler_tick()?; + } + let stuck = record_of(&world, &wedged.transaction_id)?; + assert!(!stuck.is_terminal()); + assert!(stuck.post_fencing_abort.is_none()); + assert!(stuck.last_error.is_some()); + assert!( + read_history_transactions(&world.state_store) + .iter() + .any(|transaction| transaction.command_id == command.command_id), + "the queued command was never frozen" + ); + Ok(()) + } + + #[test] + fn a_wedged_post_fence_abort_only_waits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; + let dead = transaction_at(&world, DeploymentPhase::Fencing)?; + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; + + // The abort cannot stop the unit. The abort is already durable, so + // that is a resumable error, and the tick answers every time. + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + let stuck = record_of(&world, &dead.transaction_id)?; + assert!(!stuck.is_terminal()); + assert!(stuck.last_error.is_some()); + Ok(()) + } + + /// A wedged abort whose `stop()` error carries a changing counter, as + /// systemctl output with a PID or timestamp would. + struct ChangingErrorWorkload(std::sync::atomic::AtomicU32, std::sync::atomic::AtomicBool); + + impl WorkloadPort for ChangingErrorWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + WedgedWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + WedgedWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + WedgedWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + WedgedWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + WedgedWorkload.observe(expected, activation, prior) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + let n = self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.1.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } + bail!("the unit will not stop (main pid {n})") + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } + } + + fn queue_deploy_for_nowhere(world: &EngineFixture) -> Result { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + Ok(command) + } + + #[test] + fn a_wedge_whose_error_text_changes_every_tick_does_not_starve_other_targets() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), + )))?; + transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let command = queue_deploy_for_nowhere(&world)?; + write_service_binding(&world)?; + for _ in 0..20 { + world.engine.run_scheduler_tick()?; + } + assert!( + read_history_transactions(&world.state_store) + .iter() + .any(|transaction| transaction.command_id == command.command_id), + "the queued command was starved by another target's wedge" + ); + Ok(()) + } + + #[test] + fn rewriting_last_error_is_not_progress() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let before = transaction_at(&world, DeploymentPhase::Warming)?; + let mut noted = before.clone(); + noted.last_error = Some("the unit will not stop (main pid 7)".into()); + noted.updated_at_unix_millis += 1; + assert!(!state_advanced(&before, ¬ed)); + let mut moved = noted.clone(); + moved.phase = DeploymentPhase::AwaitingReady; + assert!(state_advanced(&before, &moved)); + Ok(()) + } + + #[test] + fn a_failing_step_is_retried_after_a_backoff_not_every_tick() -> Result<()> { + let workload = Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), + )); + let world = EngineFixture::with_workload(workload.clone())?; + transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let attempts = || workload.0.load(std::sync::atomic::Ordering::SeqCst); + let before = now_millis()?; + world.engine.run_scheduler_tick()?; + let after = now_millis()?; + assert_eq!(attempts(), 1, "the step did not run"); + // The first failure waits one poll interval: the wait back_off wrote. + let poll = world.engine.options.poll_millis; + let written = world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&resident(&world)?.value.transaction_id) + .map(|backoff| backoff.not_before_unix_millis) + .expect("the failure set no backoff"); + assert!( + (before + poll..=after + poll).contains(&written), + "back_off wrote {written}, not one poll ({poll} ms) after the failure" + ); + // The wait is set from the wall clock, and a loaded host can spend it + // inside a few ticks. So the test owns the wait: held open, no tick + // retries; once due, the next tick does. + let id = resident(&world)?.value.transaction_id; + let set_due = |due: u64| { + world + .engine + .resume_backoff + .lock() + .unwrap() + .get_mut(&id) + .expect("the failure set no backoff") + .not_before_unix_millis = due; + }; + set_due(now_millis()? + RESUME_BACKOFF_CEILING_MILLIS / 2); + for _ in 0..10 { + world.engine.run_scheduler_tick()?; + } + assert_eq!(attempts(), 1, "the wedged step was retried inside its backoff"); + // Each further failure doubles the wait back_off writes. + for (attempt, polls) in [(2, 2), (3, 4)] { + set_due(1); + let before = now_millis()?; + world.engine.run_scheduler_tick()?; + let after = now_millis()?; + assert_eq!(attempts(), attempt, "the step was not retried once its backoff was due"); + let written = world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&id) + .map(|backoff| backoff.not_before_unix_millis) + .expect("the failure set no backoff"); + assert!( + (before + polls * poll..=after + polls * poll).contains(&written), + "failure {attempt} waits {written}, not {polls} polls ({poll} ms each) after it" + ); + } + Ok(()) + } + + /// A candidate that fails to advance and is dead for good. + struct DeadCandidateWorkload; + + impl WorkloadPort for DeadCandidateWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + _: &WorkloadObservation, + ) -> Result { + bail!("the candidate is gone") + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } + } + + #[test] + fn a_permanently_dead_candidate_past_the_fence_begins_its_post_fence_abort() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(DeadCandidateWorkload))?; + let dead = transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(record_of(&world, &dead.transaction_id)?.post_fencing_abort.is_none()); + for _ in 0..8 { + world.engine.resume_one_transaction()?; + } + assert!( + record_of(&world, &dead.transaction_id)? + .post_fencing_abort + .is_some(), + "a dead candidate past the fence was retried instead of aborted" + ); + Ok(()) + } + + #[test] + fn a_resume_fault_is_reported_once_and_left_in_the_record() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = transaction_at(&world, DeploymentPhase::Warming)?; + let error = anyhow!("store hiccup 1"); + world + .engine + .note_fault("could not resume transaction", &live.transaction_id, &error); + assert_eq!( + record_of(&world, &live.transaction_id)?.last_error.as_deref(), + Some("store hiccup 1") + ); + let reports = world.engine.fault_reports.lock().unwrap(); + assert_eq!(reports.len(), 1); + assert!(reports[&live.transaction_id].offer(Some("store hiccup 1".into())).is_none()); + Ok(()) + } + + #[test] + fn a_new_record_cannot_carry_the_legacy_marker() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let lifted = read_transaction_record(&envelope)?; + assert!(lifted.carries_legacy_marker()); + let error = error_text(transaction_envelope(&lifted, 1).map(|_| ())); + assert!(error.contains("legacy lift"), "{error}"); + // The migration's own path still writes it, and it reads back. + let (_, migrated) = cleanup_evidence::migrate_transaction_record(&envelope)?; + assert!(read_transaction_record(&migrated)?.carries_legacy_marker()); + Ok(()) + } + + #[test] + fn unreadable_history_does_not_skip_resident_residue_at_boot() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let first = read_transaction_record(&envelope)?; + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + let expected = first.expected.clone().context("no Expected")?; + project_candidate(&world, &first, true)?; + assert_eq!(projected_under(&world, &expected)?.len(), 2); + corrupt_history(&world)?; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted.len(), 1); + assert_eq!(projected_under(&world, &expected)?, expected_only()); + Ok(()) + } + + #[test] + fn backoff_doubles_caps_and_waits_exactly() { + let ceiling = RESUME_BACKOFF_CEILING_MILLIS; + assert_eq!(backoff_wait(500, 0, ceiling), 500); + assert_eq!(backoff_wait(500, 1, ceiling), 1000); + assert_eq!(backoff_wait(500, 2, ceiling), 2000); + assert_eq!(backoff_wait(500, 3, ceiling), 4000); + assert_eq!(backoff_wait(500, 30, ceiling), ceiling); + assert_eq!(backoff_wait(u64::MAX, 5, ceiling), ceiling); + // The ceiling is the caller's: continuity's is lower than the resume path's. + assert_eq!(backoff_wait(5_000, 5, 300_000), 160_000); + assert_eq!(backoff_wait(5_000, 6, 300_000), 300_000); + assert_eq!(backoff_wait(5_000, 20, 300_000), 300_000); + assert!(is_waiting(9, 10, 5)); + assert!(!is_waiting(10, 10, 5)); + assert!(!is_waiting(11, 10, 5)); + // Due further ahead than the longest wait the owner can set: the clock + // stepped back, so the attempt is due, not stalled for the step. + assert!(is_waiting(5, 10, 5)); + assert!(!is_waiting(4, 10, 5)); + } + + #[test] + fn a_resume_backoff_set_by_a_stepped_back_clock_is_due_and_a_real_one_waits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let id = transaction_at(&world, DeploymentPhase::Warming)?.transaction_id; + let not_before = |world: &EngineFixture| { + world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&id) + .map(|backoff| backoff.not_before_unix_millis) + }; + let set = |world: &EngineFixture, not_before_unix_millis: u64| { + world.engine.resume_backoff.lock().unwrap().insert( + id.clone(), + ResumeBackoff { + failures: 3, + not_before_unix_millis, + }, + ); + }; + + // Half a minute ahead is a wait the backoff can set: the step is left alone. + let waiting = now_millis()? + 30_000; + set(&world, waiting); + let _ = world.engine.resume_candidate(&resident(&world)?); + assert_eq!(not_before(&world), Some(waiting)); + + // A day ahead is a clock that stepped back: the step is attempted, and + // whatever it does, it replaces the backoff. + let ahead = now_millis()? + 86_400_000; + set(&world, ahead); + let _ = world.engine.resume_candidate(&resident(&world)?); + assert!(not_before(&world).is_none_or(|due| due < ahead)); + Ok(()) + } + + #[test] + fn a_persistent_fault_is_reported_once_and_again_only_after_recovery() -> Result<()> { + let workload = Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), + )); + let world = EngineFixture::with_workload(workload.clone())?; + let wedged = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let id = wedged.transaction_id; + let expire = || { + world.engine.resume_backoff.lock().unwrap().clear(); + }; + let reported = || world.engine.fault_reports.lock().unwrap().contains_key(&id); + // A fault already reported for this transaction (an Err from the + // resume itself); a handled, still-failing step must not re-arm it. + world + .engine + .note_fault("could not resume transaction", &id, &anyhow!("store hiccup")); + assert!(reported()); + world.engine.resume_one_transaction()?; + assert!(reported(), "a failing handled step re-armed the report"); + // Skipped inside the backoff: the fault is still the same fault. + world.engine.resume_one_transaction()?; + assert!(reported(), "a backoff skip re-armed the report"); + // Still failing after the backoff expires: still the same fault. + expire(); + world.engine.resume_one_transaction()?; + assert!(reported()); + // A real recovery clears it. + workload.1.store(true, std::sync::atomic::Ordering::SeqCst); + expire(); + world.engine.resume_one_transaction()?; + assert!(!reported(), "a step that ran and succeeded left the fault armed"); + Ok(()) + } + + #[test] + fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let first = read_transaction_record(&envelope)?; + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + assert!(record_of(&world, &first.transaction_id)?.carries_legacy_marker()); + corrupt_history(&world)?; + assert!(!world.engine.retire_one_terminal_transaction()?); + let noted = record_of(&world, &first.transaction_id)?; + assert!( + noted.last_error.is_some() && noted.last_error != first.last_error, + "the retire fault never reached the record" + ); + assert!(noted.carries_legacy_marker()); + Ok(()) + } + + #[test] + fn a_tick_that_only_freezes_or_only_advances_reports_progress() -> Result<()> { + // Only a freeze. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + queue_deploy_for_nowhere(&world)?; + write_service_binding(&world)?; + assert!(world.engine.run_scheduler_tick()?, "a freeze-only tick reported no progress"); + + // Only a phase advance. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(world.engine.resume_one_transaction()?, "a real advance reported none"); + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(world.engine.run_scheduler_tick()?); + Ok(()) + } + + #[test] + fn a_terminal_transaction_that_cannot_be_archived_does_not_stop_the_scheduler() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = transaction_at(&world, DeploymentPhase::Warming)?; + odin_reports_ready(&world, &live, seeded_sequence(None) + 1)?; + let (finished, command) = terminal_transaction_with_command("ghostlight")?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: finished.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, 100)?, + transaction_envelope(&finished, finished.updated_at_unix_millis)?, + ], + )? + ); + corrupt_history(&world)?; + let before = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope; + + // The finished record cannot be archived, and the live one still moves. + world.engine.run_scheduler_tick()?; + let after = ControlSnapshot::read(&world.state_store)?; + assert!( + after + .transactions + .iter() + .any(|stored| stored.value.transaction_id == finished.transaction_id) + ); + assert_ne!( + after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope, + before, + "the live transaction was not resumed" + ); + Ok(()) + } + + // ---- fix 2: supervision owns the Expected-only precondition ---- + + #[test] + fn continuity_mints_no_restart_over_a_projection_it_cannot_demote() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + + // The projection names an activation that is not the incumbent's. + let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; + let topology = world.engine.topology(); + topology.publish_expected(&incumbent.expected, &anchor)?; + let stranger = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + topology.publish_observed_activation( + &incumbent.expected, + stranger.activation.as_ref().context("no activation")?, + stranger.workload.as_ref().context("no workload")?, + )?; + // Remove the seeded stranger transaction: only the projection matters. + let seeded = resident(&world)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged( + &[ + seeded.envelope.clone(), + ControlSnapshot::read(&world.state_store)? + .commands + .into_iter() + .find(|stored| stored.value.command_id == seeded.value.command_id) + .context("seeded command")? + .envelope, + ] + )? + ); + workload.kill(); + + let before = now_millis()?; + assert!(world.engine.supervise_one_admitted_generation()?); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!( + snapshot.transactions.is_empty(), + "continuity minted a restart over a projection it could not demote" + ); + let recorded = snapshot + .supervision_for("service") + .context("the failed demotion was not recorded")? + .value + .clone(); + let deferred = recorded.continuity_deferred_until.context("no deferral time")?; + assert!(recorded.continuity_deferral_reason.is_some()); + assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); + let after = now_millis()?; + assert!( + (before + CONTINUITY_DEFERRAL_MILLIS..=after + CONTINUITY_DEFERRAL_MILLIS) + .contains(&deferred) + ); + + // Deferred: the next tick neither retries nor writes. + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(meters_of(&world, "service")?, recorded); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + + // The projection is repaired and the deferral has run out: the restart + // proceeds, from an Expected-only projection. + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_admitted(&world, &incumbent)?; + let mut ran_out = recorded.clone(); + ran_out.continuity_deferred_until = Some(1); + set_meters(&world, &ran_out)?; + assert!(world.engine.supervise_one_admitted_generation()?); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 1); + let minted = snapshot.supervision_for("service").context("no meters")?; + assert_eq!(minted.value.continuity_restarts.len(), 1); + assert_eq!(minted.value.continuity_deferred_until, None); + assert_eq!(minted.value.continuity_deferral_reason, None); + assert_eq!(snapshot.transactions[0].value.command_kind, CommandKind::Continuity); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + // ---- fix 3: one-time boot reconciliation, by exact issuer ---- + + #[test] + fn boot_demotes_the_activation_a_failed_continuity_left_and_only_that() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let failed = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + project_candidate(&world, &failed, true)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + // A failed transaction is archived the moment it finishes. + let aborted = record_of(&world, &failed.transaction_id)?; + assert!(aborted.is_terminal()); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + + // Drift a pre-B1 abort left: the failed candidate's activation stands. + project_candidate(&world, &failed, true)?; + assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); + + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!( + outcome, + ProjectionReconciliation { + demoted: vec![failed.transaction_id.clone()], + unexplained: Vec::new(), + } + ); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + + // Once: nothing is left to demote. + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + Ok(()) + } + + #[test] + fn boot_never_adopts_an_activation_no_failed_transaction_issued() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + + // The incumbent's own activation is accounted for. + project_admitted(&world, &incumbent)?; + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + + // Some other activation, issued by nothing in history, is reported and + // left exactly as it is. + let stranger = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; + let topology = world.engine.topology(); + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_candidate(&world, &stranger, true)?; + // A live transaction accounts for its own activation... + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + // ...and for nobody else's. + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + let mut other = stranger.clone(); + let other_activation = IdunnRuntimeActivationLaunch::issue( + &incumbent.expected, + runtime_instance_id("tx-someone-else")?, + now_millis()?, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + if let Some(WorkloadObservation::Systemd(observed)) = other.workload.as_mut() { + observed.runtime_instance_id = other_activation.runtime_instance_id.clone(); + } + other.activation = Some(other_activation); + project_candidate(&world, &other, true)?; + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?.unexplained, + vec!["service".to_owned()] + ); + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_candidate(&world, &stranger, true)?; + // ...and once it is gone, nothing does. + let seeded = resident(&world)?; + let command = ControlSnapshot::read(&world.state_store)? + .commands + .into_iter() + .find(|stored| stored.value.command_id == seeded.value.command_id) + .context("seeded command")?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&[seeded.envelope.clone(), command.envelope])? + ); + let projection = std::fs::read(&world.engine.options.topology_store)?; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!( + outcome, + ProjectionReconciliation { + demoted: Vec::new(), + unexplained: vec!["service".into()], + } + ); + assert_eq!(std::fs::read(&world.engine.options.topology_store)?, projection); + Ok(()) + } + + // ---- fix 4: history that cannot be read stops actuation ---- + + /// An operator binding for the target "service", so the command queue can + /// resolve selectors. + fn write_service_binding(world: &EngineFixture) -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let (head, tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{head}[brakes]{}", + tail.split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ); + std::fs::create_dir_all(&world.engine.options.bindings_dir)?; + std::fs::write(world.engine.options.bindings_dir.join("service.toml"), binding)?; + Ok(()) + } + + fn corrupt_history(world: &EngineFixture) -> Result<()> { + std::fs::write(history_store_path(&world.state_store), b"\xc1\xc1 torn")?; + Ok(()) + } + + #[test] + fn history_that_cannot_be_read_is_an_error_and_missing_history_is_empty() -> Result<()> { + let world = EngineFixture::new()?; + let history = read_history(&world.state_store)?; + assert!(history.transactions.is_empty() && history.report().is_none()); + + corrupt_history(&world)?; + let error = read_history(&world.state_store).err().context("torn history read")?; + assert!(format!("{error:#}").contains("history"), "{error:#}"); + assert!(world.engine.history_for_decision().is_none()); + // The display path still answers, with nothing. + assert!(read_history_transactions(&world.state_store).is_empty()); + Ok(()) + } + + #[test] + fn undecodable_history_names_what_it_is_missing() -> Result<()> { + let (finished, _) = terminal_transaction_with_command("ghostlight")?; + let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; + let mut torn = good.clone(); + torn.key = "tx-torn".into(); + torn.payload = vec![0xc1]; + let (transactions, undecodable) = decode_history_transactions(vec![good, torn]); + let history = HistoryRead { + transactions, + undecodable, + }; + let report = history.report().context("nothing reported")?; + assert!(report.contains("1 archived transaction"), "{report}"); + assert!(report.contains("tx-torn"), "{report}"); + + let complete = HistoryRead { + transactions: history.transactions, + undecodable: Vec::new(), + }; + assert_eq!(complete.report(), None); + Ok(()) + } + + #[test] + fn a_fault_is_said_once_while_it_lasts() { + let once = ReportOnce::default(); + assert_eq!(once.offer(None), None); + assert_eq!(once.offer(Some("torn".into())), Some("torn".into())); + assert_eq!(once.offer(Some("torn".into())), None); + assert_eq!(once.offer(Some("torn".into())), None); + assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); + assert_eq!(once.offer(None), None); + assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); + } + + #[test] + fn unreadable_history_does_not_stop_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + corrupt_history(&world)?; + + // Crash recovery reads the target's own log, never history: the dead + // workload is restarted, and the restart is counted there. + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + #[test] + fn unreadable_history_stops_command_retirement_and_freezing() -> Result<()> { + let world = EngineFixture::new()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + write_service_binding(&world)?; + corrupt_history(&world)?; + + // Whether the command was consumed is unknown, so it is not acted on. + assert!(!world.engine.freeze_one_queued_command()?); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + + // Readable, it is acted on: the unknown selector is refused. + std::fs::remove_file(history_store_path(&world.state_store))?; + assert!(world.engine.freeze_one_queued_command()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + Ok(()) + } + + #[test] + fn the_legacy_lift_owes_only_a_continuity_that_issued_an_activation() -> Result<()> { + let envelope = fixture_envelope( + FIXTURE_PRE_B1_ABORTS[0].1, + DeploymentTransaction::TYPE, + )?; + let written = lift_legacy_transaction(&envelope)?; + let owed = |transaction: &DeploymentTransaction| { + transaction + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }; + assert_eq!(owed(&written), Some(CleanupEvidence::Skipped)); + + let mut lifted = written.clone(); + cleanup_evidence::owe_legacy_continuity_projection(&mut lifted); + assert_eq!(owed(&lifted), Some(CleanupEvidence::Pending)); + + // A deployment published its own key and recorded what it owed. + let mut deploy = written.clone(); + deploy.command_kind = CommandKind::Deploy; + cleanup_evidence::owe_legacy_continuity_projection(&mut deploy); + assert_eq!(owed(&deploy), Some(CleanupEvidence::Skipped)); + + // A continuity that issued nothing owes nothing. + let mut unissued = written; + unissued.activation = None; + cleanup_evidence::owe_legacy_continuity_projection(&mut unissued); + assert_eq!(owed(&unissued), Some(CleanupEvidence::Skipped)); + Ok(()) + } + + // ---- operator incidents ---------------------------------------------- + + fn incident_path(world: &EngineFixture) -> PathBuf { + world.root.join("incidents.cc") + } + + fn incidents_of(world: &EngineFixture) -> Result> { + IncidentStore::new(&incident_path(world)).read() + } + + /// The control store never holds an incident: it would stop every older + /// binary that reads it. + fn assert_control_store_holds_no_incident(world: &EngineFixture) -> Result<()> { + ControlSnapshot::read(&world.state_store)?; + for envelope in SingleFileMessagePackBackingStore::new(&world.state_store) + .pull_all_read_only_snapshot()? + { + assert_ne!(envelope.r#type, "idunn.operator_incident"); + } + Ok(()) + } + + /// Six restarts inside the window, the last long enough ago that its wait is + /// over: only the ceiling holds. + fn spend_the_window(world: &EngineFixture) -> Result<()> { + set_meters( + world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + ) + } + + /// One entry has left the window, and the newest restart is recent enough + /// that continuity is still waiting: exhaustion is over, nothing restarts. + fn slide_the_window(world: &EngineFixture) -> Result<()> { + set_meters( + world, + &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 10_000])?, + ) + } + + #[test] + fn an_exhausted_target_opens_one_incident_across_ticks_and_restarts() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + assert!(!incident_path(&world).exists()); + + // The first pass over the dead target starts its wait: progress, no mint. + assert!(world.engine.supervise_one_admitted_generation()?); + let opened = incidents_of(&world)?; + assert_eq!(opened.len(), 1); + assert_eq!(opened[0].subject, "service"); + assert_eq!(opened[0].condition, IncidentCondition::ContinuityExhausted); + assert!(opened[0].is_open()); + let written = std::fs::read(incident_path(&world))?; + + for _ in 0..3 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + // An Idunn restart forgets its in-memory trace; the durable record + // keeps the decision from being written again. + let restarted = + Engine::open_with_systemd_workload(world.engine.options.clone(), workload.clone())?; + assert!(restarted.fault_reports.lock().unwrap().is_empty()); + assert!(!restarted.supervise_one_admitted_generation()?); + assert_eq!(std::fs::read(incident_path(&world))?, written); + assert_eq!(incidents_of(&world)?, opened); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn an_exhaustion_incident_closes_on_observed_health_not_on_the_window() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let first = incidents_of(&world)?; + assert_eq!(first.len(), 1); + + // The window slides while the workload is still dead: exhaustion is + // over, the target is not healthy, and the incident stays open. + slide_the_window(&world)?; + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(incidents_of(&world)?, first); + + // Seen running, it closes as recovered. + workload.revive(); + assert!(!world.engine.supervise_one_admitted_generation()?); + let closed = incidents_of(&world)?; + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].incident_key, first[0].incident_key); + assert!( + closed[0] + .closed_at_unix_millis + .is_some_and(|at| at >= closed[0].opened_at_unix_millis) + ); + assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + + // A later exhaustion is a new record with a new opening time. + std::thread::sleep(Duration::from_millis(5)); + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let again = incidents_of(&world)?; + assert_eq!(again.len(), 2); + let reopened = again + .iter() + .find(|record| record.is_open()) + .context("no new incident")?; + assert!(reopened.opened_at_unix_millis > first[0].opened_at_unix_millis); + assert_ne!(reopened.incident_key, first[0].incident_key); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn an_incident_whose_target_is_no_longer_admitted_closes() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert!(incidents_of(&world)?[0].is_open()); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone(); + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&[admitted])? + ); + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::NoLongerAdmitted)); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn a_tick_retires_incidents_closed_past_retention() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let store = IncidentStore::new(&incident_path(&world)); + let condition = IncidentCondition::ContinuityExhausted; + let long_ago = now_millis()? - INCIDENT_RETENTION_MILLIS - 10_000; + assert!(store.open(condition, "gone", long_ago)?); + assert!(store.close(condition, "gone", CloseReason::Recovered, long_ago + 1)?); + assert!(store.open(condition, "recent", now_millis()?)?); + assert!(store.close(condition, "recent", CloseReason::Recovered, now_millis()?)?); + + world.engine.supervise_one_admitted_generation()?; + let live = store.read()?; + assert_eq!(live.len(), 1); + assert_eq!(live[0].subject, "recent"); + assert!(world.root.join("incident-history.cc").exists()); + Ok(()) + } + + /// The same exhaustion and recovery sequence, reduced to what continuity + /// decided: each tick's return, and the transactions and restart log it left. + fn continuity_decisions( + world: &EngineFixture, + engine: &Engine, + workload: &SwitchWorkload, + ) -> Result> { + let mut decided = Vec::new(); + let tick = |decided: &mut Vec<(bool, usize, usize)>| -> Result<()> { + let moved = engine.supervise_one_admitted_generation()?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + decided.push(( + moved, + snapshot.transactions.len(), + snapshot.supervision_or_new("service").continuity_restarts.len(), + )); + Ok(()) + }; + workload.kill(); + spend_the_window(world)?; + tick(&mut decided)?; + tick(&mut decided)?; + slide_the_window(world)?; + tick(&mut decided)?; + workload.revive(); + tick(&mut decided)?; + workload.kill(); + spend_the_window(world)?; + tick(&mut decided)?; + Ok(decided) + } + + #[test] + fn an_unwritable_incident_store_changes_nothing_continuity_decides() -> Result<()> { + const CANARY: &str = "CANARY-6b1d-secret-segment"; + let healthy_workload = SwitchWorkload::new(); + let healthy = EngineFixture::with_workload(healthy_workload.clone())?; + admit_incumbent(&healthy)?; + let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; + assert!(incidents_of(&healthy)?.iter().any(|record| record.is_open())); + + // Not permission-dependent: a store path beneath a regular file cannot + // be created for any uid, root included. + let broken_workload = SwitchWorkload::new(); + let mut broken = EngineFixture::with_workload(broken_workload.clone())?; + admit_incumbent(&broken)?; + let blocker = broken.root.join(CANARY); + std::fs::write(&blocker, b"a file, not a directory")?; + let mut options = broken.engine.options.clone(); + options.incident_store = blocker.join("incidents.cc"); + broken.engine = Engine::open_with_systemd_workload(options, broken_workload.clone())?; + + let actual = continuity_decisions(&broken, &broken.engine, &broken_workload)?; + assert_eq!(actual, expected, "a broken incident store changed a continuity decision"); + assert!(!blocker.join("incidents.cc").exists()); + + // The fault is traced once, under its operation, and echoes no path. + let reports = broken.engine.fault_reports.lock().unwrap(); + let faults = reports + .iter() + .filter(|(key, _)| key.starts_with("incident-store:")) + .collect::>(); + assert_eq!( + faults.len(), + 1, + "{:?}", + faults.iter().map(|(key, _)| key).collect::>() + ); + let message = faults[0].1.last.lock().unwrap().clone().context("fault cleared")?; + assert!(message.contains("open an operator incident"), "{message}"); + assert!(!message.contains(CANARY), "{message}"); + drop(reports); + assert_control_store_holds_no_incident(&broken) + } + + #[test] + fn status_renders_incidents_and_survives_an_unreadable_store() -> Result<()> { + const CANARY: &str = "CANARY-90c2-secret-segment"; + let temp = TempDir::new()?; + let now = now_millis()?; + let mut snapshot = ControlSnapshot::default(); + let meters = TargetSupervision::new("service"); + snapshot.targets.push(Stored { + envelope: target_supervision_envelope(&meters, now)?, + value: meters, + }); + + let readable = temp.path().join("incidents.cc"); + IncidentStore::new(&readable).open(IncidentCondition::ContinuityExhausted, "service", now)?; + let lines = render_targets(&snapshot, &readable, now); + assert!(lines[0].starts_with("target service "), "{lines:?}"); + assert_eq!( + lines.last().map(String::as_str), + Some(format!(" incident continuity-exhausted opened-at {now}").as_str()) + ); + + let unreadable = temp.path().join(CANARY); + std::fs::write(&unreadable, b"not a cultcache file")?; + let lines = render_targets(&snapshot, &unreadable, now); + assert!(lines[0].starts_with("target service "), "{lines:?}"); + let last = lines.last().context("no lines")?; + assert!(last.starts_with("incidents unreadable: "), "{last}"); + assert!(!lines.iter().any(|line| line.contains(CANARY)), "{lines:?}"); + Ok(()) + } + + #[test] + fn the_incident_store_is_named_by_serve_and_status() { + let Command::Serve(options) = + parse(["serve", "--incident-store", "/tmp/i.cc"].into_iter().map(str::to_owned)) + .unwrap() + else { + panic!("expected serve") + }; + assert_eq!(options.incident_store, PathBuf::from("/tmp/i.cc")); + assert!(usage().contains("idunn status [--state-store PATH] [--incident-store PATH]")); + let Command::Status { incident_store, .. } = + parse(["status", "--incident-store", "/tmp/j.cc"].into_iter().map(str::to_owned)) + .unwrap() + else { + panic!("expected status") + }; + assert_eq!(incident_store, PathBuf::from("/tmp/j.cc")); + } + + /// A standing fault is traced once. A key is cleared only by an operation + /// that succeeded, so another site's short-circuit must not end it: a + /// target that is running but still exhausted closes nothing, and that + /// used to clear the reconcile pass's fault and print it again every pass. + #[test] + fn a_failing_reconcile_is_traced_once_while_a_running_target_stays_exhausted() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + std::fs::write(incident_path(&world), b"not an incident store")?; + spend_the_window(&world)?; + + for pass in 0..6 { + world.engine.supervise_one_admitted_generation()?; + let reports = world.engine.fault_reports.lock().unwrap(); + assert!( + reports.contains_key("incident-store:close:*"), + "pass {pass}: the standing reconcile fault was cleared, so it prints again" + ); + } + Ok(()) + } + + #[test] + fn a_release_that_runs_one_pass_after_each_restart_keeps_one_incident() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + + // Each cycle: the restarts are spent, the release dies, is seen running + // for one pass (its last restart took), and dies again. + for _ in 0..4 { + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1, "a flapping release opened another incident"); + assert!(records[0].is_open(), "running while still exhausted closed the incident"); + } + + // Real recovery: running, and the window no longer holds the restarts. + slide_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn a_target_leaving_admission_closes_only_its_own_incident() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let service = admit_incumbent(&world)?; + let store = IncidentStore::new(&incident_path(&world)); + let condition = IncidentCondition::ContinuityExhausted; + let now = now_millis()?; + for subject in ["service", "neighbour", "gone"] { + assert!(store.open(condition, subject, now)?); + } + + // Two targets are admitted: service, and a neighbour. Only the target + // that is not admitted has its incident closed. + let mut snapshot = ControlSnapshot::read(&world.state_store)?; + let envelope = snapshot + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone(); + snapshot.admitted.push(Stored { + envelope, + value: AdmittedGeneration { + target: "neighbour".into(), + ..service + }, + }); + world.engine.reconcile_incidents(&snapshot); + + let closed = |subject: &str| -> Result> { + Ok(incidents_of(&world)? + .into_iter() + .find(|record| record.subject == subject) + .context("no such incident")? + .close_reason) + }; + assert_eq!(closed("service")?, None); + assert_eq!(closed("neighbour")?, None); + assert_eq!(closed("gone")?, Some(CloseReason::NoLongerAdmitted)); + Ok(()) + } + + #[cfg(target_os = "linux")] + #[test] + fn a_held_incident_lock_changes_nothing_continuity_decides() -> Result<()> { + use crate::incident::tests::{STUCK_AFTER, hold_lock_of, run_while_held}; + + let healthy_workload = SwitchWorkload::new(); + let healthy = EngineFixture::with_workload(healthy_workload.clone())?; + admit_incumbent(&healthy)?; + let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; + + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let path = incident_path(&world); + // Three ticks reach the open of an exhausted target, one exchange each. + let actual = run_while_held(hold_lock_of(&path), 3, || { + continuity_decisions(&world, &world.engine, &workload) + })?; + assert_eq!(actual, expected, "a held incident lock changed a continuity decision"); + assert!(!path.exists(), "a contended write landed"); + + // Every refused write is a traced fault naming no path. + { + let reports = world.engine.fault_reports.lock().unwrap(); + let faults = reports + .iter() + .filter(|(key, _)| key.starts_with("incident-store:")) + .map(|(_, report)| report.last.lock().unwrap().clone().unwrap_or_default()) + .collect::>(); + assert!(!faults.is_empty()); + let root = world.root.display().to_string(); + assert!(faults.iter().all(|fault| !fault.contains(&root)), "{faults:?}"); + } + + // The lock is free again: the next pass retries and the incident lands. + // (A child another test forked while it was held can keep it a moment.) + let deadline = std::time::Instant::now() + STUCK_AFTER; + while !incidents_of(&world)?.iter().any(|record| record.is_open()) { + assert!(std::time::Instant::now() < deadline, "the incident never landed"); + world.engine.supervise_one_admitted_generation()?; + std::thread::sleep(Duration::from_millis(20)); + } + assert_control_store_holds_no_incident(&world) + } + + /// A publish that could not land is not done, at the tick: the demotion + /// continuity needs before a restart is one exchange on the topology lock, + /// and another holder of it defers the restart instead of waiting for it. + #[cfg(target_os = "linux")] + #[test] + fn a_held_topology_lock_defers_continuity_and_mints_nothing() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let store = world.engine.options.topology_store.clone(); + let projected = projected_under(&world, &incumbent.expected)?; + assert!(projected.contains(&IdunnRuntimeActivationRecord::TYPE.to_owned())); + let before = std::fs::read(&store)?; + workload.kill(); + + // One exchange: the demotion. The window is not spent, so continuity + // would restart if the projection could be demoted. + let moved = run_while_held(hold_lock_of(&store), 1, || { + world.engine.supervise_one_admitted_generation() + })?; + assert!(moved, "the deferral is a write the tick made"); + assert_eq!(std::fs::read(&store)?, before, "a contended publish changed the projection"); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!(snapshot.transactions.is_empty(), "continuity minted over a projection it could not demote"); + let recorded = snapshot.supervision_for("service").context("no deferral recorded")?.value.clone(); + assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); + let reason = recorded.continuity_deferral_reason.clone().context("no deferral reason")?; + assert!(reason.contains("locked by another holder"), "{reason}"); + assert!(!reason.contains(&world.root.display().to_string()), "{reason}"); + + // The holder is gone and the deferral has run out: the demotion lands + // and the restart proceeds. + let mut ran_out = recorded; + ran_out.continuity_deferred_until = Some(1); + set_meters(&world, &ran_out)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + /// A deployment phase records a publication digest only after the publish + /// returned: the digest is the claim that it landed. + #[cfg(target_os = "linux")] + #[test] + fn a_contended_expected_publish_is_not_recorded_as_published() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Warming)?; + let stored = resident(&world)?; + let mut starting = seeded.clone(); + starting.phase = DeploymentPhase::Starting; + starting.expected_publication_sha256 = None; + starting.activation = None; + starting.workload = None; + starting.activation_publication_sha256 = None; + starting.warming = None; + replace_transaction(&world.state_store, &stored, &starting)?; + let store = world.engine.options.topology_store.clone(); + let stored = resident(&world)?; + + let error = run_while_held(hold_lock_of(&store), 1, || { + world.engine.advance_transaction(&stored) + }) + .expect_err("a contended publish must not advance the phase"); + assert!(format!("{error:#}").contains("locked by another holder"), "{error:#}"); + assert!(!store.exists(), "a contended publish wrote the projection"); + let after = resident(&world)?.value; + assert_eq!(after.phase, DeploymentPhase::Starting); + assert_eq!(after.expected_publication_sha256, None); + + // Free again: the same step publishes and records what it published. + world.engine.advance_transaction(&resident(&world)?)?; + let expected = seeded.expected.context("seeded transaction has no Expected")?; + assert_eq!( + resident(&world)?.value.expected_publication_sha256, + Some(expected.canonical_sha256()?) + ); + Ok(()) + } + + /// A Continuity transaction parked in Starting with nothing published yet, + /// plus the stored record the tick will see. + #[cfg(target_os = "linux")] + fn parked_in_starting(world: &EngineFixture) -> Result { + let seeded = transaction_at(world, DeploymentPhase::Warming)?; + let stored = resident(world)?; + let mut starting = seeded.clone(); + starting.phase = DeploymentPhase::Starting; + starting.expected_publication_sha256 = None; + starting.activation = None; + starting.workload = None; + starting.activation_publication_sha256 = None; + starting.warming = None; + replace_transaction(&world.state_store, &stored, &starting)?; + Ok(seeded) + } + + /// A publish that lost to a held lock is transient: before the fence the + /// transaction stays in its phase with the error recorded, no abort is + /// begun, and the next tick after the release lands the publish. A + /// continuity restart is a Continuity transaction on this same path, so + /// its attempt is minted once and a contended publish spends no second one. + #[cfg(target_os = "linux")] + #[test] + fn a_contended_starting_publish_is_retried_on_a_later_tick_not_aborted() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = parked_in_starting(&world)?; + let mut meters = TargetSupervision::new("service"); + meters.continuity_restarts = vec![now_millis()?]; + set_meters(&world, &meters)?; + let store = world.engine.options.topology_store.clone(); + + for tick in 0..2 { + run_while_held(hold_lock_of(&store), 1, || world.engine.resume_one_transaction())?; + world.engine.resume_backoff.lock().unwrap().clear(); + let held = resident(&world)?.value; + assert_eq!(held.phase, DeploymentPhase::Starting, "tick {tick}"); + assert!(held.pre_fencing_abort.is_none(), "tick {tick}: a held lock aborted the transaction"); + assert!(!held.is_terminal(), "tick {tick}"); + assert_eq!(held.expected_publication_sha256, None, "tick {tick}"); + let error = held.last_error.context("a contended publish left no last_error")?; + assert!(error.contains("locked by another holder"), "{error}"); + assert!(!store.exists(), "a contended publish wrote the projection"); + } + + // Released: the next tick lands the publish and records its digest. + world.engine.resume_one_transaction()?; + let landed = resident(&world)?.value; + assert_eq!(landed.phase, DeploymentPhase::Starting); + assert!(landed.pre_fencing_abort.is_none()); + assert_eq!( + landed.expected_publication_sha256, + Some(seeded.expected.context("seeded transaction has no Expected")?.canonical_sha256()?) + ); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + /// Only contention is transient: a publish that fails for any other reason + /// still ends the transaction before the fence, as it always did. + #[cfg(target_os = "linux")] + #[test] + fn a_non_contention_publish_error_still_aborts_before_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + parked_in_starting(&world)?; + // A directory where the store should be: the snapshot cannot be read. + std::fs::create_dir_all(&world.engine.options.topology_store)?; + world.engine.resume_one_transaction()?; + let after = resident(&world)?.value; + assert!(after.pre_fencing_abort.is_some(), "a non-contention error did not abort"); + assert!( + !after.last_error.as_deref().unwrap_or_default().contains("locked by another holder"), + "{:?}", + after.last_error + ); + Ok(()) + } + + #[test] + fn status_prints_an_open_incident_under_its_target() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let opened = incidents_of(&world)?[0].opened_at_unix_millis; + + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &incident_path(&world), None)?; + let printed = String::from_utf8(printed)?; + let target = printed.find("target service ").context("no target line")?; + let incident = printed + .find(&format!(" incident continuity-exhausted opened-at {opened}")) + .context("status printed no incident")?; + assert!(target < incident, "{printed}"); + Ok(()) + } + + #[test] + fn serve_ticks_until_told_to_stop() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + let mut options = world.engine.options.clone(); + options.poll_millis = 1; + let engine = Engine::open_with_systemd_workload(options, workload.clone())?; + + let mut asked = 0; + serve_while(&engine, || { + asked += 1; + asked <= 3 + })?; + assert_eq!(asked, 4); + assert!(incidents_of(&world)?.iter().any(|record| record.is_open()), "no tick ran"); + Ok(()) + } + + #[test] + fn run_dispatches_serve_and_status() { + let temp = TempDir::new().unwrap(); + let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); + // A control store beneath a regular file: serve's boot cannot make it, + // and a serve that returned at once would say Ok instead. + let file = temp.path().join("not-a-directory"); + std::fs::write(&file, b"a file").unwrap(); + let store = file.join("control.cc"); + assert!( + run(args(&["serve", "--state-store", store.to_str().unwrap()]).into_iter()).is_err() + ); + // A control store that is not one: status reads it and refuses. + let junk = temp.path().join("junk.cc"); + std::fs::write(&junk, b"not a cultcache file").unwrap(); + assert!( + run(args(&["status", "--state-store", junk.to_str().unwrap()]).into_iter()).is_err() + ); + } + + /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment + /// waiting before Fencing. Continuity never stops and never yields a + /// deployment (budget-never-stop, handover-phase-split), so the window + /// sliding must not turn the deployment into a casualty. + #[test] + fn a_deployment_before_fencing_over_a_dead_incumbent_that_paged_is_never_yielded_and_lands() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + set_meters( + &world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + world.engine.supervise_one_admitted_generation()?; + // The oldest leaves the window: base would now yield the deployment. + set_meters( + &world, + &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + for _ in 0..3 { + let _ = world.engine.supervise_one_admitted_generation()?; + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + assert!( + after.pre_fencing_abort.is_none(), + "the deployment was yielded: {:?}", + after.pre_fencing_abort + ); + assert_eq!( + continuity_mints(&world)?, + 0, + "continuity minted beside a pre-fencing deployment" + ); + } + workload.revive(); + odin_reports_ready(&world, &redeploy, seeded_sequence(Some(&incumbent)) + 1)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + let landed = latest(&world)?; + assert_eq!(landed.transaction_id, redeploy.transaction_id); + assert_eq!(landed.phase, DeploymentPhase::Complete); + assert!(landed.pre_fencing_abort.is_none()); + Ok(()) + } + + /// The incumbent was admitted this long ago, so an anchor written "in the + /// past" is not older than its generation. + fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { + edit_incumbent(world, |generation| generation.admitted_at_unix_millis -= millis)?; + Ok(()) + } + + /// Put the target's backoff anchor `ago` milliseconds back. + fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { + let mut meters = meters_of(world, "service")?; + meters.continuity_backoff_from_unix_millis = Some(now_millis()? - ago); + set_meters(world, &meters) + } + + /// Continuity transactions minted over an incumbent (the one that admitted + /// it was seeded over nothing). + fn continuity_mints(world: &EngineFixture) -> Result { + Ok(every_transaction(world)? + .iter() + .filter(|transaction| { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.is_some() + }) + .count()) + } + + #[test] + fn continuity_keeps_restarting_past_the_paging_threshold() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + // Six charged attempts in the hour: the incident pages, nothing stops. + set_meters( + &world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + let opened = incidents_of(&world)?; + assert_eq!(opened.len(), 1); + assert!(opened[0].is_open()); + + // The streak-6 wait is 160 s. + anchor_ago(&world, 159_000)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + anchor_ago(&world, 160_001)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let after = meters_of(&world, "service")?; + assert_eq!(after.continuity_restarts.len(), 6); + assert!(after.continuity_restarts.last().is_some_and(|&last| now_millis().unwrap() - last < 60_000)); + assert_eq!(after.continuity_attempt_streak, 7); + assert_eq!(after.continuity_backoff_from_unix_millis, None); + assert_eq!(incidents_of(&world)?, opened); + Ok(()) + } + + #[test] + fn continuity_backoff_is_a_capped_doubling_of_the_attempt_streak() { + let from = 10_000_000_u64; + for (streak, seconds) in [ + (1_u32, 5_u64), + (2, 10), + (3, 20), + (4, 40), + (5, 80), + (6, 160), + (7, 300), + (8, 300), + (9, 300), + ] { + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = streak; + meters.continuity_backoff_from_unix_millis = Some(from); + // The paging log disagrees with the streak on purpose: the wait + // is a function of the streak, not of the window's count. + meters.continuity_restarts = vec![from - 10; if streak == 1 { 6 } else { 2 }]; + let due = from + seconds * 1_000; + assert_eq!(meters.backoff_due(0), Some(due), "streak {streak}"); + assert_eq!( + meters.continuity_backoff(due - 1, 0), + ContinuityBackoff::Waiting, + "streak {streak}" + ); + assert_eq!( + meters.continuity_backoff(due, 0), + ContinuityBackoff::Due { streak }, + "streak {streak}" + ); + } + // No streak, no wait. + let mut fresh = TargetSupervision::new("service"); + fresh.continuity_backoff_from_unix_millis = Some(from); + assert_eq!(fresh.continuity_backoff(from, 0), ContinuityBackoff::Due { streak: 0 }); + } + + #[test] + fn a_sustained_run_resets_the_streak_and_only_a_sustained_run() { + let now = 100_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 7; + meters.continuity_restarts = vec![1_000]; + // Lived the sustained time, and no attempt since admission. + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), + ContinuityBackoff::Due { streak: 0 } + ); + // A moment short of it. + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + 10_000), + ContinuityBackoff::Begin + ); + // Attempts after admission failed on this generation: it has not run well. + meters.continuity_restarts = vec![now - 100_000]; + assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); + // Never attempted over this log, long lived: reset. + meters.continuity_restarts = Vec::new(); + assert_eq!( + meters.continuity_backoff(now, now - 700_000), + ContinuityBackoff::Due { streak: 0 } + ); + assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); + // An anchor older than its generation decides nothing. + meters.continuity_backoff_from_unix_millis = Some(now - 800_000); + assert_eq!(meters.backoff_due(now - 700_000), None); + assert_eq!( + meters.continuity_backoff(now, now - 700_000), + ContinuityBackoff::Due { streak: 0 } + ); + assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); + // Streak zero with no history is simply due. + let fresh = TargetSupervision::new("service"); + assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); + } + + #[test] + fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + + // The last attempt was long ago, but the wait starts now: a wait + // anchored on the mint would already be over. + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // Streak 3 waits 20 s. + anchor_ago(&world, 19_000)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + anchor_ago(&world, 20_001)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 4); + Ok(()) + } + + #[test] + fn a_queued_deployment_freezes_in_the_gap_after_a_continuity_attempt() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[3_000_000])?; + meters.continuity_attempt_streak = 2; + set_meters(&world, &meters)?; + queue_declaring_redeploy(&world, "up-service-queued")?; + + for _ in 0..4 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-queued"), + "the queued deployment never froze" + ); + assert_eq!(continuity_mints(&world)?, 0); + for _ in 0..3 { + let _ = world.engine.run_scheduler_tick(); + assert_eq!(continuity_mints(&world)?, 0); + let aborts = every_transaction(&world)? + .into_iter() + .filter_map(|transaction| transaction.pre_fencing_abort) + .collect::>(); + // The deployment may fail on its own account (a root test runner + // cannot resolve its source); continuity must never be the reason. + assert!( + aborts.iter().all(|abort| !abort.error.contains("continuity")), + "the deployment was yielded: {aborts:?}" + ); + } + Ok(()) + } + + #[test] + fn a_unit_lost_to_a_host_reboot_is_rematerialized_once_with_no_backoff_step() -> Result<()> { + let fresh_world = |workload: &Arc| -> Result { + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + meters.continuity_backoff_from_unix_millis = Some(now_millis()?); + set_meters(&world, &meters)?; + Ok(world) + }; + let workload = SwitchWorkload::new(); + let world = fresh_world(&workload)?; + let before = meters_of(&world, "service")?; + + // Control: dead but not rebooted, the ordinary 20 s wait holds. + workload.kill(); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // The host rebooted under the unit: one mint, charged to nothing. + workload.reboot(); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let after = meters_of(&world, "service")?; + assert_eq!(after.continuity_restarts, before.continuity_restarts); + assert_eq!(after.continuity_attempt_streak, 3); + assert!(after.continuity_rematerialized_at_unix_millis.is_some()); + assert_eq!(after.continuity_backoff_from_unix_millis, None); + + // The bound is spent: the next death of this generation waits like any other. + clear_live_transactions(&world)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // Control: a deferred projection spaces a re-materialization too. + let workload = SwitchWorkload::new(); + let world = fresh_world(&workload)?; + let mut deferred = meters_of(&world, "service")?; + deferred.defer_continuity(now_millis()?, "demotion failed"); + set_meters(&world, &deferred)?; + workload.reboot(); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + Ok(()) + } + + fn supervision_envelope_raw(payload: Vec, schema: &str) -> Result { + Ok(CultCacheEnvelope { + key: "service".into(), + r#type: TargetSupervision::TYPE.into(), + payload, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(schema.into()), + }) + } + + #[test] + fn a_v1_target_supervision_record_is_lifted_and_migrated_once() -> Result<()> { + let world = EngineFixture::new()?; + let meters = restarted_ago(&[200_000])?; + // A record as every one on a running host was written: seven keys, v1. + let payload = rmp_serde::to_vec(&( + TARGET_SUPERVISION_SCHEMA_V1, + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + supervision_envelope_raw(payload, TARGET_SUPERVISION_SCHEMA_V1)?, + )?; + let read = meters_of(&world, "service")?; + assert_eq!(read.continuity_restarts, meters.continuity_restarts); + assert_eq!(read.continuity_attempt_streak, 0); + assert_eq!(read.continuity_backoff_from_unix_millis, None); + assert_eq!(read.continuity_rematerialized_at_unix_millis, None); + + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 0); + let stored = ControlSnapshot::read(&world.state_store)?; + let envelope = &stored.supervision_for("service").unwrap().envelope; + assert_eq!(envelope.schema_id.as_deref(), Some(TARGET_SUPERVISION_SCHEMA)); + assert_eq!(meters_of(&world, "service")?.continuity_restarts, meters.continuity_restarts); + + let mut anchored = TargetSupervision::new("service"); + anchored.continuity_backoff_from_unix_millis = Some(0); + assert!(anchored.validate().is_err()); + anchored.continuity_backoff_from_unix_millis = None; + anchored.continuity_rematerialized_at_unix_millis = Some(0); + assert!(anchored.validate().is_err()); + Ok(()) + } + + #[test] + fn one_target_supervision_schema_id_names_one_shape() -> Result<()> { + let meters = restarted_ago(&[200_000])?; + let ten = rmp_serde::to_vec(&meters)?; + let seven = rmp_serde::to_vec(&( + TARGET_SUPERVISION_SCHEMA, + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + // The current id refuses the old shape and the old id refuses the new one. + assert!( + read_supervision_record(&supervision_envelope_raw(seven, TARGET_SUPERVISION_SCHEMA)?) + .is_err() + ); + assert!( + read_supervision_record(&supervision_envelope_raw( + ten.clone(), + TARGET_SUPERVISION_SCHEMA_V1 + )?) + .is_err() + ); + assert_eq!( + read_supervision_record(&supervision_envelope_raw(ten, TARGET_SUPERVISION_SCHEMA)?)?, + meters + ); + Ok(()) + } + + #[test] + fn continuity_keeps_minting_at_every_streak_and_log_length() -> Result<()> { + // No stop at any streak, however long, with the paging log full or not. + let full = [3_000_000_u64, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000]; + for streak in 1..=12_u32 { + for log in [&[1_000_000_u64][..], &full[..]] { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(log)?; + meters.continuity_attempt_streak = streak; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the wait begins"); + anchor_ago(&world, 300_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1, "streak {streak} log {} stopped", log.len()); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, streak + 1); + } + } + Ok(()) + } + + #[test] + fn a_mint_after_a_sustained_run_stores_the_reset_streak() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + workload.kill(); + let mut meters = restarted_ago(&[3_000_000])?; + meters.continuity_attempt_streak = 7; + set_meters(&world, &meters)?; + // Ran 700 s since the last attempt: no wait, and the streak restarts at 1. + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + + #[test] + fn a_re_materialization_counts_as_the_last_attempt_for_the_sustained_reset() { + let admitted_at = 10_000_000_u64; + let now = admitted_at + 700_000; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 5; + meters.continuity_restarts = vec![admitted_at - 1_000]; + // Before admission: a sustained run, reset. + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at - 1); + assert_eq!( + meters.continuity_backoff(now, admitted_at), + ContinuityBackoff::Due { streak: 0 } + ); + // After admission: that attempt failed on this generation. + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at + 1); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); + // Even with no streak to step: the re-materialization is an attempt, + // and the next one waits the base step from its anchor. + meters.continuity_attempt_streak = 0; + meters.continuity_restarts = Vec::new(); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); + meters.continuity_backoff_from_unix_millis = Some(now); + assert_eq!(meters.backoff_due(admitted_at), Some(now + CONTINUITY_RESTART_BACKOFF_MILLIS)); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Waiting); + assert_eq!( + meters.continuity_backoff(now + CONTINUITY_RESTART_BACKOFF_MILLIS, admitted_at), + ContinuityBackoff::Due { streak: 0 } + ); + meters.continuity_rematerialized_at_unix_millis = None; + assert_eq!(meters.backoff_due(admitted_at), Some(now)); + } + + mod after_a_reboot { + use super::*; + + fn resident_continuity(world: &EngineFixture) -> Result { + Ok(ControlSnapshot::read(&world.state_store)? + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .count()) + } + + /// The attempt failed and was retired: drop continuity transactions + /// and their commands only. + fn drop_continuity(world: &EngineFixture) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let ids = snapshot + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .map(|s| s.value.command_id.clone()) + .collect::>(); + let envelopes = snapshot + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .map(|s| s.envelope.clone()) + .chain( + snapshot + .commands + .iter() + .filter(|s| ids.contains(&s.value.command_id)) + .map(|s| s.envelope.clone()), + ) + .collect::>(); + if !envelopes.is_empty() { + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&envelopes)? + ); + } + Ok(()) + } + + /// (attempts minted on tick 1, on tick 2, whether the queued deploy froze by then). + fn gap_run(reboot: bool) -> Result<(usize, usize, bool)> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + if reboot { + workload.reboot(); + } else { + workload.kill(); + } + queue_declaring_redeploy(&world, "up-gap")?; + // One tick, in the tick's order: supervise, then freeze. + world.engine.supervise_one_admitted_generation()?; + let first = resident_continuity(&world)?; + world.engine.freeze_one_queued_command()?; + // That attempt failed and was retired. + drop_continuity(&world)?; + world.engine.supervise_one_admitted_generation()?; + let second = resident_continuity(&world)?; + world.engine.freeze_one_queued_command()?; + let frozen = every_transaction(&world)?.iter().any(|t| t.command_id == "up-gap"); + Ok((first, second, frozen)) + } + + #[test] + fn a_failed_re_materialization_is_followed_by_a_gap_a_queued_deployment_can_take( + ) -> Result<()> { + // Control: an ordinary attempt after a sustained run leaves a gap. + assert_eq!(gap_run(false)?, (1, 0, true)); + // The re-materialization is an attempt too: the next pass waits. + assert_eq!(gap_run(true)?, (1, 0, true)); + Ok(()) + } + + #[test] + fn a_running_pass_ends_the_wait_so_a_later_death_after_a_sustained_run_resets( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis, None); + + // It runs on for days, then dies: no wait, and the streak restarts. + age_incumbent(&world, 200_000)?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + } + + /// A source that resolves one fixed recipe and never freezes anything. + struct FixedSource { + recipe: String, + } + + impl SourcePort for FixedSource { + fn resolve( + &self, + _binding: &OperatorBinding, + _resolution_id: &str, + _selected_at_unix_millis: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { + bail!("this source freezes nothing") + } + + fn observe_frozen( + &self, + _: &CompiledDeploymentPlan, + _: &FrozenSourceReceipt, + ) -> Result { + bail!("this source freezes nothing") + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// A Deploy transaction for an unrouted target, still Sealing, whose + /// source resolves `recipe`. One scheduler tick admits its plan or refuses it. + fn sealing_world(recipe: &str) -> Result { + use crate::deployment_plan::tests::BINDING; + let mut world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (binding_head, binding_tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + world.engine.source = Arc::new(FixedSource { + recipe: recipe.to_owned(), + }); + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )? + ); + Ok(world) + } + + fn unrouted_recipe() -> String { + crate::deployment_plan::tests::RECIPE + .split("[[dependencies]]") + .next() + .expect("the recipe has a body") + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ) + } + + #[test] + fn admission_refuses_a_recipe_that_declares_no_way_to_prove_readiness() -> Result<()> { + let recipe = unrouted_recipe(); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let refused = latest(&world)?; + assert!(refused.plan.is_none(), "the refused recipe was planned"); + assert!(refused.frozen_source.is_none(), "the refused recipe was frozen"); + let abort = refused + .pre_fencing_abort + .context("a recipe with no declared proof of readiness must be refused")?; + assert!( + abort.error.contains("target service declares no way to prove readiness"), + "{}", + abort.error + ); + Ok(()) + } + + #[test] + fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let planned = latest(&world)?; + assert!(planned.pre_fencing_abort.is_none(), "{:?}", planned.pre_fencing_abort); + assert_eq!( + planned.plan.as_ref().map(CompiledDeploymentPlan::readiness_class).transpose()?, + Some(ReadinessClass::OdinSelf) + ); + Ok(()) + } + + // --------------------------------------------------------------------- + // A held record is reported and never decided for. Supervision mints no + // continuity over a held generation, and nothing yields to a held record. + // --------------------------------------------------------------------- + + /// Rewrite the world's admitted generation into what a pre-B3 binary left + /// behind: no Odin declaration, admitted on Odin receipts. + fn undeclare_incumbent(world: &EngineFixture) -> Result { + let now = now_millis()?; + let mut failure = None; + let next = edit_incumbent(world, |generation| { + generation.expected.dependencies.clear(); + match IdunnRuntimeActivationLaunch::issue( + &generation.expected, + generation.activation.runtime_instance_id.clone(), + now, + &world.engine.idunn_signer, + ) { + Ok(launch) => generation.activation = launch.activation().clone(), + Err(error) => failure = Some(error), + } + })?; + if let Some(error) = failure { + return Err(error); + } + ControlSnapshot::read(&world.state_store).context("store with the undeclared generation")?; + assert!(matches!(next.readiness(), Err(ReadinessDisagreement::Undeclared(_)))); + Ok(next) + } + + /// Queue the operator's declaring redeploy of `service`. + fn queue_declaring_redeploy(world: &EngineFixture, command_id: &str) -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: command_id.into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now_millis()?, + }; + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, command.requested_at_unix_millis)?], + )?); + Ok(()) + } + + /// Every transaction the store knows: live, then archived. + fn every_transaction(world: &EngineFixture) -> Result> { + let mut all = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .map(|stored| stored.value) + .collect::>(); + all.extend(read_history_transactions(&world.state_store)); + Ok(all) + } + + fn is_continuity_over(transaction: &DeploymentTransaction, held: &AdmittedGeneration) -> bool { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.as_deref() == Some(held.generation_id.as_str()) + } + + #[test] + fn a_held_generation_that_dies_mints_no_continuity_and_leaves_its_target_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + world.engine.validate_durable_authority(&ControlSnapshot::read(&world.state_store)?)?; + + workload.kill(); + for _ in 0..4 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + assert!( + ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), + "supervision minted a transaction over a held generation" + ); + // Reported, once per state change: the same text is not offered again. + let reports = world.engine.fault_reports.lock().unwrap(); + let down = reports.get("generation-down:service").context("the dead hold is not reported")?; + let last = down.last.lock().unwrap().clone(); + assert!(last.is_some()); + assert!(down.offer(last).is_none()); + drop(reports); + + // The target is free: the declaring redeploy freezes and is not + // displaced by a continuity. + queue_declaring_redeploy(&world, "up-service-declared")?; + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held generation" + ); + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-declared"), + "the declaring redeploy never froze" + ); + Ok(()) + } + + #[test] + fn a_declaring_redeploy_is_not_yielded_when_the_held_incumbent_dies() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; + assert_eq!(redeploy.held_disagreement(), None); + workload.kill(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held incumbent" + ); + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + assert!( + after.pre_fencing_abort.is_none(), + "the redeploy yielded: {:?}", + after.pre_fencing_abort + ); + Ok(()) + } + + /// `seeded` with its own Expected undeclared, so Idunn holds it. + fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { + let mut bare = seeded.clone(); + // Undeclare the Expected and re-derive what is bound to its digest. + let expected = bare.expected.as_mut().context("no expected")?; + expected.dependencies.clear(); + bare.expected_publication_sha256 = Some(expected.canonical_sha256()?); + let activation = IdunnRuntimeActivationLaunch::issue( + expected, + seeded.activation.as_ref().context("no activation")?.runtime_instance_id.clone(), + now_millis()?, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); + bare.activation = Some(activation); + bare.validate()?; + assert!(bare.held_disagreement().is_some()); + Ok(bare) + } + + /// A pre-fence Deploy whose own Expected is undeclared, stored in place. + fn held_predeploy( + world: &EngineFixture, + incumbent: &AdmittedGeneration, + ) -> Result { + let seeded = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(incumbent))?; + let bare = undeclared(world, &seeded)?; + replace_transaction(&world.state_store, &resident(world)?, &bare)?; + Ok(bare) + } + + #[test] + fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + // The incumbent is declared; the Deploy in front of it is the held one. + let held = held_predeploy(&world, &incumbent)?; + + workload.kill(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; + } + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, held.transaction_id); + assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); + assert!(after.completion.is_none()); + Ok(()) + } + + #[test] + fn cancel_withdraws_a_held_predeploy_and_leaves_the_incumbent_untouched() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let held = held_predeploy(&world, &incumbent)?; + let before = incumbent_envelope(&world)?; + + cancel(&world.state_store, &held.command_id, "operator")?; + let cancelling = resident(&world)?.value; + assert_eq!( + cancelling.pre_fencing_abort.as_ref().map(|abort| abort.error.as_str()), + Some("cancelled by operator") + ); + assert!(cancelling.post_fencing_abort.is_none()); + + drive(&world, |transaction| transaction.completion.is_some())?; + // History also holds the incumbent's own admission: find ours by id. + let finished = every_transaction(&world)? + .into_iter() + .find(|transaction| transaction.transaction_id == held.transaction_id) + .context("the cancelled redeploy is nowhere")?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + // The target is released and nothing of the incumbent was touched. + assert!(!finished.blocks_new_target_mutation()); + assert_eq!(incumbent_envelope(&world)?, before); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| !is_continuity_over(transaction, &incumbent)), + "cancelling the redeploy minted a continuity" + ); + Ok(()) + } + + #[test] + fn cancel_refuses_a_predeploy_that_is_not_held_and_a_held_one_past_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + assert!(seeded.held_disagreement().is_none()); + assert!(cancel(&world.state_store, &seeded.command_id, "operator").is_err()); + assert!(resident(&world)?.value.pre_fencing_abort.is_none()); + + let mut past_the_fence = undeclared(&world, &seeded)?; + past_the_fence.enter_phase(DeploymentPhase::Fencing, now_millis()?); + assert!(past_the_fence.held_disagreement().is_some()); + assert!(!cancel_is_safe_for_held_prefence(&past_the_fence)); + Ok(()) + } + + // --------------------------------------------------------------------- + // B2: the meters, the v4 lift, and what `status` shows. + // --------------------------------------------------------------------- + + #[test] + fn the_actuation_log_slides() -> Result<()> { + let window = ROUTE_ACTUATION_WINDOW_MILLIS; + let t = 1_000_000_000_u64; + let mut meters = TargetSupervision::new("service"); + for offset in 0..12 { + meters.charge_route(t + offset, RouteActuation::Forward)?; + } + let refused = meters.charge_route(t + window - 1, RouteActuation::Forward).unwrap_err(); + assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + window)); + // Only the first entry has left the window: exactly one slot reopens. + meters.charge_route(t + window, RouteActuation::Forward)?; + let refused = meters.charge_route(t + window, RouteActuation::Forward).unwrap_err(); + assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + 1 + window)); + meters.validate()?; + Ok(()) + } + + #[test] + fn a_survival_actuation_is_recorded_but_never_counted_or_refused() -> Result<()> { + let t = 1_000_000_000_u64; + let mut meters = TargetSupervision::new("service"); + // Survivals first, past the ceiling: every one is admitted and + // recorded, and none takes a Forward slot. + for offset in 0..20 { + meters.charge_route(t + offset, RouteActuation::Survival)?; + } + assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); + assert_eq!(meters.route_survivals.last(), Some(&(t + 19))); + assert!(meters.route_actuations.is_empty()); + // Twelve Forward changes interleaved with more Survivals: the twelve + // are admitted, and the thirteenth is refused with a reopen time that + // only the oldest Forward entry decides. + for offset in 0..12 { + meters.charge_route(t + 100 + 2 * offset, RouteActuation::Forward)?; + meters.charge_route(t + 101 + 2 * offset, RouteActuation::Survival)?; + } + let refused = meters.charge_route(t + 200, RouteActuation::Forward).unwrap_err(); + assert_eq!( + (refused.used, refused.reopens_at_unix_millis), + (ROUTE_ACTUATION_CEILING, t + 100 + ROUTE_ACTUATION_WINDOW_MILLIS) + ); + // A full Forward ceiling still admits and records a Survival. + meters.charge_route(t + 201, RouteActuation::Survival)?; + assert_eq!(meters.route_survivals.last(), Some(&(t + 201))); + meters.validate()?; + Ok(()) + } + + #[test] + fn a_clock_stepped_back_stalls_nothing_past_one_window() -> Result<()> { + let t = 5_000_000_000_u64; + let now = t - 86_400_000; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = vec![t; ROUTE_ACTUATION_CEILING]; + meters.continuity_restarts = vec![t; CONTINUITY_PAGING_RESTARTS]; + meters.continuity_deferred_until = Some(t); + meters.continuity_deferral_reason = Some("demotion failed".into()); + meters.validate()?; + + // Entries from the future count as now: the meters hold for one window. + let read_only = meters.clone(); + assert!(read_only.restarts_page(now)); + let refused = meters.charge_route(now, RouteActuation::Forward).unwrap_err(); + assert!(refused.reopens_at_unix_millis <= now + ROUTE_ACTUATION_WINDOW_MILLIS); + // The clamp is written, so the window runs from the step and no longer. + assert_eq!(meters.route_actuations, vec![now; ROUTE_ACTUATION_CEILING]); + assert_eq!(meters.continuity_restarts, vec![now; CONTINUITY_PAGING_RESTARTS]); + assert!(!meters.restarts_page(now + CONTINUITY_RESTART_WINDOW_MILLIS)); + meters.charge_route(now + ROUTE_ACTUATION_WINDOW_MILLIS, RouteActuation::Forward)?; + + // Waits set for the future are due, not stalled for the length of the step. + assert!(!read_only.continuity_deferred(now)); + let mut anchored = TargetSupervision::new("service"); + anchored.continuity_attempt_streak = 7; + anchored.continuity_backoff_from_unix_millis = Some(t); + assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Due { streak: 7 }); + anchored.continuity_attempt_streak = 1; + anchored.continuity_backoff_from_unix_millis = Some(now + 10_000); + assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Waiting); + let route = RouteSupervisionState { + next_challenge_at_unix_millis: Some(t), + ..RouteSupervisionState::default() + }; + assert!(!route.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); + assert!(!is_waiting(now, t, RESUME_BACKOFF_CEILING_MILLIS)); + // The same waits, set for a moment ahead, still wait. + let ahead = RouteSupervisionState { + next_challenge_at_unix_millis: Some(now + 5_000), + ..RouteSupervisionState::default() + }; + assert!(ahead.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); + Ok(()) + } + + #[test] + fn a_route_challenge_wait_widens_to_a_cap_and_a_proof_clears_it() { + let age = DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS; + let mut state = RouteSupervisionState::default(); + let mut waits = Vec::new(); + for _ in 0..8 { + state.record_failed_challenge(1_000, age); + waits.push(state.next_challenge_at_unix_millis.unwrap() - 1_000); + } + assert_eq!(&waits[..3], &[age, age * 2, age * 4]); + assert_eq!(waits[7], ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); + assert_eq!(state.degraded_since_unix_millis, Some(1_000)); + state.record_proved_challenge(9_000); + assert_eq!(state.consecutive_failures, 0); + assert_eq!(state.next_challenge_at_unix_millis, None); + assert_eq!(state.degraded_since_unix_millis, None); + } + + #[test] + fn a_v3_generation_lifts_to_v4_once_and_reencodes_canonically() -> Result<()> { + for text in [FIXTURE_GENERATION_V3_ODIN, FIXTURE_GENERATION_V3_ROUTE_PROOF] { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; + let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; + SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(envelope.clone())?; + // The v3 bytes are not v4 bytes: only the typed lift may read them. + assert!(decode_record::(&envelope).is_err()); + + assert_eq!(migrate_control_store_to_current_schema(&store)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&store)?, 0); + let snapshot = ControlSnapshot::read(&store)?; + let migrated = snapshot.admitted.first().context("the lift lost the record")?; + assert_eq!(migrated.envelope.schema_id.as_deref(), Some(ADMITTED_GENERATION_SCHEMA)); + // After the lift the canonical re-encode accepts its own bytes. + let lifted: AdmittedGeneration = decode_record(&migrated.envelope)?; + assert_eq!(lifted, migrated.value); + + // The decisions are the ones the v3 record encoded; only the retired + // fields are gone, and no meters were invented. + assert_eq!(lifted.target, legacy.target); + assert_eq!(lifted.generation_id, legacy.generation_id); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.activation, legacy.activation); + assert_eq!(lifted.workload, legacy.workload); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.ready, legacy.ready); + assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.odin_authority, legacy.odin_authority); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!( + lifted.route_supervision, + legacy.route_supervision.map(|state| RouteSupervisionState { + last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, + consecutive_failures: state.consecutive_failures, + next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, + degraded_since_unix_millis: state.degraded_since_unix_millis, + }) + ); + assert_eq!(lifted.last_error, None); + assert!(snapshot.targets.is_empty()); + } + Ok(()) + } + + #[test] + fn a_v3_generation_that_shows_written_meters_is_refused_not_dropped() -> Result<()> { + let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; + let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; + assert!(legacy.route_supervision.is_some(), "the fixture is a routed generation"); + let refused = |change: fn(&mut LegacyAdmittedGenerationV3)| -> Result { + let mut edited = legacy.clone(); + change(&mut edited); + let mut tampered = envelope.clone(); + tampered.payload = rmp_serde::to_vec(&edited)?; + Ok(error_text(read_generation_record(&tampered).map(|_| ()))) + }; + let restarts = refused(|edited| edited.continuity_backoff.attempts = 2)?; + assert!(restarts.contains("refusing to drop"), "{restarts}"); + assert!(restarts.contains(&legacy.target), "the error names the record: {restarts}"); + let actuations = + refused(|edited| edited.route_supervision.as_mut().unwrap().actuations.count = 1)?; + assert!(actuations.contains("refusing to drop"), "{actuations}"); + + // A whole store carrying one refuses to boot, naming the record. + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let mut edited = legacy.clone(); + edited.continuity_backoff.attempts = 2; + let mut tampered = envelope.clone(); + tampered.payload = rmp_serde::to_vec(&edited)?; + SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(tampered)?; + assert!(migrate_control_store_to_current_schema(&store).is_err()); + Ok(()) + } + + #[test] + fn a_v3_generation_lifts_each_route_supervision_field_to_its_own_place() -> Result<()> { + let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; + let mut legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; + // Four distinct non-default values, so a swap cannot land on itself. + legacy.route_supervision = Some(LegacyRouteSupervisionStateV3 { + last_challenge_at_unix_millis: Some(1_111), + consecutive_failures: 3, + next_challenge_at_unix_millis: Some(2_222), + degraded_since_unix_millis: Some(3_333), + actuations: LegacyActuationWindowV3::default(), + }); + let mut edited = envelope.clone(); + edited.payload = rmp_serde::to_vec(&legacy)?; + let lifted = read_generation_record(&edited)?; + assert_eq!( + lifted.route_supervision, + Some(RouteSupervisionState { + last_challenge_at_unix_millis: Some(1_111), + consecutive_failures: 3, + next_challenge_at_unix_millis: Some(2_222), + degraded_since_unix_millis: Some(3_333), + }) + ); + Ok(()) + } + + #[test] + fn status_covers_the_admitted_targets_and_the_metered_ones() -> Result<()> { + // "service" is admitted with no meters at all; "metered" has meters and + // no generation. Neither set contains the other. + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + admit_incumbent(&world)?; + assert!(ControlSnapshot::read(&world.state_store)?.targets.is_empty()); + set_meters(&world, &TargetSupervision::new("metered"))?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!( + supervised_targets(&snapshot).into_iter().collect::>(), + ["metered", "service"] + ); + Ok(()) + } + + #[test] + fn target_supervision_reads_back_and_its_key_must_be_its_target() -> Result<()> { + let world = EngineFixture::new()?; + let meters = restarted_ago(&[5_000])?; + set_meters(&world, &meters)?; + assert_eq!(meters_of(&world, "service")?, meters); + assert_eq!(ControlSnapshot::read(&world.state_store)?.targets.len(), 1); + + let mut elsewhere = target_supervision_envelope(&meters, 5)?; + elsewhere.key = "other".into(); + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent(elsewhere)?; + let error = error_text(ControlSnapshot::read(&world.state_store).map(|_| ())); + assert!(error.contains("not its target"), "{error}"); + Ok(()) + } + + #[test] + fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { + let now = 10_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = (0..12).map(|offset| now - 5_000 + offset).collect(); + meters.continuity_restarts = vec![now - 9_000, now - 4_000]; + meters.continuity_deferred_until = Some(now + 500); + meters.continuity_deferral_reason = Some("projection would not demote".into()); + meters.continuity_attempt_streak = 2; + meters.continuity_backoff_from_unix_millis = Some(now - 4_000); + + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.last_error = Some("held".into()); + generation.admitted_at_unix_millis = 1; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains(&format!("target service {}", generation.generation_id)), "{lines}"); + assert!(lines.contains("held: held"), "{lines}"); + assert!(lines.contains(&format!( + "continuity attempts-in-hour 2 pages-at 6 streak 2 next-attempt-at {}", + now - 4_000 + 10_000 + )), "{lines}"); + assert!(!lines.contains("rematerialized-at"), "{lines}"); + meters.continuity_rematerialized_at_unix_millis = Some(now - 1_000); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" +"); + assert!(lines.contains(&format!("continuity rematerialized-at {}", now - 1_000)), "{lines}"); + // An anchor older than the generation is not shown as a pending wait. + generation.admitted_at_unix_millis = now; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" +"); + assert!(lines.contains("streak 2 next-attempt-at none"), "{lines}"); + generation.admitted_at_unix_millis = 1; + assert!(lines.contains("continuity deferred until 10000500: projection would not demote"), "{lines}"); + assert!(lines.contains(&format!( + "route actuations 12/12 in window, reopens-at {}", + now - 5_000 + ROUTE_ACTUATION_WINDOW_MILLIS + )), "{lines}"); + assert!(lines.contains("route healthy consecutive-failures 0"), "{lines}"); + assert!( + lines.contains("route survival actuations 0 in window (uncounted), last-at none"), + "{lines}" + ); + meters.route_survivals = vec![now - 7_000, now - 3_000]; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains(&format!( + "route survival actuations 2 in window (uncounted), last-at {}", + now - 3_000 + )), "{lines}"); + meters.route_survivals = (0..ROUTE_SURVIVAL_RECORD as u64).map(|n| now - 100 + n).collect(); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains("route survival actuations at least 12 in window"), "{lines}"); + // Survivals never move the ceiling's count. + assert!(lines.contains("route actuations 12/12 in window"), "{lines}"); + + // A first deployment has meters and no generation: it shows them alone. + let mut first = TargetSupervision::new("fresh"); + first.route_actuations = vec![now - 1]; + let lines = render_supervision("fresh", None, Some(&first), now).join("\n"); + assert!(lines.contains("target fresh no-admitted-generation"), "{lines}"); + assert!(lines.contains("route actuations 1/12 in window, reopens-at none"), "{lines}"); + // An unrouted, unmetered target shows no route line at all. + let lines = render_supervision("plain", Some(&unrouted_generation()?), None, now).join("\n"); + assert!(!lines.contains("route actuations"), "{lines}"); + // A generation that no longer routes, whose meters hold only survival + // actuations, still shows the record. + let mut survived = TargetSupervision::new("plain"); + survived.route_survivals = vec![now - 1]; + let lines = + render_supervision("plain", Some(&unrouted_generation()?), Some(&survived), now).join("\n"); + assert!(lines.contains("route survival actuations 1 in window"), "{lines}"); + Ok(()) + } + + fn unrouted_generation() -> Result { + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.route_supervision = None; + Ok(generation) + } + + // --------------------------------------------------------------------- + // Route-proof readiness: a routed target that declares no Odin dependency + // is admitted by Idunn's own challenges. Odin is never read. + // --------------------------------------------------------------------- + mod route_proof { + use super::*; + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + use std::time::Duration; + + /// The runtime behind both endpoints, answering Idunn's snapshot + /// challenge the way the TypeScript signer does: locally, in the + /// reported health, with `route-observation:` as the detail. + struct RuntimeStub { + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + provider: cultnet_rs::ServiceIdentitySigner, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, + state: Mutex<&'static str>, + capacity: AtomicU32, + sequence: AtomicU64, + candidate_hits: AtomicUsize, + stable_hits: AtomicUsize, + stop: AtomicBool, + /// Accept the connection and close it without a word: a process + /// that is up but not answering. Unlike a closed port, no other + /// test's listener can be handed the address. + hang_up: AtomicBool, + /// The write lease its presence claims to hold: what a candidate + /// that picked up Idunn's grant reports. + lease: Mutex>, + reply: Mutex, + /// Runs when the stable listener accepts, before it answers. + on_stable_hit: Mutex>>, + } + + /// How the runtime answers a well-formed challenge. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Reply { + Honest, + /// An answer to some other challenge. + ForeignChallengeId, + /// An HTTP error where the snapshot should be. + HttpFailure, + /// A first Odin candidate answering its Warming challenge: the + /// detail names the challenge as `idunn-warming:`. + FirstOdinWarming, + } + + impl RuntimeStub { + fn answer(&self, message_id: &str) -> Result> { + let expected = &self.expected; + let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { + schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.activation.expected_projection_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| cultnet_rs::GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: self.capacity.load(Ordering::SeqCst), + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: (*self.state.lock().unwrap()).into(), + detail: match *self.reply.lock().unwrap() { + Reply::FirstOdinWarming => format!("idunn-warming:{message_id}"), + _ => format!("route-observation:{message_id}"), + }, + write_lease_sha256: self.lease.lock().unwrap().clone(), + signer_identity_id: self.provider.entry().identity_id.clone(), + publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, + observed_at_unix_millis: now_millis()?, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation_signer.identity_id(), + activation_signature: Vec::new(), + }; + let proof = presence.canonical_proof_payload()?; + presence.signature = self + .provider + .sign::(&proof) + .signature; + presence.activation_signature = + self.activation_signer.sign_presence_proof(&presence)?; + Ok(rmp_serde::to_vec(&presence)?) + } + + fn set_state(&self, state: &'static str) { + *self.state.lock().unwrap() = state; + } + + fn serve_one(&self, mut stream: TcpStream) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(3)))?; + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let (header_end, length) = loop { + let read = stream.read(&mut chunk)?; + ensure!(read > 0, "stub client closed early"); + request.extend_from_slice(&chunk[..read]); + if let Some(index) = request.windows(4).position(|w| w == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&request[..index]).to_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .context("stub request has no Content-Length")? + .trim() + .parse::()?; + if request.len() >= index + 4 + length { + break (index + 4, length); + } + } + }; + let message = cultnet_rs::decode_cultnet_message_from_slice( + &request[header_end..header_end + length], + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + let cultnet_rs::CultNetMessage::SnapshotRequest { message_id, .. } = message + else { + bail!("stub received something other than a snapshot request"); + }; + let reply = *self.reply.lock().unwrap(); + if reply == Reply::HttpFailure { + stream.write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + )?; + return Ok(()); + } + let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id: if reply == Reply::ForeignChallengeId { + "candidate-foreign".to_owned() + } else { + message_id.clone() + }, + documents: vec![cultnet_rs::CultNetRawDocumentRecord { + schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: self.expected.target.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload: self.answer(&message_id)?, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }], + }; + let body = cultnet_rs::encode_cultnet_message_to_vec( + &response, + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + stream.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + )?; + stream.write_all(&body)?; + Ok(()) + } + + fn listen(self: &Arc, listener: TcpListener, candidate: bool) { + listener.set_nonblocking(true).expect("nonblocking listener"); + let stub = Arc::clone(self); + std::thread::spawn(move || { + while !stub.stop.load(Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + stream.set_nonblocking(false).ok(); + let hits = if candidate { + &stub.candidate_hits + } else { + &stub.stable_hits + }; + hits.fetch_add(1, Ordering::SeqCst); + if !candidate + && let Some(hook) = stub.on_stable_hit.lock().unwrap().as_ref() + { + hook(); + } + if !stub.hang_up.load(Ordering::SeqCst) { + let _ = stub.serve_one(stream); + } + } + Err(_) => std::thread::sleep(Duration::from_millis(5)), + } + } + }); + } + } + + impl Drop for RuntimeStub { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + } + } + + /// Odin, as the world it runs in sees it. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Odin { + /// Every read of the correlation store fails. + Unreachable, + /// Odin publishes a correlation that says the target is not Ready. + NotReady, + } + + struct RoutedWorld { + world: EngineFixture, + workload: Arc, + stub: Arc, + odin: Odin, + poisoned: Vec, + } + + /// A stateless, routed target that declares no Odin dependency, over + /// `minimum_capacity`, seeded at `phase` (Warming or Fencing): its + /// Warming evidence, when the phase needs it, is the candidate's own + /// answer to a real challenge. Listeners stand at both endpoints. + fn routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + provides_odin: bool, + ) -> Result { + build_routed_world( + odin, + minimum_capacity, + phase, + provides_odin, + false, + CommandKind::Continuity, + ) + } + + /// The same world owned by a deployment command instead of a + /// continuity: its route actuations are `Forward` and can be refused. + fn routed_deploy_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + ) -> Result { + build_routed_world(odin, minimum_capacity, phase, false, false, CommandKind::Deploy) + } + + /// The same world for a target with process-writable state: its + /// candidate is granted a write lease and Ready must be the candidate + /// reporting exactly that lease. + fn stateful_routed_world(odin: Odin, phase: DeploymentPhase) -> Result { + build_routed_world(odin, 1, phase, false, true, CommandKind::Continuity) + } + + fn build_routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + provides_odin: bool, + stateful: bool, + kind: CommandKind, + ) -> Result { + use crate::deployment_plan::tests::{ + BINDING, RECIPE, artifact_receipt, external_input_receipt, source, + }; + let switch = SwitchWorkload::new(); + let world = EngineFixture::with_workload(switch.clone())?; + let candidate_listener = TcpListener::bind("127.0.0.1:0")?; + let stable_listener = TcpListener::bind("127.0.0.1:0")?; + let candidate_port = candidate_listener.local_addr()?.port(); + let stable_port = stable_listener.local_addr()?.port(); + + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + let config = world.root.join("service.conf"); + let binding = BINDING + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id) + .replace("127.0.0.1:17999", &format!("127.0.0.1:{stable_port}")) + .replace( + "private_port_start = 18000", + &format!("private_port_start = {candidate_port}"), + ) + .replace( + "private_port_end = 18009", + &format!("private_port_end = {}", candidate_port + 1), + ) + .replace( + "/etc/nginx/idunn-stream-routes/service.conf", + &config.display().to_string().replace('\\', "/"), + ); + let (binding, recipe_source) = if stateful { + std::fs::create_dir_all(world.root.join("lease"))?; + ( + binding.replace( + "[brakes]", + &format!( + "[process_write_lease]\nrecord_path = \"{}\"\n\n[brakes]", + world.root.join("lease/process-write-lease.cc").display() + ), + ), + RECIPE + .replace( + "writer = \"none\"\nrecovery = \"rebuildable\"\nstartup = \"open-at-start\"", + "writer = \"process-bound-single-writer\"\nrecovery = \"preserve\"\nstartup = \"create-or-open-after-write-lease\"", + ) + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_PROCESS_WRITE_LEASE\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + ), + ) + } else { + (binding, RECIPE.to_owned()) + }; + let recipe = recipe_source + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace( + "capability = \"service.runtime\"", + &format!( + "capability = \"{}\"\ncapacity = {minimum_capacity}", + if provides_odin { + ODIN_RENDEZVOUS_CAPABILITY + } else { + "service.runtime" + } + ), + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(candidate_port), + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let expected = release.expected_projection(&plan)?; + assert!(expected.route.is_some() && expected.dependencies.is_empty()); + assert_eq!(expected.write_lease_required, stateful); + assert_eq!( + ReadinessClass::of(&expected), + Ok(if provides_odin { + ReadinessClass::OdinSelf + } else { + ReadinessClass::RouteProof + }) + ); + + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: match kind { + CommandKind::Deploy => "up-service", + CommandKind::Continuity => "continuity-service", + } + .into(), + kind, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = + DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + match kind { + CommandKind::Continuity => { + transaction.lifecycle_authorized_at_unix_millis = Some(now); + } + CommandKind::Deploy => { + transaction.frozen_source = Some(FrozenSourceReceipt { + transaction_id: transaction.transaction_id.clone(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + transaction.deployment_authorization = Some(deployment_authorization( + &world, + &expected, + &transaction.transaction_id, + now, + )?); + } + } + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&transaction.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; + let mut workload = recorded.workload.clone().context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + transaction.isolation = recorded.isolation; + transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); + transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); + transaction.sealed_release = Some(release); + transaction.expected = Some(expected.clone()); + transaction.activation = Some(activation.clone()); + transaction.workload = Some(workload); + transaction.plan = Some(plan); + + let stub = Arc::new(RuntimeStub { + expected, + activation, + provider: open_service_identity_at::( + &provider_path, + )?, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( + &seed[..], + )?, + state: Mutex::new("warming"), + capacity: AtomicU32::new(minimum_capacity), + sequence: AtomicU64::new(0), + candidate_hits: AtomicUsize::new(0), + stable_hits: AtomicUsize::new(0), + stop: AtomicBool::new(false), + hang_up: AtomicBool::new(false), + lease: Mutex::new(None), + reply: Mutex::new(Reply::Honest), + on_stable_hit: Mutex::new(None), + }); + stub.listen(candidate_listener, true); + stub.listen(stable_listener, false); + + if phase >= DeploymentPhase::Fencing && provides_odin { + // Odin's own first word about the candidate, as for any + // Odin-correlated target. + // A stateful candidate is warming, not active, until it holds + // the lease Idunn grants after the fence. + let warming = signed_correlation_as( + &world, + &transaction, + 4, + false, + if stateful { "warming" } else { "active" }, + None, + b"presence", + )?; + let authenticated = world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&world.state_store)?, + &transaction, + &warming, + None, + now, + )?; + transaction.warming = Some(WarmingEvidence::OdinTopology { + evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, + }); + transaction.odin_publisher_sequence_cursor = 4; + } else if phase >= DeploymentPhase::Fencing { + let CandidateAnswer::Answered { evidence, .. } = + world + .engine + .challenge_candidate(&transaction, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + } + if phase >= DeploymentPhase::Fencing { + transaction.route_preflight = Some(candidate_preflight(&transaction)?); + } + if stateful { + // The lease driver reads the Expected and the observed + // activation from the topology store, as Starting left them. + let plan = transaction.plan.as_ref().unwrap(); + let anchor = world.engine.provider_anchor_for_plan(plan)?; + let topology = world.engine.topology(); + topology.publish_expected(transaction.expected.as_ref().unwrap(), &anchor)?; + topology.publish_observed_activation( + transaction.expected.as_ref().unwrap(), + transaction.activation.as_ref().unwrap(), + transaction.workload.as_ref().unwrap(), + )?; + } + transaction.enter_phase(phase, now); + transaction.validate()?; + let store = SingleFileMessagePackBackingStore::new(&world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )?); + + // Odin's world. Unreachable: the correlation store is bytes no + // reader can decode, so any read of it is an error. NotReady: a + // valid correlation that says the target is not Ready. + let store = &world.engine.options.odin_correlation_store; + let poisoned = match odin { + _ if provides_odin && stateful => { + // A new warming word after the fence: what the lease + // grant waits for. + SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( + CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &transaction.target, + &transaction.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation_as( + &world, + &transaction, + 5, + false, + "warming", + None, + b"presence-5", + )?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some( + cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + ), + }, + )?; + Vec::new() + } + _ if provides_odin => { + odin_reports_ready(&world, &transaction, 5)?; + Vec::new() + } + Odin::Unreachable => { + let garbage = b"odin is not a cache".to_vec(); + std::fs::write(store, &garbage)?; + garbage + } + Odin::NotReady => { + let wrongly = signed_correlation(&world, &transaction, 9, false)?; + SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( + CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &transaction.target, + &transaction.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: wrongly, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some( + cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + ), + }, + )?; + Vec::new() + } + }; + Ok(RoutedWorld { + world, + workload: switch, + stub, + odin, + poisoned, + }) + } + + /// The preflight receipt of a candidate whose route was never + /// installed: what Warming records before Fencing, without running the + /// host's nginx. + fn candidate_preflight( + transaction: &DeploymentTransaction, + ) -> Result { + let rendered = NginxRouteDriver::new( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ) + .render(transaction.expected.as_ref().unwrap())?; + Ok(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: transaction + .activation + .as_ref() + .unwrap() + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(&rendered), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }) + } + + impl RoutedWorld { + fn step(&self) -> Result<()> { + self.world.engine.advance_transaction(&resident(&self.world)?) + } + + fn run_to_routing(&self) -> Result<()> { + for _ in 0..12 { + if self.transaction()?.phase == DeploymentPhase::Routing { + return Ok(()); + } + self.step()?; + } + bail!("the transaction never reached Routing") + } + + fn transaction(&self) -> Result { + Ok(resident(&self.world)?.value) + } + + /// Odin was never read: the poisoned store is byte for byte what + /// it was, and no step failed on it. + fn assert_odin_untouched(&self) -> Result<()> { + if self.odin == Odin::Unreachable { + assert_eq!( + std::fs::read(&self.world.engine.options.odin_correlation_store)?, + self.poisoned + ); + } + Ok(()) + } + + /// Routing runs for real: it installs the fragment through the + /// world's stub host programs and proves the stable route against + /// the stub runtime. The world only drives the steps until the + /// transaction enters Committing. + fn promote(&self) -> Result<()> { + assert_eq!(self.transaction()?.phase, DeploymentPhase::Routing); + for _ in 0..4 { + if self.transaction()?.phase == DeploymentPhase::Committing { + return Ok(()); + } + self.step()?; + } + bail!("the transaction never reached Committing") + } + } + + /// A route-proof generation from a real admission. + pub(super) fn committed_generation() -> Result { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + routed.step()?; + admitted(&routed) + } + + fn admitted(world: &RoutedWorld) -> Result { + Ok(ControlSnapshot::read(&world.world.state_store)? + .admitted + .into_iter() + .next() + .context("nothing was admitted")? + .value) + } + + #[test] + fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + // Fencing, Leasing, AwaitingReady: the candidate is still warming. + for _ in 0..12 { + routed.step()?; + if routed.transaction()?.last_error.is_some() { + break; + } + } + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.ready.is_none()); + assert_eq!( + waiting.last_error.as_deref(), + Some("candidate is still warming") + ); + // The candidate finishes warming: Ready is its own Active answer. + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.step()?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); + routed.promote()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.odin_authority.is_none()); + assert!(generation.latest_odin_observation.is_none()); + assert!(generation.route_supervision.is_some()); + // Idunn's own challenges carried every proof: the seed's warming, + // two Ready polls; the stable endpoint proved the route twice. + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 3); + assert!(routed.stub.stable_hits.load(Ordering::SeqCst) >= 2); + routed.assert_odin_untouched()?; + + // Boot and supervision read no Odin either: the topology refresh + // returns before it opens Odin's store, and durable authority + // skips a route-proof generation's receipts. + let stored = ControlSnapshot::read(&routed.world.state_store)?; + let current = stored.admitted.first().context("no admitted generation")?; + assert!(!routed.world.engine.refresh_admitted_topology(&stored, current)?); + routed.world.engine.validate_durable_authority(&stored)?; + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { + let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + routed.step()?; + let generation = admitted(&routed)?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + // The valid, not-Ready correlation was neither admitted as a + // receipt nor consulted: the record carries no Odin cursor. + assert_eq!(generation.odin_publisher_sequence_cursor, 0); + Ok(()) + } + + #[test] + fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + let warmed = routed.transaction()?; + assert_eq!(warmed.phase, DeploymentPhase::Warming); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &warmed.warming else { + bail!("Warming evidence is not the candidate's own presence"); + }; + assert!(evidence.message_id.starts_with("candidate-")); + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + routed.assert_odin_untouched()?; + + // Nothing answers: the same step waits, and says why. + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + silent.stub.hang_up.store(true, Ordering::SeqCst); + silent.step()?; + let waiting = silent.transaction()?; + assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); + assert!( + waiting + .last_error + .as_deref() + .is_some_and(|error| error.contains("candidate endpoint did not answer")), + "{:?}", + waiting.last_error + ); + Ok(()) + } + + #[test] + fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + routed.stub.capacity.store(1, Ordering::SeqCst); + let failure = routed.step().expect_err("a shortfall must be refused"); + // The shortfall is a typed value, not a sentence to search. + let disagrees = failure + .downcast_ref::() + .with_context(|| format!("the refusal is not typed: {failure:#}"))?; + let shortfall = disagrees + .disagreements + .iter() + .find(|disagreement| disagreement.code == "expected-capability-000-capacity") + .context("the capacity shortfall is not named")?; + assert!( + shortfall.expected.as_deref().is_some_and(|text| text.ends_with("capacity>=2")), + "{shortfall:?}" + ); + assert!(shortfall.observed.is_some(), "{shortfall:?}"); + assert!(admitted(&routed).is_err()); + Ok(()) + } + + #[test] + fn odin_itself_is_never_route_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let mut expected = routed.stub.expected.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + // Odin provides the rendezvous capability and declares no dependency. + expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + assert!(expected.dependencies.is_empty()); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); + + // Route-proof evidence for it is refused: its admitted generation + // is what names the Odin authority, so it must carry one. + let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; + odin.target = "odin".into(); + odin.expected = expected; + odin.ready = route_proof_evidence(); + odin.latest_odin_observation = None; + odin.odin_authority = None; + assert!(error_text(odin.validate()).contains("not route-proof")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn route_admission_installs_through_the_host_programs_without_reading_odin() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + // One real Routing step: the fragment goes in through the world's + // stub nginx, ufw and systemctl, and the stable listener proves it. + routed.step()?; + assert!(matches!( + routed.transaction()?.routing, + Some(RoutingEvidence::Promoted { .. }) + )); + assert_eq!(routed.world.route_stubs.count("systemctl reload"), 1); + assert_eq!(routed.world.route_stubs.count("ufw allow"), 1); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.warming, + Some(WarmingEvidence::RouteProofDirect { .. }) + )); + Ok(()) + } + + #[test] + fn the_lease_refresh_asks_the_candidate_for_a_new_warming_presence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let prior = routed + .world + .engine + .rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let (_, evidence, token) = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?)? + .context("a warming candidate must refresh")?; + assert!(matches!(evidence, WarmingEvidence::RouteProofDirect { .. })); + assert_ne!(token.signed_presence_sha256(), prior.signed_presence_sha256()); + // A candidate that went Active is not warming: nothing to bind. + routed.stub.set_state("active"); + let failure = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?) + .map(|_| ()) + .expect_err("an active candidate is not a warming one"); + assert!(format!("{failure:#}").contains("not warming"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn boot_reauthenticates_a_live_route_proof_transactions_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + + // Warming evidence that does not answer the challenge it names. + let current = resident(&routed.world)?; + let mut forged = current.value.clone(); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &mut forged.warming else { + bail!("no route-proof Warming evidence"); + }; + evidence.message_id = "candidate-another".into(); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Warming evidence for another challenge"); + assert!(format!("{failure:#}").contains("exact challenge"), "{failure:#}"); + + // A Ready label carried by a presence that was only warming. + let current = resident(&routed.world)?; + let CandidateAnswer::Answered { evidence, .. } = routed + .world + .engine + .challenge_candidate(¤t.value, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + let mut forged = current.value.clone(); + forged.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + forged.ready = Some(ReadinessEvidence::RouteProof { evidence }); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Ready carried by a warming presence"); + assert!(format!("{failure:#}").contains("not active"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let ready = routed.transaction()?; + assert!(ready.ready.as_ref().and_then(ReadinessEvidence::odin).is_some()); + routed.promote()?; + routed.step()?; + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Odin); + assert_eq!( + generation.odin_authority, + Some(routed.world.engine.bootstrap_odin_authority.clone()) + ); + assert!(generation.latest_odin_observation.is_some()); + // Boot re-authenticates every Odin receipt this generation carries. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + Ok(()) + } + + #[test] + fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + routed.step()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + assert_eq!( + routed.world.engine.current_odin_authority(&snapshot)?, + routed.world.engine.bootstrap_odin_authority + ); + Ok(()) + } + + impl RoutedWorld { + /// Boot would accept this store, and Odin was never read. + fn assert_boots(&self) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.world.state_store)?; + self.world.engine.validate_durable_authority(&snapshot)?; + self.assert_odin_untouched() + } + + /// Step until `done`, checking after every step that boot would + /// still accept the store. + fn drive_until(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..20 { + if done(&self.transaction()?) { + return Ok(()); + } + self.step()?; + self.assert_boots()?; + } + bail!("the transaction never reached the awaited state") + } + + fn answer_now(&self) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + match self.world.engine.challenge_candidate( + &self.transaction()?, + &["warming", "active"], + None, + )? { + CandidateAnswer::Answered { evidence, present } => Ok((evidence, present)), + CandidateAnswer::Silent(reason) => bail!("the candidate was silent: {reason}"), + } + } + } + + // ----------------------------------------------------------------- + // The stateful path, end to end through the Engine. + // ----------------------------------------------------------------- + + #[test] + fn a_stateful_route_proof_target_holds_its_lease_from_fencing_to_commit() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.assert_boots()?; + + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Prepared { .. })))?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Leasing); + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let lease_sha256 = routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + + // The candidate has not picked the lease up: still warming. + routed.step()?; + assert_eq!( + routed.transaction()?.last_error.as_deref(), + Some("candidate is still warming") + ); + assert!(routed.transaction()?.ready.is_none()); + // Active without the lease Idunn granted is not Ready. + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("an active candidate without the granted lease is refused"); + assert!( + format!("{refusal:#}").contains("exact current process write lease"), + "{refusal:#}" + ); + assert!(routed.transaction()?.ready.is_none()); + // Active holding exactly that lease is. + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.assert_boots()?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + routed.promote()?; + routed.assert_boots()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.expected.write_lease_required); + assert_eq!(generation.leasing.lease_sha256(), Some(lease_sha256.as_str())); + assert!(generation.odin_authority.is_none()); + routed.assert_boots()?; + Ok(()) + } + + // ----------------------------------------------------------------- + // Warming is one challenge, then the phase moves. + // ----------------------------------------------------------------- + + #[test] + fn warming_challenges_once_and_then_advances_on_every_tick() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + let recorded = routed + .transaction()? + .warming + .context("the first step recorded no Warming evidence")?; + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + + // The route preflight needs the host's nginx; its receipt is recorded + // here so the phase can move on. + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + next.route_preflight = Some(candidate_preflight(&next)?); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Fencing)?; + assert_eq!( + routed.stub.candidate_hits.load(Ordering::SeqCst), + 1, + "a recorded Warming answer is never asked for again" + ); + assert_eq!(routed.transaction()?.warming, Some(recorded)); + Ok(()) + } + + #[test] + fn a_stateful_candidate_may_not_answer_its_warming_challenge_active() { + let mut expected = fixture_generation(FIXTURE_GENERATION).unwrap().1.expected; + expected.write_lease_required = true; + assert_eq!(route_proof_warming_states(&expected), ["warming"]); + expected.write_lease_required = false; + assert_eq!(route_proof_warming_states(&expected), ["warming", "active"]); + } + + #[test] + fn a_stateful_candidate_that_answers_warming_active_is_refused() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("a stateful candidate cannot be active before it holds a lease"); + assert!(format!("{refusal:#}").contains("not warming"), "{refusal:#}"); + assert!(routed.transaction()?.warming.is_none()); + Ok(()) + } + + // ----------------------------------------------------------------- + // What a challenged presence must prove. + // ----------------------------------------------------------------- + + #[test] + fn a_presence_holds_the_write_lease_exactly_when_it_is_not_warming() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let lease = sha256_id(b"the granted lease"); + let other = sha256_id(b"some other lease"); + let judge = |state: &'static str, + held: Option<&str>, + current: Option<&str>, + states: &[&str]| + -> Result<()> { + let challenged = now_millis()?; + routed.stub.set_state(state); + *routed.stub.lease.lock().unwrap() = held.map(str::to_owned); + let bytes = routed.stub.answer("candidate-probe")?; + routed + .world + .engine + .authenticate_challenged_presence( + &authority, + states, + current, + "candidate-probe", + challenged, + now_millis()?, + &bytes, + ) + .map(|_| ()) + }; + let either = ["warming", "active"]; + judge("warming", None, None, &either)?; + judge("active", Some(&lease), Some(&lease), &either)?; + for (state, held, current, named) in [ + // Warming holds nothing, whatever Idunn has granted. + ("warming", Some(lease.as_str()), None, "write lease"), + ("warming", Some(lease.as_str()), Some(lease.as_str()), "write lease"), + // Anything else holds exactly the current grant. + ("active", None, Some(lease.as_str()), "exact current process write lease"), + ("active", Some(lease.as_str()), None, "exact current process write lease"), + ( + "active", + Some(other.as_str()), + Some(lease.as_str()), + "exact current process write lease", + ), + ] { + let error = judge(state, held, current, &either) + .expect_err("a presence with the wrong lease is refused"); + assert!( + format!("{error:#}").contains(named), + "{state} {held:?} {current:?}: {error:#}" + ); + } + Ok(()) + } + + #[test] + fn a_presence_minted_before_its_challenge_is_refused() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let bytes = routed.stub.answer("candidate-probe")?; + let minted = now_millis()?; + // Answered before the challenge that claims it: an old presence + // replayed for a new question. + let replay = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted + 1_000, + &bytes, + ) + .expect_err("a presence minted before its challenge"); + assert!( + format!("{replay:#}").contains("minted before its challenge"), + "{replay:#}" + ); + let early = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted, + &bytes, + ) + .expect_err("a receipt before its challenge"); + assert!(format!("{early:#}").contains("predates its challenge"), "{early:#}"); + Ok(()) + } + + #[test] + fn direct_warming_evidence_is_bound_to_the_bytes_it_authenticated() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let id = routed.transaction()?.transaction_id; + let direct = |evidence: RuntimePresenceEvidence| WarmingEvidence::RouteProofDirect { evidence }; + + let (evidence, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present)?; + + // The digest is the authenticated presence's, but the bytes are not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_bytes.push(0); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("bytes that are not the authenticated presence"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // The bytes are the authenticated presence's, but the digest is not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_sha256 = sha256_id(b"another presence"); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("a digest that is not the authenticated presence's"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // Evidence about another answer altogether. + let (other, _) = routed.answer_now()?; + let (_, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id, direct(other), present) + .expect_err("evidence about another answer"); + Ok(()) + } + + // ----------------------------------------------------------------- + // Validation: candidate-vouched evidence belongs to route-proof targets. + // ----------------------------------------------------------------- + + #[test] + fn candidate_vouched_evidence_is_refused_on_a_target_odin_reports_on() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let odin_reported = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!( + ReadinessClass::of(odin_reported.expected.as_ref().unwrap()), + Ok(ReadinessClass::OdinCorrelated) + ); + + let mut warming = odin_reported.clone(); + warming.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + assert!(error_text(warming.validate()).contains("not route-proof")); + + let mut ready = odin_reported.clone(); + ready.ready = Some(ReadinessEvidence::RouteProof { + evidence: evidence.clone(), + }); + assert!(error_text(ready.validate()).contains("not route-proof")); + + // Direct Odin warming is Odin's alone, by class: a route-proof + // target may not claim it, and a target that provides the + // rendezvous may, whatever it is named. + let mut direct = routed.transaction()?; + direct.warming = Some(WarmingEvidence::FirstOdinDirect { + evidence: evidence.clone(), + }); + assert!(error_text(direct.validate()).contains("reserved for Odin")); + let odin = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + let mut direct = odin.transaction()?; + assert_eq!(direct.target, "service"); + direct.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + direct.validate()?; + Ok(()) + } + + #[test] + fn a_route_proof_transaction_past_awaiting_ready_needs_the_candidates_ready_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let routing = routed.transaction()?; + routing.validate()?; + let mut bare = routing.clone(); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + routed.promote()?; + let mut bare = routed.transaction()?; + assert_eq!(bare.phase, DeploymentPhase::Committing); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + Ok(()) + } + + // ----------------------------------------------------------------- + // The class is the Expected's alone. Evidence collected under another + // class is held and reported, never stepped, never repaired. + // ----------------------------------------------------------------- + + /// Odin's Ready for the world's target, as pre-B3 code recorded it for + /// every routed target. + fn odin_receipt_for( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result { + odin_receipt_with(routed, transaction, sequence, true) + } + + fn odin_receipt_with( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + ) -> Result { + let now = now_millis()?; + let bytes = signed_correlation(&routed.world, transaction, sequence, ready)?; + let authenticated = routed.world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&routed.world.state_store)?, + transaction, + &bytes, + transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256), + now, + )?; + TopologyEvidence::from_authenticated(&authenticated, now) + } + + fn stored(generation: &AdmittedGeneration) -> Stored { + Stored { + value: generation.clone(), + envelope: CultCacheEnvelope { + key: generation.target.clone(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + } + } + + #[test] + fn a_pre_b3_transaction_over_a_route_proof_target_is_held_and_reported_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + + // Ready as pre-B3 Idunn wrote it: an Odin receipt. + let current = resident(&routed.world)?; + let mut legacy = current.value.clone(); + let receipt = odin_receipt_for(&routed, &legacy, 9)?; + legacy.latest_odin_observation = Some(receipt.clone()); + legacy.odin_publisher_sequence_cursor = 9; + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: receipt }); + replace_transaction(&routed.world.state_store, ¤t, &legacy)?; + + let disagreement = ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }; + assert_eq!(legacy.readiness_disagreement(), Some(disagreement.clone())); + // The record is readable and boot accepts it: held, not refused. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + + let hits = ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst), + ); + for _ in 0..4 { + assert!(!routed.world.engine.run_scheduler_tick()?); + } + let held = resident(&routed.world)?; + assert_eq!(held.value.phase, DeploymentPhase::Routing); + assert!(held.value.pre_fencing_abort.is_none()); + assert!(held.value.post_fencing_abort.is_none()); + assert_eq!(held.value.ready, legacy.ready, "nothing re-derived the evidence"); + assert_eq!( + held.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()) + ); + // Reported once: later ticks leave the record alone. + routed.world.engine.run_scheduler_tick()?; + assert_eq!(resident(&routed.world)?.envelope, held.envelope); + assert_eq!( + hits, + ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst) + ), + "a held transaction is asked nothing" + ); + routed.assert_odin_untouched()?; + + // An abort already under way is not held: it never reads the class. + let mut aborting = held.value.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + replace_transaction(&routed.world.state_store, &held, &aborting)?; + let after = resident(&routed.world)?; + let _ = routed.world.engine.resume_candidate(&after); + assert_ne!( + resident(&routed.world)?.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()), + "an abort in progress runs instead of being held" + ); + Ok(()) + } + + #[test] + fn a_pre_b3_generation_over_a_route_proof_target_boots_and_is_reported_not_refreshed() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + routed.promote()?; + routed.step()?; + + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("nothing was admitted")?; + // Receipts that would not survive being re-proved: authentic, but + // not Ready. Boot re-proves against the current Odin authority and + // requires exact semantic Ready, never what a generation stores, so + // this record is only bootable if boot does not re-prove it. + let receipt = odin_receipt_with(&routed, &at_routing, 9, false)?; + let mut legacy = current.value.clone(); + legacy.ready = ReadinessEvidence::OdinCorrelated { + evidence: receipt.clone(), + }; + legacy.latest_odin_observation = Some(receipt); + // The stored authority is present because an Odin-tagged record + // carries one; it is not what boot reads. + let stranger = cultnet_rs::enroll_service_identity_at::( + &routed.world.root.join("identities/stranger.cc"), + )?; + legacy.odin_authority = Some(AdmittedOdinAuthority::from_anchor(&stranger.trust_anchor()?)?); + legacy.odin_publisher_sequence_cursor = 9; + assert!( + SingleFileMessagePackBackingStore::new(&routed.world.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: legacy.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&legacy, now_millis()?)?], + )? + ); + + assert_eq!( + legacy.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }) + ); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("the generation is gone")?; + // The class chooses the path: no Odin receipt is re-proved at boot + // and none is refreshed, though the evidence is an Odin receipt. + routed.world.engine.validate_durable_authority(&snapshot)?; + assert!(!routed.world.engine.refresh_admitted_topology(&snapshot, current)?); + routed.assert_odin_untouched()?; + + routed.world.engine.supervise_one_admitted_generation()?; + assert!( + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .contains_key("generation:service"), + "supervision reports the disagreement" + ); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn only_a_transaction_that_still_reads_its_class_is_held_for_its_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + // What pre-B3 Idunn would have collected: an Odin receipt. + let mut legacy = at_routing.clone(); + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { + evidence: odin_receipt_for(&routed, &at_routing, 9)?, + }); + let disagreement = legacy.readiness_disagreement(); + assert!(disagreement.is_some()); + assert_eq!(legacy.held_disagreement(), disagreement); + + // Finished work, and an abort under way, never read the class. + let mut completed = legacy.clone(); + completed.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-1".into(), + }); + assert_eq!(completed.held_disagreement(), None); + let mut aborting = legacy.clone(); + aborting.pre_fencing_abort = Some(PreFencingAbort { + error: "operator abort".into(), + candidate_cleanup: CleanupEvidence::Pending, + topology_reconciliation: CleanupEvidence::Pending, + source_cleanup: CleanupEvidence::Pending, + }); + assert_eq!(aborting.held_disagreement(), None); + let mut aborting = legacy; + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + assert_eq!(aborting.held_disagreement(), None); + Ok(()) + } + + #[test] + fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { + let world = EngineFixture::new()?; + // Odin's store is unreadable: any read of it is an error. + std::fs::write(&world.engine.options.odin_correlation_store, b"odin is not a cache")?; + + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + let target = generation.expected.target.clone(); + assert_eq!(generation.readiness(), Ok(ReadinessClass::OdinCorrelated)); + generation.expected.dependencies.clear(); + generation.expected.route = None; + let undeclared = UndeclaredReadiness { target: target.clone() }; + assert_eq!( + generation.readiness(), + Err(ReadinessDisagreement::Undeclared(undeclared.clone())) + ); + assert!( + !world + .engine + .refresh_admitted_topology(&ControlSnapshot::default(), &stored(&generation))? + ); + + // The mirror: Odin declared, but the receipt is the candidate's. + let (_, mut odin_class) = fixture_generation(FIXTURE_GENERATION)?; + odin_class.ready = route_proof_evidence(); + assert_eq!( + odin_class.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: target.clone(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + + // A transaction says the same about what it has collected. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let odin_world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&odin_world, DeploymentPhase::Warming)?; + assert_eq!(seeded.readiness_disagreement(), None); + let mut collected = seeded.clone(); + collected.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + assert_eq!( + collected.readiness_disagreement(), + Some(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + let mut bare = seeded.clone(); + bare.expected.as_mut().unwrap().dependencies.clear(); + assert_eq!( + bare.readiness_disagreement(), + Some(ReadinessDisagreement::Undeclared(UndeclaredReadiness { + target: "service".into() + })) + ); + let mut no_expected = seeded; + no_expected.expected = None; + assert_eq!(no_expected.readiness_disagreement(), None); + Ok(()) + } + + // ----------------------------------------------------------------- + // Odin is whichever target provides the rendezvous. + // ----------------------------------------------------------------- + + #[test] + fn odin_is_the_target_that_provides_the_rendezvous_not_the_target_named_odin() -> Result<()> { + let world = EngineFixture::new()?; + let (_, fixture) = fixture_generation(FIXTURE_GENERATION)?; + + // A target named "odin" that merely depends on Odin is not Odin. + let mut named = fixture.clone(); + named.target = "odin".into(); + named.expected.target = "odin".into(); + let mut snapshot = ControlSnapshot::default(); + snapshot.admitted.push(stored(&named)); + assert!(snapshot.admitted_odin().is_none()); + assert_eq!( + world.engine.current_odin_authority(&snapshot)?, + world.engine.bootstrap_odin_authority + ); + + // The target that provides the rendezvous is, whatever it is called. + let mut provider = fixture.clone(); + provider.target = "verse".into(); + provider.expected.target = "verse".into(); + provider.expected.dependencies.clear(); + provide_odin(&mut provider.expected); + let authority = AdmittedOdinAuthority::from_anchor(&world.odin_signer.trust_anchor()?)?; + provider.odin_authority = Some(authority.clone()); + snapshot.admitted.push(stored(&provider)); + assert_eq!( + snapshot.admitted_odin().map(|found| found.value.target.as_str()), + Some("verse") + ); + assert_eq!( + snapshot.admitted_odin().and_then(|found| found.value.odin_authority.clone()), + Some(authority) + ); + Ok(()) + } + + #[test] + fn a_target_that_provides_the_rendezvous_warms_as_odin_whatever_it_is_named() -> Result<()> { + // Nothing else has been admitted that could observe it, so Odin + // observes itself directly -- and only a stateful incarnation may. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + assert_eq!(routed.transaction()?.target, "service"); + let refusal = routed + .step() + .expect_err("a stateless Odin cannot be observed directly"); + assert!( + format!("{refusal:#}").contains("direct Odin warming must be a stateful incarnation"), + "{refusal:#}" + ); + Ok(()) + } + + #[test] + fn direct_odin_warming_is_reserved_to_odin_by_class_not_by_name() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let mut named = routed.transaction()?; + named.target = "odin".into(); + let error = routed + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a target named odin that is not Odin"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + + #[test] + fn a_stateful_target_that_provides_the_rendezvous_is_observed_directly_whatever_it_is_named() + -> Result<()> { + let provider = build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true, true, CommandKind::Continuity)?; + let transaction = provider.transaction()?; + assert_eq!(transaction.target, "service"); + assert!(transaction.expected.as_ref().unwrap().write_lease_required); + provider.stub.set_state("warming"); + *provider.stub.reply.lock().unwrap() = Reply::FirstOdinWarming; + let challenged_at = now_millis()?; + let answer = provider.stub.answer("warming-probe")?; + provider.world.engine.authenticate_first_odin_warming_presence( + &transaction, + "warming-probe", + challenged_at, + now_millis()?, + &answer, + )?; + + // The reverse: named odin, stateful, but nothing provides the + // rendezvous. + let stateful = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let mut named = stateful.transaction()?; + assert!(named.expected.as_ref().unwrap().write_lease_required); + named.target = "odin".into(); + let error = stateful + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a stateful target named odin that does not provide the rendezvous"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + + // ----------------------------------------------------------------- + // A bad answer is an error. Only silence waits. + // ----------------------------------------------------------------- + + #[test] + fn a_bad_answer_to_a_challenge_is_an_error_and_only_silence_waits() -> Result<()> { + for reply in [Reply::ForeignChallengeId, Reply::HttpFailure] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + *routed.stub.reply.lock().unwrap() = reply; + let error = match routed.world.engine.challenge_candidate( + &routed.transaction()?, + &["warming"], + None, + ) { + Err(error) => error, + Ok(_) => bail!("a bad answer was taken for silence"), + }; + assert!( + matches!( + error.downcast_ref::(), + Some(ChallengeFailure::Refused(_)) + ), + "{error:#}" + ); + // Before the fence a candidate that answers wrongly is aborted, + // not waited on. + routed.world.engine.run_scheduler_tick()?; + let after = routed.transaction()?; + assert!(after.pre_fencing_abort.is_some(), "the bad answer aborted it"); + assert!(after.warming.is_none()); + } + + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + silent.stub.hang_up.store(true, Ordering::SeqCst); + assert!(matches!( + silent.world.engine.challenge_candidate(&silent.transaction()?, &["warming"], None)?, + CandidateAnswer::Silent(_) + )); + Ok(()) + } + + // ----------------------------------------------------------------- + // B2: the route gate, the meters and the supervision pass, through + // the Engine, over the world's stub host programs. + // ----------------------------------------------------------------- + + impl RoutedWorld { + /// Run a fresh world to an admitted generation whose transaction + /// is retired: what admitted-route supervision looks at. + fn admit(&self) -> Result<()> { + self.stub.set_state("active"); + self.run_to_routing()?; + self.promote()?; + self.drive_until(|transaction| transaction.is_terminal())?; + assert!(self.world.engine.retire_one_terminal_transaction()?); + Ok(()) + } + + fn fragment_path(&self) -> PathBuf { + self.world.root.join("service.conf") + } + + /// The target's route ledger holds `entries` recent actuations. + fn ledger(&self, entries: usize) -> Result<()> { + let now = now_millis()?; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = (0..entries as u64).map(|offset| now - 1_000 + offset).collect(); + set_meters(&self.world, &meters) + } + + #[cfg(unix)] + fn reloads(&self) -> usize { + self.world.route_stubs.count("systemctl reload") + } + + /// Something else rewrote the fragment: the admitted membership drifted. + fn drift(&self) -> Result<()> { + std::fs::write(self.fragment_path(), b"drifted bytes\n")?; + Ok(()) + } + + fn tick(&self) -> Result { + self.world.engine.supervise_one_admitted_generation() + } + } + + // A store that reads but cannot be written (ENOSPC on the state volume) + // is simulated with RLIMIT_FSIZE, which is process-wide: the test + // re-runs itself in a child process so no sibling test loses its writes. + // Returns true in the child, which runs the body; the parent returns + // false after asserting the child passed and hands back its stderr. + #[cfg(unix)] + fn in_unwritable_store_child(test: &str) -> Result> { + if std::env::var_os("IDUNN_TEST_UNWRITABLE_STORE_CHILD").is_some() { + return Ok(None); + } + let path = module_path!().split_once("::").map_or("", |(_, rest)| rest); + let output = std::process::Command::new(std::env::current_exe()?) + .args(["--exact", &format!("{path}::{test}"), "--nocapture", "--test-threads=1"]) + .env("IDUNN_TEST_UNWRITABLE_STORE_CHILD", "1") + .output()?; + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + assert!( + output.status.success(), + "child failed:\n{}\n{stderr}", + String::from_utf8_lossy(&output.stdout) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "the child ran no test: {}", + String::from_utf8_lossy(&output.stdout) + ); + Ok(Some(stderr)) + } + + /// Cap file growth one byte under the store's size: reads work, every + /// write fails. Child process only. + #[cfg(unix)] + fn break_store_writes(world: &EngineFixture) { + let size = std::fs::metadata(&world.state_store).unwrap().len(); + unsafe { + libc::signal(libc::SIGXFSZ, libc::SIG_IGN); + let limit = libc::rlimit { rlim_cur: size - 1, rlim_max: libc::RLIM_INFINITY }; + assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0); + } + assert!( + set_meters(world, &TargetSupervision::new("probe")).is_err(), + "the store is still writable" + ); + assert!(ControlSnapshot::read(&world.state_store).is_ok(), "the store is unreadable"); + } + + #[cfg(unix)] + fn challenge_due(world: &EngineFixture) -> Option { + world.engine.challenge_wait.lock().unwrap().get("service").map(|wait| wait.due_unix_millis) + } + + #[cfg(unix)] + #[test] + fn an_unwritable_store_neither_spins_a_failing_challenge_nor_reports_progress() -> Result<()> { + let name = "an_unwritable_store_neither_spins_a_failing_challenge_nor_reports_progress"; + let Some(stderr) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + routed.drift()?; + let before = routed.reloads(); + let began = now_millis()?; + break_store_writes(&routed.world); + let stored_wait = |routed: &RoutedWorld| -> Result> { + Ok(ControlSnapshot::read(&routed.world.state_store)? + .admitted + .iter() + .find_map(|stored| stored.value.route_supervision.as_ref()) + .and_then(|state| state.next_challenge_at_unix_millis)) + }; + let stored = stored_wait(&routed)?; + + // Fifty laps: one attempt, nothing reported as progress. + for _ in 0..50 { + assert!(!routed.tick()?, "a write the store refused was reported as progress"); + } + assert_eq!(routed.reloads() - before, 1, "the failing reload spun"); + assert_eq!(stored_wait(&routed)?, stored, "the record changed though writes fail"); + let due = challenge_due(&routed.world).expect("no in-memory wait after a failed record"); + let age = routed.world.engine.options.topology_maximum_age_millis; + assert!(due >= began + age, "the wait is shorter than one observation age: {due}"); + + // The wait expiring is the only thing that admits another + // attempt, and each one is again a single reload with no + // progress. The failed record is said once, not per attempt. + for attempt in 2..=4 { + routed.world.engine.challenge_wait.lock().unwrap().clear(); + assert!(!routed.tick()?); + assert!(!routed.tick()?); + assert_eq!(routed.reloads() - before, attempt); + } + return Ok(()); + }; + assert_eq!( + stderr.matches("could not record the failed route challenge of service").count(), + 1, + "{stderr}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unwritable_store_challenges_a_stale_route_once_per_wait() -> Result<()> { + let name = "an_unwritable_store_challenges_a_stale_route_once_per_wait"; + let Some(stderr) = in_unwritable_store_child(name)? else { + let mut routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.engine.options.topology_maximum_age_millis = 8_000; + std::thread::sleep(Duration::from_millis(8_300)); + break_store_writes(&routed.world); + assert!(!routed.tick()?, "a write the store refused was reported as progress"); + let first = challenge_due(&routed.world).expect("the proved challenge left no wait"); + for _ in 0..5 { + std::thread::sleep(Duration::from_millis(3)); + assert!(!routed.tick()?); + } + assert_eq!(challenge_due(&routed.world), Some(first), "the route was challenged again"); + return Ok(()); + }; + assert_eq!( + stderr.matches("could not record the proved route challenge of service").count(), + 1, + "{stderr}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_landed_challenge_record_is_progress_and_leaves_no_in_memory_wait() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + routed.drift()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.challenge_wait.lock().unwrap().insert( + "service".into(), + ChallengeWait { due_unix_millis: 0, stamp: snapshot.target_stamp("service") }, + ); + assert!(routed.tick()?, "a landed record is progress"); + assert_eq!(challenge_due(&routed.world), None); + Ok(()) + } + + /// A fragment that re-drifts after every restore, with the store + /// unwritable, is reloaded once per wait, however many laps run. + #[cfg(unix)] + #[test] + fn a_re_drifting_fragment_is_reloaded_once_per_wait_when_the_store_is_unwritable() -> Result<()> { + let name = "a_re_drifting_fragment_is_reloaded_once_per_wait_when_the_store_is_unwritable"; + let Some(_) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let before = routed.reloads(); + break_store_writes(&routed.world); + for _ in 0..30 { + routed.drift()?; + assert!(!routed.tick()?, "a refused write was reported as progress"); + } + assert_eq!(routed.reloads() - before, 1, "a re-drifting fragment reloaded every lap"); + for attempt in 2..=4 { + routed.world.engine.challenge_wait.lock().unwrap().clear(); + routed.drift()?; + assert!(!routed.tick()?); + routed.drift()?; + assert!(!routed.tick()?); + assert_eq!(routed.reloads() - before, attempt); + } + return Ok(()); + }; + Ok(()) + } + + /// The wait of a proved challenge whose record failed is one + /// observation age from the attempt. + #[cfg(unix)] + #[test] + fn a_proved_challenge_that_cannot_be_recorded_waits_one_observation_age() -> Result<()> { + let name = "a_proved_challenge_that_cannot_be_recorded_waits_one_observation_age"; + let Some(_) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let before = routed.reloads(); + break_store_writes(&routed.world); + let began = now_millis()?; + routed.drift()?; + assert!(!routed.tick()?); + assert_eq!(routed.reloads() - before, 1, "the repair did not run"); + let due = challenge_due(&routed.world).expect("no wait"); + let age = routed.world.engine.options.topology_maximum_age_millis; + assert!(due >= began + age, "the wait is shorter than one age: {}", due - began); + return Ok(()); + }; + Ok(()) + } + + /// The stored schedule sets the in-memory wait: with three failures + /// recorded, the fourth failed challenge waits eight ages (capped), and + /// no repair runs inside that wait. + #[cfg(unix)] + #[test] + fn a_failed_challenge_that_cannot_be_recorded_waits_as_long_as_the_stored_schedule() -> Result<()> { + let name = "a_failed_challenge_that_cannot_be_recorded_waits_as_long_as_the_stored_schedule"; + let Some(_) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + edit_incumbent(&routed.world, |generation| { + let state = generation.route_supervision.as_mut().unwrap(); + state.consecutive_failures = 3; + state.next_challenge_at_unix_millis = Some(1); + state.degraded_since_unix_millis = Some(1); + })?; + break_store_writes(&routed.world); + let began = now_millis()?; + routed.drift()?; + assert!(!routed.tick()?); + let due = challenge_due(&routed.world).expect("no wait"); + let age = routed.world.engine.options.topology_maximum_age_millis; + let want = (age * 8).min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); + assert!(due >= began + want, "the wait ignores the stored doubling: {}", due - began); + let reloads = routed.reloads(); + for _ in 0..10 { + routed.drift()?; + assert!(!routed.tick()?); + } + assert_eq!(routed.reloads(), reloads, "the route was repaired inside its wait"); + return Ok(()); + }; + Ok(()) + } + + /// A lost race is progress, and the next pass is governed by the + /// winner's record, not by an in-memory wait of the loser. + #[cfg(unix)] + #[test] + fn a_lost_race_leaves_no_in_memory_wait() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + let old = ControlSnapshot::read(&routed.world.state_store)?; + let stale = old.admitted_for("service").context("no generation")?; + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().last_challenge_at_unix_millis = Some(5); + })?; + routed.drift()?; + assert!(routed.world.engine.supervise_admitted_route(&old, stale)?); + assert_eq!(challenge_due(&routed.world), None, "a lost race left an in-memory wait"); + Ok(()) + } + + /// The wait left by a full disk does not outlive it: once any write for + /// the target lands, the stored schedule owns the target again and a + /// drift is repaired without waiting. + #[cfg(unix)] + #[test] + fn a_wait_left_by_a_full_disk_ends_when_any_write_for_the_target_lands() -> Result<()> { + let name = "a_wait_left_by_a_full_disk_ends_when_any_write_for_the_target_lands"; + let Some(_) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let before = routed.reloads(); + break_store_writes(&routed.world); + routed.drift()?; + assert!(!routed.tick()?); + assert!(challenge_due(&routed.world).is_some(), "no wait to outlive"); + assert_eq!(routed.reloads() - before, 1); + + // Space is freed. Nothing has landed yet, so the wait still holds. + unsafe { + let limit = libc::rlimit { rlim_cur: libc::RLIM_INFINITY, rlim_max: libc::RLIM_INFINITY }; + assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0); + } + routed.drift()?; + assert!(!routed.tick()?); + assert_eq!(routed.reloads() - before, 1, "the wait did not hold before any write landed"); + + // Something else writes for the target: the wait is over. + routed.ledger(1)?; + routed.drift()?; + assert!(routed.tick()?, "the drift was not repaired"); + assert_eq!(routed.reloads() - before, 2, "a stale wait held the repair"); + assert_eq!(challenge_due(&routed.world), None); + return Ok(()); + }; + Ok(()) + } + + /// A wait belongs to an admitted target: once the target is retired, + /// the wait is gone. + #[cfg(unix)] + #[test] + fn a_wait_ends_when_its_target_is_retired() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.challenge_wait.lock().unwrap().insert( + "service".into(), + ChallengeWait { + due_unix_millis: now_millis()? + 500_000, + stamp: snapshot.target_stamp("service"), + }, + ); + assert!(!routed.tick()?); + assert!(challenge_due(&routed.world).is_some(), "an unchanged store dropped the wait"); + assert!( + SingleFileMessagePackBackingStore::new(&routed.world.state_store) + .delete_batch_if_unchanged(&[snapshot.admitted_for("service").unwrap().envelope.clone()])? + ); + routed.tick()?; + assert_eq!(challenge_due(&routed.world), None, "a retired target kept its wait"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_full_ledger_still_restores_the_admitted_and_incumbent_route() -> Result<()> { + let full = ROUTE_ACTUATION_CEILING; + // The newest recorded Survival, or 0; the Forward ledger stays full. + let newest = |world: &EngineFixture| -> Result { + let meters = meters_of(world, "service")?; + assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); + Ok(meters.route_survivals.last().copied().unwrap_or(0)) + }; + + // (i) A post-fence abort withdraws the candidate's membership. + let aborting = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + aborting.ledger(full)?; + let before = newest(&aborting.world)?; + aborting + .world + .engine + .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; + aborting.step()?; + assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); + assert!(newest(&aborting.world)? > before, "the withdrawal was not recorded"); + + // (ii) Supervision repairs a drifted admitted fragment. + let supervised = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + supervised.admit()?; + supervised.ledger(full)?; + let (before, reloads) = (newest(&supervised.world)?, supervised.reloads()); + supervised.drift()?; + assert!(supervised.tick()?); + assert_eq!(supervised.reloads(), reloads + 1, "the repair did not run"); + assert!(newest(&supervised.world)? > before, "the repair was not recorded"); + assert!(admitted(&supervised)?.route_supervision.unwrap().degraded_since_unix_millis.is_none()); + + // (iii) A failed proof at Routing rolls the route back, though the + // install that preceded it took the last free slot. The rollback + // is recorded and leaves the Forward ledger exactly full. + let failing = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + failing.stub.set_state("active"); + failing.run_to_routing()?; + failing.ledger(full - 1)?; + failing.stub.hang_up.store(true, Ordering::SeqCst); + let before = now_millis()?; + assert!(failing.step().is_err(), "an unanswered stable route was admitted"); + assert_eq!(failing.reloads(), 2, "install then rollback"); + assert!(newest(&failing.world)? >= before, "the rollback was not recorded"); + assert!(!failing.fragment_path().exists(), "the rollback left the candidate's route"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_continuity_restart_is_never_refused_by_a_full_ledger() -> Result<()> { + // Warming: the preflight of a continuity runs and is recorded, + // and the Forward ledger is left as it was. + let warming = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + warming.step()?; + assert!(warming.transaction()?.warming.is_some()); + warming.ledger(ROUTE_ACTUATION_CEILING)?; + let ledger = meters_of(&warming.world, "service")?.route_actuations; + warming.step()?; + assert!(warming.transaction()?.route_preflight.is_some(), "the preflight was refused"); + let after = meters_of(&warming.world, "service")?; + assert_eq!(after.route_survivals.len(), 1, "the preflight was not recorded"); + assert_eq!(after.route_actuations, ledger, "a survival was counted"); + + // Routing: its install runs and is recorded, and not counted. + let routing = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routing.stub.set_state("active"); + routing.run_to_routing()?; + routing.ledger(ROUTE_ACTUATION_CEILING)?; + let ledger = meters_of(&routing.world, "service")?.route_actuations; + routing.step()?; + assert!(routing.reloads() >= 1, "the install did not run"); + assert!(routing.fragment_path().exists()); + let after = meters_of(&routing.world, "service")?; + assert_eq!(after.route_survivals.len(), 1, "the install was not recorded"); + assert_eq!(after.route_actuations, ledger, "a survival was counted"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_survival_that_cannot_be_recorded_runs_and_a_forward_is_refused() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let transaction = routed.transaction()?; + let preflight = transaction.route_preflight.clone().context("no preflight")?; + let driver = routed.world.engine.route_driver( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ); + // The ledger cannot be read, so nothing can be recorded. + std::fs::write(&routed.world.state_store, b"unreadable")?; + let engine = &routed.world.engine; + + let refused = engine + .route_gate("service", CommandKind::Deploy) + .admit(RouteActuation::Forward) + .unwrap_err(); + assert!(refused.downcast_ref::().is_none(), "{refused:#}"); + assert!(engine.charge_route_actuation("service", RouteActuation::Forward).is_err()); + + engine.charge_route_actuation("service", RouteActuation::Survival)?; + driver.withdraw_candidate_membership( + &preflight, + &engine.route_gate("service", CommandKind::Deploy), + )?; + assert_eq!(routed.reloads(), 1, "the unrecorded withdrawal did not run"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_second_failed_challenge_waits_twice_as_long_before_the_next_repair() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + let elapse = |routed: &RoutedWorld| { + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = + Some(1); + }) + }; + + routed.drift()?; + assert!(routed.tick()?); + elapse(&routed)?; + routed.drift()?; + assert!(routed.tick()?); + let state = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(state.consecutive_failures, 2); + assert_eq!( + state.next_challenge_at_unix_millis.unwrap() - state.last_challenge_at_unix_millis.unwrap(), + 2 * DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS + ); + + // The wait is longer than one observation age: no challenge, so no + // repair and no reload, however often the pass runs. + let reloads = routed.reloads(); + for _ in 0..20 { + routed.drift()?; + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "the route was repaired inside its wait"); + assert_eq!( + admitted(&routed)?.route_supervision.unwrap().consecutive_failures, + 2 + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_proved_challenge_that_cannot_be_recorded_ends_the_pass_without_failing_it() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let old = ControlSnapshot::read(&routed.world.state_store)?; + let stale = old.admitted_for("service").context("no generation")?; + // Something else writes the generation after the pass read it. + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().last_challenge_at_unix_millis = Some(5); + })?; + routed.drift()?; + assert!(routed.world.engine.supervise_admitted_route(&old, stale)?); + assert_eq!( + admitted(&routed)?.route_supervision.unwrap().last_challenge_at_unix_millis, + Some(5), + "the stale write replaced the newer generation" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_first_routed_deploy_is_metered_and_refused_at_the_ceiling() -> Result<()> { + let metered = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + metered.stub.set_state("active"); + metered.run_to_routing()?; + assert!(ControlSnapshot::read(&metered.world.state_store)?.targets.is_empty()); + metered.step()?; + assert_eq!(meters_of(&metered.world, "service")?.route_actuations.len(), 1); + + let refused = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + refused.stub.set_state("active"); + refused.run_to_routing()?; + refused.ledger(ROUTE_ACTUATION_CEILING)?; + let error = refused.step().unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert_eq!(refused.world.route_stubs.count(""), 0, "a refused install ran a program"); + assert!(!refused.fragment_path().exists()); + + // Past the fence the refusal is resumable, not an abort: the resume + // backoff spaces it and no actuation runs while it waits. + assert!(!refused.world.engine.resume_one_transaction()?); + let waiting = refused.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::Routing); + assert!(waiting.post_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|text| text.contains("ceiling"))); + assert_eq!(refused.world.route_stubs.count(""), 0); + Ok(()) + } + + #[test] + fn a_stepped_back_clock_is_settled_where_the_decision_is_made() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let ahead = now_millis()? + 86_400_000; + + // A supervision pass that would decide from the restart log settles + // it first: the write ends the pass, and nothing is minted from + // entries that only look recent because the clock stepped back. + let mut restarts = TargetSupervision::new("service"); + restarts.continuity_restarts = vec![ahead; CONTINUITY_PAGING_RESTARTS]; + set_meters(&routed.world, &restarts)?; + routed.workload.kill(); + assert!(routed.tick()?); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let settled = meters_of(&routed.world, "service")?; + assert!(settled.continuity_restarts.iter().all(|&at| at < ahead)); + + // A refused forward actuation writes the clamp, so the window runs + // from now and not from the day the clock stood on. + let mut ledger = TargetSupervision::new("service"); + ledger.route_actuations = vec![ahead; ROUTE_ACTUATION_CEILING]; + set_meters(&routed.world, &ledger)?; + let refused = routed + .world + .engine + .charge_route_actuation("service", RouteActuation::Forward) + .unwrap_err(); + assert!(refused.downcast_ref::().is_some()); + let settled = meters_of(&routed.world, "service")?; + assert!(settled.route_actuations.iter().all(|&at| at < ahead)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_refused_preflight_starts_no_private_mount_unit_and_fails_at_once_with_the_reopen_time() + -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + routed.step()?; + assert!(routed.transaction()?.warming.is_some()); + routed.ledger(ROUTE_ACTUATION_CEILING)?; + let reopens_at = meters_of(&routed.world, "service")? + .route_reopens_at(now_millis()?) + .context("the ledger is not full")?; + + // The refusal is an error of its own phase, before the fence: the + // deployment fails now, and its error carries the reopen time. + let id = routed.transaction()?.transaction_id; + routed.world.engine.resume_one_transaction()?; + let abort = routed + .transaction()? + .pre_fencing_abort + .context("the refusal did not abort the deployment")?; + assert!(abort.error.contains(&format!("reopens at unix ms {reopens_at}")), "{}", abort.error); + for _ in 0..8 { + if record_of(&routed.world, &id)?.completion.is_some() { + break; + } + routed.step()?; + } + let Some(TransactionCompletion::FailedBeforeFencing { error }) = + record_of(&routed.world, &id)?.completion + else { + let record = record_of(&routed.world, &id)?; + bail!( + "the deployment did not fail before the fence: phase {:?} completion {:?} abort {:?} error {:?}", + record.phase, record.completion, record.pre_fencing_abort, record.last_error + ); + }; + assert!(error.contains(&format!("reopens at unix ms {reopens_at}")), "{error}"); + assert_eq!(routed.world.route_stubs.count(""), 0, "a program ran"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.workload.kill(); + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + + // Tick 1: the repair charges the target's meters and the proof + // fails. Both are writes, so they end the pass: nothing else is + // decided from the snapshot they made stale. + assert!(routed.tick()?); + let state = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(state.consecutive_failures, 1); + assert!(state.degraded_since_unix_millis.is_some()); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_survivals.len(), 2); + assert!(meters.route_actuations.is_empty()); + + // Tick 2: the route waits, and the dead unit is restarted from a + // fresh read, counted in the same write that mints it. + assert!(routed.tick()?); + assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); + assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_proved_challenge_clears_the_degradation() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + assert!(routed.tick()?); + let degraded = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(degraded.consecutive_failures, 1); + assert!(degraded.degraded_since_unix_millis.is_some()); + assert!(degraded.next_challenge_at_unix_millis.is_some()); + + routed.stub.hang_up.store(false, Ordering::SeqCst); + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); + })?; + routed.drift()?; + assert!(routed.tick()?); + let healed = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(healed.consecutive_failures, 0); + assert_eq!(healed.next_challenge_at_unix_millis, None); + assert_eq!(healed.degraded_since_unix_millis, None); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failing_reload_that_puts_the_drift_back_backs_the_route_off() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + let reloads = routed.reloads(); + routed.drift()?; + for _ in 0..200 { + routed.tick()?; + } + assert_eq!(routed.reloads() - reloads, 1, "the route was reloaded again and again"); + assert_eq!(std::fs::read(routed.fragment_path())?, b"drifted bytes\n"); + assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_candidate_whose_route_proof_fails_reloads_at_most_twice_the_ceiling() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + let mut last = None; + for _ in 0..40 { + last = Some(routed.step().unwrap_err()); + } + // Every install is a counted Forward and is followed by one + // recorded rollback, so the rollbacks are bounded by the installs: + // twelve installs spend the window, each reloads twice, and every + // later attempt is refused before it runs anything. + assert_eq!(routed.reloads(), 2 * ROUTE_ACTUATION_CEILING); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_used(now_millis()?), ROUTE_ACTUATION_CEILING); + assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); + let last = last.unwrap(); + assert!(last.downcast_ref::().is_some(), "{last:#}"); + Ok(()) + } + + /// The rescue (ruling 3d A): a target whose continuity spent every + /// restart, each charging its preflight and install as Survival, is + /// still deployed: its route changes are the ceiling's first Forward. + #[cfg(unix)] + #[test] + fn a_target_that_spent_its_restarts_can_still_be_deployed() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(routed.transaction()?.warming.is_some()); + set_meters( + &routed.world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + for _ in 0..2 * CONTINUITY_PAGING_RESTARTS { + routed.world.engine.charge_route_actuation("service", RouteActuation::Survival)?; + } + let spent = meters_of(&routed.world, "service")?; + assert!(spent.restarts_page(now_millis()?)); + assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_PAGING_RESTARTS); + + // Warming's preflight and Routing's install are both admitted. + routed.step()?; + assert!(routed.transaction()?.route_preflight.is_some(), "the preflight was refused"); + routed.run_to_routing()?; + routed.promote()?; + assert!(routed.fragment_path().exists(), "the install was refused"); + assert!(routed.reloads() >= 1); + let after = meters_of(&routed.world, "service")?; + assert_eq!(after.route_actuations.len(), 2, "preflight and install are the Forward"); + Ok(()) + } + + // ----------------------------------------------------------------- + // B5: every post-fencing phase ends. + // ----------------------------------------------------------------- + + impl RoutedWorld { + /// One scheduler pass over the transaction, as the daemon runs it: + /// through the deadline resolver, with the resume backoff spent. + fn resume(&self) -> Result<()> { + self.world.engine.resume_backoff.lock().unwrap().clear(); + self.world.engine.resume_one_transaction()?; + Ok(()) + } + + fn resume_until_terminal(&self) -> Result { + for _ in 0..12 { + if latest(&self.world)?.completion.is_some() { + break; + } + self.resume()?; + } + let finished = latest(&self.world)?; + ensure!(finished.completion.is_some(), "never finished: {:?}", finished.last_error); + Ok(finished) + } + } + + /// The live transaction's phase deadline is already behind it. + fn pass_deadline(world: &EngineFixture) -> Result<()> { + let current = resident(world)?; + let mut next = current.value.clone(); + let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, ¤t, &next) + } + + /// Soul's F1 probe, committed. A continuity's route changes are + /// Survival, so no ceiling refuses its install and rollback while its + /// route proof keeps failing. The Routing deadline ends it: no attempt + /// runs after the deadline. + #[cfg(unix)] + #[test] + fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..3 { + routed.resume()?; + } + // Before the deadline each attempt installs and rolls back, and no + // resume moves the deadline. + assert_eq!(routed.reloads(), 6); + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::Routing); + assert!(waiting.post_fencing_abort.is_none(), "aborted before its deadline"); + assert!(waiting.last_error.is_some()); + assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); + + pass_deadline(&routed.world)?; + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the passed deadline did not end the phase")?; + assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + let finished = routed.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + ), + "{:?}", + finished.completion + ); + // After the deadline no attempt runs; the withdrawal writes and + // reloads once, whatever the rolled-back disk shows. + assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); + Ok(()) + } + + /// The live shape of F1. A continuity reuses the admitted Expected, so + /// its membership is the admitted fragment already on disk: while its + /// proof fails, the install and the rollback write nothing and reload + /// nothing, and neither does the withdrawal after its deadline. + #[cfg(unix)] + #[test] + fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + let binding = next.plan.as_ref().unwrap().parsed_inputs()?.1; + let admitted = NginxRouteDriver::new(binding.route.context("no route binding")?) + .render(next.expected.as_ref().context("no Expected")?)?; + let preflight = next.route_preflight.as_mut().context("no preflight")?; + preflight.incumbent_runtime_instance_id = Some(sha256_id(b"admitted-instance")); + preflight.incumbent_membership_sha256 = Some(sha256_id(&admitted)); + preflight.incumbent_configuration = Some(admitted.clone()); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + std::fs::write(routed.fragment_path(), &admitted)?; + + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..5 { + routed.resume()?; + } + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); + assert!(routed.transaction()?.last_error.is_some(), "the proof did not fail"); + assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); + + pass_deadline(&routed.world)?; + routed.resume_until_terminal()?; + assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); + assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); + Ok(()) + } + + /// The abort's withdrawal wrote the baseline back and Idunn died before + /// the reload: the disk already shows the baseline, and the retry still + /// reloads, so nginx does not keep proxying to the stopped candidate. + #[cfg(unix)] + #[test] + fn an_abort_whose_reload_a_crash_cut_off_reloads_on_retry() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + let installed = routed.reloads(); + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; + std::fs::remove_file(routed.fragment_path())?; + routed.resume()?; + assert_eq!( + routed.transaction()?.post_fencing_abort.context("no abort")?.route_restoration, + CleanupEvidence::Complete + ); + assert_eq!(routed.reloads(), installed + 1, "the withdrawal trusted unloaded bytes"); + Ok(()) + } + + /// Soul's F2 probe, committed: a post-fence withdrawal whose + /// `ufw delete` keeps failing withdraws the firewall allow first, so its + /// retries reload nginx not at all. + #[cfg(unix)] + #[test] + fn a_withdrawal_whose_firewall_step_keeps_failing_never_reloads() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + let installed = routed.reloads(); + routed.world.route_stubs.refuse_ufw_deletes()?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; + for _ in 0..20 { + routed.resume()?; + } + assert_eq!(routed.reloads(), installed, "a failing withdrawal reloaded nginx"); + assert_eq!(routed.world.route_stubs.count("ufw delete"), 20); + assert!(routed.fragment_path().exists(), "the route went before its firewall rule"); + Ok(()) + } + + /// A record lifted from before deadlines existed is timed once, from + /// the first time the resolver sees it, and never again. + #[test] + fn a_post_fence_record_without_a_deadline_is_timed_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let mut lifted = current.value.clone(); + lifted.phase_deadline = None; + replace_transaction(&routed.world.state_store, ¤t, &lifted)?; + let engine = &routed.world.engine; + + let before = now_millis()?; + assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, Some(true)); + let stamped = routed.transaction()?.phase_deadline.context("the record was not timed")?; + assert_eq!(stamped.phase, DeploymentPhase::Fencing); + assert!(stamped.entered_at_unix_millis >= before); + assert_eq!( + stamped.deadline_at_unix_millis - stamped.entered_at_unix_millis, + u64::from(lifted.plan.as_ref().unwrap().phase_deadlines().fencing_seconds) * 1000 + ); + assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, None); + assert_eq!(routed.transaction()?.phase_deadline, Some(stamped)); + Ok(()) + } + + /// `idunn expire` writes a request and nothing else; the resolver ends + /// the phase from it, and the spent request is deleted. + #[test] + fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { + let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let refused = expire(&early.world.state_store, "continuity-service", "operator") + .expect_err("a transaction before the fence was expired"); + assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); + + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let untouched = resident(&routed.world)?.envelope; + expire(&routed.world.state_store, "continuity-service", "operator")?; + assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!(snapshot.expiry_requests.len(), 1); + assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); + + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the expiry did not end the phase")?; + assert!(abort.error.contains("Fencing phase was expired by the operator"), "{}", abort.error); + routed.resume()?; + assert!( + ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), + "the spent request stayed" + ); + Ok(()) + } + + /// The `expire` verb as the CLI parses and runs it: the options land + /// where they say, and it asks only about its own command's live + /// transaction. + #[test] + fn idunn_expire_parses_its_options_and_asks_only_about_its_own_live_transaction() + -> Result<()> { + let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); + assert_eq!( + parse(args(&["expire", "up-service", "--requested-by", "alice", "--state-store", "/s/c.cc"]).into_iter())?, + Command::Expire { + state_store: PathBuf::from("/s/c.cc"), + command_id: "up-service".into(), + requested_by: "alice".into(), + } + ); + let help = parse(args(&["expire", "up-service", "--help"]).into_iter()).unwrap_err(); + assert!(format!("{help}").contains("idunn expire "), "{help}"); + assert!(parse(args(&["expire", "up-service", "--force"]).into_iter()).is_err()); + assert!(parse(args(&["expire"]).into_iter()).is_err()); + + // The verb runs: a store with nothing live refuses. + let empty = TempDir::new()?; + let store = empty.path().join("control.cc").display().to_string(); + let refused = run(args(&["expire", "up-service", "--state-store", &store]).into_iter()) + .expect_err("expired a command with no transaction"); + assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); + + // Another target's deployment in flight is not this command's. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let now = now_millis()?; + let other = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-other".into(), + kind: CommandKind::Deploy, + selector: "other".into(), + requested_by: "test".into(), + requested_at_unix_millis: now, + }; + let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; + assert!(SingleFileMessagePackBackingStore::new(&routed.world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: other.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: sealing.transaction_id.clone(), + current: None, + }, + ], + &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], + )?); + expire(&routed.world.state_store, "continuity-service", "operator")?; + let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].value.command_id, "continuity-service"); + Ok(()) + } + + /// The resolver reads a request as its own transaction's only: with two + /// live transactions, a request for the other leaves this one running. + #[test] + fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let now = now_millis()?; + let other = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-other".into(), + kind: CommandKind::Deploy, + selector: "other".into(), + requested_by: "test".into(), + requested_at_unix_millis: now, + }; + let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; + // The request names a command whose id has this transaction's + // length and first bytes, and differs only in its last. + let near = "continuity-servicx"; + assert_eq!(near.len(), "continuity-service".len()); + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: near.into(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + // And commands whose ids are a strict prefix of this one's, and + // extend it: a match is the whole id, in neither direction a part. + let prefix = ExpiryRequest { + command_id: "continuity-servic".into(), + ..request.clone() + }; + let extension = ExpiryRequest { + command_id: "continuity-service-2".into(), + ..request.clone() + }; + let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); + let absent = |r#type: &str, key: &str| CultCacheExpectedEnvelope { + r#type: r#type.into(), + key: key.into(), + current: None, + }; + assert!(store.compare_exchange( + &[ + absent(DeploymentCommand::TYPE, &other.command_id), + absent(DeploymentTransaction::TYPE, &sealing.transaction_id), + absent(ExpiryRequest::TYPE, &request.command_id), + absent(ExpiryRequest::TYPE, &prefix.command_id), + absent(ExpiryRequest::TYPE, &extension.command_id), + ], + &[ + command_envelope(&other, now)?, + transaction_envelope(&sealing, now)?, + expiry_request_envelope(&request, now)?, + expiry_request_envelope(&prefix, now)?, + expiry_request_envelope(&extension, now)?, + ], + )?); + assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.len(), 3); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!(snapshot.transactions.iter().filter(|t| !t.value.is_terminal()).count(), 2); + let mine = snapshot + .transactions + .iter() + .find(|t| t.value.command_id == "continuity-service") + .context("no continuity")?; + assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); + let after = ControlSnapshot::read(&routed.world.state_store)?; + let unchanged = after + .transactions + .iter() + .find(|t| t.value.command_id == "continuity-service") + .context("no continuity")?; + assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); + Ok(()) + } + + /// Q3 at the deadline, through the Engine. A continuity whose candidate + /// was granted a lease and never proved it held it fails as + /// `lease-not-adopted`; one whose Ready proved it restarts the admitted + /// release. A deployment whose candidate holds a lease is left to the + /// operator: nothing is aborted and its step still runs. + #[test] + fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { + let granted = |routed: &RoutedWorld| -> Result { + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + Ok(routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned()) + }; + + let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + granted(&unadopted)?; + pass_deadline(&unadopted.world)?; + unadopted.resume()?; + let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; + assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); + assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); + let finished = unadopted.resume_until_terminal()?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + + let adopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + let lease = granted(&adopted)?; + adopted.stub.set_state("active"); + *adopted.stub.lease.lock().unwrap() = Some(lease.clone()); + adopted.step()?; + let ready = adopted.transaction()?; + let Some(ReadinessEvidence::RouteProof { evidence }) = &ready.ready else { + bail!("the candidate was not Ready"); + }; + let adoption = ready.lease_adoption.clone().context("Ready recorded no adoption")?; + assert_eq!(adoption.write_lease_sha256, lease); + assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); + assert_eq!(adoption.source, AdoptionSource::Direct); + adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + pass_deadline(&adopted.world)?; + adopted.resume()?; + let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; + assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + let finished = adopted.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestartAdmitted, + .. + }) + ), + "{:?}", + finished.completion + ); + + let deploy = + build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; + granted(&deploy)?; + pass_deadline(&deploy.world)?; + for _ in 0..3 { + deploy.resume()?; + } + let held = deploy.transaction()?; + assert!(held.post_fencing_abort.is_none(), "a leased deployment was aborted"); + assert_eq!(held.phase, DeploymentPhase::AwaitingReady); + assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); + Ok(()) + } + + /// B5 Soul A. The physical grant lands before `Granted` is durable. A + /// continuity that dies in that window and passes its deadline is + /// aborted from `Prepared`; the abort still withdraws the lease on + /// disk, so the next fencing of the target is not refused. + #[test] + fn an_abort_from_a_prepared_lease_withdraws_a_grant_already_on_disk() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + let current = resident(&routed.world)?; + let mut crashed = current.value.clone(); + let Some(LeasingEvidence::Granted { lease, lease_sha256 }) = crashed.leasing.clone() else { + bail!("no granted lease"); + }; + crashed.leasing = Some(LeasingEvidence::Prepared { lease: lease.clone(), lease_sha256 }); + replace_transaction(&routed.world.state_store, ¤t, &crashed)?; + let driver = CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ); + assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); + + pass_deadline(&routed.world)?; + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + routed.resume_until_terminal()?; + assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); + // What the next fencing does to a candidate's lease path. + driver.revoke_exact(None)?; + Ok(()) + } + + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word + /// carries the granted lease, and that Ready records the adoption, + /// sourced from Odin's topology and naming the presence Odin observed. + #[test] + fn an_odin_ready_word_records_a_stateful_candidates_lease_adoption() -> Result<()> { + let routed = build_routed_world( + Odin::Unreachable, + 1, + DeploymentPhase::Fencing, + true, + true, + CommandKind::Continuity, + )?; + for _ in 0..8 { + if routed.transaction()?.phase == DeploymentPhase::AwaitingReady { + break; + } + routed.step()?; + } + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady, "{:?}", waiting.last_error); + let lease = waiting + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + assert_eq!(waiting.lease_adoption, None); + + let ready = CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &waiting.target, + &waiting.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation_as( + &routed.world, + &waiting, + waiting.odin_publisher_sequence_cursor + 1, + true, + "active", + Some(&lease), + b"presence-ready", + )?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }; + replace_odin_entry(&routed.world.engine.options.odin_correlation_store, ready)?; + // One step admits the new sequence, the next records Ready from it. + for _ in 0..3 { + if routed.transaction()?.ready.is_some() { + break; + } + routed.step()?; + } + let adopted = routed.transaction()?; + assert!( + matches!(adopted.ready, Some(ReadinessEvidence::OdinCorrelated { .. })), + "{:?}", + adopted.last_error + ); + let adoption = adopted.lease_adoption.context("Odin's Ready recorded no adoption")?; + assert_eq!(adoption.source, AdoptionSource::OdinTopology); + assert_eq!(adoption.write_lease_sha256, lease); + assert_eq!(adoption.signed_presence_sha256, sha256_id(b"presence-ready")); + Ok(()) + } + + /// F16: Commit goes on from the topology it admitted, so Odin + /// re-stamping before the step and again during the final proof does + /// not send it round again. + #[test] + fn commit_completes_while_odin_restamps_every_tick() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + let transaction = routed.transaction()?; + let cursor = transaction.odin_publisher_sequence_cursor; + let store = routed.world.engine.options.odin_correlation_store.clone(); + replace_odin_entry(&store, odin_ready_envelope(&routed.world, &transaction, cursor + 1)?)?; + let later = odin_ready_envelope(&routed.world, &transaction, cursor + 2)?; + *routed.stub.on_stable_hit.lock().unwrap() = Some(Box::new(move || { + replace_odin_entry(&store, later.clone()).expect("Odin re-stamps"); + })); + routed.step()?; + let generation = admitted(&routed)?; + assert_eq!( + generation.latest_odin_observation.context("no Odin receipt")?.publisher_sequence, + cursor + 2, + "commit did not go on from the re-stamped topology" + ); + Ok(()) + } + + fn odin_ready_envelope( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result { + let expected = transaction.expected.as_ref().context("no Expected")?; + Ok(CultCacheEnvelope { + key: crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(world, transaction, sequence, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }) + } + + fn replace_odin_entry(store: &Path, envelope: CultCacheEnvelope) -> Result<()> { + let backing = SingleFileMessagePackBackingStore::new(store); + let current = backing + .pull_all_read_only_snapshot()? + .into_iter() + .find(|entry| entry.key == envelope.key && entry.r#type == envelope.r#type); + ensure!( + backing.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: envelope.r#type.clone(), + key: envelope.key.clone(), + current, + }], + &[envelope], + )?, + "the Odin store changed under the re-stamp" + ); + Ok(()) + } + } +} From f6d2943fd3a5392ed62e068d3070070b0beb6ec4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 01:28:05 +0200 Subject: [PATCH 065/196] Restore LF line endings in control_plane.rs The previous commit rewrote the file with CRLF; this one returns it to LF so the range diff is the real change. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 40956 ++++++++++++++++++++--------------------- 1 file changed, 20478 insertions(+), 20478 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3037ab8..162e5fd 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1,20478 +1,20478 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::env; -use std::fs::{self, File, OpenOptions}; -use std::io::Write; -use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; -use std::thread; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcache_rs::{ - CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, -}; -use cultnet_rs::{ - AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, - IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, - IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, - IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, - IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, - IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, - OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, - ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, - authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, - correlate_runtime_presence_claim, derive_service_identity_id, - evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, - verify_idunn_deployment_brake_authorization, verify_runtime_authority, -}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use uuid::Uuid; - -use crate::deployment::{ - DependencyKind, OperatorBinding, RolloutStrategy, RouteBinding, WorkloadBinding, - capability_compatible, -}; -use crate::deployment_plan::{ - CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, -}; -use crate::drivers::{ - ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, - GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, - LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, - SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, - WriteLeasePort, -}; -use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; -use crate::host_actuator::{ - HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, - HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, -}; - -const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2"; -const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v4"; -const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3"; -const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2"; -const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v4"; -const ADMITTED_GENERATION_SCHEMA_V3: &str = "idunn.admitted_generation.v3"; -const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; -/// The capability whose declaration makes a target Odin-correlated. -pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; -const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v2"; -/// The seven-key layout written before continuity's streak, anchor and -/// re-materialization bound. Read and migrated by `read_supervision_record` -/// and `migrate_control_store_to_current_schema`; delete it, `LegacyTargetSupervisionV1` -/// and those two arms once every store has opened under a build that -/// migrates (the key-34 precedent's deletion line). -const TARGET_SUPERVISION_SCHEMA_V1: &str = "idunn.target_supervision.v1"; -const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1"; -/// The number of charged continuity attempts inside -/// `CONTINUITY_RESTART_WINDOW_MILLIS` at which the ContinuityExhausted incident -/// opens and is held. It gates no actuation: continuity never stops -/// (ruling budget-never-stop). -const CONTINUITY_PAGING_RESTARTS: usize = 6; -const CONTINUITY_RESTART_WINDOW_MILLIS: u64 = 3_600_000; -/// The base of the streak backoff: the wait after the first charged attempt; -/// it doubles with each further one in the streak. -const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; -/// The longest streak backoff between charged continuity attempts (ruling -/// continuity-backoff-defaults). -const CONTINUITY_BACKOFF_CEILING_MILLIS: u64 = 300_000; -/// How long an admitted generation must have run, with no attempt failing on -/// it, before the attempt streak resets (ruling continuity-backoff-defaults). -const CONTINUITY_SUSTAINED_RUN_MILLIS: u64 = 600_000; - -/// Forward route actuations (fragment write, firewall, `nginx -t`, reload, -/// private-mount validation) one target may perform per sliding window. A -/// healthy target performs none; a legitimate deployment performs one or two. -/// Only deployments count: a Survival actuation is recorded beside this -/// ceiling, never counted against it and never refused by it, so a target -/// that is being restarted can still be rescued by a deploy. -/// Survival is bounded elsewhere: a continuity's actuations by the streak -/// backoff (at most one attempt per `CONTINUITY_BACKOFF_CEILING_MILLIS` once -/// the streak is long) times what one transaction can do before its phase -/// deadline ends it (none, over its admitted route), and an admitted-route -/// repair by the challenge backoff. A withdrawal that keeps failing retries, -/// reloading each time, under the resume backoff alone: the abort has no -/// deadline. Nothing bounds -/// reloads across the host. -const ROUTE_ACTUATION_CEILING: usize = 12; -const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; -/// How many Survival route actuation times a target keeps, for `status` -/// only: they are a record, not a meter, and no decision reads them. -const ROUTE_SURVIVAL_RECORD: usize = 12; -/// The longest wait between route challenges while proofs keep failing. The -/// shortest is the observation max age. -const ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS: u64 = 600_000; - -/// How long continuity waits after it could not prepare a restart (the -/// projection would not demote) before trying again. -const CONTINUITY_DEFERRAL_MILLIS: u64 = 30_000; - -const DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS: u64 = 30_000; -const DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS: u64 = 2_000; - -/// An immutable request. Execution state deliberately does not fit in this -/// record; status is derived from the transactions created for it. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.deployment_command", - schema = "idunn.deployment_command.v2" -)] -struct DeploymentCommand { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - command_id: String, - #[cultcache(key = 2)] - kind: CommandKind, - #[cultcache(key = 3)] - selector: String, - #[cultcache(key = 4)] - requested_by: String, - #[cultcache(key = 5)] - requested_at_unix_millis: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -enum CommandKind { - Deploy, - Continuity, -} - -impl DeploymentCommand { - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == DEPLOYMENT_COMMAND_SCHEMA, - "deployment command schema is unsupported" - ); - require_id(&self.command_id, "deployment command id")?; - require_selector(&self.selector)?; - require_value(&self.requested_by, "deployment requester")?; - ensure!( - self.requested_at_unix_millis > 0, - "deployment command has no request time" - ); - match self.kind { - CommandKind::Deploy => ensure!( - !self.command_id.starts_with("continuity-"), - "operator deployment uses the continuity command namespace" - ), - CommandKind::Continuity => ensure!( - self.command_id.starts_with("continuity-") - && !self.selector.starts_with("profile:"), - "continuity command identity or selector is invalid" - ), - } - Ok(()) - } -} - -/// An operator's request that a live transaction's post-fencing phase end now -/// (`idunn expire`). The CLI only writes this record. The daemon's deadline -/// resolver reads it as a deadline that has passed, and the record is deleted -/// once its transaction no longer has a phase to end. No CLI writes a -/// transaction field. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache(type = "idunn.expiry_request", schema = "idunn.expiry_request.v1")] -struct ExpiryRequest { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - command_id: String, - #[cultcache(key = 2)] - requested_by: String, - #[cultcache(key = 3)] - requested_at_unix_millis: u64, -} - -impl ExpiryRequest { - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == EXPIRY_REQUEST_SCHEMA, - "expiry request schema is unsupported" - ); - require_id(&self.command_id, "expiry request command id")?; - require_value(&self.requested_by, "expiry requester")?; - ensure!( - self.requested_at_unix_millis > 0, - "expiry request has no request time" - ); - Ok(()) - } -} - -fn expiry_request_envelope(value: &ExpiryRequest, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.command_id, - ExpiryRequest::TYPE, - EXPIRY_REQUEST_SCHEMA, - value, - now, - ) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -#[repr(u8)] -enum DeploymentPhase { - Sealing, - Starting, - Warming, - Fencing, - Leasing, - AwaitingReady, - Routing, - Committing, - Complete, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct DeploymentAuthorization { - authorization_id: String, - brake_sha256: String, - canonical_brake_bytes: Vec, - authorized_at_unix_millis: u64, -} - -impl DeploymentAuthorization { - fn validate_shape(&self) -> Result<()> { - require_id(&self.authorization_id, "deployment authorization id")?; - ensure!( - !self.canonical_brake_bytes.is_empty() - && self.brake_sha256 == sha256_id(&self.canonical_brake_bytes) - && self.authorized_at_unix_millis > 0, - "deployment authorization receipt is incomplete" - ); - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&self.canonical_brake_bytes)?; - record.validate()?; - ensure!( - rmp_serde::to_vec(&record)? == self.canonical_brake_bytes - && record.authorization_id.as_deref() == Some(&self.authorization_id), - "deployment authorization receipt is noncanonical or mismatched" - ); - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct TopologyEvidence { - canonical_bytes: Vec, - canonical_sha256: String, - signer_identity_id: String, - publisher_sequence: u64, - admitted_at_unix_millis: u64, -} - -impl TopologyEvidence { - fn from_authenticated( - topology: &AuthenticatedOdinRuntimeTopologyCorrelation, - admitted_at_unix_millis: u64, - ) -> Result { - let record = topology.record(); - let evidence = Self { - canonical_bytes: topology.canonical_bytes().to_vec(), - canonical_sha256: sha256_id(topology.canonical_bytes()), - signer_identity_id: record.signer_identity_id.clone(), - publisher_sequence: record.publisher_sequence, - admitted_at_unix_millis, - }; - evidence.validate_shape()?; - Ok(evidence) - } - - fn validate_shape(&self) -> Result<()> { - ensure!( - !self.canonical_bytes.is_empty() - && self.canonical_sha256 == sha256_id(&self.canonical_bytes), - "topology evidence bytes or digest are invalid" - ); - require_id(&self.signer_identity_id, "Odin topology signer")?; - ensure!( - self.publisher_sequence > 0 && self.admitted_at_unix_millis > 0, - "topology evidence sequence or admission time is invalid" - ); - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct RuntimePresenceEvidence { - canonical_bytes: Vec, - canonical_sha256: String, - message_id: String, - challenged_at_unix_millis: u64, - admitted_at_unix_millis: u64, -} - -impl RuntimePresenceEvidence { - fn from_present( - present: &cultnet_rs::VerifiedRuntimePresence, - message_id: String, - challenged_at_unix_millis: u64, - admitted_at_unix_millis: u64, - ) -> Result { - let evidence = Self { - canonical_bytes: present.canonical_bytes().to_vec(), - canonical_sha256: present.signed_presence_sha256().to_owned(), - message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - }; - evidence.validate_shape()?; - Ok(evidence) - } - - fn validate_shape(&self) -> Result<()> { - ensure!( - !self.canonical_bytes.is_empty() - && self.canonical_sha256 == sha256_id(&self.canonical_bytes), - "runtime presence evidence bytes or digest are invalid" - ); - require_id(&self.message_id, "runtime presence challenge")?; - ensure!( - self.challenged_at_unix_millis > 0 - && self.admitted_at_unix_millis >= self.challenged_at_unix_millis, - "runtime presence evidence timeline is invalid" - ); - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "source", rename_all = "kebab-case", deny_unknown_fields)] -enum WarmingEvidence { - OdinTopology { evidence: TopologyEvidence }, - FirstOdinDirect { evidence: RuntimePresenceEvidence }, - /// A route-proof target's candidate answered Idunn's challenge itself. - RouteProofDirect { evidence: RuntimePresenceEvidence }, -} - -impl WarmingEvidence { - fn validate_shape(&self) -> Result<()> { - match self { - Self::OdinTopology { evidence } => evidence.validate_shape(), - Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { - evidence.validate_shape() - } - } - } - - fn voucher(&self) -> Voucher { - match self { - Self::OdinTopology { .. } | Self::FirstOdinDirect { .. } => Voucher::Odin, - Self::RouteProofDirect { .. } => Voucher::Candidate, - } - } - - /// The presence Idunn challenged for itself, when no Odin carried it. - fn direct(&self) -> Option<&RuntimePresenceEvidence> { - match self { - Self::OdinTopology { .. } => None, - Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { - Some(evidence) - } - } - } -} - -/// Opaque capability minted only after the transaction CAS stores exact -/// Warming evidence. Adjacent drivers receive only the candidate identity and -/// signed-presence digest needed to bind the write lease; they cannot inspect -/// or construct semantic admission. -#[derive(Clone, Debug)] -pub struct SequenceAdmittedWarming { - transaction_id: String, - signed_presence_sha256: String, - runtime_instance_id: String, -} - -impl SequenceAdmittedWarming { - fn from_topology( - transaction_id: String, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - ) -> Result { - let record = authenticated.record(); - let signed_presence_sha256 = record - .signed_presence_sha256 - .clone() - .context("Warming topology has no signed presence")?; - let runtime_instance_id = record - .runtime_instance_id - .clone() - .context("Warming topology has no runtime instance")?; - Ok(Self { - transaction_id, - signed_presence_sha256, - runtime_instance_id, - }) - } - - fn from_direct_presence( - transaction_id: String, - warming: WarmingEvidence, - present: cultnet_rs::VerifiedRuntimePresence, - ) -> Result { - let evidence = warming - .direct() - .context("direct Warming evidence is an Odin topology receipt")?; - ensure!( - present.canonical_bytes() == evidence.canonical_bytes - && present.signed_presence_sha256() == evidence.canonical_sha256, - "direct Warming evidence differs from its authenticated presence" - ); - let runtime_instance_id = present.record().runtime_instance_id.clone(); - Ok(Self { - transaction_id, - signed_presence_sha256: evidence.canonical_sha256.clone(), - runtime_instance_id, - }) - } - - #[cfg(test)] - pub(crate) fn for_test( - transaction_id: impl Into, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - ) -> Result { - let record = authenticated.record(); - ensure!( - record.present - && !record.ready - && record.observed_presence_state.as_deref() == Some("warming") - && record.observed_write_lease_sha256.is_none() - && record.disagreements.is_empty(), - "test receipt is not exact Warming evidence" - ); - let transaction_id = transaction_id.into(); - require_id(&transaction_id, "test transaction id")?; - let signed_presence_sha256 = record - .signed_presence_sha256 - .clone() - .context("test Warming receipt has no signed presence")?; - let runtime_instance_id = record - .runtime_instance_id - .clone() - .context("test Warming receipt has no runtime instance")?; - let token = Self::from_topology(transaction_id, authenticated)?; - ensure!( - token.signed_presence_sha256 == signed_presence_sha256 - && token.runtime_instance_id == runtime_instance_id, - "test Warming token extraction changed" - ); - Ok(token) - } - - pub(crate) fn transaction_id(&self) -> &str { - &self.transaction_id - } - - pub(crate) fn signed_presence_sha256(&self) -> &str { - &self.signed_presence_sha256 - } - - pub(crate) fn runtime_instance_id(&self) -> &str { - &self.runtime_instance_id - } -} - -/// Ready proof with the same construction boundary. Dependency planning and -/// promotion receive this token, never caller-supplied digest/sequence tuples. -#[derive(Clone, Debug)] -pub(crate) struct SequenceAdmittedReady { - transaction_id: String, - evidence: TopologyEvidence, - expected: IdunnExpectedIncarnationRecord, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, -} - -impl SequenceAdmittedReady { - #[cfg(test)] - pub(crate) fn for_test( - expected: &IdunnExpectedIncarnationRecord, - authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - admitted_at_unix_millis: u64, - ) -> Result { - expected.validate()?; - let record = authenticated.record(); - ensure!( - record.expected_projection_sha256 == expected.canonical_sha256()? - && record.target == expected.target - && record.runtime_id == expected.runtime_id - && record.present - && record.ready - && record.observed_presence_state.as_deref() == Some("active") - && record.disagreements.is_empty(), - "test receipt is not exact Ready evidence" - ); - let evidence = - TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; - Ok(Self { - transaction_id: "test-sequence-admission".into(), - evidence, - expected: expected.clone(), - authenticated, - }) - } - - pub(crate) fn authenticated(&self) -> &AuthenticatedOdinRuntimeTopologyCorrelation { - &self.authenticated - } - - pub(crate) fn expected(&self) -> &IdunnExpectedIncarnationRecord { - &self.expected - } - - pub(crate) fn transaction_id(&self) -> &str { - &self.transaction_id - } - - pub(crate) fn evidence_sha256(&self) -> &str { - &self.evidence.canonical_sha256 - } - - pub(crate) fn publisher_sequence(&self) -> u64 { - self.evidence.publisher_sequence - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] -enum FencingEvidence { - SkippedStateless, - Revoked { - incumbent_lease_sha256: Option, - candidate_lease_path_verified_empty: bool, - }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] -enum LeasingEvidence { - SkippedStateless, - Prepared { - lease: IdunnProcessWriteLeaseRecord, - lease_sha256: String, - }, - Granted { - lease: IdunnProcessWriteLeaseRecord, - lease_sha256: String, - }, -} - -impl LeasingEvidence { - fn lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { - match self { - Self::SkippedStateless | Self::Prepared { .. } => None, - Self::Granted { lease, .. } => Some(lease), - } - } - - /// The lease Idunn issued, granted or only prepared. A prepared lease may - /// already be on disk: the physical grant runs before `Granted` is durable. - fn issued_lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { - match self { - Self::SkippedStateless => None, - Self::Prepared { lease, .. } | Self::Granted { lease, .. } => Some(lease), - } - } - - fn prepared_lease(&self) -> Option<(&IdunnProcessWriteLeaseRecord, &str)> { - match self { - Self::Prepared { - lease, - lease_sha256, - } => Some((lease, lease_sha256)), - Self::SkippedStateless | Self::Granted { .. } => None, - } - } - - fn lease_sha256(&self) -> Option<&str> { - match self { - Self::SkippedStateless | Self::Prepared { .. } => None, - Self::Granted { lease_sha256, .. } => Some(lease_sha256), - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] -enum RoutingEvidence { - SkippedUnrouted, - Promoted { - observation: RouteObservation, - promoted_at_unix_millis: u64, - }, -} - -impl RoutingEvidence { - fn observation(&self) -> Option<&RouteObservation> { - match self { - Self::SkippedUnrouted => None, - Self::Promoted { observation, .. } => Some(observation), - } - } - - fn promoted_at_unix_millis(&self) -> Option { - match self { - Self::SkippedUnrouted => None, - Self::Promoted { - promoted_at_unix_millis, - .. - } => Some(*promoted_at_unix_millis), - } - } -} - -/// The states a route-proof candidate may answer its Warming challenge in. A -/// stateful one is warming until it holds a lease it cannot hold yet, and the -/// lease binds that warming presence. A stateless one may have finished -/// warming before the first challenge landed. -fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'static [&'static str] { - if expected.write_lease_required { - &["warming"] - } else { - &["warming", "active"] - } -} - -/// A challenged runtime answered as something that disagrees with current -/// authority. Each disagreement is named, so a capacity below the Expected -/// minimum reads as that and not as a generic refusal. It stays the error's -/// typed value; callers read `disagreements`, and the text is only its -/// rendering. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct PresenceDisagrees { - pub(crate) disagreements: Vec, -} - -impl std::fmt::Display for PresenceDisagrees { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let named = self - .disagreements - .iter() - .map(|disagreement| { - format!( - "{} (expected {}, observed {})", - disagreement.code, - disagreement.expected.as_deref().unwrap_or("none"), - disagreement.observed.as_deref().unwrap_or("none"), - ) - }) - .collect::>() - .join("; "); - write!( - formatter, - "route proof answered with a runtime that disagrees with current authority: {named}" - ) - } -} - -impl std::error::Error for PresenceDisagrees {} - -/// What a direct challenge to a candidate endpoint produced. -enum CandidateAnswer { - /// No answer: the reason it is still being waited on. - Silent(String), - Answered { - evidence: RuntimePresenceEvidence, - present: cultnet_rs::VerifiedRuntimePresence, - }, -} - -//// Which kind of proof admits a target as Ready, derived from what the target's -/// recipe declares and from nothing else. Idunn infers no way to prove -/// readiness: a target that provides `odin.verse-rendezvous` is Odin itself, one -/// that declares a `shared-infrastructure odin.verse-rendezvous` dependency -/// advertises into the Verse and is Odin-correlated, and a routed one that -/// declares neither answers Idunn's own challenge and is route-proof. A target -/// with no route and no Odin dependency declares no proof at all, and is -/// refused. Odin collects the same receipts an Odin-correlated target does, and -/// its admitted generation is what names the Odin authority, so it is never -/// route-proof. -/// -/// `of` is the only place a class is decided, and Expected is digest-bound, so -/// the class cannot drift from the incarnation it describes. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum ReadinessClass { - OdinSelf, - OdinCorrelated, - RouteProof, -} - -/// A recipe that declares neither a route nor an Odin dependency, and is not -/// Odin: it has said nothing about how it proves readiness. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct UndeclaredReadiness { - pub(crate) target: String, -} - -impl std::fmt::Display for UndeclaredReadiness { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - formatter, - "target {} declares no way to prove readiness: its recipe must declare a route \ - (Idunn challenges it) or a shared-infrastructure {ODIN_RENDEZVOUS_CAPABILITY} \ - dependency (Odin correlates it)", - self.target - ) - } -} - -impl std::error::Error for UndeclaredReadiness {} - -impl ReadinessClass { - /// The class of a recipe's declarations. Plan compilation and `of` both - /// call it, so admission refuses exactly what `of` would refuse. - pub(crate) fn declared( - target: &str, - provides_odin: bool, - declares_odin: bool, - routed: bool, - ) -> Result { - if provides_odin { - Ok(Self::OdinSelf) - } else if declares_odin { - Ok(Self::OdinCorrelated) - } else if routed { - Ok(Self::RouteProof) - } else { - Err(UndeclaredReadiness { - target: target.to_owned(), - }) - } - } - - pub(crate) fn of( - expected: &IdunnExpectedIncarnationRecord, - ) -> Result { - Self::declared( - &expected.target, - expected - .capabilities - .iter() - .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY), - expected.dependencies.iter().any(|dependency| { - dependency.kind == "shared-infrastructure" - && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY - }), - expected.route.is_some(), - ) - } - - /// The class of `expected`, provided every receipt collected under it was - /// vouched for by that class's voucher. The one way a stored record is - /// asked what class it is. - fn confirmed( - expected: &IdunnExpectedIncarnationRecord, - collected: impl IntoIterator, - ) -> Result { - let required = Self::of(expected).map_err(ReadinessDisagreement::Undeclared)?; - match collected - .into_iter() - .find(|voucher| *voucher != required.voucher()) - { - None => Ok(required), - Some(collected) => Err(ReadinessDisagreement::WrongVoucher { - target: expected.target.clone(), - required, - collected, - }), - } - } - - /// Who vouches for a target of this class: Odin's correlation, or the - /// candidate answering Idunn. - fn voucher(self) -> Voucher { - match self { - Self::OdinSelf | Self::OdinCorrelated => Voucher::Odin, - Self::RouteProof => Voucher::Candidate, - } - } -} - -/// Who vouched for a piece of stored readiness evidence. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Voucher { - Odin, - Candidate, -} - -/// A stored record whose readiness evidence is not what its own Expected -/// requires. A record admitted before the target's recipe declared how it -/// proves readiness looks like this. It is reported and held, never repaired: -/// re-deriving the evidence around it would make the evidence tag a second -/// authority for the class. -#[derive(Clone, Debug, PartialEq, Eq)] -enum ReadinessDisagreement { - Undeclared(UndeclaredReadiness), - WrongVoucher { - target: String, - required: ReadinessClass, - collected: Voucher, - }, -} - -impl std::fmt::Display for ReadinessDisagreement { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Undeclared(undeclared) => write!(formatter, "{undeclared}"), - Self::WrongVoucher { - target, - required, - collected, - } => write!( - formatter, - "target {target} is {required:?} but its stored readiness evidence was vouched \ - for by {collected:?}: it was collected under a different class, and is held \ - until the target is redeployed" - ), - } - } -} - -impl std::error::Error for ReadinessDisagreement {} - -/// The receipt that admitted a candidate as Ready, tagged by who vouched for -/// it. The tag records what was collected; what a target must collect is -/// `ReadinessClass::of(expected)`, and a tag that differs is a -/// `ReadinessDisagreement`, never a second opinion about the class. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)] -enum ReadinessEvidence { - OdinCorrelated { evidence: TopologyEvidence }, - RouteProof { evidence: RuntimePresenceEvidence }, -} - -impl ReadinessEvidence { - fn voucher(&self) -> Voucher { - match self { - Self::OdinCorrelated { .. } => Voucher::Odin, - Self::RouteProof { .. } => Voucher::Candidate, - } - } - - fn validate_shape(&self) -> Result<()> { - match self { - Self::OdinCorrelated { evidence } => evidence.validate_shape(), - Self::RouteProof { evidence } => evidence.validate_shape(), - } - } - - fn odin(&self) -> Option<&TopologyEvidence> { - match self { - Self::OdinCorrelated { evidence } => Some(evidence), - Self::RouteProof { .. } => None, - } - } -} - -// Signed evidence that the candidate process picked up the lease Idunn -/// granted: its presence carries `write_lease_sha256` equal to the grant. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LeaseAdoptionEvidence { - write_lease_sha256: String, - signed_presence_sha256: String, - source: AdoptionSource, - observed_at_unix_millis: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -enum AdoptionSource { - Direct, - OdinTopology, -} - -impl LeaseAdoptionEvidence { - /// The adoption a stateful candidate's Ready proves: its signed presence - /// held exactly the granted lease. `None` for a stateless candidate. - fn with_ready( - granted_lease_sha256: Option<&str>, - signed_presence_sha256: &str, - source: AdoptionSource, - observed_at_unix_millis: u64, - ) -> Option { - granted_lease_sha256.map(|lease| Self { - write_lease_sha256: lease.to_owned(), - signed_presence_sha256: signed_presence_sha256.to_owned(), - source, - observed_at_unix_millis, - }) - } - - /// Whether this evidence is about exactly the lease that was granted. - fn names(&self, leasing: &LeasingEvidence) -> bool { - leasing.lease_sha256() == Some(self.write_lease_sha256.as_str()) - } - - fn validate_shape(&self) -> Result<()> { - require_id(&self.write_lease_sha256, "adopted lease digest")?; - require_id(&self.signed_presence_sha256, "lease adoption presence digest")?; - ensure!( - self.observed_at_unix_millis > 0, - "lease adoption has no observation time" - ); - Ok(()) - } -} - -/// When the transaction entered its current post-fencing phase and when that -/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PhaseDeadline { - phase: DeploymentPhase, - entered_at_unix_millis: u64, - deadline_at_unix_millis: u64, -} - -impl PhaseDeadline { - /// `None` for a phase that has no deadline (pre-fencing and Complete). - fn entering( - phase: DeploymentPhase, - plan: &CompiledDeploymentPlan, - now: u64, - ) -> Option { - let deadlines = plan.phase_deadlines(); - let seconds = match phase { - DeploymentPhase::Fencing => deadlines.fencing_seconds, - DeploymentPhase::Leasing => deadlines.leasing_seconds, - DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, - DeploymentPhase::Routing => deadlines.routing_seconds, - DeploymentPhase::Committing => deadlines.committing_seconds, - _ => return None, - }; - Some(Self { - phase, - entered_at_unix_millis: now, - deadline_at_unix_millis: now.saturating_add(u64::from(seconds) * 1000), - }) - } -} - -/// How the deadline resolver ends a post-fencing phase. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum PhaseEnd { - /// Abort through the post-fencing path. `lease_not_adopted` names a - /// granted lease with no adoption evidence. - Abort { lease_not_adopted: bool }, - /// A stateful deployment whose candidate holds an issued lease: the phase - /// is reported and not resolved. - OperatorRequired, -} - -/// How a post-fencing failure recovers. `RestoreIncumbent` is what a -/// post-fencing abort has always done and is the default for records that -/// predate the field. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "action", rename_all = "kebab-case", deny_unknown_fields)] -enum TerminalRecovery { - RestartAdmitted, - #[default] - RestoreIncumbent, - OperatorRequired { - reason: String, - }, -} - -/// Route supervision's durable memory for one admitted routed incarnation: -/// challenge pacing and the degradation mark. It belongs to the incarnation, so -/// a new generation starts from `default()`. What a target may actuate outlives -/// every generation and lives in `TargetSupervision`. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct RouteSupervisionState { - last_challenge_at_unix_millis: Option, - consecutive_failures: u32, - next_challenge_at_unix_millis: Option, - /// Set while route proof keeps failing. Marks the route degraded so - /// dependents stop selecting it; it never authorizes a restart. - degraded_since_unix_millis: Option, -} - -impl RouteSupervisionState { - /// The next challenge is not due yet. - fn is_waiting(&self, now: u64, maximum_age_millis: u64) -> bool { - self.next_challenge_at_unix_millis.is_some_and(|next| { - is_waiting( - now, - next, - ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(maximum_age_millis), - ) - }) - } - - /// A failed proof, or a failed repair of the fragment it needs. Observation - /// state only: the route is marked degraded and the next challenge waits, - /// twice as long each time, from the observation max age up to a cap. - fn record_failed_challenge(&mut self, now: u64, maximum_age_millis: u64) { - self.consecutive_failures = self.consecutive_failures.saturating_add(1); - let doublings = self.consecutive_failures.saturating_sub(1).min(16); - let wait = maximum_age_millis - .saturating_mul(1 << doublings) - .min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS) - .max(maximum_age_millis); - self.last_challenge_at_unix_millis = Some(now); - self.next_challenge_at_unix_millis = Some(now.saturating_add(wait)); - self.degraded_since_unix_millis.get_or_insert(now); - } - - fn record_proved_challenge(&mut self, now: u64) { - self.consecutive_failures = 0; - self.last_challenge_at_unix_millis = Some(now); - self.next_challenge_at_unix_millis = None; - self.degraded_since_unix_millis = None; - } - - fn validate(&self) -> Result<()> { - if let Some(since) = self.degraded_since_unix_millis { - ensure!(since > 0, "route degradation has no start time"); - } - Ok(()) - } -} - -/// What one target has been allowed to do, kept across every generation and -/// present before the first: Forward route actuations and continuity restarts, -/// each a sliding log of the times they happened, and a record of Survival -/// route actuations. Owned by the meter primitives below; -/// the route driver only asks, and no generation write carries it. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.target_supervision", - schema = "idunn.target_supervision.v2" -)] -struct TargetSupervision { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - /// The newest `ROUTE_ACTUATION_CEILING` Forward actuation times, - /// ascending: the only entries the ceiling counts. - #[cultcache(key = 2)] - route_actuations: Vec, - /// The newest `CONTINUITY_PAGING_RESTARTS` restart times, ascending: the - /// paging threshold's log. - #[cultcache(key = 3)] - continuity_restarts: Vec, - /// A restart whose projection could not be prepared waits until this time. - #[cultcache(key = 4)] - continuity_deferred_until: Option, - #[cultcache(key = 5)] - continuity_deferral_reason: Option, - /// The newest `ROUTE_SURVIVAL_RECORD` Survival actuation times, ascending. - /// Visibility only: never counted, never refused. - #[cultcache(key = 6)] - route_survivals: Vec, - // Keys 7-9 are why this is v2: a v1 record is lifted by - // `LegacyTargetSupervisionV1` with no streak, no anchor and no - // re-materialization. - /// Charged attempts since the last sustained run. Owned by continuity - /// supervision; it decides the next attempt's backoff step. - #[cultcache(key = 7)] - continuity_attempt_streak: u32, - /// When a dead-and-free pass began waiting out the next attempt. Owned by - /// continuity supervision; it decides when the wait is over. Older than - /// the admitted generation, it decides nothing. - #[cultcache(key = 8)] - continuity_backoff_from_unix_millis: Option, - /// When the admitted generation's one uncharged re-materialization was - /// minted. Owned by continuity supervision; it can only withhold a free - /// re-materialization, never grant one. - #[cultcache(key = 9)] - continuity_rematerialized_at_unix_millis: Option, -} - -/// What continuity supervision does about the next attempt for a dead target. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ContinuityBackoff { - /// Mint now, carrying this streak into the attempt. - Due { streak: u32 }, - /// Start the wait: write the anchor and mint nothing this pass. - Begin, - /// The wait has not run out. - Waiting, -} - -/// The entries of a sliding log still inside `window` at `now`. An entry from -/// the future means the clock stepped back; it counts as `now`, so a backwards -/// step holds a meter for at most one window however far the clock moved. -fn live_entries(log: &[u64], now: u64, window: u64) -> Vec { - log.iter() - .map(|&at| at.min(now)) - .filter(|&at| now - at < window) - .collect() -} - -/// Record `now`, keeping the newest `limit` entries: all a meter needs to -/// decide "at most `limit` in any window". -fn record_entry(log: &mut Vec, now: u64, limit: usize) { - clamp_to(log, now); - log.push(now); - if log.len() > limit { - log.drain(..log.len() - limit); - } -} - -/// Make a stepped-back clock's future entries the present. `live_entries` -/// only reads them that way, so a log that is never written would keep -/// looking recent until the clock caught up; writing the clamp is what makes -/// a backwards step cost one window and no more. -fn clamp_to(log: &mut [u64], now: u64) { - for at in log.iter_mut() { - *at = (*at).min(now); - } -} - -impl TargetSupervision { - fn new(target: &str) -> Self { - Self { - schema_version: TARGET_SUPERVISION_SCHEMA.into(), - target: target.into(), - route_actuations: Vec::new(), - continuity_restarts: Vec::new(), - continuity_deferred_until: None, - continuity_deferral_reason: None, - route_survivals: Vec::new(), - continuity_attempt_streak: 0, - continuity_backoff_from_unix_millis: None, - continuity_rematerialized_at_unix_millis: None, - } - } - - /// Clamp every log to `now`. Callers that decide from a log write the - /// result back when it changed. - fn settle(&mut self, now: u64) { - clamp_to(&mut self.route_actuations, now); - clamp_to(&mut self.continuity_restarts, now); - clamp_to(&mut self.route_survivals, now); - } - - fn route_used(&self, now: u64) -> usize { - live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() - } - - /// When the oldest counted actuation leaves the window, if the ceiling is reached. - fn route_reopens_at(&self, now: u64) -> Option { - let live = live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS); - (live.len() >= ROUTE_ACTUATION_CEILING) - .then(|| live[live.len() - ROUTE_ACTUATION_CEILING] + ROUTE_ACTUATION_WINDOW_MILLIS) - } - - /// Charge one route actuation. A `Forward` change is counted against the - /// ceiling and refused at it; a `Survival` is only recorded, never - /// counted and never refused. - fn charge_route(&mut self, now: u64, kind: RouteActuation) -> Result<(), RouteActuationRefused> { - self.settle(now); - match kind { - RouteActuation::Forward => { - if let Some(reopens_at) = self.route_reopens_at(now) { - return Err(RouteActuationRefused { - target: self.target.clone(), - used: self.route_used(now), - reopens_at_unix_millis: reopens_at, - }); - } - record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING); - } - RouteActuation::Survival => { - record_entry(&mut self.route_survivals, now, ROUTE_SURVIVAL_RECORD); - } - } - Ok(()) - } - - fn survivals_in_window(&self, now: u64) -> usize { - live_entries(&self.route_survivals, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() - } - - fn restarts_used(&self, now: u64) -> usize { - live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len() - } - - /// The paging threshold of the ContinuityExhausted incident; it gates no actuation. - fn restarts_page(&self, now: u64) -> bool { - self.restarts_used(now) >= CONTINUITY_PAGING_RESTARTS - } - - /// Whether continuity must still wait out the deferral after a projection - /// that would not demote. - fn continuity_deferred(&self, now: u64) -> bool { - self.continuity_deferred_until - .is_some_and(|due| is_waiting(now, due, CONTINUITY_DEFERRAL_MILLIS)) - } - - /// The doubling steps the next attempt waits, if it waits. A streak of `n` - /// waits `n - 1` steps. The generation's one re-materialization is an - /// attempt that failed on it though it charged no streak, so the attempt - /// after it waits the base step: every attempt leaves a gap. - fn owed_steps(&self, admitted_at: u64) -> Option { - match self.continuity_attempt_streak.checked_sub(1) { - None if self.rematerialization_spent(admitted_at) => Some(0), - steps => steps, - } - } - - /// When the wait for the next charged attempt ends, from the anchor and - /// the streak; the one spelling `continuity_backoff` and `status` share. - /// An anchor older than the admitted generation decides nothing. - fn backoff_due(&self, admitted_at: u64) -> Option { - let from = self - .continuity_backoff_from_unix_millis - .filter(|&from| from >= admitted_at)?; - let wait = self.owed_steps(admitted_at).map_or(0, |steps| { - backoff_wait( - CONTINUITY_RESTART_BACKOFF_MILLIS, - steps, - CONTINUITY_BACKOFF_CEILING_MILLIS, - ) - }); - Some(from.saturating_add(wait)) - } - - /// What to do about the next attempt for the dead admitted generation - /// admitted at `admitted_at`. The streak resets only after a sustained - /// run: the generation has lived the sustained time and no attempt, charged - /// or the one re-materialization, has been minted since it was admitted. - /// A standing anchor belongs to a death the supervision pass has seen - /// through; an observed-running pass clears it, so it never hides a reset. - fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { - let streak = self.continuity_attempt_streak; - if let Some(due) = self.backoff_due(admitted_at) { - return if self.owed_steps(admitted_at).is_some() - && is_waiting(now, due, CONTINUITY_BACKOFF_CEILING_MILLIS) - { - ContinuityBackoff::Waiting - } else { - ContinuityBackoff::Due { streak } - }; - } - let sustained = self - .continuity_restarts - .last() - .into_iter() - .chain(&self.continuity_rematerialized_at_unix_millis) - .all(|&last| admitted_at >= last) - && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; - if sustained || self.owed_steps(admitted_at).is_none() { - ContinuityBackoff::Due { streak: 0 } - } else { - ContinuityBackoff::Begin - } - } - - /// One charged attempt scheduled at `streak`. It ends the wait and any - /// deferral: the projection was prepared. - fn record_restart(&mut self, now: u64, streak: u32) { - record_entry(&mut self.continuity_restarts, now, CONTINUITY_PAGING_RESTARTS); - self.continuity_attempt_streak = streak.saturating_add(1); - self.continuity_backoff_from_unix_millis = None; - self.continuity_deferred_until = None; - self.continuity_deferral_reason = None; - } - - /// The target was found dead and free: the wait for its next attempt runs from now. - fn begin_backoff(&mut self, now: u64) { - self.continuity_backoff_from_unix_millis = Some(now); - } - - /// The admitted workload was seen running: the death the anchor waited - /// out is over, and the next death waits from its own discovery. - fn end_backoff(&mut self) { - self.continuity_backoff_from_unix_millis = None; - } - - /// The one uncharged re-materialization scheduled. The log and the streak - /// are left alone. - fn record_rematerialization(&mut self, now: u64) { - self.continuity_rematerialized_at_unix_millis = Some(now); - self.continuity_backoff_from_unix_millis = None; - self.continuity_deferred_until = None; - self.continuity_deferral_reason = None; - } - - /// Whether the generation admitted at `admitted_at` has had its free re-materialization. - fn rematerialization_spent(&self, admitted_at: u64) -> bool { - self.continuity_rematerialized_at_unix_millis - .is_some_and(|at| at >= admitted_at) - } - - fn defer_continuity(&mut self, now: u64, reason: &str) { - self.continuity_deferred_until = Some(now.saturating_add(CONTINUITY_DEFERRAL_MILLIS)); - self.continuity_deferral_reason = Some(reason.to_owned()); - } - - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == TARGET_SUPERVISION_SCHEMA, - "target supervision schema is unsupported" - ); - require_id(&self.target, "supervised target")?; - for (log, limit, what) in [ - (&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"), - (&self.continuity_restarts, CONTINUITY_PAGING_RESTARTS, "continuity restart"), - (&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"), - ] { - ensure!(log.len() <= limit, "{what} log exceeds its bound"); - ensure!( - log.iter().all(|&at| at > 0) && log.windows(2).all(|pair| pair[0] <= pair[1]), - "{what} log is not an ascending list of times" - ); - } - ensure!( - self.continuity_backoff_from_unix_millis != Some(0) - && self.continuity_rematerialized_at_unix_millis != Some(0), - "continuity backoff anchor or re-materialization has no time" - ); - ensure!( - self.continuity_deferred_until.is_some() == self.continuity_deferral_reason.is_some(), - "continuity deferral has a time or a reason but not both" - ); - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "outcome", rename_all = "kebab-case", deny_unknown_fields)] -enum TransactionCompletion { - Admitted { generation_id: String }, - FailedBeforeFencing { error: String }, - FailedAfterFencing { - error: String, - #[serde(default)] - recovery: TerminalRecovery, - }, -} - -use cleanup_evidence::CleanupEvidence; - -/// Where the legacy marker lives, so that only the lift can make one. -/// -/// `LegacyLift` has a private field: outside this module no expression can -/// construct it, so no other code can write `CleanupEvidence::Legacy(..)`. -/// Decoding a stored record is the one other way in, and it stays here too. -/// `transaction_envelope` refuses a record carrying the marker; the schema -/// migration, which persists what the lift produced, writes through -/// `migrate_transaction_record`, and a fault note on an existing record through -/// `last_error_envelope`; both are defined here beside the only constructor, -/// and the encoder they share is private to this module. -mod cleanup_evidence { - use super::*; - - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub(super) struct LegacyLift(()); - - const LEGACY_MARKER: &str = "legacy-skipped-before-b1"; - - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - pub(super) enum CleanupEvidence { - Pending, - Skipped, - Complete, - /// Set only by the legacy lift, on a terminal continuity abort written - /// before the single resolution rule: it issued an activation and - /// recorded `Skipped`. Boot reconciliation, not the record, owes that - /// residue's demotion. The marker is what lets validation accept the - /// old shape without reopening a finished transaction. - Legacy(LegacyLift), - } - - impl CleanupEvidence { - pub(super) fn is_complete(self) -> bool { - !matches!(self, Self::Pending) - } - - pub(super) fn is_legacy_marker(self) -> bool { - matches!(self, Self::Legacy(_)) - } - } - - impl Serialize for CleanupEvidence { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_str(match self { - Self::Pending => "pending", - Self::Skipped => "skipped", - Self::Complete => "complete", - Self::Legacy(_) => LEGACY_MARKER, - }) - } - } - - impl<'de> Deserialize<'de> for CleanupEvidence { - fn deserialize>(deserializer: D) -> Result { - let text = String::deserialize(deserializer)?; - match text.as_str() { - "pending" => Ok(Self::Pending), - "skipped" => Ok(Self::Skipped), - "complete" => Ok(Self::Complete), - LEGACY_MARKER => Ok(Self::Legacy(LegacyLift(()))), - other => Err(serde::de::Error::unknown_variant( - other, - &["pending", "skipped", "complete", LEGACY_MARKER], - )), - } - } - } - - impl DeploymentTransaction { - pub(super) fn carries_legacy_marker(&self) -> bool { - self.pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - .into_iter() - .chain( - self.post_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation), - ) - .any(CleanupEvidence::is_legacy_marker) - } - } - - /// The schema migration's one entry point: read a stored v2, v3 or v4 - /// transaction (lifting older ones) and return its key and current-shape - /// envelope. The record may carry the marker the lift just set. - pub(super) fn migrate_transaction_record( - envelope: &CultCacheEnvelope, - ) -> Result<(String, CultCacheEnvelope)> { - let value = read_transaction_record(envelope)?; - let next = encode_transaction(&value, value.updated_at_unix_millis)?; - Ok((value.transaction_id, next)) - } - - /// The envelope for an existing record with only its `last_error` set. The - /// record may already carry the marker; this cannot add one. - pub(super) fn last_error_envelope( - value: &DeploymentTransaction, - detail: &str, - ) -> Result { - let mut next = value.clone(); - next.last_error = Some(detail.to_string()); - encode_transaction(&next, next.updated_at_unix_millis) - } - -/// A continuity abort written before the single resolution rule recorded no -/// projection cleanup although its transaction had issued an activation, and -/// so left that activation standing. -/// -/// Nothing is reopened. A record already terminal keeps its terminal phase and -/// completion and is marked the legacy marker, the one shape validation -/// accepts for a terminal abort that owes nothing yet issued an activation; -/// boot reconciliation, the single owner of that residue, demotes it by exact -/// activation. A record still in flight lifts to `Pending`: it already holds -/// its target's authority (a live record claims it before and after the lift), -/// so the lift adds no claimant, and its own abort demotes the activation. -/// Only the control store's read applies this: history describes what happened -/// and is lifted unchanged. -pub(super) fn owe_legacy_continuity_projection(transaction: &mut DeploymentTransaction) { - if transaction.command_kind != CommandKind::Continuity || transaction.activation.is_none() { - return; - } - let terminal = transaction.completion.is_some(); - if let Some(abort) = transaction.pre_fencing_abort.as_mut() - && abort.topology_reconciliation == CleanupEvidence::Skipped - { - abort.topology_reconciliation = if terminal { - CleanupEvidence::Legacy(LegacyLift(())) - } else { - CleanupEvidence::Pending - }; - } - if let Some(abort) = transaction.post_fencing_abort.as_mut() - && abort.topology_reconciliation == CleanupEvidence::Skipped - && !terminal - { - abort.topology_reconciliation = CleanupEvidence::Pending; - } -} - -} - -/// Rollback of a transaction that has already fenced the incumbent. -/// -/// Fencing stops the admitted incarnation and revokes its write lease, so a -/// candidate that then dies for good leaves the target with nothing serving it. -/// Retrying forever cannot fix that -- a transient unit with `Restart=no` will -/// not come back -- and it holds the target, leaving every later command queued -/// behind a transaction that can never finish. -/// -/// So the candidate's own artifacts are withdrawn in the order that never -/// leaves two writers: route first, then the write lease, then the process, -/// then the projection. Reconciling the projection restores the incumbent's -/// admitted Expected where there is one, which is what lets continuity bring -/// the incumbent back; where there is none, the failed Expected is withdrawn. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PostFencingAbort { - error: String, - route_restoration: CleanupEvidence, - lease_withdrawal: CleanupEvidence, - candidate_cleanup: CleanupEvidence, - topology_reconciliation: CleanupEvidence, - source_cleanup: CleanupEvidence, -} - -impl PostFencingAbort { - fn is_complete(&self) -> bool { - self.route_restoration.is_complete() - && self.lease_withdrawal.is_complete() - && self.candidate_cleanup.is_complete() - && self.topology_reconciliation.is_complete() - && self.source_cleanup.is_complete() - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PreFencingAbort { - error: String, - candidate_cleanup: CleanupEvidence, - topology_reconciliation: CleanupEvidence, - source_cleanup: CleanupEvidence, -} - -impl PreFencingAbort { - fn is_complete(&self) -> bool { - self.candidate_cleanup.is_complete() - && self.topology_reconciliation.is_complete() - && self.source_cleanup.is_complete() - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] -enum IncumbentCleanupEvidence { - SkippedNoIncumbent, - Pending { - generation_id: String, - workload: WorkloadObservation, - }, - Complete { - generation_id: String, - }, -} - -impl IncumbentCleanupEvidence { - fn is_complete(&self) -> bool { - matches!(self, Self::SkippedNoIncumbent | Self::Complete { .. }) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -enum SourceCleanupEvidence { - SkippedContinuity, - Pending, - Complete, -} - -impl SourceCleanupEvidence { - fn is_complete(self) -> bool { - !matches!(self, Self::Pending) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct PostCommitCleanup { - incumbent: IncumbentCleanupEvidence, - source: SourceCleanupEvidence, -} - -impl PostCommitCleanup { - fn is_complete(&self) -> bool { - self.incumbent.is_complete() && self.source.is_complete() - } -} - -/// The sole owner of an in-flight deployment decision. Every actuator result -/// is durable here before a later phase is entered. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v4" -)] -struct DeploymentTransaction { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - transaction_id: String, - #[cultcache(key = 2)] - command_id: String, - #[cultcache(key = 3)] - command_kind: CommandKind, - #[cultcache(key = 4)] - target: String, - #[cultcache(key = 5)] - ordinal: u32, - #[cultcache(key = 6)] - phase: DeploymentPhase, - #[cultcache(key = 7)] - created_at_unix_millis: u64, - #[cultcache(key = 8)] - updated_at_unix_millis: u64, - #[cultcache(key = 9)] - incumbent_generation_id: Option, - #[cultcache(key = 10)] - plan: Option, - #[cultcache(key = 11)] - frozen_source: Option, - #[cultcache(key = 12)] - sealed_release: Option, - #[cultcache(key = 13)] - installed_release: Option, - #[cultcache(key = 14)] - expected: Option, - #[cultcache(key = 15)] - expected_publication_sha256: Option, - #[cultcache(key = 16)] - deployment_authorization: Option, - #[cultcache(key = 17)] - lifecycle_authorized_at_unix_millis: Option, - #[cultcache(key = 18)] - activation: Option, - #[cultcache(key = 19)] - workload: Option, - #[cultcache(key = 20)] - activation_publication_sha256: Option, - #[cultcache(key = 21)] - latest_odin_observation: Option, - #[cultcache(key = 22)] - warming: Option, - #[cultcache(key = 23)] - route_preflight: Option, - #[cultcache(key = 24)] - isolation: Option, - #[cultcache(key = 25)] - fencing: Option, - #[cultcache(key = 26)] - leasing: Option, - #[cultcache(key = 27)] - ready: Option, - #[cultcache(key = 28)] - routing: Option, - #[cultcache(key = 29)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 30)] - last_error: Option, - #[cultcache(key = 31)] - completion: Option, - #[cultcache(key = 32)] - pre_fencing_abort: Option, - #[cultcache(key = 33)] - post_commit_cleanup: Option, - // because every transaction already durable predates this field: - // an absent key 34 decodes as None, which is exactly "this transaction has - // no post-fencing abort". Without it the daemon refuses to read its own - // control store and crashloops. - #[cultcache(key = 34, default)] - post_fencing_abort: Option, - /// Set by `transition` on entry to each post-fencing phase. - #[cultcache(key = 35)] - phase_deadline: Option, - #[cultcache(key = 36)] - lease_adoption: Option, -} - -impl DeploymentTransaction { - /// The one writer of `phase`. It stamps the update time and the phase - /// deadline together, so a record outside Fencing..=Committing carries no - /// deadline by construction. No other code assigns `phase`. - fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { - self.phase = phase; - self.updated_at_unix_millis = now; - self.phase_deadline = self - .plan - .as_ref() - .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); - } - - /// A live transaction past the fence that is not already being aborted: - /// the only kind whose phase a deadline, or an operator's expiry, ends. - fn resolves_by_deadline(&self) -> bool { - self.completion.is_none() - && self.pre_fencing_abort.is_none() - && self.post_fencing_abort.is_none() - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) - } - - /// What an ended post-fencing phase resolves to (Q3). A candidate that was - /// never issued a write lease cannot have written, so the transaction - /// aborts to the incumbent through the post-fencing path, stateless or not. - /// A continuity always aborts: its candidate is the admitted release, and - /// supervision restarts that release under the continuity backoff. A - /// deployment whose candidate was issued a lease may have written under a - /// new state contract, and restoring the incumbent over that is not - /// Idunn's call: it is left for the operator. - fn phase_end(&self) -> PhaseEnd { - let leasing = self.leasing.as_ref(); - let granted = leasing.and_then(LeasingEvidence::lease).is_some(); - let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); - match self.command_kind { - CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, - _ => PhaseEnd::Abort { - lease_not_adopted: granted && self.lease_adoption.is_none(), - }, - } - } - - /// The recovery a completed post-fencing abort names. A continuity whose - /// candidate adopted its lease restarts the admitted release; everything - /// else restores the incumbent. - fn terminal_recovery(&self) -> TerminalRecovery { - if self.command_kind == CommandKind::Continuity && self.lease_adoption.is_some() { - TerminalRecovery::RestartAdmitted - } else { - TerminalRecovery::RestoreIncumbent - } - } - - /// Whether this transaction's binding declares stop-then-start. A - /// transaction without a plan yet cannot, so it reads as false. - fn rollout_stops_incumbent_first(&self) -> bool { - self.plan - .as_ref() - .and_then(|plan| plan.parsed_inputs().ok()) - .is_some_and(|(_, binding)| binding.rollout.strategy == RolloutStrategy::StopThenStart) - } - - fn new( - command: &DeploymentCommand, - target: String, - ordinal: u32, - incumbent: Option<&AdmittedGeneration>, - now: u64, - ) -> Result { - let transaction = Self { - schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), - transaction_id: format!("tx-{}", Uuid::new_v4()), - command_id: command.command_id.clone(), - command_kind: command.kind, - target, - ordinal, - phase: DeploymentPhase::Sealing, - created_at_unix_millis: now, - updated_at_unix_millis: now, - incumbent_generation_id: incumbent.map(|value| value.generation_id.clone()), - plan: None, - frozen_source: None, - sealed_release: None, - installed_release: None, - expected: None, - expected_publication_sha256: None, - deployment_authorization: None, - lifecycle_authorized_at_unix_millis: None, - activation: None, - workload: None, - activation_publication_sha256: None, - latest_odin_observation: None, - warming: None, - route_preflight: None, - isolation: None, - fencing: None, - leasing: None, - ready: None, - routing: None, - odin_publisher_sequence_cursor: incumbent - .map_or(0, |value| value.odin_publisher_sequence_cursor), - last_error: None, - completion: None, - pre_fencing_abort: None, - post_fencing_abort: None, - post_commit_cleanup: None, - phase_deadline: None, - lease_adoption: None, - }; - transaction.validate()?; - Ok(transaction) - } - - fn from_continuity( - command: &DeploymentCommand, - incumbent: &AdmittedGeneration, - now: u64, - ) -> Result { - let mut transaction = - Self::new(command, incumbent.target.clone(), 0, Some(incumbent), now)?; - transaction.plan = Some(incumbent.plan.clone()); - transaction.sealed_release = Some(incumbent.sealed_release.clone()); - transaction.installed_release = Some(incumbent.installed_release.clone()); - transaction.expected = Some(incumbent.expected.clone()); - transaction.validate()?; - Ok(transaction) - } - - /// Whether an abort owes the projection a reconciliation. A deployment - /// published its own Expected key. A continuity shares the admitted key, - /// whose Expected is never withdrawn, so it owes only the demotion of the - /// activation it issued. - fn abort_topology_reconciliation(&self) -> CleanupEvidence { - let owes = match self.command_kind { - CommandKind::Deploy => self.expected_publication_sha256.is_some(), - CommandKind::Continuity => self.activation.is_some(), - }; - if owes { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - } - } - - /// Whether an abort owes the candidate's durable cleanup: it does once an - /// activation was issued or a workload was observed, either of which leaves - /// something behind. - fn candidate_cleanup_owed(&self) -> CleanupEvidence { - if self.activation.is_some() || self.workload.is_some() { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - } - } - - /// Whether an abort's recorded candidate cleanup is what - /// `candidate_cleanup_owed` says it must be: owed cleanup is pending or - /// done, unowed cleanup is skipped. - fn candidate_cleanup_recorded(&self, recorded: CleanupEvidence) -> bool { - matches!( - (self.candidate_cleanup_owed(), recorded), - ( - CleanupEvidence::Pending, - CleanupEvidence::Pending | CleanupEvidence::Complete - ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) - ) - } - - fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { - let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; - let detail = truncate(&format!("{error:#}"), 2048); - transaction.enter_phase(DeploymentPhase::Complete, now); - transaction.last_error = Some(detail.clone()); - transaction.pre_fencing_abort = Some(PreFencingAbort { - error: detail.clone(), - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Skipped, - }); - transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { error: detail }); - transaction.validate()?; - Ok(transaction) - } - - fn is_terminal(&self) -> bool { - if self.phase != DeploymentPhase::Complete { - return false; - } - match &self.completion { - Some(TransactionCompletion::FailedBeforeFencing { .. }) => self - .pre_fencing_abort - .as_ref() - .is_some_and(PreFencingAbort::is_complete), - Some(TransactionCompletion::FailedAfterFencing { .. }) => self - .post_fencing_abort - .as_ref() - .is_some_and(PostFencingAbort::is_complete), - Some(TransactionCompletion::Admitted { .. }) => self - .post_commit_cleanup - .as_ref() - .is_some_and(PostCommitCleanup::is_complete), - None => false, - } - } - - /// Complete transactions retain retryable cleanup work, but admission has - /// already transferred current-incarnation authority to AdmittedGeneration. - fn owns_target_authority(&self) -> bool { - self.phase != DeploymentPhase::Complete - } - - /// Admission has moved to the new generation at Complete, but an exact - /// draining incumbent still reserves its process and candidate endpoint. - /// No later mutation for the same target may overlap that cleanup. - fn blocks_new_target_mutation(&self) -> bool { - self.owns_target_authority() - || self - .post_commit_cleanup - .as_ref() - .is_some_and(|cleanup| !cleanup.is_complete()) - } - - /// Where what this transaction has collected disagrees with what its own - /// Expected requires. There is nothing to disagree with before Expected. - fn readiness_disagreement(&self) -> Option { - let expected = self.expected.as_ref()?; - ReadinessClass::confirmed( - expected, - self.warming - .iter() - .map(WarmingEvidence::voucher) - .chain(self.ready.iter().map(ReadinessEvidence::voucher)), - ) - .err() - } - - /// The disagreement that holds this transaction: evidence collected under - /// another class is held and reported, not advanced and not repaired, since - /// no step can make it the right kind and running one anyway only fails it - /// every tick. What is finished, or being finished (a completion, or an - /// abort under way), never reads the class, so it is left to run. - fn held_disagreement(&self) -> Option { - let finishing = self.completion.is_some() - || self.pre_fencing_abort.is_some() - || self.post_fencing_abort.is_some(); - self.readiness_disagreement().filter(|_| !finishing) - } - - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, - "deployment transaction schema is unsupported" - ); - require_id(&self.transaction_id, "deployment transaction id")?; - require_id(&self.command_id, "transaction command id")?; - require_id(&self.target, "transaction target")?; - ensure!( - self.created_at_unix_millis > 0 && self.updated_at_unix_millis > 0, - "deployment transaction timestamps are invalid" - ); - if let Some(generation) = &self.incumbent_generation_id { - require_id(generation, "incumbent generation id")?; - } - if let Some(plan) = &self.plan { - plan.validate()?; - ensure!( - plan.parsed_inputs()?.1.target == self.target, - "transaction plan belongs to another target" - ); - } - if let (Some(plan), Some(frozen)) = (&self.plan, &self.frozen_source) { - frozen.validate_against(plan)?; - } - if let (Some(plan), Some(release)) = (&self.plan, &self.sealed_release) { - release.validate_against(plan)?; - } - if let Some(expected) = &self.expected { - expected.validate()?; - ensure!(expected.target == self.target, "Expected target differs"); - if let Some(plan) = &self.plan { - ensure!(expected.plan_id == plan.plan_id, "Expected plan differs"); - } - if let Some(release) = &self.sealed_release { - ensure!( - expected.sealed_release_id == release.sealed_release_id, - "Expected release differs" - ); - } - if let Some(digest) = &self.expected_publication_sha256 { - ensure!( - digest == &expected.canonical_sha256()?, - "Expected publication receipt differs" - ); - } - } - if let (Some(expected), Some(activation)) = (&self.expected, &self.activation) { - activation.validate()?; - ensure!( - activation.expected_projection_sha256 == expected.canonical_sha256()?, - "activation belongs to another Expected projection" - ); - if let Some(workload) = &self.workload { - ensure!( - workload.runtime_instance_id() == activation.runtime_instance_id, - "workload belongs to another activation" - ); - } - if let Some(digest) = &self.activation_publication_sha256 { - ensure!( - digest == &activation.canonical_sha256()?, - "activation publication receipt differs" - ); - } - } - ensure!( - self.workload.is_none() || self.activation.is_some(), - "workload observation exists without its prepared activation" - ); - let ready_odin = self.ready.as_ref().and_then(ReadinessEvidence::odin); - for evidence in [self.latest_odin_observation.as_ref(), ready_odin] - .into_iter() - .flatten() - { - evidence.validate_shape()?; - ensure!( - evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, - "topology evidence exceeds the transaction replay cursor" - ); - } - if let Some(ready) = &self.ready { - ready.validate_shape()?; - // A candidate-vouched receipt is only ever written by this code, so - // one on a target that is not route-proof is forged. The other - // direction -- an Odin receipt on a route-proof target -- is what a - // pre-B3 record looks like, and is a ReadinessDisagreement to hold, - // not a record to refuse: refusing it would refuse to boot. - if let (Voucher::Candidate, Some(expected)) = (ready.voucher(), &self.expected) { - ensure!( - ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof), - "route-proof Ready evidence for a target that is not route-proof" - ); - } - } - if let Some(warming) = &self.warming { - warming.validate_shape()?; - match warming { - WarmingEvidence::OdinTopology { evidence } => ensure!( - evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, - "Warming topology evidence exceeds the transaction replay cursor" - ), - WarmingEvidence::FirstOdinDirect { .. } => ensure!( - self.expected.as_ref().is_some_and(|expected| { - ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::OdinSelf) - }), - "direct Warming evidence is reserved for Odin observing itself" - ), - WarmingEvidence::RouteProofDirect { .. } => ensure!( - self.expected.as_ref().is_none_or(|expected| { - ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof) - }), - "direct route-proof Warming evidence for a target that is not route-proof" - ), - } - } - if let Some(leasing) = &self.leasing { - if let Some(expected) = &self.expected { - ensure!( - matches!( - (expected.write_lease_required, leasing), - (true, LeasingEvidence::Prepared { .. }) - | (true, LeasingEvidence::Granted { .. }) - | (false, LeasingEvidence::SkippedStateless) - ), - "lease evidence differs from the Expected state contract" - ); - } - let lease_and_sha256 = match leasing { - LeasingEvidence::SkippedStateless => None, - LeasingEvidence::Prepared { - lease, - lease_sha256, - } - | LeasingEvidence::Granted { - lease, - lease_sha256, - } => Some((lease, lease_sha256)), - }; - if let Some((lease, lease_sha256)) = lease_and_sha256 { - lease.validate()?; - ensure!( - lease.canonical_sha256()? == *lease_sha256, - "lease evidence digest differs" - ); - } - } - if let (Some(expected), Some(fencing)) = (&self.expected, &self.fencing) { - match fencing { - FencingEvidence::SkippedStateless => ensure!( - !expected.write_lease_required, - "stateful candidate claims stateless fencing" - ), - FencingEvidence::Revoked { - candidate_lease_path_verified_empty, - .. - } => ensure!( - *candidate_lease_path_verified_empty == expected.write_lease_required, - "candidate lease-path fencing differs from Expected" - ), - } - } - if let Some(error) = &self.last_error { - require_detail(error, "transaction error")?; - } - if let Some(deadline) = &self.phase_deadline { - ensure!( - deadline.phase == self.phase - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&deadline.phase) - && deadline.entered_at_unix_millis > 0 - && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, - "phase deadline does not describe the current post-fencing phase" - ); - } - if let Some(adoption) = &self.lease_adoption { - adoption.validate_shape()?; - ensure!( - self.leasing - .as_ref() - .is_some_and(|leasing| adoption.names(leasing)), - "lease adoption does not name the granted lease" - ); - } - if let Some(authorization) = &self.deployment_authorization { - authorization.validate_shape()?; - } - if let Some(abort) = &self.pre_fencing_abort { - require_detail(&abort.error, "pre-fencing abort error")?; - ensure!( - self.post_commit_cleanup.is_none(), - "aborted transaction also carries post-commit cleanup" - ); - ensure!( - self.candidate_cleanup_recorded(abort.candidate_cleanup), - "abort candidate cleanup differs from its prepared activation" - ); - ensure!( - matches!( - ( - self.abort_topology_reconciliation(), - abort.topology_reconciliation - ), - ( - CleanupEvidence::Pending, - CleanupEvidence::Pending | CleanupEvidence::Complete - ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) - ) || (self.command_kind == CommandKind::Continuity - && self.phase == DeploymentPhase::Complete - && self.completion.is_some() - && abort.topology_reconciliation.is_legacy_marker()), - "abort topology cleanup differs from what the transaction projected" - ); - ensure!( - matches!( - (self.command_kind, abort.source_cleanup), - ( - CommandKind::Deploy, - CleanupEvidence::Pending | CleanupEvidence::Complete - ) | (CommandKind::Continuity, CleanupEvidence::Skipped) - ) || (self.command_kind == CommandKind::Deploy - && abort.source_cleanup == CleanupEvidence::Skipped - && self.plan.is_none() - && self.frozen_source.is_none() - && self.expected.is_none() - && self.phase == DeploymentPhase::Complete), - "abort source cleanup differs from command work" - ); - ensure!( - self.phase < DeploymentPhase::Fencing || self.phase == DeploymentPhase::Complete, - "pre-fencing abort crossed the fencing boundary" - ); - } - - if let Some(abort) = &self.post_fencing_abort { - require_detail(&abort.error, "post-fencing abort error")?; - ensure!( - self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_none(), - "post-fencing abort collides with another terminal path" - ); - ensure!( - self.phase >= DeploymentPhase::Fencing, - "post-fencing abort exists before the fence" - ); - ensure!( - self.candidate_cleanup_recorded(abort.candidate_cleanup), - "abort candidate cleanup differs from its prepared activation" - ); - } - - let failed_after_fencing = matches!( - self.completion, - Some(TransactionCompletion::FailedAfterFencing { .. }) - ); - if failed_after_fencing { - ensure!( - self.phase == DeploymentPhase::Complete, - "terminal failure is not Complete" - ); - let abort = required( - &self.post_fencing_abort, - "terminal post-fence abort evidence", - )?; - let TransactionCompletion::FailedAfterFencing { error, recovery } = - self.completion.as_ref().unwrap() - else { - unreachable!() - }; - if let TerminalRecovery::OperatorRequired { reason } = recovery { - require_detail(reason, "operator-required recovery reason")?; - } - ensure!( - abort.is_complete() && abort.error == *error, - "terminal failure lacks complete matching abort evidence" - ); - return Ok(()); - } - - let failed = matches!( - self.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - ); - if failed { - ensure!( - self.phase == DeploymentPhase::Complete, - "terminal failure is not Complete" - ); - let abort = required(&self.pre_fencing_abort, "terminal abort evidence")?; - let TransactionCompletion::FailedBeforeFencing { error } = - self.completion.as_ref().unwrap() - else { - unreachable!() - }; - ensure!( - abort.is_complete() && abort.error == *error, - "terminal failure lacks complete matching abort evidence" - ); - } else { - ensure!( - !matches!( - self.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - ), - "non-failure branch carries failure completion" - ); - if self.phase >= DeploymentPhase::Starting { - ensure!( - self.plan.is_some() - && self.sealed_release.is_some() - && self.installed_release.is_some() - && self.expected.is_some(), - "Starting transaction lacks sealed release evidence" - ); - // Expected is published as Starting's first step, after the - // brake; from the activation onward it must be there. - ensure!( - self.expected_publication_sha256.is_some() - || (self.activation.is_none() && self.phase == DeploymentPhase::Starting), - "started transaction has not published its Expected" - ); - match self.command_kind { - CommandKind::Deploy => ensure!( - self.frozen_source.is_some() && self.deployment_authorization.is_some(), - "deployment entered Starting without frozen source and brake consumption" - ), - CommandKind::Continuity => ensure!( - self.lifecycle_authorized_at_unix_millis.is_some() - && self.deployment_authorization.is_none(), - "continuity entered Starting without its lifecycle gate" - ), - } - } - if self.phase >= DeploymentPhase::Warming { - ensure!( - self.activation.is_some() - && self.workload.is_some() - && self.activation_publication_sha256.is_some(), - "Warming transaction lacks observed activation" - ); - } - if self.phase >= DeploymentPhase::Fencing { - ensure!( - self.warming.is_some() && self.isolation.is_some(), - "Fencing transaction lacks warming or isolation evidence" - ); - ensure!( - self.expected - .as_ref() - .and_then(|expected| expected.route.as_ref()) - .is_some() - == self.route_preflight.is_some(), - "route preflight does not match routed Expected" - ); - if let Some(preflight) = &self.route_preflight { - preflight.validate()?; - } - } - if self.phase >= DeploymentPhase::Leasing { - ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence"); - } - if self.phase >= DeploymentPhase::AwaitingReady { - ensure!( - matches!( - self.leasing.as_ref(), - Some(LeasingEvidence::SkippedStateless) - | Some(LeasingEvidence::Granted { .. }) - ), - "AwaitingReady lacks finalized lease evidence" - ); - } - if self.phase >= DeploymentPhase::Routing { - ensure!(self.ready.is_some(), "Routing lacks Ready evidence"); - } - if self.phase >= DeploymentPhase::Committing { - ensure!(self.routing.is_some(), "Committing lacks routing evidence"); - } - if let Some(RoutingEvidence::Promoted { - observation, - promoted_at_unix_millis, - }) = &self.routing - { - observation.validate()?; - ensure!(*promoted_at_unix_millis > 0, "route promotion has no time"); - } - if self.phase == DeploymentPhase::Complete { - ensure!( - matches!( - self.completion, - Some(TransactionCompletion::Admitted { .. }) - ), - "successful Complete transaction lacks admission receipt" - ); - ensure!( - self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_some(), - "admitted transaction lacks exclusive post-commit cleanup evidence" - ); - let TransactionCompletion::Admitted { generation_id } = - self.completion.as_ref().unwrap() - else { - unreachable!() - }; - ensure!( - generation_id == &format!("generation-{}", self.transaction_id), - "admission completion names another transaction generation" - ); - } else { - ensure!( - self.completion.is_none() && self.post_commit_cleanup.is_none(), - "non-Complete transaction has completion or post-commit state" - ); - } - } - if let Some(cleanup) = &self.post_commit_cleanup { - ensure!( - self.phase == DeploymentPhase::Complete - && matches!( - self.completion, - Some(TransactionCompletion::Admitted { .. }) - ), - "post-commit cleanup exists without an admitted Complete transaction" - ); - match (&self.incumbent_generation_id, &cleanup.incumbent) { - (None, IncumbentCleanupEvidence::SkippedNoIncumbent) => {} - ( - Some(expected), - IncumbentCleanupEvidence::Pending { generation_id, .. } - | IncumbentCleanupEvidence::Complete { generation_id }, - ) if expected == generation_id => {} - _ => bail!("post-commit incumbent cleanup names another generation"), - } - ensure!( - matches!( - (self.command_kind, cleanup.source), - (CommandKind::Deploy, SourceCleanupEvidence::Pending) - | (CommandKind::Deploy, SourceCleanupEvidence::Complete) - | ( - CommandKind::Continuity, - SourceCleanupEvidence::SkippedContinuity - ) - ), - "post-commit source cleanup differs from command kind" - ); - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct AdmittedOdinAuthority { - signer_identity_id: String, - signer_public_key: Vec, -} - -impl AdmittedOdinAuthority { - fn from_anchor(anchor: &ServiceIdentityTrustAnchor) -> Result { - ensure!( - derive_service_identity_id::(&anchor.public_key)? - == anchor.identity_id, - "Odin topology anchor identity differs from its key" - ); - Ok(Self { - signer_identity_id: anchor.identity_id.clone(), - signer_public_key: anchor.public_key.clone(), - }) - } - - fn validate(&self) -> Result<()> { - require_id(&self.signer_identity_id, "admitted Odin signer")?; - ensure!( - derive_service_identity_id::(&self.signer_public_key)? - == self.signer_identity_id, - "admitted Odin signer identity differs from its key" - ); - Ok(()) - } -} - -/// The only current-incarnation owner. Unit state, route state, topology -/// projections, and CLI output are observations of this record, never peers. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v4" -)] -struct AdmittedGeneration { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - #[cultcache(key = 2)] - generation_id: String, - #[cultcache(key = 3)] - command_id: String, - #[cultcache(key = 4)] - transaction_id: String, - #[cultcache(key = 5)] - admitted_at_unix_millis: u64, - #[cultcache(key = 6)] - plan: CompiledDeploymentPlan, - #[cultcache(key = 7)] - sealed_release: SealedRelease, - #[cultcache(key = 8)] - installed_release: InstalledReleaseObservation, - #[cultcache(key = 9)] - expected: IdunnExpectedIncarnationRecord, - #[cultcache(key = 10)] - activation: IdunnRuntimeActivationRecord, - #[cultcache(key = 11)] - workload: WorkloadObservation, - #[cultcache(key = 12)] - leasing: LeasingEvidence, - /// The receipt that admitted this incarnation, tagged by proof class. - #[cultcache(key = 13)] - ready: ReadinessEvidence, - /// Present exactly when `ready` is Odin-correlated. - #[cultcache(key = 14)] - latest_odin_observation: Option, - #[cultcache(key = 15)] - routing: RoutingEvidence, - /// Present exactly when `ready` is Odin-correlated. - #[cultcache(key = 16)] - odin_authority: Option, - /// Meaningful only for an Odin-correlated generation. - #[cultcache(key = 17)] - odin_publisher_sequence_cursor: u64, - // Keys 18 (route repair start) and 20 (continuity backoff) are retired. - // The slots stay as gaps: the layout is positional, so a slot cannot be - // reused without a bump. - /// Present exactly when `routing` is Promoted. - #[cultcache(key = 19)] - route_supervision: Option, - /// Why this generation is held or down, for `status`. Cleared when the - /// generation is replaced. - #[cultcache(key = 21)] - last_error: Option, -} - -/// The Odin-side receipts of an Odin-correlated generation. Decisions that -/// consume Odin evidence read it through here, so a route-proof generation -/// cannot be mistaken for one. -struct OdinReceipts<'a> { - ready: &'a TopologyEvidence, - latest: &'a TopologyEvidence, -} - -impl AdmittedGeneration { - /// This generation's class, or where its evidence disagrees with it. The - /// class comes from its Expected, never from the tag on its Ready receipt. - fn readiness(&self) -> Result { - ReadinessClass::confirmed(&self.expected, [self.ready.voucher()]) - } - - fn odin_receipts(&self) -> Result> { - match ( - &self.ready, - &self.latest_odin_observation, - &self.odin_authority, - ) { - (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(_)) => { - Ok(OdinReceipts { - ready: evidence, - latest, - }) - } - _ => bail!("admitted generation of {} is not Odin-correlated", self.target), - } - } - - fn from_transaction( - transaction: &DeploymentTransaction, - odin_authority: Option, - now: u64, - ) -> Result { - ensure!( - transaction.phase == DeploymentPhase::Committing, - "only Committing can create an admitted generation" - ); - let ready = required(&transaction.ready, "Ready receipt")?.clone(); - let (latest_odin_observation, odin_authority) = match &ready { - ReadinessEvidence::OdinCorrelated { .. } => ( - Some( - required(&transaction.latest_odin_observation, "latest Odin receipt")? - .clone(), - ), - Some(required(&odin_authority, "Odin authority")?.clone()), - ), - ReadinessEvidence::RouteProof { .. } => (None, None), - }; - let routing = required(&transaction.routing, "route disposition")?.clone(); - let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. }) - .then(RouteSupervisionState::default); - let generation = Self { - schema_version: ADMITTED_GENERATION_SCHEMA.into(), - target: transaction.target.clone(), - generation_id: format!("generation-{}", transaction.transaction_id), - command_id: transaction.command_id.clone(), - transaction_id: transaction.transaction_id.clone(), - admitted_at_unix_millis: now, - plan: required(&transaction.plan, "transaction plan")?.clone(), - sealed_release: required(&transaction.sealed_release, "sealed release")?.clone(), - installed_release: required(&transaction.installed_release, "installed release")? - .clone(), - expected: required(&transaction.expected, "Expected projection")?.clone(), - activation: required(&transaction.activation, "activation")?.clone(), - workload: required(&transaction.workload, "workload")?.clone(), - leasing: required(&transaction.leasing, "lease disposition")?.clone(), - ready, - latest_odin_observation, - routing, - odin_authority, - odin_publisher_sequence_cursor: transaction.odin_publisher_sequence_cursor, - route_supervision, - last_error: None, - }; - generation.validate()?; - Ok(generation) - } - - fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == ADMITTED_GENERATION_SCHEMA, - "admitted generation schema is unsupported" - ); - require_id(&self.target, "admitted target")?; - require_id(&self.generation_id, "admitted generation id")?; - require_id(&self.command_id, "admitted command id")?; - require_id(&self.transaction_id, "admitted transaction id")?; - ensure!(self.admitted_at_unix_millis > 0, "admission has no time"); - self.plan.validate()?; - self.sealed_release.validate_against(&self.plan)?; - self.expected.validate()?; - self.activation.validate()?; - self.ready.validate_shape()?; - match ( - &self.ready, - &self.latest_odin_observation, - &self.odin_authority, - ) { - (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(authority)) => { - latest.validate_shape()?; - authority.validate()?; - ensure!( - evidence.publisher_sequence <= self.odin_publisher_sequence_cursor - && latest.publisher_sequence == self.odin_publisher_sequence_cursor, - "admitted generation evidence does not describe one incarnation" - ); - } - (ReadinessEvidence::RouteProof { .. }, None, None) => ensure!( - ReadinessClass::of(&self.expected).is_ok_and(|class| class == ReadinessClass::RouteProof), - "route-proof Ready evidence for a target that is not route-proof" - ), - _ => bail!("admitted readiness evidence and Odin receipts disagree"), - } - ensure!( - self.target == self.expected.target - && self.expected.plan_id == self.plan.plan_id - && self.expected.sealed_release_id == self.sealed_release.sealed_release_id - && self.activation.expected_projection_sha256 - == self.expected.canonical_sha256()? - && self.activation.runtime_instance_id == self.workload.runtime_instance_id(), - "admitted generation evidence does not describe one incarnation" - ); - ensure!( - self.expected.write_lease_required == self.leasing.lease().is_some(), - "admitted write-lease disposition differs from Expected" - ); - ensure!( - self.expected.route.is_some() == self.routing.observation().is_some(), - "admitted route disposition differs from Expected" - ); - if let Some(route) = self.routing.observation() { - route.validate()?; - ensure!( - route.runtime_instance_id == self.activation.runtime_instance_id, - "admitted route observation belongs to another runtime instance" - ); - } - ensure!( - self.route_supervision.is_some() - == matches!(&self.routing, RoutingEvidence::Promoted { .. }), - "route supervision state exists exactly for a promoted route" - ); - if let Some(state) = &self.route_supervision { - state.validate()?; - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct RuntimeOptions { - state_store: PathBuf, - bindings_dir: PathBuf, - source_root: PathBuf, - staging_root: PathBuf, - topology_store: PathBuf, - /// Operator incidents, published world-readable beside the topology - /// projection. Not part of the control store: see `incident`. - incident_store: PathBuf, - odin_correlation_store: PathBuf, - odin_trust_anchor: PathBuf, - idunn_identity_store: PathBuf, - deployment_brake_operator_anchor: PathBuf, - source_identity: Option, - topology_maximum_age_millis: u64, - topology_maximum_future_skew_millis: u64, - poll_millis: u64, - /// Where host actuators dial in. `None` means no host-actuator workload - /// can be served; a binding that names one then fails at its first - /// driver call, not at startup, so an Idunn without hosts is unchanged. - host_actuator_bind: Option, - /// Where route drivers find nginx, systemctl, ufw and systemd-run, and - /// their scratch directory. Every route driver is built from this. - route_actuators: RouteActuators, -} - -impl Default for RuntimeOptions { - fn default() -> Self { - Self { - state_store: PathBuf::from("/var/lib/gamecult/idunn/control.cc"), - bindings_dir: PathBuf::from("/etc/gamecult/idunn/bindings"), - source_root: PathBuf::from("/var/lib/gamecult/idunn/sources"), - staging_root: PathBuf::from("/var/lib/gamecult/idunn/staging"), - topology_store: PathBuf::from("/var/lib/gamecult/idunn/topology.cc"), - incident_store: PathBuf::from("/var/lib/gamecult/idunn-projection/incidents.cc"), - odin_correlation_store: PathBuf::from( - "/var/lib/gamecult/odin/idunn-runtime-topology.cc", - ), - odin_trust_anchor: PathBuf::from("/etc/gamecult/idunn/odin-topology-anchor.cc"), - idunn_identity_store: PathBuf::from( - "/var/lib/gamecult/idunn/idunn-service-identity.cc", - ), - deployment_brake_operator_anchor: PathBuf::from( - "/etc/gamecult/idunn/deployment-brake-operator-anchor.cc", - ), - source_identity: None, - topology_maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, - topology_maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, - poll_millis: 500, - host_actuator_bind: None, - route_actuators: RouteActuators::default(), - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum Command { - Serve(RuntimeOptions), - Up { - selector: String, - requested_by: String, - state_store: PathBuf, - wait: bool, - timeout_seconds: u64, - }, - Status { - state_store: PathBuf, - incident_store: PathBuf, - command_id: Option, - }, - Cancel { - state_store: PathBuf, - command_id: String, - requested_by: String, - }, - Expire { - state_store: PathBuf, - command_id: String, - requested_by: String, - }, - Validate { - recipe: PathBuf, - binding: Option, - }, -} - -pub fn run(args: impl Iterator) -> Result<()> { - match parse(args)? { - Command::Serve(options) => serve(options), - Command::Up { - selector, - requested_by, - state_store, - wait, - timeout_seconds, - } => submit( - &state_store, - &selector, - &requested_by, - wait, - timeout_seconds, - ), - Command::Status { - state_store, - incident_store, - command_id, - } => status( - &mut std::io::stdout().lock(), - &state_store, - &incident_store, - command_id.as_deref(), - ), - Command::Cancel { - state_store, - command_id, - requested_by, - } => cancel(&state_store, &command_id, &requested_by), - Command::Expire { - state_store, - command_id, - requested_by, - } => expire(&state_store, &command_id, &requested_by), - Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), - } -} - -/// Offline admission check for authoring. Reads a recipe, optionally reads a -/// binding, and runs the same `admit` the daemon runs at load. It opens no -/// store, contacts no host, and actuates nothing, so it is safe to run against -/// a candidate binding before installing it. -fn validate(recipe_path: &Path, binding_path: Option<&Path>) -> Result<()> { - let recipe_text = fs::read_to_string(recipe_path) - .with_context(|| format!("reading recipe {}", recipe_path.display()))?; - let recipe = crate::deployment::TargetDeclaration::parse(&recipe_text) - .with_context(|| format!("validating recipe {}", recipe_path.display()))?; - println!( - "recipe ok: {} declares target {}", - recipe_path.display(), - recipe.target - ); - - let Some(binding_path) = binding_path else { - return Ok(()); - }; - let binding_text = fs::read_to_string(binding_path) - .with_context(|| format!("reading operator binding {}", binding_path.display()))?; - let binding = OperatorBinding::parse(&binding_text) - .with_context(|| format!("validating operator binding {}", binding_path.display()))?; - ensure!( - binding.target == recipe.target, - "operator binding targets {} but the recipe declares {}", - binding.target, - recipe.target - ); - binding - .admit(&recipe) - .with_context(|| format!("admitting {} against its recipe", binding.target))?; - println!( - "binding ok: {} admits target {}", - binding_path.display(), - binding.target - ); - Ok(()) -} - -fn parse(args: impl Iterator) -> Result { - let mut args = args.peekable(); - let command = args.next().ok_or_else(|| anyhow!(usage()))?; - match command.as_str() { - "serve" => parse_serve(args), - "up" => parse_up(args), - "status" => parse_status(args), - "cancel" => parse_cancel(args), - "expire" => parse_expire(args), - "validate" => parse_validate(args), - "--help" | "-h" | "help" => bail!(usage()), - _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), - } -} - -fn parse_serve(mut args: impl Iterator) -> Result { - let mut options = RuntimeOptions::default(); - let mut source_uid = None; - let mut source_gid = None; - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => options.state_store = path_value(&mut args, &argument)?, - "--bindings-dir" => options.bindings_dir = path_value(&mut args, &argument)?, - "--source-root" => options.source_root = path_value(&mut args, &argument)?, - "--staging-root" => options.staging_root = path_value(&mut args, &argument)?, - "--topology-store" => options.topology_store = path_value(&mut args, &argument)?, - "--incident-store" => options.incident_store = path_value(&mut args, &argument)?, - "--odin-correlation-store" => { - options.odin_correlation_store = path_value(&mut args, &argument)? - } - "--odin-trust-anchor" => options.odin_trust_anchor = path_value(&mut args, &argument)?, - "--idunn-identity-store" => { - options.idunn_identity_store = path_value(&mut args, &argument)? - } - "--deployment-brake-operator-anchor" => { - options.deployment_brake_operator_anchor = path_value(&mut args, &argument)? - } - "--topology-maximum-age-millis" => { - options.topology_maximum_age_millis = u64_value(&mut args, &argument)? - } - "--topology-maximum-future-skew-millis" => { - options.topology_maximum_future_skew_millis = u64_value(&mut args, &argument)? - } - "--source-uid" => source_uid = Some(u32_value(&mut args, &argument)?), - "--source-gid" => source_gid = Some(u32_value(&mut args, &argument)?), - "--poll-millis" => options.poll_millis = u64_value(&mut args, &argument)?, - "--host-actuator-bind" => { - let value = string_value(&mut args, &argument)?; - options.host_actuator_bind = Some( - value - .parse() - .with_context(|| format!("{argument} is not a socket address"))?, - ); - } - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn serve option {argument:?}"), - } - } - ensure!( - options.poll_millis > 0 && options.topology_maximum_age_millis > 0, - "poll and topology maximum age must be positive" - ); - options.source_identity = match (source_uid, source_gid) { - (Some(uid), Some(gid)) => { - ensure!( - uid > 0 && gid > 0, - "source UID and GID must be unprivileged" - ); - Some(ProcessIdentity { uid, gid }) - } - (None, None) => None, - _ => bail!("--source-uid and --source-gid must be supplied together"), - }; - Ok(Command::Serve(options)) -} - -fn parse_up(mut args: impl Iterator) -> Result { - let selector = args - .next() - .ok_or_else(|| anyhow!("idunn up requires a service or profile selector"))?; - require_selector(&selector)?; - let mut state_store = RuntimeOptions::default().state_store; - let mut requested_by = env::var("SUDO_USER") - .or_else(|_| env::var("USER")) - .or_else(|_| env::var("USERNAME")) - .unwrap_or_else(|_| "operator".into()); - let mut wait = true; - let mut timeout_seconds = 1800; - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => state_store = path_value(&mut args, &argument)?, - "--requested-by" => requested_by = string_value(&mut args, &argument)?, - "--no-wait" => wait = false, - "--timeout-seconds" => timeout_seconds = u64_value(&mut args, &argument)?, - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn up option {argument:?}"), - } - } - ensure!(timeout_seconds > 0, "--timeout-seconds must be positive"); - require_value(&requested_by, "deployment requester")?; - Ok(Command::Up { - selector, - requested_by, - state_store, - wait, - timeout_seconds, - }) -} - -fn parse_status(mut args: impl Iterator) -> Result { - let defaults = RuntimeOptions::default(); - let mut state_store = defaults.state_store; - let mut incident_store = defaults.incident_store; - let mut command_id = None; - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => state_store = path_value(&mut args, &argument)?, - "--incident-store" => incident_store = path_value(&mut args, &argument)?, - "--command" => command_id = Some(string_value(&mut args, &argument)?), - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn status option {argument:?}"), - } - } - Ok(Command::Status { - state_store, - incident_store, - command_id, - }) -} - -fn parse_expire(mut args: impl Iterator) -> Result { - let command_id = args - .next() - .ok_or_else(|| anyhow!("idunn expire requires a command id"))?; - require_id(&command_id, "command id")?; - let mut state_store = RuntimeOptions::default().state_store; - let mut requested_by = env::var("SUDO_USER") - .or_else(|_| env::var("USER")) - .or_else(|_| env::var("USERNAME")) - .unwrap_or_else(|_| "operator".into()); - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => state_store = path_value(&mut args, &argument)?, - "--requested-by" => requested_by = string_value(&mut args, &argument)?, - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn expire option {argument:?}"), - } - } - require_value(&requested_by, "expiry requester")?; - Ok(Command::Expire { - state_store, - command_id, - requested_by, - }) -} - -/// Ask the daemon to end a live transaction's post-fencing phase now. This -/// writes a request and nothing else: the daemon's deadline resolver ends the -/// phase exactly as it ends one whose deadline has passed. -fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { - let snapshot = ControlSnapshot::read(store_path)?; - let mut live = snapshot - .transactions - .iter() - .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); - let current = live - .next() - .context("command has no live transaction to expire")?; - ensure!( - live.next().is_none(), - "command has multiple live transactions" - ); - ensure!( - current.value.resolves_by_deadline(), - "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" - ); - let now = now_millis()?; - let request = ExpiryRequest { - schema_version: EXPIRY_REQUEST_SCHEMA.into(), - command_id: command_id.to_owned(), - requested_by: requested_by.to_owned(), - requested_at_unix_millis: now, - }; - ensure!( - SingleFileMessagePackBackingStore::new(store_path) - .insert_entry_if_absent(expiry_request_envelope(&request, now)?)?, - "an expiry request for this command is already waiting" - ); - println!("{command_id} expiry requested; Idunn ends its phase as an expired deadline"); - Ok(()) -} - -fn parse_cancel(mut args: impl Iterator) -> Result { - let command_id = args - .next() - .ok_or_else(|| anyhow!("idunn cancel requires a deployment command id"))?; - require_id(&command_id, "deployment command id")?; - let mut state_store = RuntimeOptions::default().state_store; - let mut requested_by = env::var("SUDO_USER") - .or_else(|_| env::var("USER")) - .or_else(|_| env::var("USERNAME")) - .unwrap_or_else(|_| "operator".into()); - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => state_store = path_value(&mut args, &argument)?, - "--requested-by" => requested_by = string_value(&mut args, &argument)?, - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn cancel option {argument:?}"), - } - } - require_value(&requested_by, "cancellation requester")?; - Ok(Command::Cancel { - state_store, - command_id, - requested_by, - }) -} - -/// Withdraw queued work, a held deployment that has not fenced, or ask Idunn to -/// clean up a live stateless candidate. Stateful candidates past fencing cannot be cancelled because their prior -/// writer may already be stopped and its state boundary needs explicit repair. -fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { - let snapshot = ControlSnapshot::read(store_path)?; - let command = snapshot - .commands - .iter() - .find(|stored| stored.value.command_id == command_id) - .context("deployment command is unknown or already retired")?; - ensure!( - command.value.kind == CommandKind::Deploy, - "only deployment commands can be cancelled; continuity is Idunn's own" - ); - let mut live = snapshot - .transactions - .iter() - .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); - if let Some(current) = live.next() { - ensure!( - live.next().is_none(), - "deployment command has multiple live transactions" - ); - ensure!( - cancel_is_safe_for_live_stateless(¤t.value) - || cancel_is_safe_for_held_prefence(¤t.value), - "live deployment can only be cancelled before fencing when Idunn holds it, or after fencing when its candidate is stateless" - ); - let error = format!("cancelled by {requested_by}"); - let mut next = current.value.clone(); - if cancel_is_safe_for_held_prefence(&next) { - next.pre_fencing_abort = Some(pre_fencing_abort_intent(&next, &error)); - } else { - next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); - } - next.last_error = Some(error); - next.updated_at_unix_millis = now_millis()?; - replace_transaction(store_path, current, &next)?; - println!("{command_id} cancellation requested; Idunn is performing exact candidate cleanup"); - return Ok(()); - } - let now = now_millis()?; - let refusal = DeploymentTransaction::rejected( - &command.value, - anyhow!("cancelled by {requested_by} before freezing"), - now, - )?; - let envelope = transaction_envelope(&refusal, now)?; - ensure!( - SingleFileMessagePackBackingStore::new(store_path).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command_id.to_owned(), - current: Some(command.envelope.clone()), - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: refusal.transaction_id.clone(), - current: None, - }, - ], - std::slice::from_ref(&envelope), - )?, - "deployment command changed before it could be cancelled" - ); - archive_terminal_transaction(store_path, &envelope)?; - println!("{command_id} cancelled"); - Ok(()) -} - -/// A deployment Idunn holds before it fenced anything has touched nothing of -/// the incumbent, so an operator may withdraw it. Idunn never does so itself. -fn cancel_is_safe_for_held_prefence(transaction: &DeploymentTransaction) -> bool { - transaction.command_kind == CommandKind::Deploy - && transaction.phase < DeploymentPhase::Fencing - && transaction.held_disagreement().is_some() -} - -fn cancel_is_safe_for_live_stateless(transaction: &DeploymentTransaction) -> bool { - transaction.command_kind == CommandKind::Deploy - && transaction.phase >= DeploymentPhase::Fencing - && matches!( - transaction.fencing.as_ref(), - Some(FencingEvidence::SkippedStateless) - ) - && transaction.completion.is_none() - && transaction.pre_fencing_abort.is_none() - && transaction.post_fencing_abort.is_none() - && transaction.post_commit_cleanup.is_none() -} - -fn parse_validate(mut args: impl Iterator) -> Result { - let mut recipe = None; - let mut binding = None; - while let Some(argument) = args.next() { - match argument.as_str() { - "--recipe" => recipe = Some(path_value(&mut args, &argument)?), - "--binding" => binding = Some(path_value(&mut args, &argument)?), - "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn validate option {argument:?}"), - } - } - let recipe = recipe.ok_or_else(|| anyhow!("validate requires --recipe"))?; - Ok(Command::Validate { recipe, binding }) -} - -#[derive(Clone)] -struct Stored { - envelope: CultCacheEnvelope, - value: T, -} - -#[derive(Default)] -struct ControlSnapshot { - commands: Vec>, - transactions: Vec>, - admitted: Vec>, - targets: Vec>, - expiry_requests: Vec>, -} - -impl ControlSnapshot { - fn read(path: &Path) -> Result { - if !path.exists() { - return Ok(Self::default()); - } - let mut snapshot = Self::default(); - for envelope in SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot() - .context("reading Idunn control snapshot")? - { - match envelope.r#type.as_str() { - DeploymentCommand::TYPE => { - ensure!( - envelope.schema_id.as_deref() == Some(DEPLOYMENT_COMMAND_SCHEMA), - "Idunn control store contains an unsupported command" - ); - let value: DeploymentCommand = decode_record(&envelope)?; - value.validate()?; - ensure!( - envelope.key == value.command_id, - "deployment command key differs from its identity" - ); - snapshot.commands.push(Stored { envelope, value }); - } - DeploymentTransaction::TYPE => { - let value = read_transaction_record(&envelope)?; - ensure!( - envelope.key == value.transaction_id, - "deployment transaction key differs from its identity" - ); - snapshot.transactions.push(Stored { envelope, value }); - } - AdmittedGeneration::TYPE => { - let value = read_generation_record(&envelope)?; - ensure!( - envelope.key == value.target, - "admitted generation key is not its target" - ); - snapshot.admitted.push(Stored { envelope, value }); - } - TargetSupervision::TYPE => { - let value = read_supervision_record(&envelope)?; - ensure!( - envelope.key == value.target, - "target supervision key is not its target" - ); - snapshot.targets.push(Stored { envelope, value }); - } - ExpiryRequest::TYPE => { - ensure!( - envelope.schema_id.as_deref() == Some(EXPIRY_REQUEST_SCHEMA), - "Idunn control store contains an unsupported expiry request" - ); - let value: ExpiryRequest = decode_record(&envelope)?; - value.validate()?; - ensure!( - envelope.key == value.command_id, - "expiry request key is not its command" - ); - snapshot.expiry_requests.push(Stored { envelope, value }); - } - _ => bail!("Idunn control store contains a foreign document"), - } - } - snapshot.validate_relations()?; - Ok(snapshot) - } - - fn validate_relations(&self) -> Result<()> { - let command_ids = self - .commands - .iter() - .map(|stored| stored.value.command_id.as_str()) - .collect::>(); - ensure!( - command_ids.len() == self.commands.len(), - "Idunn control store contains duplicate commands" - ); - let transaction_ids = self - .transactions - .iter() - .map(|stored| stored.value.transaction_id.as_str()) - .collect::>(); - ensure!( - transaction_ids.len() == self.transactions.len(), - "Idunn control store contains duplicate transactions" - ); - let admitted_targets = self - .admitted - .iter() - .map(|stored| stored.value.target.as_str()) - .collect::>(); - ensure!( - admitted_targets.len() == self.admitted.len(), - "Idunn control store contains duplicate current generations" - ); - ensure!( - self.targets - .iter() - .map(|stored| stored.value.target.as_str()) - .collect::>() - .len() - == self.targets.len(), - "Idunn control store contains duplicate target supervision records" - ); - for transaction in &self.transactions { - let command = self - .commands - .iter() - .find(|candidate| candidate.value.command_id == transaction.value.command_id) - .context("deployment transaction has no immutable command")?; - ensure!( - command.value.kind == transaction.value.command_kind, - "transaction kind differs from its immutable command" - ); - } - let mut live_targets = BTreeSet::new(); - for transaction in self - .transactions - .iter() - .filter(|stored| stored.value.owns_target_authority()) - { - ensure!( - live_targets.insert(transaction.value.target.as_str()), - "multiple transactions claim current-incarnation authority for one target" - ); - if let Some(fencing) = transaction - .value - .fencing - .as_ref() - .filter(|_| transaction.value.phase < DeploymentPhase::Complete) - { - let incumbent = self.admitted_for(&transaction.value.target); - let incumbent_matches = - match (&transaction.value.incumbent_generation_id, incumbent) { - (None, None) => true, - (Some(expected), Some(current)) => current.value.generation_id == *expected, - _ => false, - }; - ensure!( - incumbent_matches, - "fenced transaction incumbent differs from current admission" - ); - let incumbent_lease_sha256 = incumbent - .and_then(|stored| stored.value.leasing.lease()) - .map(IdunnProcessWriteLeaseRecord::canonical_sha256) - .transpose()?; - match fencing { - FencingEvidence::SkippedStateless => ensure!( - incumbent_lease_sha256.is_none(), - "stateless fencing omitted the admitted incumbent lease" - ), - FencingEvidence::Revoked { - incumbent_lease_sha256: recorded, - .. - } => { - ensure!( - recorded == &incumbent_lease_sha256, - "fencing evidence names another incumbent lease" - ); - ensure!( - incumbent_lease_sha256.is_some() - || transaction - .value - .expected - .as_ref() - .is_some_and(|expected| expected.write_lease_required), - "stateless transaction did not use its explicit fencing skip" - ); - } - } - } - } - let mut authorizations = BTreeSet::new(); - for authorization in self - .transactions - .iter() - .filter_map(|stored| stored.value.deployment_authorization.as_ref()) - { - ensure!( - authorizations.insert(authorization.authorization_id.as_str()), - "deployment authorization was consumed more than once" - ); - } - Ok(()) - } - - /// Every stored record that concerns `target`, as read: the generation, the - /// meters and its transactions. Two reads with equal stamps saw no write - /// for the target between them. - fn target_stamp(&self, target: &str) -> Vec { - let mut stamp = self - .admitted - .iter() - .filter(|stored| stored.value.target == target) - .map(|stored| stored.envelope.clone()) - .chain( - self.targets - .iter() - .filter(|stored| stored.value.target == target) - .map(|stored| stored.envelope.clone()), - ) - .chain( - self.transactions - .iter() - .filter(|stored| stored.value.target == target) - .map(|stored| stored.envelope.clone()), - ) - .collect::>(); - stamp.sort_by(|a, b| (&a.r#type, &a.key).cmp(&(&b.r#type, &b.key))); - stamp - } - - fn admitted_for(&self, target: &str) -> Option<&Stored> { - self.admitted - .iter() - .find(|stored| stored.value.target == target) - } - - fn supervision_for(&self, target: &str) -> Option<&Stored> { - self.targets - .iter() - .find(|stored| stored.value.target == target) - } - - /// The target's meters, or empty ones: a target that was never metered has - /// used nothing. - fn supervision_or_new(&self, target: &str) -> TargetSupervision { - self.supervision_for(target) - .map_or_else(|| TargetSupervision::new(target), |stored| stored.value.clone()) - } - - /// The admitted generation of the target that provides - /// `odin.verse-rendezvous`: Odin is whichever target says so, not - /// whichever is named "odin". - fn admitted_odin(&self) -> Option<&Stored> { - self.admitted.iter().find(|stored| { - ReadinessClass::of(&stored.value.expected) - .is_ok_and(|class| class == ReadinessClass::OdinSelf) - }) - } - - fn transaction_for_command(&self, command_id: &str) -> Vec<&DeploymentTransaction> { - let mut values = self - .transactions - .iter() - .filter(|stored| stored.value.command_id == command_id) - .map(|stored| &stored.value) - .collect::>(); - values.sort_by_key(|value| value.ordinal); - values - } - - fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool { - self.transactions.iter().any(|stored| { - stored.value.owns_target_authority() - && stored.value.command_id == transaction.command_id - && stored.value.ordinal < transaction.ordinal - }) - } - - /// The highest Odin publisher sequence this target has ever *acted on*. - /// - /// A transaction that failed contributes nothing. Its observations were - /// never built upon -- the deployment was abandoned and its projection - /// withdrawn -- so treating them as admitted history only raises the bar - /// for the next attempt. Odin does not reuse a sequence, so nothing is - /// weakened by forgetting the readings of an attempt that came to nothing; - /// what would be weakened is refusing every retry after a failure, which is - /// how a target ends up unable to deploy at all. - fn max_odin_sequence(&self, target: &str, signer_identity_id: &str) -> u64 { - let transaction_max = self - .transactions - .iter() - .filter_map(|stored| { - stored - .value - .latest_odin_observation - .as_ref() - .filter(|evidence| { - stored.value.target == target - && evidence.signer_identity_id == signer_identity_id - && !matches!( - stored.value.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - | Some(TransactionCompletion::FailedAfterFencing { .. }) - ) - }) - .map(|evidence| evidence.publisher_sequence) - }) - .max() - .unwrap_or(0); - let admitted_max = self - .admitted - .iter() - .filter(|stored| { - stored.value.target == target - && stored - .value - .odin_authority - .as_ref() - .is_some_and(|authority| authority.signer_identity_id == signer_identity_id) - }) - .map(|stored| stored.value.odin_publisher_sequence_cursor) - .max() - .unwrap_or(0); - transaction_max.max(admitted_max) - } -} - -pub(crate) fn decode_record(envelope: &CultCacheEnvelope) -> Result -where - T: for<'de> Deserialize<'de> + Serialize, -{ - let value: T = rmp_serde::from_slice(&envelope.payload)?; - ensure!( - rmp_serde::to_vec(&value)? == envelope.payload, - "Idunn control store contains a noncanonical record" - ); - Ok(value) -} - -fn command_envelope(value: &DeploymentCommand, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.command_id, - DeploymentCommand::TYPE, - DEPLOYMENT_COMMAND_SCHEMA, - value, - now, - ) -} - -/// The transaction layout of v2 and v3 records: `ready` is a bare Odin receipt, -/// and there are no phase deadline or lease adoption slots. It exists only to -/// decode records written before v4. `migrate_control_store_to_current_schema` -/// rewrites every one in `control.cc` at boot, but `history.cc` is never -/// migrated or pruned, so this struct dies only when history is pruned or -/// migrated. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v3" -)] -struct LegacyDeploymentTransaction { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - transaction_id: String, - #[cultcache(key = 2)] - command_id: String, - #[cultcache(key = 3)] - command_kind: CommandKind, - #[cultcache(key = 4)] - target: String, - #[cultcache(key = 5)] - ordinal: u32, - #[cultcache(key = 6)] - phase: DeploymentPhase, - #[cultcache(key = 7)] - created_at_unix_millis: u64, - #[cultcache(key = 8)] - updated_at_unix_millis: u64, - #[cultcache(key = 9)] - incumbent_generation_id: Option, - #[cultcache(key = 10)] - plan: Option, - #[cultcache(key = 11)] - frozen_source: Option, - #[cultcache(key = 12)] - sealed_release: Option, - #[cultcache(key = 13)] - installed_release: Option, - #[cultcache(key = 14)] - expected: Option, - #[cultcache(key = 15)] - expected_publication_sha256: Option, - #[cultcache(key = 16)] - deployment_authorization: Option, - #[cultcache(key = 17)] - lifecycle_authorized_at_unix_millis: Option, - #[cultcache(key = 18)] - activation: Option, - #[cultcache(key = 19)] - workload: Option, - #[cultcache(key = 20)] - activation_publication_sha256: Option, - #[cultcache(key = 21)] - latest_odin_observation: Option, - #[cultcache(key = 22)] - warming: Option, - #[cultcache(key = 23)] - route_preflight: Option, - #[cultcache(key = 24)] - isolation: Option, - #[cultcache(key = 25)] - fencing: Option, - #[cultcache(key = 26)] - leasing: Option, - #[cultcache(key = 27)] - ready: Option, - #[cultcache(key = 28)] - routing: Option, - #[cultcache(key = 29)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 30)] - last_error: Option, - #[cultcache(key = 31)] - completion: Option, - #[cultcache(key = 32)] - pre_fencing_abort: Option, - #[cultcache(key = 33)] - post_commit_cleanup: Option, - // A v2 record has no key 34: it predates post-fencing aborts. - #[cultcache(key = 34, default)] - post_fencing_abort: Option, -} - -impl LegacyDeploymentTransaction { - /// Every legacy Ready receipt is an Odin receipt: no route-proof class - /// existed. The new slots start empty, which is what an older transaction - /// had. - fn into_current(self) -> DeploymentTransaction { - DeploymentTransaction { - schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), - transaction_id: self.transaction_id, - command_id: self.command_id, - command_kind: self.command_kind, - target: self.target, - ordinal: self.ordinal, - phase: self.phase, - created_at_unix_millis: self.created_at_unix_millis, - updated_at_unix_millis: self.updated_at_unix_millis, - incumbent_generation_id: self.incumbent_generation_id, - plan: self.plan, - frozen_source: self.frozen_source, - sealed_release: self.sealed_release, - installed_release: self.installed_release, - expected: self.expected, - expected_publication_sha256: self.expected_publication_sha256, - deployment_authorization: self.deployment_authorization, - lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, - activation: self.activation, - workload: self.workload, - activation_publication_sha256: self.activation_publication_sha256, - latest_odin_observation: self.latest_odin_observation, - warming: self.warming, - route_preflight: self.route_preflight, - isolation: self.isolation, - fencing: self.fencing, - leasing: self.leasing, - ready: self - .ready - .map(|evidence| ReadinessEvidence::OdinCorrelated { evidence }), - routing: self.routing, - odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, - last_error: self.last_error, - completion: self.completion, - pre_fencing_abort: self.pre_fencing_abort, - post_commit_cleanup: self.post_commit_cleanup, - post_fencing_abort: self.post_fencing_abort, - phase_deadline: None, - lease_adoption: None, - } - } -} - -/// The v2 admitted-generation layout. Same lifetime rule as -/// `LegacyDeploymentTransaction`. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v2" -)] -struct LegacyAdmittedGeneration { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - #[cultcache(key = 2)] - generation_id: String, - #[cultcache(key = 3)] - command_id: String, - #[cultcache(key = 4)] - transaction_id: String, - #[cultcache(key = 5)] - admitted_at_unix_millis: u64, - #[cultcache(key = 6)] - plan: CompiledDeploymentPlan, - #[cultcache(key = 7)] - sealed_release: SealedRelease, - #[cultcache(key = 8)] - installed_release: InstalledReleaseObservation, - #[cultcache(key = 9)] - expected: IdunnExpectedIncarnationRecord, - #[cultcache(key = 10)] - activation: IdunnRuntimeActivationRecord, - #[cultcache(key = 11)] - workload: WorkloadObservation, - #[cultcache(key = 12)] - leasing: LeasingEvidence, - #[cultcache(key = 13)] - ready: TopologyEvidence, - #[cultcache(key = 14)] - latest_odin_observation: TopologyEvidence, - #[cultcache(key = 15)] - routing: RoutingEvidence, - #[cultcache(key = 16)] - odin_authority: AdmittedOdinAuthority, - #[cultcache(key = 17)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 18)] - route_repair_started_at_unix_millis: Option, -} - -impl LegacyAdmittedGeneration { - /// Every v2 generation was admitted on Odin receipts, so it migrates as - /// Odin-correlated with those receipts unchanged. Route supervision state - /// starts empty for a promoted route. The v2 repair start decided nothing - /// after S1 and is not carried. - fn into_current(self) -> AdmittedGeneration { - let route_supervision = matches!(&self.routing, RoutingEvidence::Promoted { .. }) - .then(RouteSupervisionState::default); - AdmittedGeneration { - schema_version: ADMITTED_GENERATION_SCHEMA.into(), - target: self.target, - generation_id: self.generation_id, - command_id: self.command_id, - transaction_id: self.transaction_id, - admitted_at_unix_millis: self.admitted_at_unix_millis, - plan: self.plan, - sealed_release: self.sealed_release, - installed_release: self.installed_release, - expected: self.expected, - activation: self.activation, - workload: self.workload, - leasing: self.leasing, - ready: ReadinessEvidence::OdinCorrelated { - evidence: self.ready, - }, - latest_odin_observation: Some(self.latest_odin_observation), - routing: self.routing, - odin_authority: Some(self.odin_authority), - odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, - route_supervision, - last_error: None, - } - } -} - -/// The v3 admitted-generation layout. Its meters (`actuations`, the continuity -/// backoff, the repair start) are retired: no released Idunn ever wrote a -/// nonzero one, and a record that shows one is refused instead of lifted. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.admitted_generation", - schema = "idunn.admitted_generation.v3" -)] -struct LegacyAdmittedGenerationV3 { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - #[cultcache(key = 2)] - generation_id: String, - #[cultcache(key = 3)] - command_id: String, - #[cultcache(key = 4)] - transaction_id: String, - #[cultcache(key = 5)] - admitted_at_unix_millis: u64, - #[cultcache(key = 6)] - plan: CompiledDeploymentPlan, - #[cultcache(key = 7)] - sealed_release: SealedRelease, - #[cultcache(key = 8)] - installed_release: InstalledReleaseObservation, - #[cultcache(key = 9)] - expected: IdunnExpectedIncarnationRecord, - #[cultcache(key = 10)] - activation: IdunnRuntimeActivationRecord, - #[cultcache(key = 11)] - workload: WorkloadObservation, - #[cultcache(key = 12)] - leasing: LeasingEvidence, - #[cultcache(key = 13)] - ready: ReadinessEvidence, - #[cultcache(key = 14)] - latest_odin_observation: Option, - #[cultcache(key = 15)] - routing: RoutingEvidence, - #[cultcache(key = 16)] - odin_authority: Option, - #[cultcache(key = 17)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 18)] - route_repair_started_at_unix_millis: Option, - #[cultcache(key = 19)] - route_supervision: Option, - #[cultcache(key = 20)] - continuity_backoff: LegacyContinuityBackoffV3, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LegacyRouteSupervisionStateV3 { - last_challenge_at_unix_millis: Option, - consecutive_failures: u32, - next_challenge_at_unix_millis: Option, - degraded_since_unix_millis: Option, - actuations: LegacyActuationWindowV3, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LegacyActuationWindowV3 { - window_started_at_unix_millis: u64, - count: u32, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LegacyContinuityBackoffV3 { - window_started_at_unix_millis: Option, - attempts: u32, - next_restart_at_unix_millis: Option, -} - -impl LegacyAdmittedGenerationV3 { - /// Drops the retired meters. The lift creates no `TargetSupervision`: every - /// target starts unmetered, which is what no released binary ever changed. - fn into_current(self) -> Result { - ensure!( - self.continuity_backoff.attempts == 0 - && self - .route_supervision - .as_ref() - .is_none_or(|state| state.actuations.count == 0), - "admitted generation of {} records route actuations or continuity restarts that no released Idunn wrote; refusing to drop them", - self.target - ); - Ok(AdmittedGeneration { - schema_version: ADMITTED_GENERATION_SCHEMA.into(), - target: self.target, - generation_id: self.generation_id, - command_id: self.command_id, - transaction_id: self.transaction_id, - admitted_at_unix_millis: self.admitted_at_unix_millis, - plan: self.plan, - sealed_release: self.sealed_release, - installed_release: self.installed_release, - expected: self.expected, - activation: self.activation, - workload: self.workload, - leasing: self.leasing, - ready: self.ready, - latest_odin_observation: self.latest_odin_observation, - routing: self.routing, - odin_authority: self.odin_authority, - odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, - route_supervision: self.route_supervision.map(|state| RouteSupervisionState { - last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, - consecutive_failures: state.consecutive_failures, - next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, - degraded_since_unix_millis: state.degraded_since_unix_millis, - }), - last_error: None, - }) - } -} - -/// Decode a v2 or v3 transaction and lift it to the current shape, without -/// validating it. History reads stop here; the control store validates. -fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result { - let schema = match envelope.schema_id.as_deref() { - Some(schema @ (DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3)) => { - schema - } - _ => bail!("Idunn control store contains an unsupported transaction"), - }; - let value: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload) - .with_context(|| format!("decoding a {schema} deployment transaction"))?; - ensure!( - value.schema_version == schema, - "stored transaction schema differs from its envelope" - ); - ensure!( - schema != DEPLOYMENT_TRANSACTION_SCHEMA_V2 || value.post_fencing_abort.is_none(), - "a v2 transaction cannot carry post-fencing abort evidence" - ); - Ok(value.into_current()) -} - -/// Read one stored transaction, lifting a v2 or v3 record to the current shape. -/// -/// The control store refuses noncanonical bytes: a record must re-encode to -/// exactly what is stored. That check is what makes tampering visible, and it -/// also means adding a field is a schema change: an older record re-encodes -/// with extra keys and would be read as tampered. -/// -/// So an older record is decoded by its own layout and lifted. Byte-exactness -/// cannot apply across a version boundary -- the bytes are a different schema -/// by definition -- so the lift leans on the full semantic `validate()` -/// instead, and everything downstream sees only the current shape. -fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result { - match envelope.schema_id.as_deref() { - Some(DEPLOYMENT_TRANSACTION_SCHEMA) => { - let value: DeploymentTransaction = decode_record(envelope)?; - value.validate()?; - Ok(value) - } - _ => { - let mut value = lift_legacy_transaction(envelope)?; - cleanup_evidence::owe_legacy_continuity_projection(&mut value); - value.validate()?; - Ok(value) - } - } -} - -/// Read one stored admitted generation, lifting a v2 or v3 record to the -/// current shape by the same rule as `read_transaction_record`. -fn read_generation_record(envelope: &CultCacheEnvelope) -> Result { - match envelope.schema_id.as_deref() { - Some(ADMITTED_GENERATION_SCHEMA) => { - let value: AdmittedGeneration = decode_record(envelope)?; - value.validate()?; - Ok(value) - } - Some(ADMITTED_GENERATION_SCHEMA_V3) => { - let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload) - .context("decoding a v3 admitted generation")?; - ensure!( - legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V3, - "stored generation schema differs from its envelope" - ); - let value = legacy.into_current()?; - value.validate()?; - Ok(value) - } - Some(ADMITTED_GENERATION_SCHEMA_V2) => { - let legacy: LegacyAdmittedGeneration = rmp_serde::from_slice(&envelope.payload) - .context("decoding a v2 admitted generation")?; - ensure!( - legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V2, - "stored generation schema differs from its envelope" - ); - let value = legacy.into_current(); - value.validate()?; - Ok(value) - } - _ => bail!("Idunn control store contains an unsupported admitted generation"), - } -} - -/// The v1 target-supervision layout: the first seven keys of the current one. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.target_supervision", - schema = "idunn.target_supervision.v1" -)] -struct LegacyTargetSupervisionV1 { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - target: String, - #[cultcache(key = 2)] - route_actuations: Vec, - #[cultcache(key = 3)] - continuity_restarts: Vec, - #[cultcache(key = 4)] - continuity_deferred_until: Option, - #[cultcache(key = 5)] - continuity_deferral_reason: Option, - #[cultcache(key = 6)] - route_survivals: Vec, -} - -impl LegacyTargetSupervisionV1 { - /// No streak, no anchor, no re-materialization: the same defaults a fresh - /// record starts with. - fn into_current(self) -> TargetSupervision { - TargetSupervision { - schema_version: TARGET_SUPERVISION_SCHEMA.into(), - target: self.target, - route_actuations: self.route_actuations, - continuity_restarts: self.continuity_restarts, - continuity_deferred_until: self.continuity_deferred_until, - continuity_deferral_reason: self.continuity_deferral_reason, - route_survivals: self.route_survivals, - continuity_attempt_streak: 0, - continuity_backoff_from_unix_millis: None, - continuity_rematerialized_at_unix_millis: None, - } - } -} - -/// Read one stored target supervision record, lifting a v1 record to the -/// current shape by the same rule as `read_generation_record`. One schema id -/// names one shape: a v1 record that carries ten keys, or a v2 record that -/// carries seven, is refused. -fn read_supervision_record(envelope: &CultCacheEnvelope) -> Result { - let value = match envelope.schema_id.as_deref() { - Some(TARGET_SUPERVISION_SCHEMA) => decode_record::(envelope)?, - Some(TARGET_SUPERVISION_SCHEMA_V1) => { - let legacy: LegacyTargetSupervisionV1 = decode_record(envelope) - .context("decoding a v1 target supervision record")?; - ensure!( - legacy.schema_version == TARGET_SUPERVISION_SCHEMA_V1, - "stored supervision schema differs from its envelope" - ); - legacy.into_current() - } - _ => bail!("Idunn control store contains an unsupported target supervision record"), - }; - value.validate()?; - ensure!( - envelope.key == value.target, - "target supervision key is not its target" - ); - Ok(value) -} - -/// Rewrite every older record as current, once, before the engine runs. -/// -/// The read path lifts older records on its own, so this is convergence rather -/// than correctness: without it the store keeps records in several shapes for -/// as long as the oldest one survives. Each rewrite is a compare-exchange -/// against the exact stored envelope, so a record that changed underneath is -/// left alone rather than clobbered. -fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { - if !store_path.exists() { - return Ok(0); - } - let store = SingleFileMessagePackBackingStore::new(store_path); - let stale = store - .pull_all_read_only_snapshot() - .context("reading Idunn control snapshot for migration")? - .into_iter() - .filter(|envelope| { - let schema = envelope.schema_id.as_deref(); - (envelope.r#type == DeploymentTransaction::TYPE - && matches!( - schema, - Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) - )) - || (envelope.r#type == AdmittedGeneration::TYPE - && matches!( - schema, - Some(ADMITTED_GENERATION_SCHEMA_V2 | ADMITTED_GENERATION_SCHEMA_V3) - )) - || (envelope.r#type == TargetSupervision::TYPE - && schema == Some(TARGET_SUPERVISION_SCHEMA_V1)) - }) - .collect::>(); - let mut migrated = 0; - for envelope in stale { - let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { - let (key, next) = cleanup_evidence::migrate_transaction_record(&envelope)?; - (DeploymentTransaction::TYPE, key, next) - } else if envelope.r#type == TargetSupervision::TYPE { - let value = read_supervision_record(&envelope)?; - let next = target_supervision_envelope(&value, now_millis()?)?; - (TargetSupervision::TYPE, value.target, next) - } else { - let value = read_generation_record(&envelope)?; - let next = admitted_envelope(&value, value.admitted_at_unix_millis)?; - (AdmittedGeneration::TYPE, value.target, next) - }; - next.stored_at = envelope.stored_at.clone(); - ensure!( - store.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: record_type.into(), - key, - current: Some(envelope.clone()), - }], - &[next], - )?, - "control record changed during schema migration" - ); - migrated += 1; - } - Ok(migrated) -} - -fn transaction_envelope(value: &DeploymentTransaction, now: u64) -> Result { - ensure!( - !value.carries_legacy_marker(), - "only the legacy lift may produce a legacy-marked record; it cannot be written as a new one" - ); - encode_transaction(value, now) -} - -fn encode_transaction(value: &DeploymentTransaction, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.transaction_id, - DeploymentTransaction::TYPE, - DEPLOYMENT_TRANSACTION_SCHEMA, - value, - now, - ) -} - -fn admitted_envelope(value: &AdmittedGeneration, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.target, - AdmittedGeneration::TYPE, - ADMITTED_GENERATION_SCHEMA, - value, - now, - ) -} - -fn target_supervision_envelope(value: &TargetSupervision, now: u64) -> Result { - value.validate()?; - typed_envelope( - &value.target, - TargetSupervision::TYPE, - TARGET_SUPERVISION_SCHEMA, - value, - now, - ) -} - -pub(crate) fn typed_envelope( - key: &str, - record_type: &str, - schema: &str, - value: &T, - now: u64, -) -> Result { - Ok(CultCacheEnvelope { - key: key.into(), - r#type: record_type.into(), - payload: rmp_serde::to_vec(value)?, - stored_at: rfc3339_millis(now)?, - schema_id: Some(schema.into()), - }) -} - -/// `history.cc`, beside the control store. Terminal transactions live here so -/// the control store holds only what still gates a decision. -fn history_store_path(state_store: &Path) -> PathBuf { - state_store.with_file_name("history.cc") -} - -/// Move one finished transaction out of the live set. -/// -/// History first, then the live copy: a crash between the two leaves the -/// record in both, which reads as still-resident and is archived again on the -/// next attempt. The other order can lose it, and a completion nobody can -/// observe is the failure that actually costs an operator something. -/// -/// `insert_entry_if_absent` makes the repeat a no-op, and the delete is a -/// compare-and-swap on the exact envelope archived, so a transaction that -/// changed underneath us is left alone rather than dropped. -fn archive_terminal_transaction(state_store: &Path, envelope: &CultCacheEnvelope) -> Result<()> { - SingleFileMessagePackBackingStore::new(&history_store_path(state_store)) - .insert_entry_if_absent(envelope.clone()) - .context("archiving a finished transaction to history")?; - SingleFileMessagePackBackingStore::new(state_store) - .delete_batch_if_unchanged(std::slice::from_ref(envelope)) - .context("retiring a finished transaction from the control store")?; - // The command goes with its last transaction. A command is consumed by - // being frozen once; left resident after its transaction retired, it read - // as queued again and was frozen again on the next tick, which is how one - // `idunn up` became an unbounded series of deployment attempts. - let transaction = read_transaction_record(envelope)?; - let snapshot = ControlSnapshot::read(state_store)?; - if !snapshot - .transaction_for_command(&transaction.command_id) - .is_empty() - { - return Ok(()); - } - if let Some(command) = snapshot - .commands - .iter() - .find(|stored| stored.value.command_id == transaction.command_id) - { - SingleFileMessagePackBackingStore::new(&history_store_path(state_store)) - .insert_entry_if_absent(command.envelope.clone()) - .context("archiving a consumed command to history")?; - SingleFileMessagePackBackingStore::new(state_store) - .delete_batch_if_unchanged(std::slice::from_ref(&command.envelope)) - .context("retiring a consumed command from the control store")?; - } - Ok(()) -} - -/// What `history.cc` yielded: the transactions it decoded and the entries it -/// could not, each with its key and reason. -struct HistoryRead { - transactions: Vec, - undecodable: Vec<(String, String)>, -} - -impl HistoryRead { - /// What is missing from `transactions`, or `None` when nothing is. - fn report(&self) -> Option { - if self.undecodable.is_empty() { - return None; - } - let keys = self - .undecodable - .iter() - .map(|(key, error)| format!("{key} ({error})")) - .collect::>() - .join("; "); - Some(format!( - "Idunn history: {} archived transaction(s) could not be decoded and are not counted \ - by status or continuity backoff: {keys}", - self.undecodable.len() - )) - } -} - -/// Finished transactions. `control.cc` keeps the byte-exact canonical check -/// because its records still gate decisions. History describes what already -/// happened, so an undecodable entry is reported and skipped rather than -/// allowed to refuse the read -- that asymmetry is the whole reason the two -/// files are separate. A file that cannot be read at all is an error: the -/// caller decides whether an empty answer is safe. -fn read_history(state_store: &Path) -> Result { - let path = history_store_path(state_store); - if !path.exists() { - return Ok(HistoryRead { - transactions: Vec::new(), - undecodable: Vec::new(), - }); - } - let envelopes = SingleFileMessagePackBackingStore::new(&path) - .pull_all_read_only_snapshot() - .with_context(|| format!("Idunn cannot read {}", path.display()))?; - let (transactions, undecodable) = decode_history_transactions(envelopes); - Ok(HistoryRead { - transactions, - undecodable, - }) -} - -/// History for display only (status). An unreadable file shows as empty and -/// says so; decisions use `Engine::history_for_decision`, which does not. -fn read_history_transactions(state_store: &Path) -> Vec { - match read_history(state_store) { - Ok(history) => { - if let Some(report) = history.report() { - eprintln!("{report}"); - } - history.transactions - } - Err(error) => { - eprintln!("{error:#}; every archived transaction is invisible to status"); - Vec::new() - } - } -} - -/// How long to wait after `failures` consecutive failures: one poll interval -/// after the first, doubling each time, never past the ceiling. -fn backoff_wait(base_millis: u64, failures: u32, ceiling_millis: u64) -> u64 { - base_millis - .saturating_mul(1u64 << failures.min(20)) - .min(ceiling_millis) -} - -/// Whether an attempt due at `not_before` must still wait at `now`. A due time -/// further ahead than any wait its owner can set means the clock stepped back, -/// so the attempt is due: a step stalls nothing longer than the wait itself. -fn is_waiting(now: u64, not_before: u64, longest_wait: u64) -> bool { - now < not_before && not_before - now <= longest_wait -} - -/// Whether a transaction moved between two reads. An error note (`last_error`) -/// and its timestamp are commentary on a stuck step, not a step. -fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) -> bool { - let mut after = after.clone(); - after.last_error.clone_from(&before.last_error); - after.updated_at_unix_millis = before.updated_at_unix_millis; - &after != before -} - -/// Says a fault once on stderr while it lasts. A scheduler tick that hits the same -/// unreadable file every half second must not print it every half second. -/// -/// It is a trace and owns nothing else: it is in memory, keyed by free-form -/// strings, and an Idunn restart forgets it. Whether the operator is told -/// belongs to the incident record (`incident`), never to this map. -#[derive(Default)] -struct ReportOnce { - last: Mutex>, -} - -impl ReportOnce { - /// The text to print now: `report` when it differs from the last one - /// offered, nothing while the same fault stands or once it has cleared. - fn offer(&self, report: Option) -> Option { - let mut last = self.last.lock().expect("report mutex"); - if *last == report { - return None; - } - last.clone_from(&report); - report - } -} - -/// Decode every archived transaction. An entry that fails is returned with its -/// key and reason instead of vanishing, so the reader can say how many records -/// its answer is missing. -/// -/// `LegacyDeploymentTransaction` decodes the v2 and v3 layouts. `history.cc` is -/// never migrated or pruned, so it dies only when history is pruned or -/// migrated, not at boot. -fn decode_history_transactions( - envelopes: Vec, -) -> (Vec, Vec<(String, String)>) { - let mut transactions = Vec::new(); - let mut undecodable = Vec::new(); - for envelope in envelopes - .into_iter() - .filter(|envelope| envelope.r#type == DeploymentTransaction::TYPE) - { - let decoded = match envelope.schema_id.as_deref() { - Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) => { - lift_legacy_transaction(&envelope) - } - _ => rmp_serde::from_slice::(&envelope.payload) - .map_err(anyhow::Error::from), - }; - match decoded { - Ok(transaction) => transactions.push(transaction), - Err(error) => undecodable.push((envelope.key, format!("{error:#}"))), - } - } - (transactions, undecodable) -} - -/// Consumed commands, read as leniently as their transactions. -fn read_history_commands(state_store: &Path) -> Vec { - let path = history_store_path(state_store); - if !path.exists() { - return Vec::new(); - } - let Ok(envelopes) = SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot() - else { - return Vec::new(); - }; - envelopes - .into_iter() - .filter(|envelope| envelope.r#type == DeploymentCommand::TYPE) - .filter_map(|envelope| rmp_serde::from_slice::(&envelope.payload).ok()) - .collect() -} - -fn replace_transaction( - store_path: &Path, - current: &Stored, - next: &DeploymentTransaction, -) -> Result<()> { - next.validate()?; - ensure!( - current.value.transaction_id == next.transaction_id, - "transaction replacement changes identity" - ); - let envelope = transaction_envelope(next, next.updated_at_unix_millis)?; - ensure!( - SingleFileMessagePackBackingStore::new(store_path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: current.value.transaction_id.clone(), - current: Some(current.envelope.clone()), - }], - std::slice::from_ref(&envelope), - )?, - "deployment transaction changed before its compare-exchange" - ); - if next.is_terminal() { - archive_terminal_transaction(store_path, &envelope)?; - } - Ok(()) -} - -#[derive(Clone)] -struct LoadedBinding { - binding: OperatorBinding, - bytes: Vec, -} - -fn load_bindings(directory: &Path) -> Result> { - let mut paths = fs::read_dir(directory) - .with_context(|| format!("reading Idunn binding directory {}", directory.display()))? - .map(|entry| entry.map(|entry| entry.path())) - .collect::>>()?; - paths.retain(|path| path.extension() == Some(std::ffi::OsStr::new("toml"))); - paths.sort(); - let mut bindings = BTreeMap::new(); - for path in paths { - let bytes = fs::read(&path) - .with_context(|| format!("reading operator binding {}", path.display()))?; - let text = std::str::from_utf8(&bytes) - .with_context(|| format!("operator binding {} is not UTF-8", path.display()))?; - let binding = OperatorBinding::parse(text) - .with_context(|| format!("validating operator binding {}", path.display()))?; - let target = binding.target.clone(); - ensure!( - bindings - .insert(target.clone(), LoadedBinding { binding, bytes }) - .is_none(), - "operator binding target {target} is duplicated" - ); - } - ensure!(!bindings.is_empty(), "Idunn binding directory is empty"); - validate_route_bindings(&bindings)?; - Ok(bindings) -} - -fn validate_route_bindings(bindings: &BTreeMap) -> Result<()> { - let routes = bindings - .iter() - .filter_map(|(target, loaded)| { - loaded - .binding - .route - .as_ref() - .map(|route| (target.as_str(), route)) - }) - .collect::>(); - validate_route_binding_set(&routes) -} - -fn validate_route_binding_set(routes: &[(&str, &RouteBinding)]) -> Result<()> { - for (index, (target, route)) in routes.iter().enumerate() { - let (stable_host, stable_port) = route.stable_socket()?; - let private_host = route - .private_host - .parse::() - .context("validated route private host stopped being an IP address")?; - ensure!( - stable_host != private_host - || !(route.private_port_start..=route.private_port_end).contains(&stable_port), - "route {target} stable socket overlaps its candidate port range" - ); - - for (other_target, other) in &routes[index + 1..] { - ensure!( - route.route_id != other.route_id, - "route id {} is shared by targets {target} and {other_target}", - route.route_id - ); - ensure!( - route.config_path != other.config_path, - "route fragment {} is shared by targets {target} and {other_target}", - route.config_path.display() - ); - let other_stable = other.stable_socket()?; - ensure!( - route.driver != other.driver || (stable_host, stable_port) != other_stable, - "stable route socket {}:{} is shared by targets {target} and {other_target}", - stable_host, - stable_port - ); - if route.driver != other.driver { - continue; - } - let other_private_host = other - .private_host - .parse::() - .context("validated route private host stopped being an IP address")?; - let ranges_overlap = route.private_host == other.private_host - && route.private_port_start <= other.private_port_end - && other.private_port_start <= route.private_port_end; - ensure!( - !ranges_overlap, - "candidate port ranges overlap for targets {target} and {other_target}" - ); - ensure!( - stable_host != other_private_host - || !(other.private_port_start..=other.private_port_end).contains(&stable_port), - "stable route socket for {target} overlaps {other_target}'s candidate range" - ); - ensure!( - other_stable.0 != private_host - || !(route.private_port_start..=route.private_port_end) - .contains(&other_stable.1), - "stable route socket for {other_target} overlaps {target}'s candidate range" - ); - } - } - Ok(()) -} - -fn resolve_selector( - bindings: &BTreeMap, - selector: &str, -) -> Result> { - let mut targets = if let Some(profile) = selector.strip_prefix("profile:") { - bindings - .values() - .filter(|binding| binding.binding.profiles.contains(profile)) - .map(|binding| binding.binding.target.clone()) - .collect::>() - } else { - ensure!( - bindings.contains_key(selector), - "unknown deployment target {selector}" - ); - vec![selector.to_owned()] - }; - ensure!( - !targets.is_empty(), - "unknown or empty deployment profile {selector}" - ); - targets.sort_by(|left, right| { - (left != "odin") - .cmp(&(right != "odin")) - .then_with(|| left.cmp(right)) - }); - Ok(targets) -} - -fn submit( - store_path: &Path, - selector: &str, - requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { - if let Some(parent) = store_path.parent() { - fs::create_dir_all(parent)?; - } - let now = now_millis()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), - kind: CommandKind::Deploy, - selector: selector.into(), - requested_by: requested_by.into(), - requested_at_unix_millis: now, - }; - command.validate()?; - ensure!( - SingleFileMessagePackBackingStore::new(store_path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, now)?], - )?, - "deployment command id collided" - ); - println!("{}", command.command_id); - if !wait { - return Ok(()); - } - let deadline = now.saturating_add(timeout_seconds.saturating_mul(1000)); - loop { - let snapshot = ControlSnapshot::read(store_path)?; - ensure!( - snapshot - .commands - .iter() - .any(|stored| stored.value.command_id == command.command_id), - "submitted deployment command disappeared" - ); - let transactions = snapshot.transaction_for_command(&command.command_id); - if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { - if let Some(error) = - transactions - .iter() - .find_map(|transaction| match &transaction.completion { - Some(TransactionCompletion::FailedBeforeFencing { error }) - | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => { - Some(error.as_str()) - } - _ => None, - }) - { - bail!("deployment failed: {error}") - } - println!("succeeded {} target(s)", transactions.len()); - return Ok(()); - } - if now_millis()? >= deadline { - bail!("deployment command timed out") - } - thread::sleep(Duration::from_millis(250)); - } -} - -fn status( - out: &mut impl Write, - store_path: &Path, - incident_store: &Path, - command_id: Option<&str>, -) -> Result<()> { - let snapshot = ControlSnapshot::read(store_path)?; - // Commands stay resident; their finished transactions do not. Without - // history a completed command would report as though it had never run, - // which is the operator surface R11 refused to trade away for a smaller - // live set. - let archived = read_history_transactions(store_path); - let archived_commands = read_history_commands(store_path); - let mut commands = snapshot - .commands - .iter() - .map(|stored| &stored.value) - .chain(archived_commands.iter()) - .collect::>(); - commands.sort_by_key(|command| command.requested_at_unix_millis); - if let Some(command_id) = command_id { - commands.retain(|command| command.command_id == command_id); - ensure!(!commands.is_empty(), "deployment command is unknown"); - } - for command in commands { - let mut transactions = snapshot.transaction_for_command(&command.command_id); - transactions.extend( - archived - .iter() - .filter(|value| value.command_id == command.command_id), - ); - transactions.sort_by_key(|value| value.ordinal); - let (state, detail) = derived_command_status(&transactions); - writeln!( - out, - "{} {} {} {}", - command.command_id, command.selector, state, detail - )?; - // Naming one command asks about that command, so print what a stuck - // transaction is actually waiting on. A gate reason lives in - // `last_error` and was never rendered anywhere, which left "Sealing" - // looking identical whether the brake had not authorized the - // transaction or the phase was simply slow. The release and deployment - // ids are here because they are exactly what a brake release must name. - if command_id.is_some() { - for transaction in transactions { - writeln!(out, " transaction {}", transaction.transaction_id)?; - writeln!( - out, - " target {} phase {:?}", - transaction.target, transaction.phase - )?; - if let Some(expected) = &transaction.expected { - writeln!(out, " runtime {}", expected.runtime_id)?; - writeln!(out, " release {}", expected.sealed_release_id)?; - } - writeln!( - out, - " odin publisher cursor {}", - transaction.odin_publisher_sequence_cursor - )?; - if let Some(evidence) = &transaction.latest_odin_observation { - writeln!( - out, - " latest odin observation sequence {}", - evidence.publisher_sequence - )?; - } - if let Some(reason) = &transaction.last_error { - writeln!(out, " waiting on {reason}")?; - } - // A Complete transaction that is not terminal still owns its - // target through unfinished cleanup. Without this an operator - // sees "Complete" and a stale reason and cannot tell what is - // holding the target. - if let Some(completion) = &transaction.completion { - writeln!(out, " completion {completion:?}")?; - } - if let Some(abort) = &transaction.pre_fencing_abort { - writeln!(out, " pre-fencing abort {abort:?}")?; - } - if let Some(abort) = &transaction.post_fencing_abort { - writeln!(out, " post-fencing abort {abort:?}")?; - } - if let Some(cleanup) = &transaction.post_commit_cleanup { - writeln!(out, " post-commit cleanup {cleanup:?}")?; - } - writeln!( - out, - " terminal {} owns-target {}", - transaction.is_terminal(), - transaction.blocks_new_target_mutation() - )?; - } - } - } - // What supervision holds for each target: the union of the admitted - // generations and the metered targets, so a target that has only been - // deployed to (no generation yet) still shows what it has used. - for line in render_targets(&snapshot, incident_store, now_millis()?) { - writeln!(out, "{line}")?; - } - Ok(()) -} - -/// Every supervised target with its meters and its incidents. An incident -/// store that cannot be read costs the incident lines and nothing else. -fn render_targets(snapshot: &ControlSnapshot, incident_store: &Path, now: u64) -> Vec { - let incidents = IncidentStore::new(incident_store).read(); - let mut lines = Vec::new(); - for target in supervised_targets(snapshot) { - lines.extend(render_supervision( - target, - snapshot.admitted_for(target).map(|stored| &stored.value), - snapshot.supervision_for(target).map(|stored| &stored.value), - now, - )); - if let Ok(records) = &incidents { - lines.extend(render_incidents(records, target, now)); - } - } - if let Err(error) = &incidents { - lines.push(format!("incidents unreadable: {}", error.root_cause())); - } - lines -} - -/// Every target supervision holds something for: the admitted generations -/// and the metered targets. -fn supervised_targets(snapshot: &ControlSnapshot) -> BTreeSet<&str> { - snapshot - .admitted - .iter() - .map(|stored| stored.value.target.as_str()) - .chain(snapshot.targets.iter().map(|stored| stored.value.target.as_str())) - .collect() -} - -/// One target's meters and route pacing, one line each, times in unix -/// milliseconds. The counts are what is still inside the window at `now`. -fn render_supervision( - target: &str, - generation: Option<&AdmittedGeneration>, - supervision: Option<&TargetSupervision>, - now: u64, -) -> Vec { - let at = |time: Option| time.map_or("none".to_owned(), |millis| millis.to_string()); - let meters = supervision.cloned().unwrap_or_else(|| TargetSupervision::new(target)); - let mut lines = vec![format!( - "target {target} {}", - generation.map_or("no-admitted-generation", |value| value.generation_id.as_str()) - )]; - if let Some(reason) = generation.and_then(|value| value.last_error.as_deref()) { - lines.push(format!(" held: {reason}")); - } - lines.push(format!( - " continuity attempts-in-hour {} pages-at {} streak {} next-attempt-at {}", - meters.restarts_used(now), - CONTINUITY_PAGING_RESTARTS, - meters.continuity_attempt_streak, - at(generation.and_then(|value| meters.backoff_due(value.admitted_at_unix_millis))) - )); - if let Some(rematerialized) = meters.continuity_rematerialized_at_unix_millis { - lines.push(format!(" continuity rematerialized-at {rematerialized}")); - } - if let (Some(until), Some(reason)) = ( - meters.continuity_deferred_until, - &meters.continuity_deferral_reason, - ) { - lines.push(format!(" continuity deferred until {until}: {reason}")); - } - if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { - lines.push(format!( - " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", - match route.degraded_since_unix_millis { - Some(since) => format!("degraded-since {since}"), - None => "healthy".to_owned(), - }, - route.consecutive_failures, - at(route.last_challenge_at_unix_millis), - at(route.next_challenge_at_unix_millis) - )); - } - if generation.is_none_or(|value| value.route_supervision.is_some()) - || !meters.route_actuations.is_empty() - || !meters.route_survivals.is_empty() - { - lines.push(format!( - " route actuations {}/{} in window, reopens-at {}", - meters.route_used(now), - ROUTE_ACTUATION_CEILING, - at(meters.route_reopens_at(now)) - )); - // A record, not a meter: only the newest are kept, so a saturated - // count says "at least". - let survivals = meters.survivals_in_window(now); - lines.push(format!( - " route survival actuations {}{} in window (uncounted), last-at {}", - if survivals == ROUTE_SURVIVAL_RECORD { "at least " } else { "" }, - survivals, - at(meters.route_survivals.last().copied()) - )); - } - lines -} - -fn derived_command_status(transactions: &[&DeploymentTransaction]) -> (&'static str, String) { - if transactions.is_empty() { - return ("queued", String::new()); - } - // A live transaction is the command's present tense. Reporting an older - // attempt's failure while a new one is sealing hid a running deployment - // behind the word "failed". - let current = transactions - .iter() - .filter(|transaction| !transaction.is_terminal()) - .map(|transaction| format!("{}:{:?}", transaction.target, transaction.phase)) - .collect::>(); - if !current.is_empty() { - return ("running", current.join(",")); - } - if let Some(error) = transactions - .iter() - .find_map(|transaction| match &transaction.completion { - Some(TransactionCompletion::FailedBeforeFencing { error }) - | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => Some(error.clone()), - _ => None, - }) - { - return ("failed", error); - } - if transactions.iter().all(|transaction| { - transaction.is_terminal() - && matches!( - transaction.completion, - Some(TransactionCompletion::Admitted { .. }) - ) - }) { - return ( - "succeeded", - transactions - .iter() - .map(|transaction| transaction.target.as_str()) - .collect::>() - .join(","), - ); - } - ("complete", String::new()) -} - -struct ProcessLock { - file: File, - #[cfg(not(unix))] - path: PathBuf, -} - -impl ProcessLock { - fn acquire(store_path: &Path) -> Result { - let path = sibling_path(store_path, ".daemon.lock"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - #[cfg(unix)] - { - use std::os::fd::AsRawFd; - let file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(&path) - .with_context(|| format!("opening Idunn daemon lock {}", path.display()))?; - let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; - ensure!(result == 0, "another Idunn daemon owns this control store"); - Ok(Self { file }) - } - #[cfg(not(unix))] - { - let file = OpenOptions::new() - .create_new(true) - .read(true) - .write(true) - .open(&path) - .with_context(|| format!("another Idunn daemon owns {}", path.display()))?; - Ok(Self { file, path }) - } - } -} - -impl Drop for ProcessLock { - fn drop(&mut self) { - #[cfg(unix)] - { - use std::os::fd::AsRawFd; - let _ = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; - } - #[cfg(not(unix))] - { - let _ = fs::remove_file(&self.path); - } - } -} - -/// One target's in-memory challenge wait: when it is due, and the stored -/// records of the target as they were when the record failed to land. -struct ChallengeWait { - due_unix_millis: u64, - stamp: Vec, -} - -struct Engine { - options: RuntimeOptions, - idunn_signer: ServiceIdentitySigner, - idunn_anchor: ServiceIdentityTrustAnchor, - bootstrap_odin_authority: AdmittedOdinAuthority, - source: Arc, - docker_runner: DockerRunnerDriver, - systemd_workload: Arc, - host_actuators: Option, - history_report: ReportOnce, - /// One `ReportOnce` per faulting record, so one wedged transaction says - /// its fault once and never drowns another's. - fault_reports: Mutex>, - /// Retry pacing for a transaction whose step keeps failing, by - /// transaction id: consecutive failures and the earliest next attempt. - /// Process-local on purpose: a restart retries at once, and no schema - /// carries a clock that only pacing reads. - resume_backoff: Mutex>, - /// When each routed target's next challenge is due, held only when the - /// stored `RouteSupervisionState` could not record the last one. The stored - /// value owns the schedule and survives restarts; this spaces attempts - /// while the record did not land. An entry ends the moment any write for - /// its target lands (its stamp no longer matches the store) or the target - /// is retired or withdrawn: it never outlives the episode that made it. - challenge_wait: Mutex>, -} - -/// The route driver's door to the actuation ceiling: every group of host -/// mutations a driver makes for `target` is charged through here. The gate -/// owns which charge a request is: a driver asks for the change it is making, -/// and the command that owns the transaction decides whether that change is -/// deployment (`Forward`, refusable) or the continuity of the admitted -/// generation (`Survival`, never refused). -struct EngineRouteGate<'a> { - engine: &'a Engine, - target: &'a str, - command: CommandKind, -} - -impl RouteActuationGate for EngineRouteGate<'_> { - fn admit(&self, requested: RouteActuation) -> Result<()> { - let kind = match self.command { - CommandKind::Deploy => requested, - CommandKind::Continuity => RouteActuation::Survival, - }; - self.engine.charge_route_actuation(self.target, kind) - } -} - -#[derive(Clone, Copy)] -struct ResumeBackoff { - failures: u32, - not_before_unix_millis: u64, -} - -/// Longest wait between attempts at a failing transaction step. -const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; - -impl Engine { - fn open(options: RuntimeOptions) -> Result { - Self::open_with_systemd_workload( - options, - Arc::new(SystemdTransientWorkloadDriver::default()), - ) - } - - /// `open` with the systemd workload port supplied. It is the seam that - /// lets the phase engine run past Fencing without a systemd to talk to. - fn open_with_systemd_workload( - options: RuntimeOptions, - systemd_workload: Arc, - ) -> Result { - let idunn_signer = - open_service_identity_at::(&options.idunn_identity_store) - .context("opening Idunn activation identity")?; - let idunn_anchor = idunn_signer.trust_anchor()?; - let odin_anchor = read_trust_anchor::(&options.odin_trust_anchor) - .context("reading bootstrap Odin topology anchor")?; - let bootstrap_odin_authority = AdmittedOdinAuthority::from_anchor(&odin_anchor)?; - let source: Arc = Arc::new(GitSourceDriver::new( - &options.source_root, - options.staging_root.join("frozen-sources"), - options.source_identity, - )); - let host_actuators = options - .host_actuator_bind - .map(|bind| HostActuatorHub::bind(bind).map(|hub| Arc::new(Mutex::new(hub)))) - .transpose()?; - if let Some(hub) = &host_actuators { - eprintln!( - "Idunn host actuator hub listening on {}", - hub.lock().expect("hub mutex").local_addr()? - ); - let service_signer = - open_service_identity_at::(&options.idunn_identity_store) - .context("opening Idunn identity for the host actuator hub thread")?; - let bindings_dir = options.bindings_dir.clone(); - spawn_hub_service(Arc::clone(hub), service_signer, move || { - host_anchors_from(&bindings_dir) - }); - } - Ok(Self { - options, - idunn_signer, - idunn_anchor, - bootstrap_odin_authority, - source, - docker_runner: DockerRunnerDriver::default(), - systemd_workload, - host_actuators, - history_report: ReportOnce::default(), - fault_reports: Mutex::default(), - resume_backoff: Mutex::default(), - challenge_wait: Mutex::default(), - }) - } - - fn host_anchors(&self) -> BTreeMap { - host_anchors_from(&self.options.bindings_dir) - } - - /// Every route driver comes from here, so the actuator paths are the - /// options' and nothing else's. - fn route_driver(&self, binding: RouteBinding) -> NginxRouteDriver { - NginxRouteDriver::with_actuators(binding, &self.options.route_actuators) - } - - fn route_gate<'a>(&'a self, target: &'a str, command: CommandKind) -> EngineRouteGate<'a> { - EngineRouteGate { - engine: self, - target, - command, - } - } - - /// Charge one route actuation to the target's `TargetSupervision` before - /// the driver acts. The record is created by the first charge, so a first - /// deployment is metered like any other. A `Forward` is counted against - /// the sliding ceiling; a refused one is the typed `RouteActuationRefused`, - /// and one that cannot be recorded is refused too: no charge, no - /// actuation. A `Survival` is never counted or refused, only recorded when - /// it can be, and runs regardless when it cannot: an unwritable store must - /// not stop a rollback, a withdrawal or a repair. - fn charge_route_actuation(&self, target: &str, kind: RouteActuation) -> Result<()> { - match self.record_route_actuation(target, kind) { - Ok(charged) => charged.map_err(anyhow::Error::new), - Err(error) if kind == RouteActuation::Survival => { - eprintln!( - "Idunn could not record a survival route actuation of {target}; it proceeds unrecorded: {error:#}" - ); - Ok(()) - } - Err(error) => Err(error), - } - } - - /// The outer error is the ledger failing to read or write; the inner one - /// is the ceiling refusing. - fn record_route_actuation( - &self, - target: &str, - kind: RouteActuation, - ) -> Result> { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let seen = snapshot.supervision_for(target); - let mut next = snapshot.supervision_or_new(target); - let now = now_millis()?; - let charged = next.charge_route(now, kind); - // A refusal writes nothing unless the clock had stepped back and the - // log needed settling. - if charged.is_ok() || seen.is_some_and(|stored| stored.value != next) { - self.write_target_supervision(seen, &next, now)?; - } - Ok(charged) - } - - /// Replace a target's meters, or create them when `seen` is `None`, by - /// compare-exchange against exactly what was read. - fn write_target_supervision( - &self, - seen: Option<&Stored>, - next: &TargetSupervision, - now: u64, - ) -> Result<()> { - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: TargetSupervision::TYPE.into(), - key: next.target.clone(), - current: seen.map(|stored| stored.envelope.clone()), - }], - &[target_supervision_envelope(next, now)?], - )?, - "target supervision changed before its meters were written" - ); - Ok(()) - } - - /// Replace an admitted generation by compare-exchange against the envelope - /// that was read. `Ok(false)` is a lost race: another writer moved the - /// record, so the caller's snapshot is stale. `Err` is a write the store - /// refused, which changed nothing. - fn replace_generation( - &self, - seen: &Stored, - next: &AdmittedGeneration, - now: u64, - ) -> Result { - Ok(SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: seen.value.target.clone(), - current: Some(seen.envelope.clone()), - }], - &[admitted_envelope(next, now)?], - )?) - } - - /// Say a fault once while it lasts, under `key`. - fn report_once(&self, key: &str, message: String) { - let offered = self - .fault_reports - .lock() - .expect("fault report mutex") - .entry(key.to_string()) - .or_default() - .offer(Some(message)); - if let Some(message) = offered { - eprintln!("{message}"); - } - } - - fn incidents(&self) -> IncidentStore { - IncidentStore::new(&self.options.incident_store) - } - - /// Settle the outcome of one incident operation. A failure never leaves - /// this function: it is traced once per episode, under the operation and - /// the subject it was decided for (`*` for a pass over the whole store), - /// and continuity decides exactly as it would have with a healthy store. - /// Each decision site has its own key, so one site's success never ends - /// another site's episode. The message names the operation and the root - /// cause and never a path or a value. Incident writes are never progress. - fn record_incident(&self, operation: &str, subject: &str, outcome: Result) { - let key = format!("incident-store:{operation}:{subject}"); - match outcome { - Ok(_) => self.clear_fault(&key), - Err(error) => self.report_once( - &key, - format!( - "Idunn could not {operation} an operator incident: {}", - error.root_cause() - ), - ), - } - } - - /// Close every open incident whose subject is a target that is no longer - /// admitted, then retire what has been closed past retention. - fn reconcile_incidents(&self, snapshot: &ControlSnapshot) { - let store = self.incidents(); - let close_unadmitted = || -> Result<()> { - let now = now_millis()?; - for record in store.read()? { - if record.is_open() - && record.condition.ends_with_admission() - && snapshot.admitted_for(&record.subject).is_none() - { - store.close( - record.condition, - &record.subject, - CloseReason::NoLongerAdmitted, - now, - )?; - } - } - Ok(()) - }; - self.record_incident("close", "*", close_unadmitted()); - self.record_incident("retire", "*", now_millis().and_then(|now| store.retire_closed(now))); - } - - fn host_access(&self) -> Result> { - let hub = self - .host_actuators - .as_ref() - .context("this Idunn serves no host actuators (no --host-actuator-bind)")?; - Ok(HostActuatorAccess { - hub: Arc::clone(hub), - anchors: self.host_anchors(), - signer: &self.idunn_signer, - }) - } -} - -/// The trust anchor of every host a binding names. A binding whose anchor -/// cannot be read leaves its host unattachable and is logged; the other -/// hosts are unaffected. -fn host_anchors_from(bindings_dir: &Path) -> BTreeMap { - { - let mut anchors = BTreeMap::new(); - let bindings = match load_bindings(bindings_dir) { - Ok(bindings) => bindings, - Err(error) => { - eprintln!("Idunn cannot read bindings for host anchors: {error:#}"); - return anchors; - } - }; - for loaded in bindings.values() { - let WorkloadBinding::HostActuator(workload) = &loaded.binding.workload else { - continue; - }; - match read_trust_anchor::(&workload.host_trust_anchor_store) - { - Ok(anchor) => { - anchors.insert(workload.host.clone(), anchor); - } - Err(error) => eprintln!( - "Idunn cannot read the actuator anchor for host {}: {error:#}", - workload.host - ), - } - } - anchors - } -} - -impl Engine { - /// The runner the plan's binding declares. Every runner in one binding - /// is of one kind; the binding validated that. - fn runner_for(&self, plan: &CompiledDeploymentPlan) -> Result> { - let (_, binding) = plan.parsed_inputs()?; - Ok(match &binding.workload { - WorkloadBinding::SystemdTransient(_) => Box::new(self.docker_runner.clone()), - WorkloadBinding::HostActuator(_) => Box::new(HostActuatorRunnerDriver { - access: self.host_access()?, - }), - }) - } - - /// The workload driver the plan's binding declares. - fn workload_for(&self, plan: &CompiledDeploymentPlan) -> Result> { - let (_, binding) = plan.parsed_inputs()?; - Ok(match &binding.workload { - WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), - WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { - access: self.host_access()?, - }), - }) - } - - /// History for a decision that would be wrong on an empty answer: the - /// continuity restart ceiling and the check that a command is consumed. - /// `None` means `history.cc` cannot be read, so the decision is not made; - /// a history that merely has no entries is `Some`. The fault is printed - /// once while it lasts, not on every tick. - fn history_for_decision(&self) -> Option> { - let (report, transactions) = match read_history(&self.options.state_store) { - Ok(history) => (history.report(), Some(history.transactions)), - Err(error) => ( - Some(format!( - "{error:#}; continuity and command retirement stop until it is readable" - )), - None, - ), - }; - if let Some(text) = self.history_report.offer(report) { - eprintln!("{text}"); - } - transactions - } - - fn topology(&self) -> CultCacheTopologyDriver { - CultCacheTopologyDriver { - projection_store: self.options.topology_store.clone(), - correlation_store: self.options.odin_correlation_store.clone(), - } - } - - fn current_odin_authority(&self, snapshot: &ControlSnapshot) -> Result { - // The bootstrap key stands only until Odin itself is admitted. Once it - // is, its own generation names the authority; an admitted Odin that - // carries none has no authority to name, and falling back to the - // bootstrap key would let a stale key vouch for the Verse. - let authority = match snapshot.admitted_odin() { - None => self.bootstrap_odin_authority.clone(), - Some(stored) => stored.value.odin_authority.clone().with_context(|| { - "admitted Odin generation carries no Odin authority (route-proof readiness)" - })?, - }; - authority.validate()?; - Ok(authority) - } - - fn trusted_topology_context(&self, now: u64) -> OdinTopologyAuthenticationContext { - OdinTopologyAuthenticationContext { - trusted_received_at_unix_millis: now, - maximum_age_millis: self.options.topology_maximum_age_millis, - maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, - } - } - - /// Re-prove the evidence that still gates a decision. - /// - /// Terminal transactions are skipped: they describe what already happened - /// and authorize nothing further, so re-proving them only creates ways for - /// history to refuse a boot. On yggdrasil that is 259 of 260 records. - /// - /// The operator anchor is read on demand rather than up front. Reading it - /// unconditionally meant an absent brake artifact refused startup against - /// an *empty* store -- a deployment brake gating Idunn itself, which - /// `F:\Projects\CLAUDE.md` forbids outright: a target brake "may never gate - /// Idunn itself or unrelated service lifecycle". - fn validate_durable_authority(&self, snapshot: &ControlSnapshot) -> Result<()> { - let mut operator_anchor = None; - for stored in &snapshot.transactions { - let transaction = &stored.value; - if transaction.is_terminal() { - continue; - } - if let Some(authorization) = &transaction.deployment_authorization { - authorization.validate_shape()?; - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; - if operator_anchor.is_none() { - operator_anchor = - Some(read_trust_anchor::( - &self.options.deployment_brake_operator_anchor, - )?); - } - verify_idunn_deployment_brake_authorization( - &record, - operator_anchor.as_ref().expect("read directly above"), - )?; - let expected = required(&transaction.expected, "authorized Expected projection")?; - ensure!( - record.authorized_release_id.as_deref() - == Some(expected.sealed_release_id.as_str()) - && record.authorized_deployment_id.as_deref() - == Some(transaction.transaction_id.as_str()) - && record.runtime_id == expected.runtime_id, - "durable deployment authorization names another release or transaction" - ); - } - let lease = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - if let Some(evidence) = &transaction.latest_odin_observation { - let authenticated = self.authenticate_topology_bytes( - snapshot, - transaction, - &evidence.canonical_bytes, - lease, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(evidence, &authenticated)?; - } - if let Some(evidence) = &transaction.warming { - match evidence { - WarmingEvidence::OdinTopology { evidence } => { - let authenticated = self.authenticate_topology_bytes( - snapshot, - transaction, - &evidence.canonical_bytes, - None, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(evidence, &authenticated)?; - let incumbent_lease_sha256 = - self.incumbent_lease_sha256_for_warming(snapshot, transaction)?; - ensure!( - is_semantic_warming( - required(&transaction.expected, "Warming Expected projection",)?, - required(&transaction.activation, "Warming activation")?, - incumbent_lease_sha256.as_deref(), - &authenticated, - )?, - "durable Warming gate is not supported by current runtime evidence" - ); - } - WarmingEvidence::FirstOdinDirect { evidence } => { - self.authenticate_first_odin_warming_presence( - transaction, - &evidence.message_id, - evidence.challenged_at_unix_millis, - evidence.admitted_at_unix_millis, - &evidence.canonical_bytes, - )?; - } - WarmingEvidence::RouteProofDirect { evidence } => { - self.reauthenticate_route_proof( - transaction, - evidence, - route_proof_warming_states(required( - &transaction.expected, - "Warming Expected projection", - )?), - None, - evidence.admitted_at_unix_millis, - )?; - } - } - } - if let Some(ReadinessEvidence::RouteProof { evidence }) = &transaction.ready { - self.reauthenticate_route_proof( - transaction, - evidence, - &["active"], - lease, - evidence.admitted_at_unix_millis, - )?; - } - if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) { - let authenticated = self.authenticate_topology_bytes( - snapshot, - transaction, - &evidence.canonical_bytes, - lease, - evidence.admitted_at_unix_millis, - )?; - validate_authenticated_evidence(evidence, &authenticated)?; - ensure!( - is_semantic_ready(&authenticated), - "durable Ready label is not exact semantic Ready" - ); - } - } - let odin_authority = self.current_odin_authority(snapshot)?; - for stored in &snapshot.admitted { - let generation = &stored.value; - // Only a generation whose own Expected says Odin vouches for it has - // Odin receipts to re-prove. One whose evidence disagrees with its - // class is held and reported by supervision, not re-proved here. - if !matches!(generation.readiness(), Ok(class) if class != ReadinessClass::RouteProof) - { - continue; - } - let odin = generation.odin_receipts()?; - let authority = self.runtime_authority_parts( - &generation.plan, - &generation.expected, - &generation.activation, - )?; - let latest = authenticate_odin_runtime_topology_correlation( - &odin.latest.canonical_bytes, - &authority, - generation.leasing.lease_sha256(), - &odin_authority.signer_public_key, - self.trusted_topology_context(odin.latest.admitted_at_unix_millis), - )?; - validate_authenticated_evidence(odin.latest, &latest)?; - let ready = authenticate_odin_runtime_topology_correlation( - &odin.ready.canonical_bytes, - &authority, - generation.leasing.lease_sha256(), - &odin_authority.signer_public_key, - self.trusted_topology_context(odin.ready.admitted_at_unix_millis), - )?; - validate_authenticated_evidence(odin.ready, &ready)?; - ensure!( - is_semantic_ready(&ready), - "admitted generation Ready label is not exact semantic Ready" - ); - } - Ok(()) - } -} - -/// What the boot reconciliation of failed continuity projections found. -#[derive(Debug, Default, PartialEq, Eq)] -struct ProjectionReconciliation { - /// Failed continuity transactions whose exact activation was demoted. - demoted: Vec, - /// Targets whose projection names an activation that neither the admitted - /// generation, a live transaction, nor a failed issuer accounts for. They - /// are reported and left alone. - unexplained: Vec, -} - -impl Engine { - /// Once at boot: demote the activation a failed continuity left standing, - /// where the projection still names exactly the activation that failed - /// transaction issued. A continuity abort before the single resolution - /// rule left such residue (and the lift owes the same demotion to any - /// abort still resident). The issuer is identified by exact activation, - /// from history and from resident terminal records not yet retired to it. - /// - /// Nothing is adopted: an activation with no failed issuer is never - /// demoted or kept on anyone's behalf, only reported. - fn reconcile_failed_continuity_projections(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut outcome = ProjectionReconciliation::default(); - // Resident issuers need no history; only the history issuers wait on it. - let history = self.history_for_decision(); - if history.is_none() { - eprintln!( - "Idunn cannot read history at boot: failed continuities archived there are not \ - reconciled, resident ones still are" - ); - } - let topology = self.topology(); - let issuers = history - .iter() - .flatten() - .chain(snapshot.transactions.iter().map(|stored| &stored.value)) - .filter(|transaction| { - transaction.command_kind == CommandKind::Continuity - && matches!( - transaction.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - | Some(TransactionCompletion::FailedAfterFencing { .. }) - ) - }); - for issuer in issuers { - let (Some(expected), Some(activation), Some(plan)) = - (&issuer.expected, &issuer.activation, &issuer.plan) - else { - continue; - }; - let demoted = (|| -> Result { - if topology.projected_activation(expected)?.as_ref() != Some(activation) { - return Ok(false); - } - let provider_anchor = self.provider_anchor_for_plan(plan)?; - topology.demote_to_expected_only(expected, &provider_anchor, activation, None)?; - Ok(true) - })(); - match demoted { - Ok(true) => outcome.demoted.push(issuer.transaction_id.clone()), - Ok(false) => {} - Err(error) => eprintln!( - "Idunn could not reconcile the projection of failed continuity {}: {error:#}", - issuer.transaction_id - ), - } - } - for generation in &snapshot.admitted { - let projected = match topology.projected_activation(&generation.value.expected) { - Ok(projected) => projected, - Err(error) => { - eprintln!( - "Idunn cannot read the projected activation of {}: {error:#}", - generation.value.target - ); - continue; - } - }; - let Some(projected) = projected else { - continue; - }; - let accounted = projected == generation.value.activation - || snapshot.transactions.iter().any(|stored| { - stored.value.target == generation.value.target - && !stored.value.is_terminal() - && stored.value.activation.as_ref() == Some(&projected) - }); - if !accounted { - outcome.unexplained.push(generation.value.target.clone()); - } - } - Ok(outcome) - } -} - -/// Everything the daemon does once, before its loop: lock, migrate, validate, -/// reconcile the projection. The lock is returned so the caller holds it for as -/// long as the engine runs. -fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { - for path in [ - &options.state_store, - &options.topology_store, - &options.staging_root, - ] { - let directory = if path.extension().is_some() { - path.parent() - .context("configured Idunn path has no parent")? - } else { - path.as_path() - }; - fs::create_dir_all(directory) - .with_context(|| format!("creating Idunn directory {}", directory.display()))?; - } - let lock = ProcessLock::acquire(&options.state_store)?; - let migrated = migrate_control_store_to_current_schema(&options.state_store) - .context("migrating Idunn control records to the current schema")?; - if migrated > 0 { - println!( - "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA}" - ); - } - ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; - let engine = Engine::open(options)?; - engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; - match engine.reconcile_failed_continuity_projections() { - Ok(outcome) => { - for transaction_id in &outcome.demoted { - println!("demoted the activation failed continuity {transaction_id} left projected"); - } - for target in &outcome.unexplained { - eprintln!( - "Idunn found an activation projected for {target} that no admitted generation, \ - live transaction or failed continuity accounts for; left as it is" - ); - } - } - Err(error) => eprintln!("Idunn boot projection reconciliation failed: {error:#}"), - } - Ok((lock, engine)) -} - -fn serve(options: RuntimeOptions) -> Result<()> { - let (_lock, engine) = boot(options)?; - serve_while(&engine, || true) -} - -/// The daemon's loop: tick, sleep, and go again while `again` says so. `serve` -/// says so forever; a test says so a few times. -fn serve_while(engine: &Engine, mut again: impl FnMut() -> bool) -> Result<()> { - while again() { - match engine.run_scheduler_tick() { - Ok(true) => continue, - Ok(false) => {} - // A fault in one transaction is that transaction's problem. Idunn - // runs under Restart=always, so propagating it here turns a single - // unreadable record into a crashloop that takes every unrelated - // target's continuity down with it -- the daemon-survival organ - // killed by the thing it exists to survive. Log against the tick - // and keep going; the fault recurs every poll until it is fixed, - // which is louder than a restart loop and cheaper than an outage. - Err(error) => eprintln!("Idunn scheduler tick failed: {error:#}"), - } - thread::sleep(Duration::from_millis(engine.options.poll_millis)); - } - Ok(()) -} - -impl Engine { - /// One resident terminal transaction goes to history per tick. - /// - /// Transactions that finished before finished transactions travelled to - /// history are still resident with their commands, gating nothing and - /// consuming their commands correctly, but growing the store every decision - /// reads. They leave the same way a transaction finishing today does. - fn retire_one_terminal_transaction(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - // A record that cannot be archived (history unreadable, a torn write) - // is that record's fault: the next terminal record is still tried, and - // the tick goes on to supervise and freeze. - for stored in snapshot - .transactions - .iter() - .filter(|stored| stored.value.is_terminal()) - { - match archive_terminal_transaction(&self.options.state_store, &stored.envelope) { - Ok(()) => { - eprintln!( - "Idunn retired resident terminal transaction {} to history", - stored.value.transaction_id - ); - return Ok(true); - } - Err(error) => self.note_fault( - "could not retire terminal transaction", - &stored.value.transaction_id, - &error, - ), - } - } - Ok(false) - } - - /// One pass of the scheduler. An unreadable control store is the tick's - /// fault and is returned; anything one transaction or one admitted - /// generation does wrong is logged against it and never stops the others. - fn run_scheduler_tick(&self) -> Result { - if self.retire_one_terminal_transaction()? { - return Ok(true); - } - let transaction_progress = self.resume_one_transaction()?; - let continuity_progress = self.supervise_one_admitted_generation()?; - // Freezing is not gated on the others' progress: a queued command - // whose target is free must not wait behind an unrelated target's - // wedge. `freeze_command` already refuses a busy target, so this is - // still at most one freeze per tick and never a second claimant. - let froze = self.freeze_one_queued_command()?; - Ok(transaction_progress || continuity_progress || froze) - } - - /// Say a per-record fault once while it lasts and leave it in the - /// record's `last_error`, which status renders. Recording is best effort: - /// a store too broken to take the note is already in stderr. - fn note_fault(&self, what: &str, transaction_id: &str, error: &anyhow::Error) { - let detail = truncate(&format!("{error:#}"), 2048); - let offered = { - let mut reports = self.fault_reports.lock().expect("fault report mutex"); - reports - .entry(transaction_id.to_string()) - .or_default() - .offer(Some(detail.clone())) - }; - let recorded = self.record_last_error(transaction_id, &detail); - if offered.is_some() { - eprintln!("Idunn {what} {transaction_id}: {detail}"); - if let Err(error) = recorded { - eprintln!("Idunn could not leave that in the record of {transaction_id}: {error:#}"); - } - } - } - - fn clear_fault(&self, transaction_id: &str) { - self.fault_reports - .lock() - .expect("fault report mutex") - .remove(transaction_id); - } - - /// Write `detail` as the record's `last_error`, touching nothing else and - /// not archiving: a terminal record whose archive is the fault stays put. - fn record_last_error(&self, transaction_id: &str, detail: &str) -> Result<()> { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let Some(stored) = snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == transaction_id) - else { - return Ok(()); - }; - if stored.value.last_error.as_deref() == Some(detail) { - return Ok(()); - } - let envelope = cleanup_evidence::last_error_envelope(&stored.value, detail)?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction_id.to_string(), - current: Some(stored.envelope.clone()), - }], - &[envelope], - )?, - "transaction changed before its fault note" - ); - Ok(()) - } - - /// Startup and every later loop use the same order: unfinished ownership - /// work first, admitted-body continuity second, new commands last. A - /// waiting transaction yields without relaxing ordinal order inside its - /// own command, so one target brake cannot suspend unrelated continuity. - fn resume_one_transaction(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; - let mut candidates = snapshot - .transactions - .iter() - .filter(|stored| !stored.value.is_terminal()) - .collect::>(); - candidates.sort_by_key(|stored| { - ( - stored.value.created_at_unix_millis, - stored.value.ordinal, - stored.value.transaction_id.as_str(), - ) - }); - for current in candidates { - if snapshot.has_earlier_authority_sibling(¤t.value) { - continue; - } - match self.resume_candidate(current) { - Ok(moved) => progressed |= moved, - Err(error) => { - self.note_fault( - "could not resume transaction", - ¤t.value.transaction_id, - &error, - ); - } - } - } - Ok(progressed) - } - - /// Advance one transaction, turning its failure into the durable record - /// its phase calls for. Returns whether its record changed. - /// - /// "Moved" means the transaction's state advanced: a phase change, a - /// completion, or a durable abort step. Rewriting `last_error` to say the - /// same step failed again in new words is not movement, or a wedged - /// transaction would report progress every tick. - fn resume_candidate(&self, current: &Stored) -> Result { - let id = current.value.transaction_id.as_str(); - if let Some(moved) = self.resolve_phase_deadline(current)? { - return Ok(moved); - } - if self - .resume_backoff - .lock() - .expect("resume backoff mutex") - .get(id) - .is_some_and(|backoff| { - now_millis().is_ok_and(|now| { - is_waiting(now, backoff.not_before_unix_millis, RESUME_BACKOFF_CEILING_MILLIS) - }) - }) - { - return Ok(false); - } - if let Some(disagreement) = current.value.held_disagreement() { - self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); - return Ok(false); - } - let advanced = self.advance_transaction(current); - if advanced.is_ok() { - // The step ran and succeeded: the fault, if any, has recovered, so - // a later failure is news. A skipped or failed attempt clears nothing. - self.resume_backoff - .lock() - .expect("resume backoff mutex") - .remove(id); - self.clear_fault(id); - } - if let Err(error) = advanced { - let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; - let latest = latest_snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while recording an execution error")?; - if latest.value.is_terminal() { - self.resume_backoff - .lock() - .expect("resume backoff mutex") - .remove(id); - return Ok(true); - } - self.back_off(id)?; - if latest.value.phase < DeploymentPhase::Fencing { - // Before the fence an abort that cannot finish a step is - // resumable, never a post-fence abort: that path refuses a - // pre-fence phase, so choosing it only wedged the tick. - // - // A publish that lost to a held lock is not a failed step: the - // holder lets go, so the transaction stays in its phase with - // the error recorded and the backoff above retries it. The - // phase deadline, not an abort, bounds the wait. - if latest.value.pre_fencing_abort.is_none() - && error.downcast_ref::().is_none() - { - self.begin_pre_fencing_abort(latest, error)?; - } else { - self.record_resumable_error(latest, &error)?; - } - } else if latest.value.post_fencing_abort.is_none() - && self.candidate_is_permanently_stopped(&latest.value)? - { - // Past the fence an error is resumable while the candidate - // can still recover. This one cannot: its transient unit - // has failed and carries Restart=no, so retrying would hold - // the target forever behind a transaction that can never - // finish. - self.begin_post_fencing_abort(latest, error)?; - } else { - // Resumable only until the phase deadline: the resolver ends - // the phase before the next step once it has passed. - self.record_resumable_error(latest, &error)?; - } - } - let after = ControlSnapshot::read(&self.options.state_store)?; - let live = after - .transactions - .iter() - .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while checking scheduler progress")?; - Ok(state_advanced(¤t.value, &live.value)) - } - - /// Wait twice as long after each consecutive failure of a transaction's - /// step, from one poll interval up to a ceiling. - fn back_off(&self, transaction_id: &str) -> Result<()> { - let now = now_millis()?; - let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); - let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); - let wait = backoff_wait(self.options.poll_millis, failures, RESUME_BACKOFF_CEILING_MILLIS); - all.insert( - transaction_id.to_string(), - ResumeBackoff { - failures: failures.saturating_add(1), - not_before_unix_millis: now.saturating_add(wait), - }, - ); - Ok(()) - } - - fn freeze_one_queued_command(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let live_commands = snapshot - .transactions - .iter() - .map(|stored| stored.value.command_id.clone()) - .collect::>(); - let mut candidates = snapshot - .commands - .iter() - .filter(|stored| { - stored.value.kind == CommandKind::Deploy - && !live_commands.contains(&stored.value.command_id) - }) - .collect::>(); - if candidates.is_empty() { - return Ok(false); - } - // A command is consumed by any transaction, live or retired. Commands - // whose transactions were retired to history before commands travelled - // with them are still resident; they are not queued, they are - // history that never moved. Retire them here and never freeze them. - // With history unreadable, whether a command was consumed is unknown: - // freezing it again could run a finished deployment twice. - let Some(history) = self.history_for_decision() else { - return Ok(false); - }; - let historical_commands = history - .into_iter() - .map(|transaction| transaction.command_id) - .collect::>(); - let mut retired = false; - candidates.retain(|stored| { - if !historical_commands.contains(&stored.value.command_id) { - return true; - } - match SingleFileMessagePackBackingStore::new(&history_store_path( - &self.options.state_store, - )) - .insert_entry_if_absent(stored.envelope.clone()) - .and_then(|_| { - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .delete_batch_if_unchanged(std::slice::from_ref(&stored.envelope)) - }) { - Ok(_) => { - eprintln!( - "Idunn retired consumed command {}: its transactions are history", - stored.value.command_id - ); - retired = true; - } - Err(error) => eprintln!( - "Idunn left consumed command {} resident: {error:#}", - stored.value.command_id - ), - } - false - }); - if retired { - return Ok(true); - } - candidates.sort_by_key(|stored| stored.value.requested_at_unix_millis); - // Oldest first, but a command whose target is busy does not block the - // ones behind it for other targets. - for command in candidates { - if self.freeze_command(&snapshot, command)? { - return Ok(true); - } - } - Ok(false) - } - - /// Freeze one queued command into its transactions. `Ok(false)` means its - /// target is busy and nothing was written. - fn freeze_command( - &self, - snapshot: &ControlSnapshot, - command: &Stored, - ) -> Result { - let bindings = match load_bindings(&self.options.bindings_dir) { - Ok(bindings) => bindings, - Err(error) => { - eprintln!( - "Idunn left command {} queued because operator bindings are invalid: {error:#}", - command.value.command_id - ); - return Ok(false); - } - }; - let targets = match resolve_selector(&bindings, &command.value.selector) { - Ok(targets) => targets, - Err(error) => { - let now = now_millis()?; - let rejected = DeploymentTransaction::rejected(&command.value, error, now)?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: rejected.transaction_id.clone(), - current: None, - }], - &[transaction_envelope(&rejected, now)?], - )?, - "bad selector changed before refusal was recorded" - ); - return Ok(true); - } - }; - let busy = snapshot - .transactions - .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) - .map(|stored| stored.value.target.as_str()) - .collect::>(); - if targets.iter().any(|target| busy.contains(target.as_str())) { - return Ok(false); - } - let now = now_millis()?; - let transactions = targets - .into_iter() - .enumerate() - .map(|(ordinal, target)| { - DeploymentTransaction::new( - &command.value, - target.clone(), - u32::try_from(ordinal)?, - snapshot.admitted_for(&target).map(|stored| &stored.value), - now, - ) - }) - .collect::>>()?; - let expected = transactions - .iter() - .map(|transaction| CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }) - .collect::>(); - let next = transactions - .iter() - .map(|transaction| transaction_envelope(transaction, now)) - .collect::>>()?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .compare_exchange(&expected, &next)?, - "queued command lost its ordered transaction creation CAS" - ); - Ok(true) - } - - fn supervise_one_admitted_generation(&self) -> Result { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - // A wait belongs to the failed record it spaces. Once any write for its - // target has landed, or the target is no longer admitted, the stored - // schedule owns the target again. - self.challenge_wait - .lock() - .expect("challenge wait mutex") - .retain(|target, wait| snapshot.target_stamp(target) == wait.stamp); - self.reconcile_incidents(&snapshot); - let mut progressed = false; - let mut admitted = snapshot.admitted.iter().collect::>(); - admitted.sort_by_key(|stored| stored.value.target.as_str()); - for current in admitted { - if let Err(disagreement) = current.value.readiness() { - self.note_fault( - "holds admitted generation", - &format!("generation:{}", current.value.target), - &anyhow::Error::new(disagreement), - ); - } - let blocker = snapshot.transactions.iter().find(|stored| { - stored.value.target == current.value.target - && stored.value.blocks_new_target_mutation() - }); - if blocker.is_some_and(|stored| { - let owns_from = if stored.value.rollout_stops_incumbent_first() { - // The incumbent is stopped at Starting by design; a - // continuity restart in that window would fight the - // candidate for the same host resources. - DeploymentPhase::Starting - } else { - DeploymentPhase::Fencing - }; - stored.value.phase >= owns_from && stored.value.phase < DeploymentPhase::Complete - }) { - continue; - } - match self.supervise_admitted_route(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn rejected admitted {} route continuity: {error:#}", - current.value.target - ), - } - match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved admitted {} after rejecting topology observation: {error:#}", - current.value.target - ), - } - match self.retire_stale_incarnations(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved a stale {} incarnation projection: {error:#}", - current.value.target - ), - } - // Restore the admitted Expected before anything is asked of the - // workload, and only the Expected. - // - // A target that cannot see its own Expected cannot start, and an - // aborted deployment leaves exactly that gap, so this cannot wait - // for an observation the dead process will never provide. The - // activation is deliberately not restored: it names one running - // incarnation, every restart issues a fresh one, and republishing - // the admitted one leaves the projection naming an incarnation that - // no longer exists -- which the workload rightly refuses to adopt. - // That record is published from observation once the process is up. - if blocker.is_none() - || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) - { - let repair = (|| -> Result<()> { - let topology = self.topology(); - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - if !topology.admitted_expected_projection_is_exact( - ¤t.value.expected, - &provider_anchor, - )? { - ensure!( - topology.publish_expected(¤t.value.expected, &provider_anchor)? - == current.value.expected.canonical_sha256()?, - "admitted Expected projection repair differs" - ); - } - Ok(()) - })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target - ); - } - } - let mut operational_error = None; - let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { - workload.observe( - ¤t.value.expected, - ¤t.value.activation, - ¤t.value.workload, - ) - }) { - Ok(observation) => { - let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { - let lease_health = (|| -> Result { - let lease_path = current - .value - .plan - .parsed_inputs()? - .1 - .process_write_lease - .context("admitted lease has no operator binding")? - .record_path; - let driver = - CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - if driver.observe_exact(lease)? { - return Ok(true); - } - if driver.observe_empty()? { - return Ok(false); - } - bail!("write-lease store contains unexpected authority") - })(); - match lease_health { - Ok(true) => false, - Ok(false) => { - operational_error = - Some(anyhow!("admitted physical write lease is missing")); - true - } - Err(error) => { - eprintln!( - "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", - current.value.target - ); - continue; - } - } - } else { - false - }; - if lease_is_missing { - None - } else { - Some(observation) - } - } - Err(error) if error.downcast_ref::().is_some() => { - // The host cannot be asked right now (its actuator is - // between sessions, or Idunn just restarted). That is - // silence, not a death; counting it as one burned every - // continuity attempt in the reattach window on - // 2026-09-11 and left a healthy Muninn demoted. - eprintln!( - "Idunn cannot observe admitted {} this tick: {error:#}", - current.value.target - ); - continue; - } - Err(error) => { - operational_error = Some(error); - None - } - }; - if observation.is_some() { - let seen = snapshot.supervision_for(¤t.value.target); - if let Some(stored) = seen.filter(|stored| { - stored.value.continuity_backoff_from_unix_millis.is_some() - }) { - let mut next = stored.value.clone(); - next.end_backoff(); - self.write_target_supervision(Some(stored), &next, now_millis()?)?; - } - // The admitted workload was seen running: the only evidence - // that ends an exhaustion incident, once the window no longer holds it. - // The window sliding alone is not health and never closes one. - self.record_incident( - "close", - ¤t.value.target, - now_millis().and_then(|now| { - // Running is not enough while the window still holds - // the restarts that exhausted it: a release that - // survives one poll after each restart is still the - // same incident. - if snapshot - .supervision_or_new(¤t.value.target) - .restarts_page(now) - { - return Ok(false); - } - self.incidents().close( - IncidentCondition::ContinuityExhausted, - ¤t.value.target, - CloseReason::Recovered, - now, - ) - }), - ); - continue; - } - let workload_error = operational_error - .context("admitted operational state has neither observation nor error")?; - // A held generation is a record the recipe never declared - // readiness for. Idunn reports it and does not decide for it: a - // continuity minted over it copies the undeclared Expected, is - // held at once, and owns the target forever, so the declaring - // redeploy that would clear the hold could never freeze. The dead - // held target stays down and free. - if current.value.readiness().is_err() { - self.note_fault( - "holds a dead admitted generation down; declare readiness in its recipe and redeploy", - &format!("generation-down:{}", current.value.target), - &anyhow!("admitted generation {} is not running", current.value.generation_id), - ); - continue; - } - // Continuity restarts a release; it cannot repair one, and it - // never stops (budget-never-stop). Every charged attempt is spaced - // by a doubling, capped wait, so a queued deployment always gets - // a free gap (map B10). The log outlives every generation and only - // pages. Nothing here reads history: an unreadable history file - // must not stop crash recovery. - let now = now_millis()?; - let supervision = snapshot.supervision_or_new(¤t.value.target); - // A stepped-back clock leaves entries in the future. Settle them - // before deciding from them; the write ends this target's pass. - let mut settled = supervision.clone(); - settled.settle(now); - if settled != supervision { - self.write_target_supervision( - snapshot.supervision_for(¤t.value.target), - &settled, - now, - )?; - progressed = true; - continue; - } - let continuity_key = format!("continuity:{}", current.value.target); - - // Continuity defers to any live transaction for its target and - // never aborts one (handover-phase-split). - if blocker.is_some() { - continue; - } - if supervision.restarts_page(now) { - // The record comes first and the trace after it: the trace is - // this record's shadow, never its substitute. - self.record_incident( - "open", - ¤t.value.target, - self.incidents().open( - IncidentCondition::ContinuityExhausted, - ¤t.value.target, - now, - ), - ); - self.report_once( - &continuity_key, - format!( - "Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff (streak {}): {workload_error:#}", - current.value.target, - supervision.restarts_used(now), - supervision.continuity_attempt_streak, - ), - ); - } else { - self.clear_fault(&continuity_key); - } - if supervision.continuity_deferred(now) { - continue; - } - // A unit the host lost to a reboot is re-materialized once per - // generation, uncharged and with no wait; the driver owns the - // evidence and supervision owns the budget. An unanswerable - // driver is not evidence of a reboot. - let admitted_at = current.value.admitted_at_unix_millis; - let rematerialize = !supervision.rematerialization_spent(admitted_at) - && match self.workload_for(¤t.value.plan).and_then(|workload| { - workload.lost_to_host_reboot(¤t.value.workload, admitted_at) - }) { - Ok(lost) => lost, - Err(error) => { - self.note_fault( - "could not tell whether the host rebooted under", - &format!("reboot:{}", current.value.target), - &error, - ); - false - } - }; - let mut streak = 0; - if !rematerialize { - match supervision.continuity_backoff(now, admitted_at) { - ContinuityBackoff::Waiting => continue, - ContinuityBackoff::Begin => { - let mut began = supervision.clone(); - began.begin_backoff(now); - self.write_target_supervision( - snapshot.supervision_for(¤t.value.target), - &began, - now, - )?; - progressed = true; - continue; - } - ContinuityBackoff::Due { streak: due } => streak = due, - } - } - - // The projected activation names an incarnation that is gone. Every - // restart is issued a fresh one, so leaving the old record standing - // makes the projection describe a process that no longer exists -- - // and a workload that reads it refuses to adopt an activation that - // is not its own, which is a restart loop rather than a recovery. - // Demote to Expected-only, the same shape an aborted deployment - // leaves, and let the restart publish its own activation once it is - // observed. - let demotion = (|| -> Result<()> { - let topology = self.topology(); - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - if !topology.projected_activation_is_present(¤t.value.expected)? { - return Ok(()); - } - topology.demote_to_expected_only( - ¤t.value.expected, - &provider_anchor, - ¤t.value.activation, - current.value.leasing.lease(), - )?; - Ok(()) - })(); - // Expected-only is the precondition of a restart: a continuity - // minted over a projection that still names an activation would - // fail between preparing and publishing its own, and its abort - // could not resolve. So a failed demotion mints nothing. The - // failure is written to the generation, and the next attempt is - // not before the deferral ends. - if let Err(error) = demotion { - eprintln!( - "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", - current.value.target - ); - let mut deferred = supervision.clone(); - deferred.defer_continuity(now, &truncate(&format!("{error:#}"), 2048)); - self.write_target_supervision( - snapshot.supervision_for(¤t.value.target), - &deferred, - now, - )?; - progressed = true; - continue; - } - // An engaged lifecycle brake means no continuity transaction is - // minted at all. A transaction that exists and waits on the brake - // owns the target, and a target owned by a parked restart accepts - // no deployment -- which is the one thing a lifecycle brake must - // never gate. The Sealing-phase check remains as the guard for a - // transaction minted just before the brake was engaged. - if !self.lifecycle_allows_generation(current, now)? { - continue; - } - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("continuity-{}", Uuid::new_v4()), - kind: CommandKind::Continuity, - selector: current.value.target.clone(), - requested_by: "idunn-continuity".into(), - requested_at_unix_millis: now, - }; - command.validate()?; - let transaction = - DeploymentTransaction::from_continuity(&command, ¤t.value, now)?; - // The attempt is charged, or recorded as the one re-materialization, - // in the same CAS that mints it, so a crash cannot schedule one the - // meters never saw. - let mut counted = supervision; - if rematerialize { - counted.record_rematerialization(now); - } else { - counted.record_restart(now, streak); - } - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: TargetSupervision::TYPE.into(), - key: counted.target.clone(), - current: snapshot - .supervision_for(&counted.target) - .map(|stored| stored.envelope.clone()), - }, - ], - &[ - command_envelope(&command, now)?, - transaction_envelope(&transaction, now)?, - target_supervision_envelope(&counted, now)?, - ], - )?, - "continuity scheduling lost its command/transaction/meter CAS" - ); - progressed = true; - } - Ok(progressed) - } - - fn supervise_admitted_route( - &self, - snapshot: &ControlSnapshot, - current: &Stored, - ) -> Result { - let Some(expected_route) = current.value.expected.route.as_ref() else { - ensure!( - matches!(¤t.value.routing, RoutingEvidence::SkippedUnrouted), - "unrouted admitted generation carries route authority" - ); - return Ok(false); - }; - let RoutingEvidence::Promoted { - observation, - promoted_at_unix_millis, - } = ¤t.value.routing - else { - bail!("routed admitted generation has no promoted route receipt") - }; - let now = now_millis()?; - // A route whose proofs keep failing is challenged on a widening - // schedule, not every tick. - if current - .value - .route_supervision - .as_ref() - .is_some_and(|state| state.is_waiting(now, self.options.topology_maximum_age_millis)) - { - return Ok(false); - } - if self - .challenge_wait - .lock() - .expect("challenge wait mutex") - .get(¤t.value.target) - .is_some_and(|wait| { - is_waiting( - now, - wait.due_unix_millis, - ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(self.options.topology_maximum_age_millis), - ) - }) - { - return Ok(false); - } - ensure!( - observation.route_id == expected_route.route_id - && observation.runtime_instance_id == current.value.activation.runtime_instance_id, - "admitted route receipt names another incarnation" - ); - let binding = current.value.plan.parsed_inputs()?.1; - let lease_driver = if let Some(lease) = current.value.leasing.lease() { - let lease_path = &binding - .process_write_lease - .as_ref() - .context("stateful admitted generation has no write-lease binding")? - .record_path; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - ensure!( - driver.observe_exact(lease)?, - "admitted process write lease is no longer exact" - ); - Some(driver) - } else { - None - }; - let route_binding = binding - .route - .context("routed admitted generation has no operator route binding")?; - let driver = self.route_driver(route_binding); - let gate = self.route_gate(¤t.value.target, CommandKind::Continuity); - - if driver.observe_membership(¤t.value.expected, &observation.membership_sha256)? - && route_observation_is_current( - observation.observed_at_unix_millis, - now, - self.options.topology_maximum_age_millis, - self.options.topology_maximum_future_skew_millis, - ) - { - return Ok(false); - } - let route_key = format!("route:{}", current.value.target); - // A failed repair or a failed proof changes observation state only: it - // marks the route degraded and widens the wait. That record is written - // like a proved one: when it lands the pass ends (`Ok(true)`), because - // the caller's snapshot no longer matches the generation; when the store - // refuses it the pass reports no progress and the in-memory wait below - // spaces the next attempt. - let challenge = (|| -> Result { - // Actuates only when the fragment on disk differs from the - // admitted membership; an exact fragment makes this a no-op. It is - // survival: restoring the admitted route is never refused. - driver.restore_admitted_membership( - ¤t.value.expected, - &observation.membership_sha256, - &gate, - )?; - let authority = self.runtime_authority_parts( - ¤t.value.plan, - ¤t.value.expected, - ¤t.value.activation, - )?; - let refreshed = self.prove_stable_route_against( - ¤t.value.expected, - ¤t.value.activation, - &authority, - current.value.leasing.lease_sha256(), - &driver, - observation.membership_sha256.clone(), - )?; - ensure!( - driver.observe_membership(¤t.value.expected, &refreshed.membership_sha256)?, - "admitted route membership changed during its continuity challenge" - ); - if let (Some(lease_driver), Some(lease)) = (&lease_driver, current.value.leasing.lease()) - { - ensure!( - lease_driver.observe_exact(lease)?, - "admitted process write lease changed during its continuity challenge" - ); - } - Ok(refreshed) - })(); - let mut next = current.value.clone(); - // The wait the record carries, also held in memory until a record lands. - let (recorded, wait_until) = match challenge { - Ok(refreshed) => { - next.routing = RoutingEvidence::Promoted { - observation: refreshed, - promoted_at_unix_millis: *promoted_at_unix_millis, - }; - if let Some(state) = next.route_supervision.as_mut() { - state.record_proved_challenge(now); - } - self.clear_fault(&route_key); - ( - "proved", - now.saturating_add(self.options.topology_maximum_age_millis), - ) - } - Err(error) => { - if let Some(state) = next.route_supervision.as_mut() { - state.record_failed_challenge(now, self.options.topology_maximum_age_millis); - } - self.report_once( - &route_key, - format!( - "Idunn could not prove admitted {} on its stable route: {error:#}", - current.value.target - ), - ); - let due = next - .route_supervision - .as_ref() - .and_then(|state| state.next_challenge_at_unix_millis) - .unwrap_or(now); - ("failed", due) - } - }; - // Progress is a store that changed: our record landed, or another - // writer beat it (the snapshot is stale either way, so the pass ends). - // A write the store refused changed nothing: no progress, the scheduler - // sleeps if nothing else moved, and the in-memory wait spaces the next - // challenge. - let record_key = format!("route-record:{}", current.value.target); - match self.replace_generation(current, &next, now) { - Ok(_) => { - self.challenge_wait - .lock() - .expect("challenge wait mutex") - .remove(¤t.value.target); - self.clear_fault(&record_key); - Ok(true) - } - Err(record) => { - self.challenge_wait - .lock() - .expect("challenge wait mutex") - .insert( - current.value.target.clone(), - ChallengeWait { - due_unix_millis: wait_until, - stamp: snapshot.target_stamp(¤t.value.target), - }, - ); - self.report_once( - &record_key, - format!( - "Idunn could not record the {recorded} route challenge of {}: {}", - current.value.target, - // The root cause only: the error chain names a fresh temp file each - // attempt, which would defeat once-per-fault. - record.root_cause() - ), - ); - Ok(false) - } - } - } - - fn refresh_admitted_topology( - &self, - snapshot: &ControlSnapshot, - current: &Stored, - ) -> Result { - if !matches!(current.value.readiness(), Ok(class) if class != ReadinessClass::RouteProof) { - return Ok(false); - } - let odin = current.value.odin_receipts()?; - let Some(received) = self.topology().receive( - ¤t.value.target, - ¤t.value.expected.canonical_sha256()?, - )? - else { - return Ok(false); - }; - ensure!( - received.target == current.value.target, - "topology transport substituted admitted target" - ); - let now = now_millis()?; - let authority = self.runtime_authority_parts( - ¤t.value.plan, - ¤t.value.expected, - ¤t.value.activation, - )?; - let odin_authority = self.current_odin_authority(snapshot)?; - let authenticated = match authenticate_odin_runtime_topology_correlation( - &received.canonical_bytes, - &authority, - current.value.leasing.lease_sha256(), - &odin_authority.signer_public_key, - self.trusted_topology_context(now), - ) { - Ok(authenticated) => authenticated, - // An aged-out correlation is nothing new about the admitted - // generation, not a rejected observation. - Err(error) if is_stale_observation(&error) => return Ok(false), - Err(error) => return Err(error), - }; - let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; - if !sequence_requires_admission( - Some(odin.latest), - snapshot.max_odin_sequence(¤t.value.target, &evidence.signer_identity_id), - &evidence, - )? { - return Ok(false); - } - let mut next = current.value.clone(); - next.latest_odin_observation = Some(evidence.clone()); - next.odin_publisher_sequence_cursor = evidence.publisher_sequence; - if is_semantic_ready(&authenticated) { - next.ready = ReadinessEvidence::OdinCorrelated { evidence }; - } - next.validate()?; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: current.value.target.clone(), - current: Some(current.envelope.clone()), - }], - &[admitted_envelope(&next, now)?], - )?, - "admitted generation changed before topology sequence CAS" - ); - Ok(true) - } - - fn advance_transaction(&self, current: &Stored) -> Result<()> { - if current.value.post_fencing_abort.is_some() && current.value.completion.is_none() { - return self.advance_post_fencing_abort(current); - } - if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() { - return self.advance_pre_fencing_abort(current); - } - match current.value.phase { - DeploymentPhase::Sealing => self.advance_sealing(current), - DeploymentPhase::Starting => self.advance_starting(current), - DeploymentPhase::Warming => self.advance_warming(current), - DeploymentPhase::Fencing => self.advance_fencing(current), - DeploymentPhase::Leasing => self.advance_leasing(current), - DeploymentPhase::AwaitingReady => self.advance_awaiting_ready(current), - DeploymentPhase::Routing => self.advance_routing(current), - DeploymentPhase::Committing => self.advance_committing(current), - DeploymentPhase::Complete => self.advance_post_commit_cleanup(current), - } - } - - fn advance_sealing(&self, current: &Stored) -> Result<()> { - if current.value.plan.is_none() { - ensure!( - current.value.command_kind == CommandKind::Deploy, - "continuity transaction lost its admitted plan" - ); - let bindings = load_bindings(&self.options.bindings_dir)?; - let loaded = bindings - .get(¤t.value.target) - .context("operator binding disappeared before sealing")?; - let now = now_millis()?; - let resolved = - self.source - .resolve(&loaded.binding, ¤t.value.transaction_id, now)?; - let provider_snapshot = ControlSnapshot::read(&self.options.state_store)?; - let providers = self.current_ready_provider_tokens(&provider_snapshot)?; - let candidate_port = self.select_candidate_port(&loaded.binding)?; - let plan = compile_deployment_plan( - &resolved.recipe_bytes, - &loaded.bytes, - resolved.facts, - format!("incarnation-{}", current.value.transaction_id), - candidate_port, - now, - &providers, - )?; - // Idunn infers no way to prove readiness. A recipe that declares - // none is refused here, before anything is frozen, built or installed. - plan.readiness_class()?; - return self.persist_same_phase(current, |next| { - next.plan = Some(plan); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - - if current.value.command_kind == CommandKind::Deploy - && current.value.frozen_source.is_none() - { - let now = now_millis()?; - let plan = required(¤t.value.plan, "transaction plan")?; - let receipt = self.source.freeze(¤t.value.transaction_id, plan)?; - return self.persist_same_phase(current, |next| { - next.frozen_source = Some(receipt); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - - if current.value.sealed_release.is_none() - || current.value.installed_release.is_none() - || current.value.expected.is_none() - { - ensure!( - current.value.command_kind == CommandKind::Deploy, - "continuity transaction lost its admitted release evidence" - ); - let now = now_millis()?; - let plan = required(¤t.value.plan, "transaction plan")?; - let frozen_receipt = required(¤t.value.frozen_source, "frozen source")?; - let frozen = self.source.observe_frozen(plan, frozen_receipt)?; - let materialized = self.runner_for(plan)?.materialize( - &frozen, - plan, - &self.options.staging_root, - now, - )?; - let installed = self.workload_for(plan)?.install(plan, &materialized)?; - let expected = materialized.release.expected_projection(plan)?; - return self.persist_same_phase(current, |next| { - next.sealed_release = Some(materialized.release); - next.installed_release = Some(installed); - next.expected = Some(expected); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - - // The candidate's Expected is published in Starting, after the brake - // has admitted this exact transaction. Publishing it is the first - // Verse-visible change a deployment makes, and the brake gates changes. - let now = now_millis()?; - let mut next = current.value.clone(); - match current.value.command_kind { - CommandKind::Deploy => { - let Some(authorization) = self.deployment_authorization(current, now)? else { - return self.record_gate_wait( - current, - "deployment brake has not released this exact transaction", - ); - }; - next.deployment_authorization = Some(authorization); - } - CommandKind::Continuity => { - if !self.lifecycle_allows(current, now)? { - return self - .record_gate_wait(current, "lifecycle brake denies continuity restart"); - } - next.lifecycle_authorized_at_unix_millis = Some(now); - } - } - validate_live_providers_for_deploy(current.value.command_kind, || { - self.validate_selected_providers_current(required( - ¤t.value.plan, - "transaction plan", - )?) - })?; - next.enter_phase(DeploymentPhase::Starting, now); - next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) - } - - fn advance_starting(&self, current: &Stored) -> Result<()> { - let expected = required(¤t.value.expected, "Expected projection")?; - let plan = required(¤t.value.plan, "transaction plan")?; - let release = required(¤t.value.sealed_release, "sealed release")?; - let installed = required(¤t.value.installed_release, "installed release")?; - if current.value.expected_publication_sha256.is_none() { - // Published under the candidate's own incarnation key, beside - // whatever the target's admitted incarnation currently projects. - // Nothing the incumbent reads about itself changes here. - let now = now_millis()?; - let provider_anchor = self.provider_anchor_for_plan(plan)?; - let digest = self - .topology() - .publish_expected(expected, &provider_anchor)?; - return self.persist_same_phase(current, |next| { - next.expected_publication_sha256 = Some(digest); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - if current.value.activation.is_none() { - validate_live_providers_for_deploy(current.value.command_kind, || { - self.validate_selected_providers_current(plan) - })?; - let now = now_millis()?; - let runtime_instance_id = runtime_instance_id(¤t.value.transaction_id)?; - let launch = IdunnRuntimeActivationLaunch::issue( - expected, - runtime_instance_id, - now, - &self.idunn_signer, - )?; - let activation = self - .workload_for(plan)? - .prepare_activation(plan, expected, launch)?; - return self.persist_same_phase(current, |next| { - next.activation = Some(activation); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - if current.value.workload.is_none() { - let now = now_millis()?; - let activation = required(¤t.value.activation, "activation")?; - if current.value.rollout_stops_incumbent_first() { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? { - self.workload_for(&incumbent.value.plan)? - .stop(&incumbent.value.workload) - .context("stopping the incumbent before the candidate starts")?; - } - } - let observation = self - .workload_for(plan)? - .start_prepared(plan, release, installed, expected, activation)?; - return self.persist_same_phase(current, |next| { - next.workload = Some(observation); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - if current.value.activation_publication_sha256.is_none() { - let now = now_millis()?; - let activation = required(¤t.value.activation, "activation")?; - let workload = required(¤t.value.workload, "workload")?; - self.workload_for(required(¤t.value.plan, "transaction plan")?)? - .observe(expected, activation, workload)?; - let digest = self - .topology() - .publish_observed_activation(expected, activation, workload)?; - return self.persist_same_phase(current, |next| { - next.activation_publication_sha256 = Some(digest); - next.updated_at_unix_millis = now; - next.last_error = None; - Ok(()) - }); - } - let mut next = current.value.clone(); - next.enter_phase(DeploymentPhase::Warming, now_millis()?); - next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) - } - - fn advance_warming(&self, current: &Stored) -> Result<()> { - let expected = required(¤t.value.expected, "Expected projection")?; - let activation = required(¤t.value.activation, "activation")?; - let workload = required(¤t.value.workload, "workload")?; - self.workload_for(required(¤t.value.plan, "transaction plan")?)? - .observe(expected, activation, workload)?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent_lease_sha256 = - self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; - let class = ReadinessClass::of(expected)?; - if current.value.warming.is_none() && class == ReadinessClass::RouteProof { - // A route-proof target's warming is Idunn's own challenge to the - // candidate endpoint. No Odin is read: the target declared no - // reason to be aware of one. - return match self.challenge_candidate( - ¤t.value, - route_proof_warming_states(expected), - None, - )? { - CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), - CandidateAnswer::Answered { evidence, present } => { - let warming = WarmingEvidence::RouteProofDirect { evidence }; - let _token = SequenceAdmittedWarming::from_direct_presence( - current.value.transaction_id.clone(), - warming.clone(), - present, - )?; - self.persist_same_phase(current, |next| { - next.warming = Some(warming); - Ok(()) - }) - } - }; - } - if current.value.warming.is_none() { - // Odin's warming is observed directly when nothing else can observe - // it. That is the first bootstrap, and it is also every continuity - // restart of Odin: the incumbent that would do the observing is the - // very process being restarted, so it cannot report on its own - // return. A deployment of Odin is not included -- there the healthy - // incumbent observes the candidate, which is the point. - // Odin's warming is observed directly exactly when no Odin can - // observe it: the first bootstrap, a continuity restart of Odin - // itself, or any moment the admitted Odin has no usable correlation - // to report through. The rule is the same one that justified the - // bootstrap case -- "observed through Odin" is not available -- and - // it stays scoped to this one target. Every other target keeps - // waiting for Odin, which is what makes Odin the root of the chain. - // - // Asked once. Admitting the latest topology persists the sequence - // cursor, so asking twice with the same record fails the second - // time as "transaction changed before topology admission". - let is_odin = class == ReadinessClass::OdinSelf; - let odin_observation = if is_odin - && snapshot.admitted_odin().is_some() - && current.value.command_kind != CommandKind::Continuity - { - self.admit_latest_topology(current, None)? - } else { - None - }; - let odin_observes_itself = is_odin && odin_observation.is_none(); - if odin_observes_itself { - ensure!( - expected.write_lease_required, - "direct Odin warming must be a stateful incarnation" - ); - let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let warming = WarmingEvidence::FirstOdinDirect { evidence }; - let _token = SequenceAdmittedWarming::from_direct_presence( - current.value.transaction_id.clone(), - warming.clone(), - present, - )?; - return self.persist_same_phase(current, |next| { - next.warming = Some(warming); - Ok(()) - }); - } - let observation = match odin_observation { - Some(observation) => Some(observation), - None => self.admit_latest_topology(current, None)?, - }; - let Some((admitted, authenticated)) = observation else { - // Only the first-Odin bootstrap above observes presence - // directly. Every other target's warming presence arrives as - // Odin's authenticated runtime topology correlation, so until - // Odin is admitted and publishing, a candidate warms forever - // with nothing to say why. - return self.record_gate_wait( - current, - "no authenticated Odin topology correlation yet: warming presence for a \ - non-Odin target is observed through Odin, which must be admitted first", - ); - }; - if admitted.envelope != current.envelope { - return Ok(()); - } - let semantic_warming = is_semantic_warming( - expected, - activation, - incumbent_lease_sha256.as_deref(), - &authenticated, - )?; - let latest = required( - &admitted.value.latest_odin_observation, - "sequence-admitted warming evidence", - )?; - if semantic_warming { - let evidence = latest.clone(); - let _token = SequenceAdmittedWarming::from_topology( - admitted.value.transaction_id.clone(), - authenticated, - )?; - return self.persist_same_phase(&admitted, |next| { - next.warming = Some(WarmingEvidence::OdinTopology { evidence }); - Ok(()) - }); - } else { - return Ok(()); - } - } - let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; - ensure!( - warming.transaction_id() == current.value.transaction_id - && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), - "durable Warming evidence belongs to another candidate" - ); - - if expected.route.is_some() && current.value.route_preflight.is_none() { - let now = now_millis()?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; - let incumbent_route = - incumbent.and_then(|generation| generation.value.routing.observation().cloned()); - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let route_binding = binding - .route - .context("routed Expected lost route binding")?; - let driver = self.route_driver(route_binding); - let receipt = driver.preflight( - expected, - &activation.runtime_instance_id, - incumbent_route.as_ref(), - &self.route_gate(¤t.value.target, current.value.command_kind), - )?; - return self.persist_same_phase(current, |next| { - next.route_preflight = Some(receipt); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - - if current.value.isolation.is_none() { - let now = now_millis()?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; - // Isolation is a property of two processes running at once. A - // stopped incumbent has already released its DynamicUser UID, and - // systemd is free to hand that same UID to the candidate -- so - // comparing the candidate against the incumbent's *recorded* - // identity turns ordinary UID reuse into a permanent refusal to - // fence. There is nothing left to be isolated from. - let incumbent_workload = match incumbent { - Some(value) - if self - .workload_for(&value.value.plan)? - .is_permanently_stopped(&value.value.workload)? => - { - None - } - other => other.map(|value| &value.value.workload), - }; - let isolation = WorkloadObservation::prove_isolation(workload, incumbent_workload)?; - return self.persist_same_phase(current, |next| { - next.isolation = Some(isolation); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - - self.transition(current, DeploymentPhase::Fencing) - } - - fn advance_fencing(&self, current: &Stored) -> Result<()> { - if current.value.fencing.is_none() { - let now = now_millis()?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; - let incumbent_lease = incumbent.and_then(|generation| generation.value.leasing.lease()); - let expected = required(¤t.value.expected, "Expected projection")?; - let evidence = if expected.write_lease_required || incumbent_lease.is_some() { - let incumbent_lease_path = incumbent_lease - .map(|_| -> Result { - Ok(incumbent - .context("incumbent lease has no admitted generation")? - .value - .plan - .parsed_inputs()? - .1 - .process_write_lease - .context("incumbent lease has no admitted binding")? - .record_path) - }) - .transpose()?; - if let (Some(lease), Some(path)) = (incumbent_lease, &incumbent_lease_path) { - let incumbent = incumbent.context("incumbent lease lost its generation")?; - self.workload_for(&incumbent.value.plan)? - .stop(&incumbent.value.workload) - .context("stopping the exact incumbent before revoking its lifetime-held write lease")?; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); - driver.revoke_exact(Some(lease))?; - ensure!( - driver.observe_empty()?, - "incumbent write lease remained after exact fencing" - ); - self.topology().withdraw_process_write_lease( - &incumbent.value.expected, - &incumbent.value.activation, - Some(lease), - )?; - } - let candidate_lease_path = if expected.write_lease_required { - Some( - current - .value - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1 - .process_write_lease - .context("stateful candidate has no write-lease binding")? - .record_path, - ) - } else { - None - }; - if let Some(path) = &candidate_lease_path { - if incumbent_lease_path.as_ref() != Some(path) { - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); - driver.revoke_exact(None)?; - ensure!( - driver.observe_empty()?, - "candidate write-lease path was not empty before grant" - ); - } - } - self.topology().withdraw_process_write_lease( - expected, - required(¤t.value.activation, "candidate activation")?, - None, - )?; - FencingEvidence::Revoked { - incumbent_lease_sha256: incumbent_lease - .map(IdunnProcessWriteLeaseRecord::canonical_sha256) - .transpose()?, - candidate_lease_path_verified_empty: candidate_lease_path.is_some(), - } - } else { - FencingEvidence::SkippedStateless - }; - return self.persist_same_phase(current, |next| { - next.fencing = Some(evidence); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - self.transition(current, DeploymentPhase::Leasing) - } - - fn advance_leasing(&self, current: &Stored) -> Result<()> { - let expected = required(¤t.value.expected, "Expected projection")?; - if !expected.write_lease_required { - if current.value.leasing.is_none() { - let now = now_millis()?; - return self.persist_same_phase(current, |next| { - next.leasing = Some(LeasingEvidence::SkippedStateless); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - return self.transition(current, DeploymentPhase::AwaitingReady); - } - - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let lease_path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - - if matches!( - current.value.leasing.as_ref(), - Some(LeasingEvidence::Granted { .. }) - ) { - let activation = required(¤t.value.activation, "activation")?; - let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; - let lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease) - .context("granted Leasing evidence has no write lease")?; - let recorded_sha256 = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("granted Leasing evidence has no lease digest")?; - ensure!( - driver.observe_exact(lease)?, - "physical write lease disappeared after Granted became durable" - ); - self.workload_for(required(¤t.value.plan, "transaction plan")?)? - .observe( - expected, - activation, - required(¤t.value.workload, "candidate workload")?, - )?; - ensure!( - driver.grant(expected, activation, &warming, lease)? == recorded_sha256, - "replayed physical write lease differs from Granted evidence" - ); - ensure!( - self.topology() - .publish_process_write_lease(expected, activation, lease)? - == recorded_sha256, - "replayed write-lease projection differs from Granted evidence" - ); - return self.transition(current, DeploymentPhase::AwaitingReady); - } - - if let Some((lease, prepared_sha256)) = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::prepared_lease) - { - let now = now_millis()?; - let activation = required(¤t.value.activation, "activation")?; - let historical_warming = self.rehydrate_warming_token(¤t.value, now, false)?; - let physical_is_exact = driver.observe_exact(lease)?; - let warming = if physical_is_exact { - historical_warming - } else { - ensure!( - driver.observe_empty()?, - "candidate write-lease store contains authority other than its durable prepared lease" - ); - match self.rehydrate_warming_token(¤t.value, now, true) { - Ok(warming) => warming, - Err(_) => { - let Some((admitted, fresh_evidence, fresh_warming)) = - self.fresh_warming_for_lease(current, now)? - else { - return Ok(()); - }; - let replacement = self.prepare_candidate_write_lease( - &admitted.value, - &fresh_warming, - now, - )?; - let replacement_sha256 = replacement.canonical_sha256()?; - return self.persist_same_phase(&admitted, |next| { - next.warming = Some(fresh_evidence); - next.leasing = Some(LeasingEvidence::Prepared { - lease: replacement, - lease_sha256: replacement_sha256, - }); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - } - }; - self.workload_for(required(¤t.value.plan, "transaction plan")?)? - .observe( - expected, - activation, - required(¤t.value.workload, "candidate workload")?, - )?; - if !physical_is_exact { - ensure!( - driver.observe_empty()?, - "candidate write-lease store changed before physical grant" - ); - } - let granted_sha256 = driver.grant(expected, activation, &warming, lease)?; - ensure!( - granted_sha256 == prepared_sha256, - "granted write lease differs from the durable prepared lease" - ); - let projected_sha256 = self - .topology() - .publish_process_write_lease(expected, activation, lease)?; - ensure!( - projected_sha256 == granted_sha256, - "projected write lease differs from the granted process authority" - ); - let lease = lease.clone(); - return self.persist_same_phase(current, |next| { - next.leasing = Some(LeasingEvidence::Granted { - lease, - lease_sha256: projected_sha256, - }); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - - ensure!( - current.value.leasing.is_none(), - "stateful Leasing phase carries invalid lease evidence" - ); - ensure!( - driver.observe_empty()?, - "candidate write-lease store contains authority before lease preparation" - ); - let now = now_millis()?; - let Some((admitted, fresh_evidence, fresh_warming)) = - self.fresh_warming_for_lease(current, now)? - else { - return Ok(()); - }; - let lease = self.prepare_candidate_write_lease(&admitted.value, &fresh_warming, now)?; - let lease_sha256 = lease.canonical_sha256()?; - self.persist_same_phase(&admitted, |next| { - next.warming = Some(fresh_evidence); - next.leasing = Some(LeasingEvidence::Prepared { - lease, - lease_sha256, - }); - next.updated_at_unix_millis = now; - Ok(()) - }) - } - - fn advance_awaiting_ready(&self, current: &Stored) -> Result<()> { - let now = now_millis()?; - let expected = required(¤t.value.expected, "Expected projection")?; - self.observe_candidate_before_waiting(¤t.value)?; - if expected.write_lease_required { - let activation = required(¤t.value.activation, "activation")?; - let warming = self.rehydrate_warming_token(¤t.value, now, false)?; - let lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease) - .context("stateful transaction has no process write lease")?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let path = binding - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - ensure!( - CultCacheWriteLeaseDriver::new(¤t.value.target, path) - .observe(expected, activation, &warming, lease)?, - "candidate process write lease is no longer exact" - ); - } - if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { - // Ready is the candidate answering Idunn's own challenge Active, - // holding the exact lease Idunn granted. Once recorded it is - // history; the stable route's challenge is what keeps proving it. - if matches!( - current.value.ready, - Some(ReadinessEvidence::RouteProof { .. }) - ) { - return self.transition(current, DeploymentPhase::Routing); - } - let lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - return match self.challenge_candidate(¤t.value, &["warming", "active"], lease)? { - CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), - CandidateAnswer::Answered { present, .. } if present.record().state == "warming" => { - self.record_gate_wait(current, "candidate is still warming") - } - CandidateAnswer::Answered { evidence, .. } => { - let adoption = LeaseAdoptionEvidence::with_ready( - lease, - &evidence.canonical_sha256, - AdoptionSource::Direct, - evidence.admitted_at_unix_millis, - ); - self.persist_same_phase(current, |next| { - next.ready = Some(ReadinessEvidence::RouteProof { evidence }); - next.lease_adoption = adoption; - Ok(()) - }) - } - }; - } - { - let current_lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let Some((admitted, authenticated)) = - self.admit_latest_topology(current, current_lease)? - else { - return Ok(()); - }; - if admitted.envelope != current.envelope { - return Ok(()); - } - let semantic_ready = is_semantic_ready(&authenticated); - let latest = required( - &admitted.value.latest_odin_observation, - "sequence-admitted Ready evidence", - )?; - if current.value.ready.as_ref().and_then(ReadinessEvidence::odin) == Some(latest) { - ensure!(semantic_ready, "stored Ready receipt changed meaning"); - } else if semantic_ready { - let evidence = latest.clone(); - // A Ready stateful correlation carries exactly the current - // lease (the authenticator refuses one that does not). - let adoption = authenticated - .record() - .signed_presence_sha256 - .as_deref() - .and_then(|presence| { - LeaseAdoptionEvidence::with_ready( - current_lease, - presence, - AdoptionSource::OdinTopology, - evidence.admitted_at_unix_millis, - ) - }); - let _token = SequenceAdmittedReady { - transaction_id: admitted.value.transaction_id.clone(), - evidence: evidence.clone(), - expected: expected.clone(), - authenticated, - }; - return self.persist_same_phase(&admitted, |next| { - next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence }); - next.lease_adoption = adoption; - Ok(()) - }); - } else { - return Ok(()); - } - } - self.transition(current, DeploymentPhase::Routing) - } - - fn advance_routing(&self, current: &Stored) -> Result<()> { - let expected = required(¤t.value.expected, "Expected projection")?; - let activation = required(¤t.value.activation, "activation")?; - self.observe_candidate_before_waiting(¤t.value)?; - if current.value.routing.is_none() { - let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { - // Ready is the candidate's own proof, recorded in AwaitingReady. - // Validation refuses Routing without a Ready receipt, and a - // receipt the other party vouched for is held before any step - // runs. No Odin correlation is read to admit a route. - current.clone() - } else { - let current_lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let Some((admitted, authenticated)) = - self.admit_latest_topology(current, current_lease)? - else { - return Ok(()); - }; - if admitted.envelope != current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at route admission" - ); - let latest = required( - &admitted.value.latest_odin_observation, - "current route-admission topology evidence", - )?; - if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) - != Some(latest) - { - let latest = latest.clone(); - return self.persist_same_phase(&admitted, |next| { - next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); - Ok(()) - }); - } - let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; - ensure!( - ready.transaction_id() == admitted.value.transaction_id, - "Ready token belongs to another transaction" - ); - admitted - }; - validate_live_providers_for_deploy(admitted.value.command_kind, || { - self.validate_selected_providers_current(required( - &admitted.value.plan, - "transaction plan", - )?) - })?; - self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; - let evidence = if expected.route.is_some() { - let binding = admitted.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let route_binding = binding.route.context("routed plan has no route binding")?; - let preflight = required(&admitted.value.route_preflight, "route preflight")?; - let driver = self.route_driver(route_binding); - let gate = self.route_gate(&admitted.value.target, admitted.value.command_kind); - ensure!( - preflight.candidate_runtime_instance_id == activation.runtime_instance_id, - "route preflight belongs to another runtime instance" - ); - let rollback_allowed = may_rollback_route_after_failed_proof(required( - &admitted.value.fencing, - "route fencing evidence", - )?); - let membership_sha256 = driver.install( - expected, - &activation.runtime_instance_id, - preflight, - rollback_allowed, - &gate, - )?; - let observation = match self.prove_stable_route( - &admitted.value, - &driver, - membership_sha256, - ) { - Ok(observation) => observation, - Err(proof_error) if !rollback_allowed => { - return Err(proof_error).context( - "incumbent authority was fenced; candidate route remains installed for fail-closed retry", - ); - } - Err(proof_error) => { - return match driver.rollback( - expected, - &activation.runtime_instance_id, - preflight, - &gate, - ) { - Ok(()) => Err(proof_error) - .context("candidate did not answer its stable route challenge"), - Err(rollback_error) => Err(proof_error).context(format!( - "candidate did not answer its stable route challenge; exact route rollback also failed: {rollback_error:#}" - )), - }; - } - }; - ensure!( - driver.observe_membership(expected, &observation.membership_sha256)?, - "route membership changed during its signed stable-listener observation" - ); - self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; - let promoted_at_unix_millis = observation.observed_at_unix_millis; - RoutingEvidence::Promoted { - observation, - promoted_at_unix_millis, - } - } else { - RoutingEvidence::SkippedUnrouted - }; - let now = now_millis()?; - return self.persist_same_phase(&admitted, |next| { - next.routing = Some(evidence); - next.updated_at_unix_millis = now; - Ok(()) - }); - } - self.transition(current, DeploymentPhase::Committing) - } - - fn advance_committing(&self, current: &Stored) -> Result<()> { - self.observe_candidate_before_waiting(¤t.value)?; - let route_proof = ReadinessClass::of(required( - ¤t.value.expected, - "Expected projection", - )?)? == ReadinessClass::RouteProof; - let ready_current = if route_proof { - // The candidate's Ready proof is durable, and the stable route's - // challenge below is its currency. No Odin is read to commit. - current.clone() - } else { - let current_lease_sha256 = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - // A newer sequence admitted here changed the record, and commit - // goes on from that record rather than returning: returning let - // Odin's next re-stamp abort every attempt (F16). - let Some((ready_current, authenticated)) = - self.admit_latest_topology(current, current_lease_sha256.as_deref())? - else { - return Ok(()); - }; - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at admission commit" - ); - // Not require_current: the Ready receipt is durable evidence, so - // authenticating it against `now` asks a stored record to be live and - // refuses it once it ages past the 30s observation window. Route - // admission (:3906) refreshes the receipt to the latest observation and - // can therefore demand currency; by Committing the receipt is history. - // Currency here is answered by the latest observation, authenticated - // and checked for semantic readiness immediately above. - self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; - ready_current - }; - validate_live_providers_for_deploy(ready_current.value.command_kind, || { - self.validate_selected_providers_current(required( - &ready_current.value.plan, - "transaction plan", - )?) - })?; - - let expected = required(&ready_current.value.expected, "Expected projection")?; - let activation = required(&ready_current.value.activation, "activation")?; - let workload = required(&ready_current.value.workload, "workload")?; - self.workload_for(required(&ready_current.value.plan, "transaction plan")?)? - .observe(expected, activation, workload)?; - self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; - if let Some(route) = ready_current - .value - .routing - .as_ref() - .and_then(RoutingEvidence::observation) - { - let binding = ready_current - .value - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1; - let driver = - self.route_driver(binding.route.context("routed plan has no route binding")?); - ensure!( - driver.observe_membership(expected, &route.membership_sha256)?, - "route membership changed before admission commit" - ); - let current_route = self.prove_stable_route( - &ready_current.value, - &driver, - route.membership_sha256.clone(), - )?; - ensure!( - current_route.route_id == route.route_id - && current_route.runtime_instance_id == route.runtime_instance_id, - "stable route changed incarnation before admission commit" - ); - ensure!( - driver.observe_membership(expected, &route.membership_sha256)?, - "route membership changed during the final signed admission challenge" - ); - self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; - } - - let commit_current = if route_proof { - ready_current - } else { - let current_lease_sha256 = ready_current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - // The final proof's own presence makes Odin re-stamp, so this - // admission usually writes. Commit is from the record it wrote. - let Some((commit_current, authenticated)) = - self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? - else { - return Ok(()); - }; - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready after the final admission challenge" - ); - self.rehydrate_ready_token(&commit_current.value, now_millis()?, false)?; - commit_current - }; - let now = now_millis()?; - validate_live_providers_for_deploy(commit_current.value.command_kind, || { - self.validate_selected_providers_current(required( - &commit_current.value.plan, - "transaction plan", - )?) - })?; - self.workload_for(required(&commit_current.value.plan, "transaction plan")?)? - .observe( - required(&commit_current.value.expected, "Expected projection")?, - required(&commit_current.value.activation, "activation")?, - required(&commit_current.value.workload, "workload")?, - )?; - self.ensure_transaction_write_lease_current(&commit_current.value, now)?; - - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; - let odin_authority = (!route_proof) - .then(|| self.current_odin_authority(&snapshot)) - .transpose()?; - let generation = AdmittedGeneration::from_transaction( - &commit_current.value, - odin_authority, - now, - )?; - let post_commit_cleanup = PostCommitCleanup { - incumbent: match incumbent { - Some(incumbent) - if incumbent_was_stopped_during_fencing(required( - &commit_current.value.fencing, - "commit fencing evidence", - )?) => - { - IncumbentCleanupEvidence::Complete { - generation_id: incumbent.value.generation_id.clone(), - } - } - Some(incumbent) => IncumbentCleanupEvidence::Pending { - generation_id: incumbent.value.generation_id.clone(), - workload: incumbent.value.workload.clone(), - }, - None => IncumbentCleanupEvidence::SkippedNoIncumbent, - }, - source: match commit_current.value.command_kind { - CommandKind::Deploy => SourceCleanupEvidence::Pending, - CommandKind::Continuity => SourceCleanupEvidence::SkippedContinuity, - }, - }; - let mut complete = commit_current.value.clone(); - complete.enter_phase(DeploymentPhase::Complete, now); - complete.last_error = None; - complete.completion = Some(TransactionCompletion::Admitted { - generation_id: generation.generation_id.clone(), - }); - complete.post_commit_cleanup = Some(post_commit_cleanup); - complete.validate()?; - - let admitted_expected = CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: generation.target.clone(), - current: incumbent.map(|stored| stored.envelope.clone()), - }; - ensure!( - SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: commit_current.value.transaction_id.clone(), - current: Some(commit_current.envelope.clone()), - }, - admitted_expected, - ], - &[ - transaction_envelope(&complete, now)?, - admitted_envelope(&generation, now)?, - ], - )?, - "incumbent or transaction changed before atomic admission commit" - ); - Ok(()) - } - - fn exact_incumbent<'a>( - &self, - snapshot: &'a ControlSnapshot, - transaction: &DeploymentTransaction, - ) -> Result>> { - let current = snapshot.admitted_for(&transaction.target); - match (&transaction.incumbent_generation_id, current) { - (None, None) => Ok(None), - (Some(expected), Some(current)) if current.value.generation_id == *expected => { - Ok(Some(current)) - } - _ => bail!("target incumbent changed after transaction creation"), - } - } - - fn incumbent_lease_sha256_for_warming( - &self, - snapshot: &ControlSnapshot, - transaction: &DeploymentTransaction, - ) -> Result> { - if let Some(fencing) = &transaction.fencing { - return match fencing { - FencingEvidence::SkippedStateless => Ok(None), - FencingEvidence::Revoked { - incumbent_lease_sha256, - .. - } => Ok(incumbent_lease_sha256.clone()), - }; - } - self.exact_incumbent(snapshot, transaction)? - .and_then(|incumbent| incumbent.value.leasing.lease()) - .map(IdunnProcessWriteLeaseRecord::canonical_sha256) - .transpose() - } - - fn fresh_warming_for_lease( - &self, - current: &Stored, - now: u64, - ) -> Result< - Option<( - Stored, - WarmingEvidence, - SequenceAdmittedWarming, - )>, - > { - let prior = self.rehydrate_warming_token(¤t.value, now, false)?; - match required(¤t.value.warming, "pre-fence Warming evidence")?.clone() { - WarmingEvidence::FirstOdinDirect { .. } => { - // Whether this transaction may observe Odin directly was - // settled when the Warming evidence was recorded, and that - // decision is durable in the transaction. Re-deciding it here - // with the narrower "no admitted Odin" test contradicted the - // caller and failed the refresh. What the refresh owes is - // freshness, and the replay check below is what provides it. - // That the target is Odin was already required by the - // rehydration above, which authenticates the recorded answer. - let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let warming = WarmingEvidence::FirstOdinDirect { evidence }; - let token = SequenceAdmittedWarming::from_direct_presence( - current.value.transaction_id.clone(), - warming.clone(), - present, - )?; - ensure!( - token.signed_presence_sha256() != prior.signed_presence_sha256(), - "first Odin replayed its pre-fence Warming presence" - ); - Ok(Some((current.clone(), warming, token))) - } - WarmingEvidence::RouteProofDirect { .. } => { - // A stateful route-proof candidate is warming until it holds - // the lease this refresh is about to bind, so the fresh - // presence must be warming again, and must be a new one. - let CandidateAnswer::Answered { evidence, present } = - self.challenge_candidate(¤t.value, &["warming"], None)? - else { - return Ok(None); - }; - let warming = WarmingEvidence::RouteProofDirect { evidence }; - let token = SequenceAdmittedWarming::from_direct_presence( - current.value.transaction_id.clone(), - warming.clone(), - present, - )?; - ensure!( - token.signed_presence_sha256() != prior.signed_presence_sha256(), - "route-proof candidate replayed its pre-fence Warming presence" - ); - Ok(Some((current.clone(), warming, token))) - } - WarmingEvidence::OdinTopology { - evidence: prior_evidence, - } => { - let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)? - else { - return Ok(None); - }; - let fresh_evidence = required( - &admitted.value.latest_odin_observation, - "post-fence Odin Warming observation", - )? - .clone(); - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let incumbent_lease_sha256 = - self.incumbent_lease_sha256_for_warming(&snapshot, &admitted.value)?; - if !is_semantic_warming( - required(&admitted.value.expected, "Warming Expected projection")?, - required(&admitted.value.activation, "Warming activation")?, - incumbent_lease_sha256.as_deref(), - &authenticated, - )? { - return Ok(None); - } - let token = SequenceAdmittedWarming::from_topology( - admitted.value.transaction_id.clone(), - authenticated, - )?; - if !provider_warming_advanced( - prior_evidence.publisher_sequence, - prior.signed_presence_sha256(), - fresh_evidence.publisher_sequence, - token.signed_presence_sha256(), - ) { - return Ok(None); - } - Ok(Some(( - admitted, - WarmingEvidence::OdinTopology { - evidence: fresh_evidence, - }, - token, - ))) - } - } - } - - fn prepare_candidate_write_lease( - &self, - transaction: &DeploymentTransaction, - warming: &SequenceAdmittedWarming, - now: u64, - ) -> Result { - let expected = required(&transaction.expected, "Expected projection")?; - let activation = required(&transaction.activation, "activation")?; - ensure!( - expected.write_lease_required - && warming.transaction_id() == transaction.transaction_id - && warming.runtime_instance_id() == activation.runtime_instance_id, - "fresh Warming token does not own this stateful candidate" - ); - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let epoch = self - .exact_incumbent(&snapshot, transaction)? - .and_then(|generation| generation.value.leasing.lease()) - .map_or(1, |lease| lease.lease_epoch.saturating_add(1)); - let lease = IdunnProcessWriteLeaseRecord { - schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: expected.canonical_sha256()?, - plan_id: expected.plan_id.clone(), - incarnation_id: expected.incarnation_id.clone(), - sealed_release_id: expected.sealed_release_id.clone(), - activation_witness_sha256: activation.canonical_sha256()?, - state_schema_generation: expected - .state_schema_generation - .clone() - .context("stateful Expected has no schema generation")?, - state_contract_sha256: expected - .state_contract_sha256 - .clone() - .context("stateful Expected has no state contract")?, - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: activation.runtime_instance_id.clone(), - warming_presence_sha256: warming.signed_presence_sha256().to_owned(), - lease_epoch: epoch, - issued_at_unix_millis: now, - }; - lease.validate()?; - Ok(lease) - } - - fn ensure_transaction_write_lease_current( - &self, - transaction: &DeploymentTransaction, - now: u64, - ) -> Result<()> { - let expected = required(&transaction.expected, "Expected projection")?; - let lease = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease); - ensure!( - expected.write_lease_required == lease.is_some(), - "transaction write-lease disposition differs from Expected" - ); - let Some(lease) = lease else { - return Ok(()); - }; - let activation = required(&transaction.activation, "activation")?; - let warming = self.rehydrate_warming_token(transaction, now, false)?; - let lease_path = required(&transaction.plan, "transaction plan")? - .parsed_inputs()? - .1 - .process_write_lease - .context("stateful target has no write-lease binding")? - .record_path; - ensure!( - CultCacheWriteLeaseDriver::new(&transaction.target, lease_path) - .observe(expected, activation, &warming, lease,)?, - "process write lease changed across the authority boundary" - ); - Ok(()) - } - - fn select_candidate_port(&self, binding: &OperatorBinding) -> Result> { - let Some(route) = &binding.route else { - return Ok(None); - }; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut used = BTreeSet::new(); - for plan in snapshot - .admitted - .iter() - .map(|stored| &stored.value.plan) - .chain( - snapshot - .transactions - .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) - .filter_map(|stored| stored.value.plan.as_ref()), - ) - { - if let Some(port) = plan.candidate_port { - used.insert(port); - } - } - (route.private_port_start..=route.private_port_end) - .find(|port| !used.contains(port)) - .map(Some) - .context("no private candidate port remains in the operator range") - } - - fn deployment_authorization( - &self, - current: &Stored, - now: u64, - ) -> Result> { - let expected = required(¤t.value.expected, "Expected projection")?; - let plan = required(¤t.value.plan, "transaction plan")?; - let binding = plan.parsed_inputs()?.1; - let Some((record, canonical_bytes)) = - read_deployment_brake(&binding.brakes.deployment_store)? - else { - return Ok(None); - }; - let anchor = read_trust_anchor::( - &self.options.deployment_brake_operator_anchor, - )?; - if evaluate_idunn_deployment_brake( - IdunnDeploymentBrakeObservation::Present(&record), - &anchor, - &expected.runtime_id, - &expected.sealed_release_id, - ¤t.value.transaction_id, - now, - ) - .is_err() - { - return Ok(None); - } - let authorization_id = record - .authorization_id - .clone() - .context("released brake has no authorization id")?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - ensure!( - !snapshot.transactions.iter().any(|stored| { - stored - .value - .deployment_authorization - .as_ref() - .is_some_and(|used| used.authorization_id == authorization_id) - }), - "deployment authorization was already consumed" - ); - Ok(Some(DeploymentAuthorization { - authorization_id, - brake_sha256: sha256_id(&canonical_bytes), - canonical_brake_bytes: canonical_bytes, - authorized_at_unix_millis: now, - })) - } - - fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { - let plan = required(¤t.value.plan, "continuity plan")?; - let binding = plan.parsed_inputs()?.1; - let expected = required(¤t.value.expected, "continuity Expected")?; - match read_lifecycle_brake(&binding.brakes.lifecycle_store) { - Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart( - IdunnLifecycleBrakeObservation::Present(&record), - &expected.runtime_id, - ¤t.value.target, - now, - ) - .is_ok()), - Ok(None) => Ok(evaluate_idunn_continuity_restart( - IdunnLifecycleBrakeObservation::Missing, - &expected.runtime_id, - ¤t.value.target, - now, - ) - .is_ok()), - Err(_) => Ok(false), - } - } - - /// A phase that waits on Odin's evidence about the candidate must first - /// confirm the candidate still exists. Otherwise a candidate that died - /// while its correlation was pending is never observed again: the phase - /// returns early on "no evidence yet" every tick, the error that would have - /// triggered the post-fencing abort never happens, and the transaction - /// owns the target forever behind a process that is gone. - fn observe_candidate_before_waiting(&self, transaction: &DeploymentTransaction) -> Result<()> { - self.workload_for(required(&transaction.plan, "transaction plan")?)? - .observe( - required(&transaction.expected, "Expected projection")?, - required(&transaction.activation, "activation")?, - required(&transaction.workload, "candidate workload")?, - ) - .map(|_| ()) - } - - /// Withdraw one projected incarnation of this target that nothing owns: - /// not the admitted generation, not any live transaction. A replaced - /// incumbent, an aborted candidate whose reconciliation was skipped, or a - /// record left by the projection migration all end here. One per tick; - /// the projection is a shared file and each withdrawal is one CAS. - fn retire_stale_incarnations( - &self, - snapshot: &ControlSnapshot, - current: &Stored, - ) -> Result { - let topology = self.topology(); - let target = current.value.target.as_str(); - let owned = snapshot - .transactions - .iter() - .filter(|stored| stored.value.target == target && !stored.value.is_terminal()) - .filter_map(|stored| stored.value.expected.as_ref()) - .map(IdunnExpectedIncarnationRecord::canonical_sha256) - .collect::>>()?; - let admitted_sha256 = current.value.expected.canonical_sha256()?; - let Some(stale) = topology - .projected_incarnations(target)? - .into_iter() - .find(|expected| { - expected - .canonical_sha256() - .is_ok_and(|sha256| sha256 != admitted_sha256 && !owned.contains(&sha256)) - }) - else { - return Ok(false); - }; - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - topology.withdraw_stale_incarnation(&stale, &provider_anchor)?; - eprintln!( - "Idunn withdrew stale {} incarnation {} from the projection", - target, stale.incarnation_id - ); - Ok(true) - } - - /// The lifecycle brake as it applies to restarting one admitted - /// generation: the same evaluation `lifecycle_allows` makes for a - /// continuity transaction, asked before one is minted. - fn lifecycle_allows_generation( - &self, - current: &Stored, - now: u64, - ) -> Result { - let binding = current.value.plan.parsed_inputs()?.1; - let expected = ¤t.value.expected; - match read_lifecycle_brake(&binding.brakes.lifecycle_store) { - Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart( - IdunnLifecycleBrakeObservation::Present(&record), - &expected.runtime_id, - ¤t.value.target, - now, - ) - .is_ok()), - Ok(None) => Ok(evaluate_idunn_continuity_restart( - IdunnLifecycleBrakeObservation::Missing, - &expected.runtime_id, - ¤t.value.target, - now, - ) - .is_ok()), - Err(_) => Ok(false), - } - } - - fn record_gate_wait( - &self, - current: &Stored, - detail: &str, - ) -> Result<()> { - if current.value.last_error.as_deref() == Some(detail) { - return Ok(()); - } - let mut next = current.value.clone(); - next.last_error = Some(detail.into()); - next.updated_at_unix_millis = now_millis()?; - replace_transaction(&self.options.state_store, current, &next) - } - - fn runtime_authority( - &self, - transaction: &DeploymentTransaction, - ) -> Result { - self.runtime_authority_parts( - required(&transaction.plan, "transaction plan")?, - required(&transaction.expected, "Expected projection")?, - required(&transaction.activation, "activation")?, - ) - } - - /// The stable route's proof: a challenged presence that is Active. - fn authenticate_routed_presence( - &self, - authority: &cultnet_rs::VerifiedRuntimeAuthority, - current_write_lease_sha256: Option<&str>, - message_id: &str, - challenged_at_unix_millis: u64, - received_at_unix_millis: u64, - canonical_presence: &[u8], - ) -> Result<(String, u64)> { - let present = self.authenticate_challenged_presence( - authority, - &["active"], - current_write_lease_sha256, - message_id, - challenged_at_unix_millis, - received_at_unix_millis, - canonical_presence, - )?; - Ok(( - present.signed_presence_sha256().to_owned(), - received_at_unix_millis, - )) - } - - /// A presence Idunn challenged for itself, from the stable endpoint or the - /// candidate's: signed by the provider and the launch's activation key, - /// bound to current authority, minted after the challenge, answering that - /// exact challenge, in one of `states`. A warming presence holds no write - /// lease; a presence in any other state holds exactly the current one. - /// Every disagreement with authority is named, so a capacity below the - /// Expected minimum reads as that and not as a generic refusal. - fn authenticate_challenged_presence( - &self, - authority: &cultnet_rs::VerifiedRuntimeAuthority, - states: &[&str], - current_write_lease_sha256: Option<&str>, - message_id: &str, - challenged_at_unix_millis: u64, - received_at_unix_millis: u64, - canonical_presence: &[u8], - ) -> Result { - ensure!( - received_at_unix_millis >= challenged_at_unix_millis, - "route proof predates its challenge" - ); - let authenticated = authenticate_runtime_presence_claim( - canonical_presence, - authority, - RuntimePresenceAuthenticationContext { - trusted_received_at_unix_millis: received_at_unix_millis, - maximum_age_millis: self.options.topology_maximum_age_millis, - maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, - }, - )?; - let correlation = correlate_runtime_presence_claim(authenticated, authority)?; - if !correlation.disagreements().is_empty() { - return Err(PresenceDisagrees { - disagreements: correlation.disagreements().to_vec(), - } - .into()); - } - let present = correlation.into_undisputed_present()?; - let presence = present.record(); - ensure!( - presence.observed_at_unix_millis >= challenged_at_unix_millis, - "route proof returned a presence minted before its challenge" - ); - ensure!( - states.contains(&presence.state.as_str()), - "route proof runtime is {}, not {}", - presence.state, - states.join(" or ") - ); - ensure!( - presence.detail == format!("route-observation:{message_id}"), - "route proof response is not bound to the exact challenge" - ); - // A warming presence is exempt from the lease comparison: it is - // Idunn's own lease that a warming candidate cannot hold yet, and the - // authenticator above already refuses one that claims a lease. - let lease_is_current = presence.state == "warming" - || presence.write_lease_sha256.as_deref() == current_write_lease_sha256; - ensure!( - lease_is_current, - "route proof runtime does not hold the exact current process write lease" - ); - Ok(present) - } - - fn authenticate_first_odin_warming_presence( - &self, - transaction: &DeploymentTransaction, - message_id: &str, - challenged_at_unix_millis: u64, - received_at_unix_millis: u64, - canonical_presence: &[u8], - ) -> Result { - // Whether direct observation is warranted at all is decided in - // advance_warming, which asks the question that matters: can any Odin - // report on this one. Re-deciding it here with the narrower "has an - // Odin ever been admitted" test only contradicted that caller. What is - // still checked is what this function itself depends on -- the target - // is Odin, and the incarnation is stateful. - let expected = required(&transaction.expected, "Odin Expected")?; - ensure!( - ReadinessClass::of(expected)? == ReadinessClass::OdinSelf - && expected.write_lease_required, - "direct Warming presence is reserved for a stateful Odin incarnation" - ); - ensure!( - received_at_unix_millis >= challenged_at_unix_millis, - "first Odin Warming observation predates its challenge" - ); - let authority = self.runtime_authority(transaction)?; - let authenticated = authenticate_runtime_presence_claim( - canonical_presence, - &authority, - RuntimePresenceAuthenticationContext { - trusted_received_at_unix_millis: received_at_unix_millis, - maximum_age_millis: self.options.topology_maximum_age_millis, - maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, - }, - )?; - let correlation = correlate_runtime_presence_claim(authenticated, &authority)?; - ensure!( - correlation.disagreements().is_empty(), - "first Odin Warming presence disagrees with its Expected activation" - ); - let present = correlation.into_undisputed_present()?; - let presence = present.record(); - ensure!( - presence.observed_at_unix_millis >= challenged_at_unix_millis - && presence.state == "warming" - && presence.write_lease_sha256.is_none() - && presence.detail == format!("idunn-warming:{message_id}"), - "first Odin candidate did not return exact fresh pre-lease Warming evidence" - ); - Ok(present) - } - - fn observe_first_odin_warming( - &self, - transaction: &DeploymentTransaction, - ) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { - let expected = required(&transaction.expected, "first Odin Expected")?; - let binding = required(&transaction.plan, "first Odin plan")? - .parsed_inputs()? - .1; - let driver = self.route_driver( - binding - .route - .context("first Odin bootstrap has no candidate route binding")?, - ); - let message_id = format!("warming-{}", Uuid::new_v4().simple()); - let challenged_at_unix_millis = now_millis()?; - let response = driver.request_candidate_runtime_presence(expected, &message_id)?; - ensure!( - response.message_id == message_id, - "first Odin candidate transport substituted its challenge identity" - ); - let admitted_at_unix_millis = now_millis()?; - let present = self.authenticate_first_odin_warming_presence( - transaction, - &message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - &response.canonical_presence, - )?; - let evidence = RuntimePresenceEvidence::from_present( - &present, - message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - )?; - Ok((evidence, present)) - } - - fn prove_stable_route( - &self, - transaction: &DeploymentTransaction, - driver: &NginxRouteDriver, - membership_sha256: String, - ) -> Result { - let expected = required(&transaction.expected, "route Expected projection")?; - let activation = required(&transaction.activation, "route activation")?; - let authority = self.runtime_authority(transaction)?; - let current_write_lease_sha256 = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - self.prove_stable_route_against( - expected, - activation, - &authority, - current_write_lease_sha256, - driver, - membership_sha256, - ) - } - - fn prove_stable_route_against( - &self, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - authority: &cultnet_rs::VerifiedRuntimeAuthority, - current_write_lease_sha256: Option<&str>, - driver: &NginxRouteDriver, - membership_sha256: String, - ) -> Result { - let message_id = format!("route-{}", Uuid::new_v4().simple()); - let challenged_at_unix_millis = now_millis()?; - let response = driver.request_runtime_presence(expected, &message_id)?; - ensure!( - response.message_id == message_id, - "route transport substituted its challenge identity" - ); - let received_at_unix_millis = now_millis()?; - let (signed_presence_sha256, observed_at_unix_millis) = self.authenticate_routed_presence( - authority, - current_write_lease_sha256, - &message_id, - challenged_at_unix_millis, - received_at_unix_millis, - &response.canonical_presence, - )?; - let observation = RouteObservation { - route_id: driver.binding.route_id.clone(), - runtime_instance_id: activation.runtime_instance_id.clone(), - membership_sha256, - signed_presence_sha256, - observed_at_unix_millis, - }; - observation.validate()?; - Ok(observation) - } - - fn runtime_authority_parts( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - let provider_anchor = self.provider_anchor_for_plan(plan)?; - verify_runtime_authority( - expected, - activation, - &self.idunn_anchor, - &provider_anchor.public_key, - ) - } - - fn provider_anchor_for_plan( - &self, - plan: &CompiledDeploymentPlan, - ) -> Result { - let binding = plan.parsed_inputs()?.1; - let provider_anchor = read_trust_anchor::( - &binding.runtime_identity.trust_anchor_store, - )?; - ensure!( - provider_anchor.schema_version - == ::TRUST_ANCHOR_SCHEMA, - "provider runtime presence trust anchor schema is unsupported" - ); - Ok(provider_anchor) - } - - fn authenticate_topology_bytes( - &self, - snapshot: &ControlSnapshot, - transaction: &DeploymentTransaction, - canonical_bytes: &[u8], - current_write_lease_sha256: Option<&str>, - trusted_received_at: u64, - ) -> Result { - let authority = self.runtime_authority(transaction)?; - let odin_authority = self.current_odin_authority(snapshot)?; - authenticate_odin_runtime_topology_correlation( - canonical_bytes, - &authority, - current_write_lease_sha256, - &odin_authority.signer_public_key, - self.trusted_topology_context(trusted_received_at), - ) - } - - /// Persist every newly authenticated publisher sequence before deciding - /// whether it means Warming, Ready, degraded, or disagreement. - fn admit_latest_topology( - &self, - current: &Stored, - current_write_lease_sha256: Option<&str>, - ) -> Result< - Option<( - Stored, - AuthenticatedOdinRuntimeTopologyCorrelation, - )>, - > { - // Odin keeps one correlation per incarnation, so the transaction's own - // Expected selects exactly the evidence about it. A correlation about - // the incarnation this one is replacing lives under another key and is - // never handed back here. - let expected_sha256 = - required(¤t.value.expected, "Expected projection")?.canonical_sha256()?; - let Some(received) = self - .topology() - .receive(¤t.value.target, &expected_sha256)? - else { - return Ok(None); - }; - ensure!( - received.target == current.value.target, - "topology transport substituted target" - ); - let now = now_millis()?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let live = snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared before topology admission")?; - ensure!( - live.envelope == current.envelope, - "transaction changed before topology admission" - ); - let authenticated = match self.authenticate_topology_bytes( - &snapshot, - &live.value, - &received.canonical_bytes, - current_write_lease_sha256, - now, - ) { - Ok(authenticated) => authenticated, - // Odin re-stamps a correlation only when its facts change, and a - // live presence changes them every heartbeat. A correlation that - // has aged out therefore says one thing: nothing about this - // incarnation has arrived lately. That is absence to wait on, not - // evidence to refuse; it failed every Heimdall deployment and the - // first observed Odin deployment as a fault. - Err(error) if is_stale_observation(&error) => return Ok(None), - Err(error) => return Err(error), - }; - let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; - if !sequence_requires_admission( - live.value.latest_odin_observation.as_ref(), - snapshot.max_odin_sequence(&live.value.target, &evidence.signer_identity_id), - &evidence, - )? { - return Ok(Some((live.clone(), authenticated))); - } - let mut next = live.value.clone(); - next.latest_odin_observation = Some(evidence.clone()); - next.odin_publisher_sequence_cursor = evidence.publisher_sequence; - next.updated_at_unix_millis = now; - next.last_error = None; - replace_transaction(&self.options.state_store, live, &next)?; - let admitted = ControlSnapshot::read(&self.options.state_store)? - .transactions - .into_iter() - .find(|stored| stored.value.transaction_id == next.transaction_id) - .context("sequence-admitted transaction disappeared")?; - ensure!( - admitted.value.latest_odin_observation.as_ref() == Some(&evidence) - && admitted.value.odin_publisher_sequence_cursor == evidence.publisher_sequence, - "topology sequence was not durably admitted" - ); - Ok(Some((admitted, authenticated))) - } - - fn rehydrate_warming_token( - &self, - transaction: &DeploymentTransaction, - now: u64, - require_current: bool, - ) -> Result { - let evidence = required(&transaction.warming, "durable warming evidence")?.clone(); - match evidence { - WarmingEvidence::OdinTopology { evidence } => { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let authenticated = self.authenticate_topology_bytes( - &snapshot, - transaction, - &evidence.canonical_bytes, - None, - if require_current { - now - } else { - evidence.admitted_at_unix_millis - }, - )?; - validate_authenticated_evidence(&evidence, &authenticated)?; - let incumbent_lease_sha256 = - self.incumbent_lease_sha256_for_warming(&snapshot, transaction)?; - ensure!( - is_semantic_warming( - required(&transaction.expected, "Warming Expected projection")?, - required(&transaction.activation, "Warming activation")?, - incumbent_lease_sha256.as_deref(), - &authenticated, - )?, - "durable warming evidence no longer satisfies the Warming gate" - ); - SequenceAdmittedWarming::from_topology( - transaction.transaction_id.clone(), - authenticated, - ) - } - WarmingEvidence::FirstOdinDirect { evidence } => { - let present = self.authenticate_first_odin_warming_presence( - transaction, - &evidence.message_id, - evidence.challenged_at_unix_millis, - if require_current { - now - } else { - evidence.admitted_at_unix_millis - }, - &evidence.canonical_bytes, - )?; - SequenceAdmittedWarming::from_direct_presence( - transaction.transaction_id.clone(), - WarmingEvidence::FirstOdinDirect { evidence }, - present, - ) - } - WarmingEvidence::RouteProofDirect { evidence } => { - let present = self.reauthenticate_route_proof( - transaction, - &evidence, - route_proof_warming_states(required( - &transaction.expected, - "Warming Expected projection", - )?), - None, - if require_current { - now - } else { - evidence.admitted_at_unix_millis - }, - )?; - SequenceAdmittedWarming::from_direct_presence( - transaction.transaction_id.clone(), - WarmingEvidence::RouteProofDirect { evidence }, - present, - ) - } - } - } - - /// A route-proof presence Idunn recorded, authenticated again as of `at`. - fn reauthenticate_route_proof( - &self, - transaction: &DeploymentTransaction, - evidence: &RuntimePresenceEvidence, - states: &[&str], - current_write_lease_sha256: Option<&str>, - at_unix_millis: u64, - ) -> Result { - let authority = self.runtime_authority(transaction)?; - self.authenticate_challenged_presence( - &authority, - states, - current_write_lease_sha256, - &evidence.message_id, - evidence.challenged_at_unix_millis, - at_unix_millis, - &evidence.canonical_bytes, - ) - } - - /// One direct challenge to a route-proof target's candidate endpoint. - /// Failing to reach or hear the candidate is only silence: a process that - /// is still starting does not answer, and that is waiting, not a fault. - /// What does answer is authenticated, and a bad answer is an error, as is - /// a challenge Idunn could not make: those stay a `ChallengeFailure` in the - /// error's chain. - fn challenge_candidate( - &self, - transaction: &DeploymentTransaction, - states: &[&str], - current_write_lease_sha256: Option<&str>, - ) -> Result { - let expected = required(&transaction.expected, "candidate Expected")?; - let binding = required(&transaction.plan, "candidate plan")? - .parsed_inputs()? - .1; - let driver = self.route_driver( - binding - .route - .context("route-proof candidate has no route binding")?, - ); - let message_id = format!("candidate-{}", Uuid::new_v4().simple()); - let challenged_at_unix_millis = now_millis()?; - let response = match driver.request_candidate_runtime_presence(expected, &message_id) { - Ok(response) => response, - Err(ChallengeFailure::Silent(error)) => { - return Ok(CandidateAnswer::Silent(format!( - "candidate endpoint did not answer its challenge: {error:#}" - ))); - } - Err(bad @ ChallengeFailure::Refused(_)) => return Err(bad.into()), - }; - let admitted_at_unix_millis = now_millis()?; - let authority = self.runtime_authority(transaction)?; - let present = self.authenticate_challenged_presence( - &authority, - states, - current_write_lease_sha256, - &message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - &response.canonical_presence, - )?; - let evidence = RuntimePresenceEvidence::from_present( - &present, - message_id, - challenged_at_unix_millis, - admitted_at_unix_millis, - )?; - Ok(CandidateAnswer::Answered { evidence, present }) - } - - fn rehydrate_ready_token( - &self, - transaction: &DeploymentTransaction, - now: u64, - require_current: bool, - ) -> Result { - let evidence = required(&transaction.ready, "durable Ready evidence")? - .odin() - .context("durable Ready evidence is not an Odin receipt")? - .clone(); - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let current_lease = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let authenticated = self.authenticate_topology_bytes( - &snapshot, - transaction, - &evidence.canonical_bytes, - current_lease, - if require_current { - now - } else { - evidence.admitted_at_unix_millis - }, - )?; - validate_authenticated_evidence(&evidence, &authenticated)?; - ensure!( - is_semantic_ready(&authenticated), - "durable Ready evidence no longer authenticates as Ready" - ); - Ok(SequenceAdmittedReady { - transaction_id: transaction.transaction_id.clone(), - evidence, - expected: required(&transaction.expected, "Expected projection")?.clone(), - authenticated, - }) - } - - fn current_ready_provider_tokens( - &self, - snapshot: &ControlSnapshot, - ) -> Result> { - let mut providers = Vec::new(); - for stored in &snapshot.admitted { - // No pre-filter on ready == latest: a provider that published again - // after going ready is still ready. rehydrate_admitted_ready owns - // that judgment now, and reports why when it refuses. - match self.rehydrate_admitted_ready(snapshot, &stored.value) { - Ok(provider) => providers.push(provider), - Err(error) => eprintln!( - "Idunn excluded non-current provider {}: {error:#}", - stored.value.target - ), - } - } - Ok(providers) - } - - fn rehydrate_admitted_ready( - &self, - snapshot: &ControlSnapshot, - generation: &AdmittedGeneration, - ) -> Result { - let odin = generation.odin_receipts()?; - let authority = self.runtime_authority_parts( - &generation.plan, - &generation.expected, - &generation.activation, - )?; - let odin_authority = self.current_odin_authority(snapshot)?; - let current_lease = generation.leasing.lease_sha256(); - let authenticated = authenticate_odin_runtime_topology_correlation( - &odin.ready.canonical_bytes, - &authority, - current_lease, - &odin_authority.signer_public_key, - self.trusted_topology_context(odin.ready.admitted_at_unix_millis), - )?; - validate_authenticated_evidence(odin.ready, &authenticated)?; - ensure!( - is_semantic_ready(&authenticated), - "admitted provider's Ready receipt is not Ready at its admission time" - ); - // A provider that published after going ready is the ordinary case, not - // a stale one, so the cursor is authenticated on its own terms rather - // than compared to the receipt. Authentication is what binds it to this - // incarnation -- it requires the record's current_activation_sha256 and - // runtime_instance_id to match this generation's activation -- so a - // separate identity check here would only restate it more weakly. - // Note this is the cursor as frozen at admission, not a live read of - // Odin. Both receipts are historical proofs; current workload health - // is owned by continuity and route observation, not receipt age. - let authenticated_latest = authenticate_odin_runtime_topology_correlation( - &odin.latest.canonical_bytes, - &authority, - current_lease, - &odin_authority.signer_public_key, - self.trusted_topology_context(odin.latest.admitted_at_unix_millis), - )?; - validate_authenticated_evidence(odin.latest, &authenticated_latest)?; - ensure!( - is_semantic_ready(&authenticated_latest), - "admitted provider's latest receipt is not Ready at its admission time" - ); - Ok(SequenceAdmittedReady { - transaction_id: generation.transaction_id.clone(), - evidence: odin.ready.clone(), - expected: generation.expected.clone(), - authenticated, - }) - } - - fn validate_selected_providers_current( - &self, - plan: &CompiledDeploymentPlan, - ) -> Result<()> { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - for selection in &plan.dependencies { - let Some(provider) = &selection.provider else { - ensure!( - selection.requirement.kind == DependencyKind::Optional, - "required dependency has no provider" - ); - continue; - }; - let DependencyProviderAuthority::ManagedReady { - target, - incarnation_id, - plan_id, - sealed_release_id, - expected_projection_sha256, - odin_topology_correlation_sha256, - odin_topology_publisher_sequence, - } = &provider.authority - else { - continue; - }; - let admitted = snapshot - .admitted_for(target) - .context("selected managed dependency is no longer admitted")?; - let token = self.rehydrate_admitted_ready(&snapshot, &admitted.value)?; - ensure!( - admitted.value.expected.incarnation_id == *incarnation_id - && admitted.value.plan.plan_id == *plan_id - && admitted.value.sealed_release.sealed_release_id == *sealed_release_id - && admitted.value.expected.canonical_sha256()? == *expected_projection_sha256 - && token.publisher_sequence() >= *odin_topology_publisher_sequence, - "selected dependency authority changed before actuation" - ); - ensure!( - token.publisher_sequence() != *odin_topology_publisher_sequence - || token.evidence_sha256() == odin_topology_correlation_sha256, - "selected dependency Odin sequence changed evidence" - ); - ensure!( - token - .authenticated() - .record() - .observed_capabilities - .iter() - .any(|capability| { - capability_compatible( - &selection.requirement.capability, - &selection.requirement.schema, - &selection.requirement.compatibility, - &capability.capability, - &capability.schema, - &capability.compatibility, - ) && capability.capacity >= selection.requirement.minimum_capacity - }), - "selected dependency no longer provides its required capability" - ); - } - Ok(()) - } - - fn persist_same_phase( - &self, - current: &Stored, - mutation: F, - ) -> Result<()> - where - F: FnOnce(&mut DeploymentTransaction) -> Result<()>, - { - let mut next = current.value.clone(); - mutation(&mut next)?; - ensure!( - next.phase == current.value.phase, - "same-phase evidence update changed phase" - ); - next.updated_at_unix_millis = now_millis()?; - next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) - } - - fn transition( - &self, - current: &Stored, - next_phase: DeploymentPhase, - ) -> Result<()> { - ensure!( - next_phase as u8 == current.value.phase as u8 + 1, - "deployment phase transition is not adjacent" - ); - let mut next = current.value.clone(); - next.enter_phase(next_phase, now_millis()?); - next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) - } - - /// Every post-fencing phase ends. Before a live transaction past the - /// fence runs its step, a phase whose deadline has passed, or whose - /// command an operator asked to expire, is resolved as `phase_end` says. - /// Returns `Some(moved)` when it decided the tick, `None` to run the step. - /// - /// The deadline is written only by `enter_phase`, so nothing here or in a - /// resume path extends it. A record lifted from before deadlines existed - /// has none; its phase is timed once, from the first time this sees it. - fn resolve_phase_deadline( - &self, - current: &Stored, - ) -> Result> { - let transaction = ¤t.value; - if !transaction.resolves_by_deadline() { - return Ok(None); - } - let now = now_millis()?; - let Some(deadline) = transaction.phase_deadline else { - let mut next = transaction.clone(); - next.enter_phase(transaction.phase, now); - if next.phase_deadline.is_none() { - return Ok(None); - } - replace_transaction(&self.options.state_store, current, &next)?; - return Ok(Some(true)); - }; - let requested = ControlSnapshot::read(&self.options.state_store)? - .expiry_requests - .iter() - .any(|request| request.value.command_id == transaction.command_id); - let cause = if now >= deadline.deadline_at_unix_millis { - format!( - "the {:?} phase passed its deadline at unix ms {}", - transaction.phase, deadline.deadline_at_unix_millis - ) - } else if requested { - format!("the {:?} phase was expired by the operator", transaction.phase) - } else { - return Ok(None); - }; - match transaction.phase_end() { - PhaseEnd::Abort { lease_not_adopted } => { - let error = if lease_not_adopted { - anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") - } else { - anyhow!("{cause}") - }; - self.begin_post_fencing_abort(current, error)?; - Ok(Some(true)) - } - PhaseEnd::OperatorRequired => { - self.report_once( - &format!("deadline:{}", transaction.transaction_id), - format!( - "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", - transaction.transaction_id - ), - ); - Ok(None) - } - } - } - - /// Delete expiry requests whose command has no phase left to end: it was - /// resolved, it finished, or its command is gone. - fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { - let spent = snapshot - .expiry_requests - .iter() - .filter(|request| { - !snapshot.transactions.iter().any(|stored| { - stored.value.command_id == request.value.command_id - && stored.value.resolves_by_deadline() - }) - }) - .map(|request| request.envelope.clone()) - .collect::>(); - if spent.is_empty() { - return Ok(false); - } - SingleFileMessagePackBackingStore::new(&self.options.state_store) - .delete_batch_if_unchanged(&spent) - } - - fn record_resumable_error( - &self, - current: &Stored, - error: &anyhow::Error, - ) -> Result<()> { - let detail = truncate(&format!("{error:#}"), 2048); - if current.value.last_error.as_deref() == Some(detail.as_str()) { - return Ok(()); - } - let mut next = current.value.clone(); - next.last_error = Some(detail); - next.updated_at_unix_millis = now_millis()?; - replace_transaction(&self.options.state_store, current, &next) - } - - fn begin_pre_fencing_abort( - &self, - current: &Stored, - error: anyhow::Error, - ) -> Result<()> { - ensure!( - current.value.phase < DeploymentPhase::Fencing, - "post-fence transaction cannot terminal-fail" - ); - ensure!( - current.value.pre_fencing_abort.is_none(), - "pre-fencing abort intent is already durable" - ); - let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); - self.persist_same_phase(current, |next| { - next.pre_fencing_abort = Some(abort); - Ok(()) - }) - } - - /// Whether this transaction's candidate can no longer run at all. - /// - /// A transaction that has never started one has no candidate to be dead, so - /// it is not permanently stopped -- it is simply not there yet. - fn candidate_is_permanently_stopped( - &self, - transaction: &DeploymentTransaction, - ) -> Result { - let Some(workload) = &transaction.workload else { - return Ok(false); - }; - self.workload_for(required(&transaction.plan, "transaction plan")?)? - .is_permanently_stopped(workload) - } - - fn begin_post_fencing_abort( - &self, - current: &Stored, - error: anyhow::Error, - ) -> Result<()> { - ensure!( - current.value.phase >= DeploymentPhase::Fencing, - "pre-fence transaction must abort through the pre-fencing path" - ); - ensure!( - current.value.post_fencing_abort.is_none(), - "post-fencing abort intent is already durable" - ); - let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); - self.persist_same_phase(current, |next| { - next.post_fencing_abort = Some(abort); - Ok(()) - }) - } - - /// Withdraw the candidate in the one order that never leaves two writers: - /// take away its route, then its write lease, then the process itself, and - /// only then reconcile the projection. - fn advance_post_fencing_abort(&self, current: &Stored) -> Result<()> { - let abort = required( - ¤t.value.post_fencing_abort, - "post-fencing abort intent", - )?; - if abort.route_restoration == CleanupEvidence::Pending { - let preflight = required(¤t.value.route_preflight, "route preflight receipt")?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let driver = - self.route_driver(binding.route.context("routed plan has no route binding")?); - driver - .withdraw_candidate_membership( - preflight, - &self.route_gate(¤t.value.target, current.value.command_kind), - ) - .context("restoring the route the candidate found")?; - return self.persist_same_phase(current, |next| { - next.post_fencing_abort.as_mut().unwrap().route_restoration = - CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.lease_withdrawal == CleanupEvidence::Pending { - let expected = required(¤t.value.expected, "Expected projection")?; - let activation = required(¤t.value.activation, "activation")?; - // A prepared lease is withdrawn too: its physical grant may have - // landed before `Granted` was durable. Revoking an absent lease, - // and withdrawing an absent projection, are no-ops. - let lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::issued_lease) - .context("post-fencing abort lost the lease it must withdraw")?; - let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; - let lease_path = binding - .process_write_lease - .context("leased target has no write-lease binding")? - .record_path; - let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - driver - .revoke_exact(Some(lease)) - .context("revoking the abandoned candidate write lease")?; - ensure!( - driver.observe_empty()?, - "candidate write lease remained after post-fencing withdrawal" - ); - self.topology() - .withdraw_process_write_lease(expected, activation, Some(lease)) - .context("withdrawing the abandoned candidate write-lease projection")?; - return self.persist_same_phase(current, |next| { - next.post_fencing_abort.as_mut().unwrap().lease_withdrawal = - CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.candidate_cleanup == CleanupEvidence::Pending { - let plan = required(¤t.value.plan, "abandoned candidate plan")?; - if let Some(workload) = ¤t.value.workload { - self.workload_for(plan)? - .stop(workload) - .context("stopping the abandoned candidate")?; - } - self.workload_for(plan)? - .discard_prepared( - required(¤t.value.plan, "abandoned candidate plan")?, - required(¤t.value.expected, "abandoned Expected projection")?, - required(¤t.value.activation, "abandoned activation")?, - ) - .context("discarding the abandoned activation material")?; - return self.persist_same_phase(current, |next| { - next.post_fencing_abort.as_mut().unwrap().candidate_cleanup = - CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.topology_reconciliation == CleanupEvidence::Pending { - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - // The failed candidate's projection is resolved first. For a - // deployment, fencing also stopped the incumbent and revoked its - // lease, so the incumbent is demoted to Expected-only under its - // key with its own exact activation, which is what lets - // continuity bring it back. A continuity candidate is the - // incumbent's key: it has nothing further to demote. - let topology = self.topology(); - let failed_provider_anchor = self.provider_anchor_for_plan(required( - ¤t.value.plan, - "failed transaction plan", - )?)?; - reconcile_failed_candidate_projection( - &topology, - ¤t.value, - &failed_provider_anchor, - )?; - if let Some(incumbent) = self - .exact_incumbent(&snapshot, ¤t.value)? - .filter(|_| current.value.command_kind == CommandKind::Deploy) - { - let admitted_provider_anchor = - self.provider_anchor_for_plan(&incumbent.value.plan)?; - let expected_sha256 = topology.demote_to_expected_only( - &incumbent.value.expected, - &admitted_provider_anchor, - &incumbent.value.activation, - incumbent.value.leasing.lease(), - )?; - ensure!( - expected_sha256 == incumbent.value.expected.canonical_sha256()?, - "restored incumbent Expected receipt differs" - ); - } - return self.persist_same_phase(current, |next| { - next.post_fencing_abort - .as_mut() - .unwrap() - .topology_reconciliation = CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.source_cleanup == CleanupEvidence::Pending { - self.source - .cleanup( - ¤t.value.transaction_id, - current.value.frozen_source.as_ref(), - ) - .context("cleaning the abandoned source")?; - return self.persist_same_phase(current, |next| { - next.post_fencing_abort.as_mut().unwrap().source_cleanup = - CleanupEvidence::Complete; - Ok(()) - }); - } - ensure!( - abort.is_complete(), - "post-fencing abort cleanup is incomplete" - ); - let mut next = current.value.clone(); - next.enter_phase(DeploymentPhase::Complete, now_millis()?); - next.last_error = Some(abort.error.clone()); - next.completion = Some(TransactionCompletion::FailedAfterFencing { - error: abort.error.clone(), - recovery: current.value.terminal_recovery(), - }); - replace_transaction(&self.options.state_store, current, &next) - } - - fn advance_pre_fencing_abort(&self, current: &Stored) -> Result<()> { - ensure!( - current.value.phase < DeploymentPhase::Fencing, - "pre-fencing abort crossed the fencing boundary" - ); - let abort = required(¤t.value.pre_fencing_abort, "pre-fencing abort intent")?; - if abort.candidate_cleanup == CleanupEvidence::Pending { - let plan = required(¤t.value.plan, "transaction plan")?; - if let Some(workload) = ¤t.value.workload { - self.workload_for(plan)? - .stop(workload) - .context("stopping exact pre-fence candidate")?; - } - self.workload_for(plan)? - .discard_prepared( - required(¤t.value.plan, "pre-fencing candidate plan")?, - required(¤t.value.expected, "pre-fencing Expected projection")?, - required(¤t.value.activation, "pre-fencing activation")?, - ) - .context("discarding exact pre-fence activation material")?; - return self.persist_same_phase(current, |next| { - next.pre_fencing_abort.as_mut().unwrap().candidate_cleanup = - CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.topology_reconciliation == CleanupEvidence::Pending { - // Before fencing a deployment never touched its incumbent: it is - // still running under its own key with its activation and lease, - // so only the candidate's records are withdrawn. A continuity - // candidate shares the admitted key and demotes what it issued. - let plan = required(¤t.value.plan, "failed transaction plan")?; - let provider_anchor = self.provider_anchor_for_plan(plan)?; - reconcile_failed_candidate_projection( - &self.topology(), - ¤t.value, - &provider_anchor, - )?; - return self.persist_same_phase(current, |next| { - next.pre_fencing_abort - .as_mut() - .unwrap() - .topology_reconciliation = CleanupEvidence::Complete; - Ok(()) - }); - } - if abort.source_cleanup == CleanupEvidence::Pending { - self.source - .cleanup( - ¤t.value.transaction_id, - current.value.frozen_source.as_ref(), - ) - .context("cleaning failed pre-fence source")?; - return self.persist_same_phase(current, |next| { - next.pre_fencing_abort.as_mut().unwrap().source_cleanup = CleanupEvidence::Complete; - Ok(()) - }); - } - ensure!( - abort.is_complete(), - "pre-fencing abort cleanup is incomplete" - ); - let mut next = current.value.clone(); - next.enter_phase(DeploymentPhase::Complete, now_millis()?); - next.last_error = Some(abort.error.clone()); - next.completion = Some(TransactionCompletion::FailedBeforeFencing { - error: abort.error.clone(), - }); - replace_transaction(&self.options.state_store, current, &next) - } - - fn advance_post_commit_cleanup(&self, current: &Stored) -> Result<()> { - let cleanup = required( - ¤t.value.post_commit_cleanup, - "post-commit cleanup evidence", - )?; - if let IncumbentCleanupEvidence::Pending { - generation_id, - workload, - } = &cleanup.incumbent - { - let binding = required(¤t.value.plan, "committed transaction plan")? - .parsed_inputs()? - .1; - if let Some(promoted_at_unix_millis) = required( - ¤t.value.routing, - "committed transaction routing evidence", - )? - .promoted_at_unix_millis() - { - let retire_not_before = - route_drain_deadline(promoted_at_unix_millis, binding.rollout.drain_seconds)?; - if now_millis()? < retire_not_before { - return Ok(()); - } - } - // The incumbent is the same target under the same binding kind; - // an incumbent of another kind is refused by the driver, not - // guessed at. - self.workload_for(required(¤t.value.plan, "committed transaction plan")?)? - .stop(workload) - .with_context(|| format!("retiring admitted incumbent {generation_id}"))?; - let generation_id = generation_id.clone(); - return self.persist_same_phase(current, |next| { - next.post_commit_cleanup.as_mut().unwrap().incumbent = - IncumbentCleanupEvidence::Complete { generation_id }; - Ok(()) - }); - } - if cleanup.source == SourceCleanupEvidence::Pending { - self.source - .cleanup( - ¤t.value.transaction_id, - current.value.frozen_source.as_ref(), - ) - .context("cleaning committed deployment source")?; - return self.persist_same_phase(current, |next| { - next.post_commit_cleanup.as_mut().unwrap().source = SourceCleanupEvidence::Complete; - Ok(()) - }); - } - ensure!(cleanup.is_complete(), "post-commit cleanup is incomplete"); - Ok(()) - } -} - -fn route_drain_deadline(promoted_at_unix_millis: u64, drain_seconds: u32) -> Result { - promoted_at_unix_millis - .checked_add( - u64::from(drain_seconds) - .checked_mul(1_000) - .context("route drain duration overflows milliseconds")?, - ) - .context("route drain deadline overflows Unix milliseconds") -} - -/// CultLib refuses a correlation older than the trusted window with this -/// text. Idunn reads that one refusal as "nothing new", everywhere it waits -/// on Odin's evidence; every other refusal stays a refusal. -fn is_stale_observation(error: &anyhow::Error) -> bool { - format!("{error:#}").contains("outside the trusted observation window") -} - -fn route_observation_is_current( - observed_at_unix_millis: u64, - now_unix_millis: u64, - maximum_age_millis: u64, - maximum_future_skew_millis: u64, -) -> bool { - observed_at_unix_millis <= now_unix_millis.saturating_add(maximum_future_skew_millis) - && now_unix_millis.saturating_sub(observed_at_unix_millis) <= maximum_age_millis -} - -fn provider_warming_advanced( - prior_odin_sequence: u64, - prior_signed_presence_sha256: &str, - candidate_odin_sequence: u64, - candidate_signed_presence_sha256: &str, -) -> bool { - candidate_odin_sequence > prior_odin_sequence - && candidate_signed_presence_sha256 != prior_signed_presence_sha256 -} - -/// Resolve the projection a failed candidate left. A deployment candidate owns -/// its own incarnation key and withdraws it whole. A continuity candidate -/// shares the admitted key: the admitted Expected is never withdrawn, and the -/// only thing demoted is the activation this transaction issued, exactly. -fn reconcile_failed_candidate_projection( - topology: &CultCacheTopologyDriver, - transaction: &DeploymentTransaction, - provider_anchor: &ServiceIdentityTrustAnchor, -) -> Result<()> { - let expected = required(&transaction.expected, "failed Expected projection")?; - match transaction.command_kind { - CommandKind::Deploy => topology - .withdraw_incarnation( - expected, - provider_anchor, - transaction.activation.as_ref(), - None, - ) - .context("withdrawing the failed candidate projection"), - CommandKind::Continuity => topology - .demote_to_expected_only( - expected, - provider_anchor, - required(&transaction.activation, "failed continuity activation")?, - None, - ) - .map(drop) - .context("demoting the failed continuity activation"), - } -} - -fn may_rollback_route_after_failed_proof(fencing: &FencingEvidence) -> bool { - matches!(fencing, FencingEvidence::SkippedStateless) -} - -fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { - matches!( - fencing, - FencingEvidence::Revoked { - incumbent_lease_sha256: Some(_), - .. - } - ) -} - -fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { - PreFencingAbort { - error: truncate(error, 2048), - candidate_cleanup: transaction.candidate_cleanup_owed(), - topology_reconciliation: transaction.abort_topology_reconciliation(), - source_cleanup: if transaction.command_kind == CommandKind::Deploy { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - } -} - -fn post_fencing_abort_intent( - transaction: &DeploymentTransaction, - error: &str, -) -> PostFencingAbort { - PostFencingAbort { - error: truncate(error, 2048), - route_restoration: if transaction.route_preflight.is_some() { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - lease_withdrawal: if transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::issued_lease) - .is_some() - { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - candidate_cleanup: transaction.candidate_cleanup_owed(), - topology_reconciliation: transaction.abort_topology_reconciliation(), - source_cleanup: if transaction.command_kind == CommandKind::Deploy { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - } -} - -fn sequence_requires_admission( - latest: Option<&TopologyEvidence>, - maximum_admitted_sequence: u64, - candidate: &TopologyEvidence, -) -> Result { - candidate.validate_shape()?; - if latest.is_some_and(|existing| { - existing.canonical_bytes == candidate.canonical_bytes - && existing.canonical_sha256 == candidate.canonical_sha256 - && existing.signer_identity_id == candidate.signer_identity_id - && existing.publisher_sequence == candidate.publisher_sequence - }) { - return Ok(false); - } - ensure!( - candidate.publisher_sequence > maximum_admitted_sequence, - "Odin topology publisher sequence was replayed or reordered" - ); - Ok(true) -} - -fn validate_live_providers_for_deploy(command_kind: CommandKind, validation: F) -> Result<()> -where - F: FnOnce() -> Result<()>, -{ - match command_kind { - CommandKind::Deploy => validation(), - CommandKind::Continuity => Ok(()), - } -} - -fn validate_authenticated_evidence( - evidence: &TopologyEvidence, - authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, -) -> Result<()> { - let record = authenticated.record(); - ensure!( - authenticated.canonical_bytes() == evidence.canonical_bytes.as_slice() - && sha256_id(authenticated.canonical_bytes()) == evidence.canonical_sha256 - && record.signer_identity_id == evidence.signer_identity_id - && record.publisher_sequence == evidence.publisher_sequence, - "durable topology evidence differs from its authenticated receipt" - ); - Ok(()) -} - -fn is_semantic_warming( - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - incumbent_lease_sha256: Option<&str>, - authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, -) -> Result { - let record = authenticated.record(); - if !record.present || record.observed_write_lease_sha256.is_some() { - return Ok(false); - } - let expected_projection_detail = format!( - "expected:{};activation:{}", - expected.canonical_sha256()?, - activation.canonical_sha256()? - ); - if !warming_disagreements_match_incumbent( - &expected_projection_detail, - incumbent_lease_sha256, - &record.disagreements, - ) { - return Ok(false); - } - Ok(if expected.write_lease_required { - !record.ready && record.observed_presence_state.as_deref() == Some("warming") - } else { - matches!( - record.observed_presence_state.as_deref(), - Some("warming" | "active") - ) - }) -} - -fn warming_disagreements_match_incumbent( - expected_projection_detail: &str, - incumbent_lease_sha256: Option<&str>, - disagreements: &[OdinTopologyDisagreement], -) -> bool { - if disagreements.is_empty() { - true - } else if let (Some(incumbent_lease_sha256), [disagreement]) = - (incumbent_lease_sha256, disagreements) - { - disagreement.code == "projected-write-lease" - && disagreement.expected.as_deref() == Some(expected_projection_detail) - && disagreement.observed.as_deref() == Some(incumbent_lease_sha256) - } else { - false - } -} - -fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool { - let record = authenticated.record(); - record.present - && record.ready - && record.observed_presence_state.as_deref() == Some("active") - && record.disagreements.is_empty() -} - -fn read_deployment_brake(path: &Path) -> Result)>> { - let Some(envelope) = read_single_envelope(path)? else { - return Ok(None); - }; - ensure!( - envelope.r#type == IdunnDeploymentBrakeRecord::TYPE - && envelope.schema_id.as_deref() == Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA), - "deployment brake store contains a foreign record" - ); - let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&envelope.payload)?; - record.validate()?; - ensure!( - rmp_serde::to_vec(&record)? == envelope.payload && envelope.key == record.brake_id, - "deployment brake is noncanonical or keyed by another authority" - ); - Ok(Some((record, envelope.payload))) -} - -fn read_lifecycle_brake(path: &Path) -> Result> { - let Some(envelope) = read_single_envelope(path)? else { - return Ok(None); - }; - ensure!( - envelope.r#type == IdunnLifecycleBrakeRecord::TYPE - && envelope.schema_id.as_deref() == Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA), - "lifecycle brake store contains a foreign record" - ); - IdunnLifecycleBrakeRecord::decode_canonical(&envelope.payload).map(Some) -} - -fn read_single_envelope(path: &Path) -> Result> { - if !path.exists() { - return Ok(None); - } - let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - match entries.as_slice() { - [] => Ok(None), - [entry] => Ok(Some(entry.clone())), - _ => bail!("single-record authority store is ambiguous"), - } -} - -pub(crate) fn read_trust_anchor( - path: &Path, -) -> Result { - let envelope = read_single_envelope(path)? - .with_context(|| format!("service identity trust anchor {} is absent", path.display()))?; - ensure!( - envelope.r#type == P::TRUST_ANCHOR_TYPE - && envelope.key == P::TRUST_ANCHOR_KEY - && envelope.schema_id.as_deref() == Some(P::TRUST_ANCHOR_SCHEMA), - "service identity trust anchor belongs to another profile" - ); - let anchor: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&envelope.payload)?; - ensure!( - rmp_serde::to_vec(&anchor)? == envelope.payload - && derive_service_identity_id::

(&anchor.public_key)? == anchor.identity_id, - "service identity trust anchor is noncanonical or self-inconsistent" - ); - Ok(anchor) -} - -fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { - value.as_ref().with_context(|| format!("{label} is absent")) -} - -fn now_millis() -> Result { - Ok(SystemTime::now() - .duration_since(UNIX_EPOCH) - .context("system clock predates Unix epoch")? - .as_millis() - .try_into()?) -} - -fn rfc3339_millis(millis: u64) -> Result { - chrono::DateTime::from_timestamp_millis(i64::try_from(millis)?) - .context("timestamp is out of range") - .map(|value| value.to_rfc3339()) -} - -fn sibling_path(path: &Path, suffix: &str) -> PathBuf { - let mut value = path.as_os_str().to_os_string(); - value.push(suffix); - PathBuf::from(value) -} - -fn string_value(args: &mut impl Iterator, name: &str) -> Result { - args.next() - .ok_or_else(|| anyhow!("{name} requires a value")) -} - -fn path_value(args: &mut impl Iterator, name: &str) -> Result { - Ok(PathBuf::from(string_value(args, name)?)) -} - -fn u32_value(args: &mut impl Iterator, name: &str) -> Result { - string_value(args, name)? - .parse() - .with_context(|| format!("{name} requires a u32")) -} - -fn u64_value(args: &mut impl Iterator, name: &str) -> Result { - string_value(args, name)? - .parse() - .with_context(|| format!("{name} requires a u64")) -} - -fn require_selector(value: &str) -> Result<()> { - let value = value.strip_prefix("profile:").unwrap_or(value); - require_id(value, "deployment selector") -} - -pub(crate) fn require_id(value: &str, label: &str) -> Result<()> { - ensure!( - !value.is_empty() - && value.len() <= 256 - && value.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/') - }), - "{label} is invalid" - ); - Ok(()) -} - -fn require_value(value: &str, label: &str) -> Result<()> { - ensure!( - !value.trim().is_empty() - && value == value.trim() - && value.len() <= 1024 - && !value.contains('\0'), - "{label} is invalid" - ); - Ok(()) -} - -fn require_detail(value: &str, label: &str) -> Result<()> { - ensure!( - !value.is_empty() && value.len() <= 2048 && !value.contains('\0'), - "{label} is invalid" - ); - Ok(()) -} - -fn truncate(value: &str, length: usize) -> String { - value.chars().take(length).collect() -} - -fn sha256_id(bytes: &[u8]) -> String { - let digest = Sha256::digest(bytes); - let mut encoded = String::with_capacity(71); - encoded.push_str("sha256-"); - for byte in digest { - use std::fmt::Write as _; - write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); - } - encoded -} - -fn runtime_instance_id(transaction_id: &str) -> Result { - require_id(transaction_id, "runtime transaction id")?; - Ok(sha256_id( - format!("gamecult.idunn.runtime-instance.v1:{transaction_id}").as_bytes(), - )) -} - -fn usage() -> &'static str { - "Idunn deployment, admission, and continuity control plane\n\n\ - idunn serve [runtime options] [--host-actuator-bind ADDR]\n\ - idunn up [--state-store PATH] [--no-wait]\n\ - idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ - idunn cancel [--state-store PATH]\n\ - idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ - Recipes describe capability and process requirements. Idunn seals exact\n\ - source and artifacts, admits one incarnation, and delegates execution to\n\ - systemd and routing mechanics to the configured proxy." -} - -#[cfg(test)] -mod tests { - use cultnet_rs::{ - GameCultProviderHealthIdentity, OdinRuntimeTopologyCorrelationPurpose, - OdinRuntimeTopologyCorrelationRecord, enroll_service_identity_at, - export_service_identity_trust_anchor, - }; - use tempfile::TempDir; - - use super::*; - use crate::drivers::{SystemdWorkloadObservation, incarnation_key}; - use crate::incident::INCIDENT_RETENTION_MILLIS; - - /// Fixed clock for the signing fixture; correlations must land inside - /// DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS of it to authenticate. - const NOW: u64 = 1_700_000_000_000; - - fn digest(byte: char) -> String { - byte.to_string().repeat(64) - } - - fn route_binding( - route_id: &str, - stable_port: u16, - private_port_start: u16, - private_port_end: u16, - ) -> RouteBinding { - RouteBinding { - driver: crate::deployment::RouteDriver::NginxStreamTcp, - route_id: route_id.into(), - stable_endpoint: format!("tcp://127.0.0.1:{stable_port}"), - private_host: "127.0.0.1".into(), - private_port_start, - private_port_end, - config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), - reload_unit: "nginx.service".into(), - } - } - - fn command(kind: CommandKind) -> DeploymentCommand { - DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: match kind { - CommandKind::Deploy => "up-test".into(), - CommandKind::Continuity => "continuity-test".into(), - }, - kind, - selector: "ghostlight".into(), - requested_by: "operator".into(), - requested_at_unix_millis: 100, - } - } - - /// Drives the replacement gate against the real `control.cc`. The point is - /// not that the unit tests pass but that the record actually stuck on the - /// host would now commit. Run with: - /// IDUNN_LIVE_CONTROL=/path/to/control.cc cargo test live_committing -- --ignored --nocapture - #[test] - #[ignore] - fn live_committing_record_is_unstuck_by_the_replacement_gate() -> Result<()> { - let Ok(path) = std::env::var("IDUNN_LIVE_CONTROL") else { - eprintln!("IDUNN_LIVE_CONTROL unset"); - return Ok(()); - }; - let snapshot = ControlSnapshot::read(Path::new(&path))?; - for stored in &snapshot.transactions { - let value = &stored.value; - if value.completion.is_some() { - continue; - } - let (Some(ready), Some(latest)) = ( - value.ready.as_ref().and_then(ReadinessEvidence::odin), - &value.latest_odin_observation, - ) else { - continue; - }; - println!( - "LIVE {} target={} phase={:?} ready_seq={} latest_seq={}", - value.transaction_id, - value.target, - value.phase, - ready.publisher_sequence, - latest.publisher_sequence - ); - println!( - " old gate (receipt == cursor): {}", - if ready == latest { "PASS" } else { "REFUSE" } - ); - let (latest_record, _) = - OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( - &latest.canonical_bytes, - )?; - println!( - " cursor says: projection={} instance={:?} ready={} present={} disagreements={}", - latest_record.expected_projection_sha256, - latest_record.runtime_instance_id, - latest_record.ready, - latest_record.present, - latest_record.disagreements.len() - ); - let (Some(expected), Some(activation)) = (&value.expected, &value.activation) else { - println!(" transaction has not declared an incarnation yet"); - continue; - }; - println!( - " declared: projection={} instance={}", - expected.canonical_sha256()?, - activation.runtime_instance_id - ); - let binds = latest_record.expected_projection_sha256 == expected.canonical_sha256()? - && latest_record.runtime_instance_id.as_deref() - == Some(activation.runtime_instance_id.as_str()); - println!(" binds declared incarnation (what authentication enforces): {binds}"); - } - Ok(()) - } - - /// The smallest real `Engine` the scheduler needs. Idunn had no - /// Engine-level test at all, which is why "does one bad record kill the - /// daemon" was arguable rather than measurable. - struct EngineFixture { - _temp: TempDir, - engine: Engine, - state_store: PathBuf, - odin_signer: ServiceIdentitySigner, - root: PathBuf, - /// The host programs every route driver of this Engine actuates - /// through: logged stubs, so nothing touches the workstation's nginx. - #[cfg(unix)] - route_stubs: RouteStubs, - } - - /// Stub nginx, systemd-run, systemctl and ufw that append each call to one - /// log. `reload-fails` makes `systemctl reload` exit non-zero. - #[cfg(unix)] - struct RouteStubs { - calls: PathBuf, - reload_fails: PathBuf, - ufw_delete_fails: PathBuf, - } - - #[cfg(unix)] - impl RouteStubs { - fn new(root: &Path) -> Result<(RouteActuators, Self)> { - use std::os::unix::fs::PermissionsExt; - - let dir = root.join("route-stubs"); - std::fs::create_dir_all(&dir)?; - let calls = dir.join("calls"); - let reload_fails = dir.join("reload-fails"); - let ufw_delete_fails = dir.join("ufw-delete-fails"); - let program = |name: &str, refuse: &str| -> Result { - let path = dir.join(name); - std::fs::write( - &path, - format!( - "#!/bin/sh\necho \"{name} $*\" >> '{}'\n{refuse}exit 0\n", - calls.display() - ), - )?; - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))?; - Ok(path) - }; - let actuators = RouteActuators { - nginx: program("nginx", "")?, - systemd_run: program("systemd-run", "")?, - systemctl: program( - "systemctl", - &format!( - "if [ \"$1\" = reload ] && [ -e '{}' ]; then exit 1; fi\n", - reload_fails.display() - ), - )?, - ufw: program( - "ufw", - &format!( - "if [ \"$1\" = delete ] && [ -e '{}' ]; then exit 1; fi\n", - ufw_delete_fails.display() - ), - )?, - preflight_root: root.join("route-preflight"), - }; - Ok(( - actuators, - Self { - calls, - reload_fails, - ufw_delete_fails, - }, - )) - } - - fn refuse_reloads(&self) -> Result<()> { - std::fs::write(&self.reload_fails, b"x")?; - Ok(()) - } - - /// `ufw delete` exits non-zero with an error that is not "non-existent". - fn refuse_ufw_deletes(&self) -> Result<()> { - std::fs::write(&self.ufw_delete_fails, b"x")?; - Ok(()) - } - - /// How many logged calls begin with `prefix`, e.g. `systemctl reload`. - fn count(&self, prefix: &str) -> usize { - std::fs::read_to_string(&self.calls) - .unwrap_or_default() - .lines() - .filter(|line| line.starts_with(prefix)) - .count() - } - } - - impl EngineFixture { - fn new() -> Result { - Self::build(None) - } - - /// An Engine whose systemd workload port is `workload`, so the phase - /// machine can run without systemd. This is the whole seam: nginx, - /// systemctl and the network are never reached on an unrouted, - /// stateless target, and the Odin evidence is signed by the key this - /// fixture enrolled. - fn with_workload(workload: Arc) -> Result { - Self::build(Some(workload)) - } - - fn build(workload: Option>) -> Result { - let temp = TempDir::new()?; - let root = temp.path(); - std::fs::create_dir_all(root.join("identities"))?; - let idunn = root.join("identities/idunn.cc"); - enroll_service_identity_at::(&idunn)?; - let odin_private = root.join("identities/odin.cc"); - let odin_signer = enroll_service_identity_at::(&odin_private)?; - let odin_anchor = root.join("identities/odin-anchor.cc"); - export_service_identity_trust_anchor(&odin_signer, &odin_anchor)?; - - let state_store = root.join("control.cc"); - #[cfg(unix)] - let (route_actuators, route_stubs) = RouteStubs::new(root)?; - let options = RuntimeOptions { - state_store: state_store.clone(), - bindings_dir: root.join("bindings"), - source_root: root.join("sources"), - staging_root: root.join("staging"), - topology_store: root.join("topology.cc"), - incident_store: root.join("incidents.cc"), - odin_correlation_store: root.join("odin-correlation.cc"), - odin_trust_anchor: odin_anchor, - idunn_identity_store: idunn, - deployment_brake_operator_anchor: root.join("brake-anchor.cc"), - #[cfg(unix)] - route_actuators, - ..RuntimeOptions::default() - }; - let engine = match workload { - Some(workload) => Engine::open_with_systemd_workload(options, workload)?, - None => Engine::open(options)?, - }; - Ok(Self { - root: root.to_path_buf(), - _temp: temp, - engine, - state_store, - odin_signer, - #[cfg(unix)] - route_stubs, - }) - } - } - - fn terminal_transaction(target: &str) -> Result { - Ok(terminal_transaction_with_command(target)?.0) - } - - fn terminal_transaction_with_command( - target: &str, - ) -> Result<(DeploymentTransaction, DeploymentCommand)> { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{target}"), - kind: CommandKind::Deploy, - selector: target.into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let mut transaction = DeploymentTransaction::new(&command, target.into(), 0, None, 100)?; - transaction.phase = DeploymentPhase::Complete; - transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { - error: "sealed source vanished".into(), - }); - transaction.pre_fencing_abort = Some(PreFencingAbort { - error: "sealed source vanished".into(), - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Complete, - }); - assert!(transaction.is_terminal(), "fixture must be terminal"); - Ok((transaction, command)) - } - - #[test] - fn a_finished_transaction_leaves_the_live_set_and_stays_observable() -> Result<()> { - let world = EngineFixture::new()?; - let history = history_store_path(&world.state_store); - - let (live, command) = terminal_transaction_with_command("ghostlight")?; - // A transaction is only readable beside its immutable command. - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope( - &command, - command.requested_at_unix_millis - )?], - )? - ); - let mut opening = live.clone(); - opening.phase = DeploymentPhase::Sealing; - opening.completion = None; - opening.pre_fencing_abort = None; - let opening_envelope = transaction_envelope(&opening, opening.updated_at_unix_millis)?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: opening.transaction_id.clone(), - current: None, - }], - std::slice::from_ref(&opening_envelope), - )? - ); - let stored = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .next() - .context("seeded transaction")?; - - replace_transaction(&world.state_store, &stored, &live)?; - - // Gone from the set that still gates decisions... - assert!( - ControlSnapshot::read(&world.state_store)? - .transactions - .is_empty(), - "a finished transaction stayed resident" - ); - // ...and still answerable, which is what R11 refused to trade away. - let archived = read_history_transactions(&world.state_store); - assert_eq!(archived.len(), 1); - assert_eq!(archived[0].transaction_id, live.transaction_id); - assert!(history.exists()); - // The command went with it. Left resident it would read as queued and - // be frozen again next tick; consumed, it is history beside its - // transaction and `status` still answers for it. - assert!( - ControlSnapshot::read(&world.state_store)? - .commands - .is_empty(), - "a consumed command stayed resident" - ); - let archived_commands = read_history_commands(&world.state_store); - assert_eq!(archived_commands.len(), 1); - assert_eq!(archived_commands[0].command_id, command.command_id); - assert!(!world.engine.freeze_one_queued_command()?); - Ok(()) - } - - #[test] - fn a_resident_command_whose_transactions_are_history_is_retired_not_frozen() -> Result<()> { - // Before commands travelled to history with their transactions, a - // command outlived its retired transaction in the control store and - // read as queued again. Freeze must retire it, never re-freeze it. - let world = EngineFixture::new()?; - let (finished, command) = terminal_transaction_with_command("ghostlight")?; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( - command_envelope(&command, command.requested_at_unix_millis)?, - )?; - SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) - .insert_entry_if_absent(transaction_envelope( - &finished, - finished.updated_at_unix_millis, - )?)?; - assert_eq!(ControlSnapshot::read(&world.state_store)?.commands.len(), 1); - - assert!(world.engine.freeze_one_queued_command()?); - - assert!( - ControlSnapshot::read(&world.state_store)? - .commands - .is_empty(), - "consumed command stayed resident" - ); - assert_eq!(read_history_commands(&world.state_store).len(), 1); - assert!( - ControlSnapshot::read(&world.state_store)? - .transactions - .is_empty(), - "a consumed command was frozen again" - ); - assert!(!world.engine.freeze_one_queued_command()?); - Ok(()) - } - - #[test] - fn a_resident_terminal_transaction_is_retired_by_the_tick_with_its_command() -> Result<()> { - let world = EngineFixture::new()?; - let (finished, command) = terminal_transaction_with_command("ghostlight")?; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( - command_envelope(&command, command.requested_at_unix_millis)?, - )?; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( - transaction_envelope(&finished, finished.updated_at_unix_millis)?, - )?; - assert!(world.engine.retire_one_terminal_transaction()?); - let live = ControlSnapshot::read(&world.state_store)?; - assert!(live.transactions.is_empty() && live.commands.is_empty()); - assert_eq!(read_history_transactions(&world.state_store).len(), 1); - assert_eq!(read_history_commands(&world.state_store).len(), 1); - assert!(!world.engine.retire_one_terminal_transaction()?); - Ok(()) - } - - #[test] - fn re_archiving_after_a_crash_window_is_a_no_op() -> Result<()> { - // History is written before the live copy is removed, so a crash - // between the two leaves the record in both. The repeat must not - // duplicate it or fail. - let world = EngineFixture::new()?; - let live = terminal_transaction("odin")?; - let envelope = transaction_envelope(&live, live.updated_at_unix_millis)?; - - archive_terminal_transaction(&world.state_store, &envelope)?; - archive_terminal_transaction(&world.state_store, &envelope)?; - - let archived = read_history_transactions(&world.state_store); - assert_eq!(archived.len(), 1, "the repeat duplicated the record"); - Ok(()) - } - - #[test] - fn an_unreadable_history_entry_is_skipped_rather_than_fatal() -> Result<()> { - // The asymmetry that justifies two files: control.cc refuses a record - // it cannot re-encode byte for byte, because those records still gate - // decisions. History describes what already happened and must never be - // able to refuse a read. - let world = EngineFixture::new()?; - let live = terminal_transaction("voidbot")?; - let envelope = transaction_envelope(&live, live.updated_at_unix_millis)?; - archive_terminal_transaction(&world.state_store, &envelope)?; - - let history = history_store_path(&world.state_store); - assert!( - SingleFileMessagePackBackingStore::new(&history).insert_entry_if_absent( - CultCacheEnvelope { - key: "tx-corrupt".into(), - r#type: DeploymentTransaction::TYPE.into(), - payload: vec![0xc1], - stored_at: rfc3339_millis(1)?, - schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), - } - )? - ); - - let archived = read_history_transactions(&world.state_store); - assert_eq!(archived.len(), 1, "the readable record must survive"); - assert_eq!(archived[0].transaction_id, live.transaction_id); - Ok(()) - } - - #[test] - fn an_absent_brake_anchor_does_not_gate_an_empty_store() -> Result<()> { - // `F:\Projects\CLAUDE.md`: a target brake "may never gate Idunn itself - // or unrelated service lifecycle". Reading the operator anchor up front - // meant a missing brake artifact refused startup against a store with - // nothing in it -- the deployment authority held hostage by an artifact - // describing one target's consent. - let world = EngineFixture::new()?; - assert!( - !world - .engine - .options - .deployment_brake_operator_anchor - .exists(), - "fixture must not have written a brake anchor" - ); - let snapshot = ControlSnapshot::read(&world.state_store)?; - world.engine.validate_durable_authority(&snapshot)?; - Ok(()) - } - - #[test] - fn one_unreadable_record_faults_its_tick_without_touching_the_store() -> Result<()> { - let world = EngineFixture::new()?; - - // An empty store schedules nothing and must not fault. - assert!(!world.engine.run_scheduler_tick()?); - - // A record the control store cannot read. Idunn runs under - // Restart=always, so before run_scheduler_tick existed this - // propagated out of the loop and crashlooped the daemon -- taking - // every unrelated target's continuity with it. - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: "tx-unreadable".into(), - current: None, - }], - &[CultCacheEnvelope { - key: "tx-unreadable".into(), - r#type: DeploymentTransaction::TYPE.into(), - // 0xc1 is msgpack's never-used byte: decodable by nothing. - payload: vec![0xc1], - stored_at: rfc3339_millis(1)?, - schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), - }], - )? - ); - let poisoned = std::fs::read(&world.state_store)?; - - for _ in 0..3 { - assert!( - world.engine.run_scheduler_tick().is_err(), - "the tick must report the fault rather than swallow it" - ); - } - - // Faulting is not the same as flailing: a tick that cannot read must - // not rewrite, truncate or repair the store behind the operator. - assert_eq!( - std::fs::read(&world.state_store)?, - poisoned, - "a faulting tick rewrote the control store" - ); - Ok(()) - } - - /// A real signing chain for topology gates. Until this existed no test in - /// this file could construct a signed Odin correlation -- every topology - /// fixture was `vec![byte]` -- so every gate between Idunn and Odin was - /// asserted against bytes that could not be authenticated. That is how a - /// comparison against Idunn's own wall clock survived in the admission - /// path. Modelled on `odin-daemon`'s TestWorld, which already had to build - /// this chain to test the other side of the same contract. - struct TopologyFixture { - _temp: TempDir, - idunn_anchor: ServiceIdentityTrustAnchor, - odin_signer: ServiceIdentitySigner, - provider_public_key: Vec, - expected: IdunnExpectedIncarnationRecord, - activation: IdunnRuntimeActivationRecord, - } - - impl TopologyFixture { - fn new(target: &str) -> Result { - let temp = TempDir::new()?; - let root = temp.path().join("identities"); - std::fs::create_dir_all(&root)?; - let idunn_signer = - enroll_service_identity_at::(&root.join("idunn.cc"))?; - let idunn_anchor = idunn_signer.trust_anchor()?; - let odin_signer = - enroll_service_identity_at::(&root.join("odin.cc"))?; - let provider_signer = enroll_service_identity_at::( - &root.join("provider.cc"), - )?; - - let expected = IdunnExpectedIncarnationRecord { - schema_version: cultnet_rs::IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), - target: target.into(), - plan_id: sha256_id(b"plan"), - incarnation_id: format!("{target}/generation-1"), - sealed_release_id: sha256_id(b"release"), - source_repository: format!("github.com/GameCult/{target}"), - source_revision: "3".repeat(40), - recipe_sha256: sha256_id(b"recipe"), - runtime_id: format!("{target}-runtime"), - expected_signer_identity_id: provider_signer.entry().identity_id.clone(), - health_contract: format!("{target}.runtime-health.v1"), - artifact_sha256: sha256_id(b"artifact"), - state_schema_generation: None, - state_contract_sha256: None, - write_lease_required: false, - route: None, - capabilities: Vec::new(), - dependencies: Vec::new(), - }; - expected.validate()?; - - // Idunn-signed, exactly as the real activation path issues it: the - // authority chain is what makes a correlation authenticable at all. - let launch = IdunnRuntimeActivationLaunch::issue( - &expected, - sha256_id(b"artifact-witness"), - NOW - 20, - &idunn_signer, - )?; - let activation = launch.activation().clone(); - - Ok(Self { - _temp: temp, - idunn_anchor, - odin_signer, - provider_public_key: provider_signer.entry().public_key.clone(), - expected, - activation, - }) - } - - fn authority(&self) -> Result { - verify_runtime_authority( - &self.expected, - &self.activation, - &self.idunn_anchor, - &self.provider_public_key, - ) - } - - fn correlation( - &self, - sequence: u64, - ready: bool, - ) -> Result { - Ok(OdinRuntimeTopologyCorrelationRecord { - schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - target: self.expected.target.clone(), - expected_projection_sha256: self.expected.canonical_sha256()?, - expected: true, - current_activation_sha256: Some(self.activation.canonical_sha256()?), - signed_presence_sha256: Some(sha256_id(b"presence")), - observed_presence_state: Some("active".into()), - observed_presence_publisher_sequence: Some(sequence), - observed_write_lease_sha256: None, - observed_capabilities: Vec::new(), - runtime_id: self.expected.runtime_id.clone(), - runtime_instance_id: Some(self.activation.runtime_instance_id.clone()), - present: true, - ready, - dependencies: Vec::new(), - disagreements: Vec::new(), - signer_identity_id: self.odin_signer.entry().identity_id.clone(), - publisher_sequence: sequence, - observed_at_unix_millis: NOW, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - }) - } - - fn sign(&self, record: &mut OdinRuntimeTopologyCorrelationRecord) -> Result> { - record.signature = self - .odin_signer - .sign::( - &record.unsigned_signature_payload()?, - ) - .signature; - record.canonical_bytes() - } - - fn authenticate( - &self, - canonical: &[u8], - ) -> Result { - authenticate_odin_runtime_topology_correlation( - canonical, - &self.authority()?, - None, - &self.odin_signer.entry().public_key, - OdinTopologyAuthenticationContext { - trusted_received_at_unix_millis: NOW, - maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, - maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, - }, - ) - } - } - - #[test] - fn the_fixture_actually_authenticates() -> Result<()> { - // First: prove the chain can REFUSE. Without this the tests below - // establish nothing -- a fixture that authenticates anything would - // make every gate look satisfied. - let world = TopologyFixture::new("odin")?; - let mut record = world.correlation(297_765, true)?; - let canonical = world.sign(&mut record)?; - world.authenticate(&canonical)?; - - // Re-encoded with the old signature over changed content. - let mut forged = record.clone(); - forged.publisher_sequence = 297_766; - assert!(world.authenticate(&forged.canonical_bytes()?).is_err()); - - let mut unsigned = world.correlation(297_765, true)?; - unsigned.signature = vec![0; 64]; - assert!(world.authenticate(&unsigned.canonical_bytes()?).is_err()); - - // Stale beyond the trusted observation window. - let mut old = world.correlation(297_765, true)?; - old.observed_at_unix_millis = NOW - DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS - 1; - let old_canonical = world.sign(&mut old)?; - assert!(world.authenticate(&old_canonical).is_err()); - Ok(()) - } - - #[test] - fn a_signed_observation_far_past_the_receipt_admits_this_incarnation() -> Result<()> { - // The yggdrasil wedge, with a real signature rather than vec![byte]: - // the cursor has run hundreds of thousands of sequences past the Ready - // receipt and still describes the incarnation the transaction declared. - let world = TopologyFixture::new("odin")?; - let mut record = world.correlation(501_713, true)?; - let canonical = world.sign(&mut record)?; - let authenticated = world.authenticate(&canonical)?; - - // Authenticating at all is the incarnation check: the authority is - // built from the declared expected and activation, and the correlation - // had to bind both to get here. Nothing further to compare. - assert!(is_semantic_ready(&authenticated)); - assert_eq!(authenticated.record().publisher_sequence, 501_713); - Ok(()) - } - - #[test] - fn authentication_binds_a_correlation_to_the_declared_incarnation() -> Result<()> { - // The failure that matters: Odin's signature is valid and the record is - // fresh, but the process it describes is not the one we activated. - // Authentication already refuses this -- the authority is built from - // this transaction's own expected and activation, and the correlation - // must bind both. Pinned here because a redundant identity gate sat on - // top of this for want of a test that could reach it. - let world = TopologyFixture::new("odin")?; - let mut restarted = world.correlation(501_713, true)?; - restarted.runtime_instance_id = Some(sha256_id(b"another-instance")); - let canonical = world.sign(&mut restarted)?; - let error = world - .authenticate(&canonical) - .expect_err("correlation must bind the current activation"); - assert!( - format!("{error:#}").contains("does not bind the current activation"), - "{error:#}" - ); - - // Same for a correlation about another Expected projection: the - // authority carries this transaction's declaration, so a correlation - // that is not about it cannot authenticate against it. - let mut reprojected = world.correlation(501_714, true)?; - reprojected.expected_projection_sha256 = sha256_id(b"another-projection"); - let canonical = world.sign(&mut reprojected)?; - assert!(world.authenticate(&canonical).is_err()); - Ok(()) - } - - #[test] - fn a_signed_observation_that_is_not_ready_does_not_admit() -> Result<()> { - let world = TopologyFixture::new("odin")?; - let mut degraded = world.correlation(501_713, false)?; - let canonical = world.sign(&mut degraded)?; - assert!(!is_semantic_ready(&world.authenticate(&canonical)?)); - - // Ready alongside a disagreement is not merely "not ready" -- the - // correlation contract refuses to encode it at all, so a publisher - // cannot assert readiness over its own dissent. Worth pinning: it is - // why is_semantic_ready's disagreement clause is belt to the schema's - // braces rather than the only thing standing there. - let mut disputed = world.correlation(501_714, true)?; - disputed.disagreements.push(OdinTopologyDisagreement { - code: "route-membership-differs".into(), - expected: Some("a".into()), - observed: Some("b".into()), - }); - assert!(world.sign(&mut disputed).is_err()); - Ok(()) - } - - // --------------------------------------------------------------------- - // Admission audit (soul/admission-audit). Each test below is a negative - // check on one claim made by c09993c..8448f83; none asserts that the - // happy path works, the tests above already do that. - // --------------------------------------------------------------------- - - impl TopologyFixture { - /// Authenticate with an explicit trusted time, against this fixture's - /// own authority and Odin key. Mirrors `rehydrate_ready_token`'s two - /// callers: `now` when require_current, `admitted_at` otherwise. - fn authenticate_at( - &self, - canonical: &[u8], - trusted_received_at_unix_millis: u64, - ) -> Result { - authenticate_odin_runtime_topology_correlation( - canonical, - &self.authority()?, - None, - &self.odin_signer.entry().public_key, - OdinTopologyAuthenticationContext { - trusted_received_at_unix_millis, - maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, - maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, - }, - ) - } - } - - /// Claim 1: deleting `observation_describes_declared_incarnation` opened - /// nothing because authentication binds the same identity. The strongest - /// attack is a correlation that is internally perfect for ANOTHER - /// incarnation of the same target -- its own projection, its own activation - /// digest, its own instance id -- signed by OUR admitted Odin, so the only - /// thing left to refuse it is the incarnation binding itself. - #[test] - fn a_perfect_correlation_about_another_incarnation_of_this_target_is_refused() -> Result<()> { - let ours = TopologyFixture::new("odin")?; - let theirs = TopologyFixture::new("odin")?; - assert_ne!( - ours.expected.canonical_sha256()?, - theirs.expected.canonical_sha256()?, - "fixtures must declare distinct incarnations for this test to mean anything" - ); - // Sanity: it is a real, Ready correlation for the other incarnation. - let mut genuine = theirs.correlation(501_713, true)?; - let genuine = theirs.sign(&mut genuine)?; - assert!(is_semantic_ready(&theirs.authenticate(&genuine)?)); - - let mut record = theirs.correlation(501_713, true)?; - let canonical = ours.sign(&mut record)?; - let error = ours - .authenticate(&canonical) - .expect_err("another incarnation's correlation must not authenticate against ours"); - assert!( - format!("{error:#}").contains("substitutes or omits Expected authority"), - "{error:#}" - ); - Ok(()) - } - - /// Claim 1, second face: same Expected, different activation. This is the - /// case the deleted gate could not see at all -- it compared instance ids - /// only -- and the one that matters when a process is re-activated under - /// the same projection. Authentication compares the activation digest. - #[test] - fn a_correlation_bound_to_another_activation_of_the_same_expected_is_refused() -> Result<()> { - let world = TopologyFixture::new("odin")?; - let other = TopologyFixture::new("odin")?; - - let mut reactivated = world.correlation(501_713, true)?; - reactivated.current_activation_sha256 = Some(other.activation.canonical_sha256()?); - let canonical = world.sign(&mut reactivated)?; - let error = world - .authenticate(&canonical) - .expect_err("activation digest must bind"); - assert!( - format!("{error:#}").contains("does not bind the current activation"), - "{error:#}" - ); - - // A Present record with no activation cannot even be encoded: the - // schema refuses it upstream of authentication ("Present topology - // state lacks an authenticated runtime session"). Either refusal is - // the right answer; what must not happen is an absent field reading as - // a match for ours. - let mut anonymous = world.correlation(501_714, true)?; - anonymous.current_activation_sha256 = None; - let error = world - .sign(&mut anonymous) - .and_then(|canonical| world.authenticate(&canonical)) - .expect_err("absent activation must not read as ours"); - assert!( - format!("{error:#}").contains("lacks an authenticated runtime session"), - "{error:#}" - ); - - let mut nameless = world.correlation(501_715, true)?; - nameless.runtime_instance_id = None; - let error = world - .sign(&mut nameless) - .and_then(|canonical| world.authenticate(&canonical)) - .expect_err("absent instance id must not read as ours"); - // Likewise refused at encoding: a Present record must carry its - // instance id ("runtime instance identity and observation evidence are - // partial"). - assert!(format!("{error:#}").contains("are partial"), "{error:#}"); - Ok(()) - } - - /// Claim 1, third face: the deleted gate also compared `target`. The - /// projection digest already covers the target, but a record can carry a - /// foreign `target` field beside our projection digest; prove - /// `validate_against_expected` refuses that on its own. - #[test] - fn a_correlation_naming_another_target_beside_our_projection_is_refused() -> Result<()> { - let world = TopologyFixture::new("odin")?; - let mut stranger = world.correlation(501_713, true)?; - stranger.target = "ghostlight".into(); - let canonical = world.sign(&mut stranger)?; - let error = world - .authenticate(&canonical) - .expect_err("target must bind"); - assert!( - format!("{error:#}").contains("substitutes or omits Expected authority"), - "{error:#}" - ); - Ok(()) - } - - /// Claim 4: the fixture's chain can refuse at every link, not only at the - /// correlation signature. If any of these passed, the tests above would be - /// asserting against a rubber stamp. - #[test] - fn the_fixture_chain_refuses_at_every_link() -> Result<()> { - let ours = TopologyFixture::new("odin")?; - let theirs = TopologyFixture::new("odin")?; - - // Authority link: activation must bind this Expected ... - assert!( - verify_runtime_authority( - &ours.expected, - &theirs.activation, - &ours.idunn_anchor, - &ours.provider_public_key - ) - .is_err(), - "another incarnation's activation verified against our Expected" - ); - // ... be signed by the Idunn we trust ... - assert!( - verify_runtime_authority( - &ours.expected, - &ours.activation, - &theirs.idunn_anchor, - &ours.provider_public_key - ) - .is_err(), - "activation verified against a foreign Idunn anchor" - ); - // ... and name the provider key the Expected selected. - assert!( - verify_runtime_authority( - &ours.expected, - &ours.activation, - &ours.idunn_anchor, - &theirs.provider_public_key - ) - .is_err(), - "a foreign provider key was accepted as the Expected signer" - ); - - // Odin link: a correlation honestly signed by another Odin identity - // (its own id, its own signature) is refused against our admitted key. - let mut record = ours.correlation(501_713, true)?; - record.signer_identity_id = theirs.odin_signer.entry().identity_id.clone(); - let canonical = theirs.sign(&mut record)?; - let error = ours - .authenticate(&canonical) - .expect_err("foreign Odin signer accepted"); - assert!( - format!("{error:#}").contains("names a different identity"), - "{error:#}" - ); - - // Odin link: our identity id, their signature bytes. - let mut record = ours.correlation(501_713, true)?; - record.signature = theirs - .odin_signer - .sign::(&record.unsigned_signature_payload()?) - .signature; - let error = ours - .authenticate(&record.canonical_bytes()?) - .expect_err("forged signature accepted"); - assert!( - format!("{error:#}").contains("signature verification failed"), - "{error:#}" - ); - Ok(()) - } - - /// Claim 2: `rehydrate_ready_token(.., require_current = false)` replays the - /// receipt's authentication at its own `admitted_at`, so the window it - /// widens is exactly the one the receipt already passed at admission. Pin - /// both halves: at `admitted_at` the receipt authenticates however old it - /// is now; at `now` past the window it does not. An `admitted_at` earlier - /// than the observation is refused by the future-skew bound, so the field - /// cannot be pulled backwards to launder a later observation either. - #[test] - fn a_durable_receipt_replays_at_its_admission_time_and_not_at_now() -> Result<()> { - let world = TopologyFixture::new("odin")?; - let mut record = world.correlation(3_841, true)?; - let canonical = world.sign(&mut record)?; - let admitted_at = NOW + 5; - - world.authenticate_at(&canonical, admitted_at)?; - let much_later = admitted_at + DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS + 1; - assert!( - world.authenticate_at(&canonical, much_later).is_err(), - "require_current would refuse" - ); - world.authenticate_at(&canonical, admitted_at)?; - - let before_observation = NOW - DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS - 1; - assert!( - world - .authenticate_at(&canonical, before_observation) - .is_err(), - "an admitted_at earlier than the observation must not authenticate" - ); - Ok(()) - } - - /// Claim 5 companion: what the live store holds, not only whether the - /// stuck record would commit. Run with: - /// IDUNN_LIVE_CONTROL=/path/to/control.cc cargo test live_store_inventory -- --ignored --nocapture - #[test] - #[ignore] - fn live_store_inventory() -> Result<()> { - let Ok(path) = std::env::var("IDUNN_LIVE_CONTROL") else { - eprintln!("IDUNN_LIVE_CONTROL unset"); - return Ok(()); - }; - let snapshot = ControlSnapshot::read(Path::new(&path))?; - println!( - "INVENTORY commands={} transactions={} admitted={}", - snapshot.commands.len(), - snapshot.transactions.len(), - snapshot.admitted.len() - ); - for stored in &snapshot.transactions { - let value = &stored.value; - println!( - "TX {} target={} kind={:?} phase={:?} complete={} ready_seq={:?} latest_seq={:?} incumbent={:?}", - value.transaction_id, - value.target, - value.command_kind, - value.phase, - value.completion.is_some(), - value - .ready - .as_ref() - .and_then(ReadinessEvidence::odin) - .map(|evidence| evidence.publisher_sequence), - value - .latest_odin_observation - .as_ref() - .map(|evidence| evidence.publisher_sequence), - value.incumbent_generation_id, - ); - } - for stored in &snapshot.admitted { - let value = &stored.value; - println!( - "ADMITTED target={} generation={} tx={} admitted_at={} ready_seq={:?} ready_admitted_at={:?} latest_seq={:?} latest_admitted_at={:?} instance={} {}", - value.target, - value.generation_id, - value.transaction_id, - value.admitted_at_unix_millis, - value.ready.odin().map(|evidence| evidence.publisher_sequence), - value.ready.odin().map(|evidence| evidence.admitted_at_unix_millis), - value - .latest_odin_observation - .as_ref() - .map(|evidence| evidence.publisher_sequence), - value - .latest_odin_observation - .as_ref() - .map(|evidence| evidence.admitted_at_unix_millis), - value.activation.runtime_instance_id, - value.workload.describe(), - ); - } - Ok(()) - } - - fn topology(sequence: u64, byte: u8) -> TopologyEvidence { - let canonical_bytes = vec![byte]; - TopologyEvidence { - canonical_sha256: sha256_id(&canonical_bytes), - canonical_bytes, - signer_identity_id: "odin-signer".into(), - publisher_sequence: sequence, - admitted_at_unix_millis: 100, - } - } - - fn workload(uid: u32, pid_namespace_id: u64, mount_namespace_id: u64) -> WorkloadObservation { - WorkloadObservation::Systemd(SystemdWorkloadObservation { - unit: format!("idunn-{uid}.service"), - unit_description: format!("Idunn test {uid}"), - invocation_id: format!("invocation-{uid}"), - exec_main_start_timestamp_monotonic: 1, - service_type: "exec".into(), - restart_policy: "always".into(), - kill_mode: "control-group".into(), - dynamic_user: true, - systemd_user: format!("u{uid}"), - systemd_group: format!("u{uid}"), - supplementary_groups: String::new(), - capability_bounding_set: String::new(), - ambient_capabilities: String::new(), - private_mounts: true, - private_pids: true, - protect_proc: "invisible".into(), - proc_subset: "pid".into(), - no_new_privileges: true, - umask: "0077".into(), - inaccessible_paths: String::new(), - load_credential: String::new(), - main_pid: uid, - process_start_time: 1, - process_uids: [uid; 4], - process_gids: [uid; 4], - process_groups: vec![uid], - process_cap_inheritable: 0, - process_cap_permitted: 0, - process_cap_effective: 0, - process_cap_bounding: 0, - process_cap_ambient: 0, - process_no_new_privileges: true, - process_namespace_pids: vec![1], - mount_namespace_id, - pid_namespace_id, - executable: PathBuf::from("/opt/test/bin/service"), - executable_device: 1, - executable_inode: 1, - executable_sha256: sha256_id(&[1]), - runtime_instance_id: sha256_id(&uid.to_be_bytes()), - working_directory: PathBuf::from("/opt/test"), - runtime_bundle: PathBuf::from("/run/test"), - command_line_sha256: sha256_id(&[2]), - environment_names: Vec::new(), - environment_contract_sha256: sha256_id(&[3]), - control_group: format!("/system.slice/idunn-{uid}.service"), - credentials_directory: None, - parent_only_file_descriptors: Vec::new(), - activation_signer_identity_id: "activation".into(), - activation_signer_public_key: vec![1; 32], - service_credentials: Vec::new(), - }) - } - - fn write_lease() -> IdunnProcessWriteLeaseRecord { - IdunnProcessWriteLeaseRecord { - schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), - target: "ghostlight".into(), - expected_projection_sha256: sha256_id(&[1]), - plan_id: sha256_id(&[2]), - incarnation_id: "incarnation-test".into(), - sealed_release_id: sha256_id(&[3]), - activation_witness_sha256: sha256_id(&[4]), - state_schema_generation: "ghostlight-state-v1".into(), - state_contract_sha256: sha256_id(&[5]), - runtime_id: "ghostlight".into(), - runtime_instance_id: sha256_id(&[6]), - warming_presence_sha256: sha256_id(&[7]), - lease_epoch: 1, - issued_at_unix_millis: 100, - } - } - - #[test] - fn validate_is_offline_and_declarative() { - // It must accept a recipe alone, accept a recipe plus a binding, and - // refuse to be handed anything executable. It opens no store and - // actuates nothing, which is what makes it safe to run on a candidate - // binding before that binding is installed. - let parsed = parse( - ["validate", "--recipe", "/tmp/recipe.toml"] - .into_iter() - .map(str::to_owned), - ) - .unwrap(); - assert!(matches!(parsed, Command::Validate { binding: None, .. })); - - assert!( - parse( - ["validate", "--recipe", "/r.toml", "--binding", "/b.toml"] - .into_iter() - .map(str::to_owned) - ) - .is_ok() - ); - assert!(parse(["validate"].into_iter().map(str::to_owned)).is_err()); - for rejected in [ - vec![ - "validate", - "--recipe", - "/r.toml", - "--deploy-command", - "sh -c bad", - ], - vec!["validate", "--recipe", "/r.toml", "--state-store", "/s.cc"], - ] { - assert!(parse(rejected.into_iter().map(str::to_owned)).is_err()); - } - } - - #[test] - fn cli_exposes_only_declarative_commands() { - for arguments in [ - vec!["up", "ghostlight", "--deploy-command", "sh -c bad"], - vec!["serve", "--swarm-profile", "yggdrasil-local"], - vec!["serve", "--restart-command", "bad"], - vec!["validate-runtime-admission"], - ] { - assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); - } - } - - #[test] - fn garden_path_accepts_service_and_profile_selectors() { - for selector in ["ghostlight", "profile:aetheria", "profile:full-gamecult"] { - let parsed = - parse(["up", selector, "--no-wait"].into_iter().map(str::to_owned)).unwrap(); - let Command::Up { - selector: actual, .. - } = parsed - else { - panic!("expected up command") - }; - assert_eq!(actual, selector); - } - } - - #[test] - fn route_bindings_are_one_global_socket_and_candidate_authority_map() { - let first = route_binding("first", 4103, 14103, 14111); - let second = route_binding("second", 8831, 18831, 18839); - validate_route_binding_set(&[("first", &first), ("second", &second)]).unwrap(); - - let mut duplicate_id = second.clone(); - duplicate_id.route_id = first.route_id.clone(); - assert!( - validate_route_binding_set(&[("first", &first), ("second", &duplicate_id)]).is_err() - ); - - let mut overlapping_candidates = second.clone(); - overlapping_candidates.private_port_start = 14111; - overlapping_candidates.private_port_end = 14120; - assert!( - validate_route_binding_set(&[("first", &first), ("second", &overlapping_candidates),]) - .is_err() - ); - - let mut stable_inside_other_range = second; - stable_inside_other_range.stable_endpoint = "tcp://127.0.0.1:14105".into(); - assert!( - validate_route_binding_set(&[ - ("first", &first), - ("second", &stable_inside_other_range), - ]) - .is_err() - ); - } - - #[test] - fn deployment_command_is_immutable_positional_fact() -> Result<()> { - let value = command(CommandKind::Deploy); - value.validate()?; - let encoded = rmp_serde::to_vec(&value)?; - assert_eq!(encoded[0], 0x96); - assert!(!encoded.windows(7).any(|window| window == b"running")); - assert!(!encoded.windows(5).any(|window| window == b"owner")); - Ok(()) - } - - #[test] - fn phases_are_one_exact_forward_chain() { - let phases = [ - DeploymentPhase::Sealing, - DeploymentPhase::Starting, - DeploymentPhase::Warming, - DeploymentPhase::Fencing, - DeploymentPhase::Leasing, - DeploymentPhase::AwaitingReady, - DeploymentPhase::Routing, - DeploymentPhase::Committing, - DeploymentPhase::Complete, - ]; - assert!( - phases - .windows(2) - .all(|pair| pair[1] as u8 == pair[0] as u8 + 1) - ); - } - - #[test] - fn only_live_stateless_deployments_can_be_cancelled_after_fencing() { - let command = command(CommandKind::Deploy); - let mut stateless = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100).unwrap(); - stateless.phase = DeploymentPhase::Routing; - stateless.fencing = Some(FencingEvidence::SkippedStateless); - assert!(cancel_is_safe_for_live_stateless(&stateless)); - - let mut stateful = stateless.clone(); - stateful.fencing = Some(FencingEvidence::Revoked { - incumbent_lease_sha256: None, - candidate_lease_path_verified_empty: true, - }); - assert!(!cancel_is_safe_for_live_stateless(&stateful)); - - let mut unfenced = stateless.clone(); - unfenced.phase = DeploymentPhase::Sealing; - assert!(!cancel_is_safe_for_live_stateless(&unfenced)); - - let mut committed = stateless; - committed.completion = Some(TransactionCompletion::Admitted { - generation_id: "generation-test".into(), - }); - assert!(!cancel_is_safe_for_live_stateless(&committed)); - } - - #[test] - fn every_new_odin_sequence_must_advance_and_exact_retry_is_idempotent() -> Result<()> { - let first = topology(7, 1); - assert!(sequence_requires_admission(None, 6, &first)?); - assert!(!sequence_requires_admission( - Some(&first), - 7, - &topology(7, 1) - )?); - assert!(sequence_requires_admission( - Some(&first), - 7, - &topology(8, 2) - )?); - assert!(sequence_requires_admission(Some(&first), 7, &topology(7, 2)).is_err()); - assert!(sequence_requires_admission(Some(&first), 7, &topology(6, 3)).is_err()); - Ok(()) - } - - #[test] - fn odin_sequence_cursors_are_scoped_by_target_and_signer() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut ghostlight = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - ghostlight.latest_odin_observation = Some(topology(7, 1)); - ghostlight.odin_publisher_sequence_cursor = 7; - let mut odin = DeploymentTransaction::new(&command, "odin".into(), 1, None, 100)?; - odin.latest_odin_observation = Some(topology(41, 2)); - odin.odin_publisher_sequence_cursor = 41; - let envelope = |key: &str| CultCacheEnvelope { - key: key.into(), - r#type: DeploymentTransaction::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), - }; - let snapshot = ControlSnapshot { - expiry_requests: Vec::new(), - commands: Vec::new(), - transactions: vec![ - Stored { - envelope: envelope("ghostlight"), - value: ghostlight, - }, - Stored { - envelope: envelope("odin"), - value: odin, - }, - ], - admitted: Vec::new(), - targets: Vec::new(), - }; - - assert_eq!(snapshot.max_odin_sequence("ghostlight", "odin-signer"), 7); - assert_eq!(snapshot.max_odin_sequence("odin", "odin-signer"), 41); - assert_eq!(snapshot.max_odin_sequence("ghostlight", "other-signer"), 0); - - // A transaction that failed was never built upon, so its readings must - // not raise the bar for the next attempt. Counting them is how a target - // that has failed once becomes a target that can never deploy. - let mut abandoned = snapshot; - abandoned.transactions[0].value.completion = - Some(TransactionCompletion::FailedBeforeFencing { - error: "sealed source was rejected".into(), - }); - assert_eq!(abandoned.max_odin_sequence("ghostlight", "odin-signer"), 0); - abandoned.transactions[1].value.completion = - Some(TransactionCompletion::FailedAfterFencing { - error: "candidate died after the fence".into(), - recovery: TerminalRecovery::RestoreIncumbent, - }); - assert_eq!(abandoned.max_odin_sequence("odin", "odin-signer"), 0); - Ok(()) - } - - #[test] - fn stale_odin_provider_receipt_cannot_gate_continuity_restart() -> Result<()> { - let touched = std::cell::Cell::new(false); - validate_live_providers_for_deploy(CommandKind::Continuity, || { - touched.set(true); - bail!("Odin provider receipt is stale") - })?; - assert!(!touched.get()); - assert!( - validate_live_providers_for_deploy(CommandKind::Deploy, || { - bail!("Odin provider receipt is stale") - }) - .is_err() - ); - Ok(()) - } - - #[test] - fn prepared_write_lease_is_durable_but_not_admitted_authority() -> Result<()> { - let lease = write_lease(); - lease.validate()?; - let prepared = LeasingEvidence::Prepared { - lease: lease.clone(), - lease_sha256: lease.canonical_sha256()?, - }; - let encoded = rmp_serde::to_vec(&prepared)?; - let decoded: LeasingEvidence = rmp_serde::from_slice(&encoded)?; - assert_eq!(decoded, prepared); - assert!(decoded.lease().is_none()); - assert!(decoded.lease_sha256().is_none()); - assert_eq!(decoded.prepared_lease().unwrap().0, &lease); - Ok(()) - } - - #[test] - fn runtime_instance_identity_is_stable_for_activation_prepare_replay() -> Result<()> { - let first = runtime_instance_id("tx-one")?; - assert_eq!(runtime_instance_id("tx-one")?, first); - assert_ne!(runtime_instance_id("tx-two")?, first); - Ok(()) - } - - #[test] - fn pre_fencing_abort_is_durable_before_cleanup_and_cannot_resume_as_success() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut transaction = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - transaction.pre_fencing_abort = Some(PreFencingAbort { - error: "sealed source was rejected".into(), - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Pending, - }); - transaction.validate()?; - assert!(!transaction.is_terminal()); - - transaction - .pre_fencing_abort - .as_mut() - .unwrap() - .source_cleanup = CleanupEvidence::Complete; - transaction.phase = DeploymentPhase::Complete; - transaction.last_error = Some("sealed source was rejected".into()); - transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { - error: "sealed source was rejected".into(), - }); - transaction.validate()?; - assert!(transaction.is_terminal()); - Ok(()) - } - - #[test] - fn post_fencing_abort_is_terminal_only_when_every_cleanup_is_complete() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut transaction = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - transaction.phase = DeploymentPhase::Complete; - transaction.post_fencing_abort = Some(PostFencingAbort { - error: "candidate died after the fence".into(), - route_restoration: CleanupEvidence::Skipped, - lease_withdrawal: CleanupEvidence::Pending, - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Pending, - }); - transaction.last_error = Some("candidate died after the fence".into()); - transaction.completion = Some(TransactionCompletion::FailedAfterFencing { - error: "candidate died after the fence".into(), - recovery: TerminalRecovery::RestoreIncumbent, - }); - // The evidence exists so that a target is released only once the - // candidate's route, lease, process and projection are actually gone. - assert!(transaction.validate().is_err()); - assert!(!transaction.is_terminal()); - - let abort = transaction.post_fencing_abort.as_mut().unwrap(); - abort.lease_withdrawal = CleanupEvidence::Complete; - abort.source_cleanup = CleanupEvidence::Complete; - transaction.validate()?; - assert!(transaction.is_terminal()); - - // The recorded error and the completion must name the same failure. - transaction.completion = Some(TransactionCompletion::FailedAfterFencing { - error: "a different story".into(), - recovery: TerminalRecovery::RestoreIncumbent, - }); - assert!(transaction.validate().is_err()); - Ok(()) - } - - #[test] - fn a_pre_fence_transaction_cannot_carry_a_post_fencing_abort() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut transaction = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - assert!(transaction.phase < DeploymentPhase::Fencing); - transaction.post_fencing_abort = Some(PostFencingAbort { - error: "candidate died after the fence".into(), - route_restoration: CleanupEvidence::Skipped, - lease_withdrawal: CleanupEvidence::Skipped, - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Skipped, - }); - assert!(transaction.validate().is_err()); - Ok(()) - } - #[test] - fn candidate_cleanup_is_owed_when_an_activation_or_a_workload_exists() -> Result<()> { - // (activation issued, workload observed, expected cleanup) - let rows = [ - (false, false, CleanupEvidence::Skipped), - (true, false, CleanupEvidence::Pending), - (false, true, CleanupEvidence::Pending), - (true, true, CleanupEvidence::Pending), - ]; - for (has_activation, has_workload, expected) in rows { - let world = ContinuityProjection::after_candidate_observed()?; - let mut transaction = world.continuity_transaction()?; - if !has_activation { - transaction.activation = None; - } - transaction.workload = has_workload.then(|| workload(1000, 1, 2)); - assert_eq!( - transaction.candidate_cleanup_owed(), - expected, - "activation {has_activation}, workload {has_workload}" - ); - assert_eq!( - pre_fencing_abort_intent(&transaction, "x").candidate_cleanup, - expected, - "pre-fencing intent, activation {has_activation}, workload {has_workload}" - ); - assert_eq!( - post_fencing_abort_intent(&transaction, "x").candidate_cleanup, - expected, - "post-fencing intent, activation {has_activation}, workload {has_workload}" - ); - } - Ok(()) - } - - #[test] - fn a_stored_abort_whose_candidate_cleanup_disagrees_with_the_owed_rule_is_refused() -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - // (activation issued, recorded cleanup, accepted). - // Rows with a workload are omitted: a transaction holding one is refused on - // other grounds, so they could not speak for this rule. The workload operand - // of the owed rule is pinned by the method-level test above. - let rows = [ - (false, CleanupEvidence::Skipped, true), - (false, CleanupEvidence::Pending, false), - (false, CleanupEvidence::Complete, false), - (true, CleanupEvidence::Skipped, false), - (true, CleanupEvidence::Pending, true), - (true, CleanupEvidence::Complete, true), - ]; - for (has_activation, recorded, accepted) in rows { - let label = format!("activation {has_activation}, {recorded:?}"); - let shape = |mut transaction: DeploymentTransaction| -> DeploymentTransaction { - transaction.activation = has_activation.then(|| world.candidate_activation.clone()); - transaction - }; - - let mut pre = shape(world.continuity_transaction()?); - pre.pre_fencing_abort = Some(PreFencingAbort { - error: "candidate refused".into(), - candidate_cleanup: recorded, - topology_reconciliation: pre.abort_topology_reconciliation(), - source_cleanup: CleanupEvidence::Skipped, - }); - assert_eq!(pre.validate().is_ok(), accepted, "pre-fencing, {label}"); - - let mut post = shape(DeploymentTransaction::new( - &command(CommandKind::Deploy), - "ghostlight".into(), - 0, - None, - 100, - )?); - post.phase = DeploymentPhase::Complete; - post.post_fencing_abort = Some(PostFencingAbort { - error: "candidate died after the fence".into(), - route_restoration: CleanupEvidence::Skipped, - lease_withdrawal: CleanupEvidence::Skipped, - candidate_cleanup: recorded, - topology_reconciliation: post.abort_topology_reconciliation(), - source_cleanup: CleanupEvidence::Complete, - }); - post.last_error = Some("candidate died after the fence".into()); - post.completion = Some(TransactionCompletion::FailedAfterFencing { - error: "candidate died after the fence".into(), - recovery: TerminalRecovery::RestoreIncumbent, - }); - // A Pending cleanup is not terminal, so this completed shape is - // only judged for the recorded values a terminal abort can carry. - if recorded != CleanupEvidence::Pending { - assert_eq!(post.validate().is_ok(), accepted, "post-fencing, {label}"); - } - } - Ok(()) - } - - #[test] - fn bad_selector_becomes_one_terminal_refusal_record() -> Result<()> { - let command = command(CommandKind::Deploy); - let transaction = DeploymentTransaction::rejected( - &command, - anyhow!("deployment selector is unknown"), - 100, - )?; - assert!(transaction.is_terminal()); - assert_eq!(transaction.target, command.selector); - assert!(matches!( - transaction.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); - Ok(()) - } - - #[test] - fn post_commit_cleanup_keeps_exact_incumbent_work_owned_until_retired() -> Result<()> { - let incumbent = workload(1001, 40, 50); - let cleanup = PostCommitCleanup { - incumbent: IncumbentCleanupEvidence::Pending { - generation_id: "generation-old".into(), - workload: incumbent.clone(), - }, - source: SourceCleanupEvidence::Pending, - }; - let encoded = rmp_serde::to_vec(&cleanup)?; - let mut replayed: PostCommitCleanup = rmp_serde::from_slice(&encoded)?; - assert_eq!(replayed, cleanup); - assert!(!replayed.is_complete()); - replayed.incumbent = IncumbentCleanupEvidence::Complete { - generation_id: "generation-old".into(), - }; - assert!(!replayed.is_complete()); - replayed.source = SourceCleanupEvidence::Complete; - assert!(replayed.is_complete()); - Ok(()) - } - - #[test] - fn routed_incumbent_retirement_waits_for_the_declared_drain_deadline() -> Result<()> { - assert_eq!(route_drain_deadline(1_000, 30)?, 31_000); - assert!(route_drain_deadline(u64::MAX, 1).is_err()); - Ok(()) - } - - #[test] - fn admitted_route_receipt_refreshes_when_stale_or_implausibly_future_dated() { - assert!(route_observation_is_current(900, 1_000, 100, 10)); - assert!(route_observation_is_current(1_010, 1_000, 100, 10)); - assert!(!route_observation_is_current(899, 1_000, 100, 10)); - assert!(!route_observation_is_current(1_011, 1_000, 100, 10)); - } - - #[test] - fn lease_warming_requires_both_new_odin_and_new_provider_evidence() { - let old = digest('1'); - let fresh = digest('2'); - assert!(!provider_warming_advanced(7, &old, 7, &fresh)); - assert!(!provider_warming_advanced(7, &old, 8, &old)); - assert!(provider_warming_advanced(7, &old, 8, &fresh)); - } - - #[test] - fn failed_route_proof_never_restores_any_fenced_incumbent() { - assert!(!may_rollback_route_after_failed_proof( - &FencingEvidence::Revoked { - incumbent_lease_sha256: Some(digest('1')), - candidate_lease_path_verified_empty: false, - } - )); - assert!(!may_rollback_route_after_failed_proof( - &FencingEvidence::Revoked { - incumbent_lease_sha256: None, - candidate_lease_path_verified_empty: true, - } - )); - assert!(may_rollback_route_after_failed_proof( - &FencingEvidence::SkippedStateless - )); - } - - #[test] - fn fenced_writer_is_already_retired_before_post_commit_cleanup() { - assert!(incumbent_was_stopped_during_fencing( - &FencingEvidence::Revoked { - incumbent_lease_sha256: Some(digest('1')), - candidate_lease_path_verified_empty: false, - } - )); - assert!(!incumbent_was_stopped_during_fencing( - &FencingEvidence::Revoked { - incumbent_lease_sha256: None, - candidate_lease_path_verified_empty: true, - } - )); - assert!(!incumbent_was_stopped_during_fencing( - &FencingEvidence::SkippedStateless - )); - } - - #[test] - fn warming_accepts_only_the_exact_projected_incumbent_lease_disagreement() { - let expected_detail = format!("expected:{};activation:{}", digest('1'), digest('2')); - let incumbent = digest('3'); - let exact = OdinTopologyDisagreement { - code: "projected-write-lease".into(), - expected: Some(expected_detail.clone()), - observed: Some(incumbent.clone()), - }; - - assert!(warming_disagreements_match_incumbent( - &expected_detail, - Some(&incumbent), - &[exact.clone()], - )); - assert!(!warming_disagreements_match_incumbent( - &expected_detail, - None, - &[exact.clone()], - )); - - let mut substituted = exact.clone(); - substituted.observed = Some(digest('4')); - assert!(!warming_disagreements_match_incumbent( - &expected_detail, - Some(&incumbent), - &[substituted], - )); - assert!(!warming_disagreements_match_incumbent( - &expected_detail, - Some(&incumbent), - &[exact.clone(), exact], - )); - } - - #[test] - fn candidate_and_incumbent_must_differ_in_all_three_native_boundaries() { - let incumbent = workload(1001, 40, 50); - let prove = |candidate: &WorkloadObservation| { - WorkloadObservation::prove_isolation(candidate, Some(&incumbent)) - }; - assert!(prove(&workload(1002, 41, 51)).is_ok()); - assert!(prove(&workload(1001, 41, 51)).is_err()); - assert!(prove(&workload(1002, 40, 51)).is_err()); - assert!(prove(&workload(1002, 41, 50)).is_err()); - } - - #[test] - fn exactly_one_inflight_transaction_may_own_a_target() -> Result<()> { - let command = command(CommandKind::Deploy); - let one = DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - let two = DeploymentTransaction::new(&command, "ghostlight".into(), 1, None, 100)?; - let snapshot = ControlSnapshot { - expiry_requests: Vec::new(), - commands: vec![Stored { - envelope: command_envelope(&command, 100)?, - value: command, - }], - transactions: vec![ - Stored { - envelope: transaction_envelope(&one, 100)?, - value: one, - }, - Stored { - envelope: transaction_envelope(&two, 100)?, - value: two, - }, - ], - admitted: Vec::new(), - targets: Vec::new(), - }; - assert!(snapshot.validate_relations().is_err()); - Ok(()) - } - - #[test] - fn command_ordinals_do_not_turn_one_target_wait_into_a_global_brake() -> Result<()> { - let first_command = command(CommandKind::Deploy); - let first = DeploymentTransaction::new(&first_command, "odin".into(), 0, None, 100)?; - let second = DeploymentTransaction::new(&first_command, "ghostlight".into(), 1, None, 100)?; - let mut unrelated_command = command(CommandKind::Deploy); - unrelated_command.command_id = "up-unrelated".into(); - let unrelated = - DeploymentTransaction::new(&unrelated_command, "huginn".into(), 0, None, 101)?; - let snapshot = ControlSnapshot { - expiry_requests: Vec::new(), - commands: Vec::new(), - transactions: [&first, &second, &unrelated] - .into_iter() - .map(|value| Stored { - envelope: transaction_envelope(value, 100).unwrap(), - value: value.clone(), - }) - .collect(), - admitted: Vec::new(), - targets: Vec::new(), - }; - assert!(!snapshot.has_earlier_authority_sibling(&first)); - assert!(snapshot.has_earlier_authority_sibling(&second)); - assert!(!snapshot.has_earlier_authority_sibling(&unrelated)); - Ok(()) - } - - #[test] - fn complete_cleanup_remains_retryable_without_owning_the_current_incarnation() -> Result<()> { - let command = command(CommandKind::Deploy); - let mut transaction = - DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; - transaction.phase = DeploymentPhase::Complete; - transaction.completion = Some(TransactionCompletion::Admitted { - generation_id: format!("generation-{}", transaction.transaction_id), - }); - transaction.post_commit_cleanup = Some(PostCommitCleanup { - incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, - source: SourceCleanupEvidence::Pending, - }); - - assert!(!transaction.is_terminal()); - assert!(!transaction.owns_target_authority()); - assert!(transaction.blocks_new_target_mutation()); - Ok(()) - } - - #[test] - fn legacy_mutable_command_schema_is_rejected() -> Result<()> { - let temporary = tempfile::tempdir()?; - let path = temporary.path().join("control.cc"); - let legacy = CultCacheEnvelope { - key: "up-legacy".into(), - r#type: DeploymentCommand::TYPE.into(), - payload: rmp_serde::to_vec(&(DEPLOYMENT_COMMAND_SCHEMA, "up-legacy", "running"))?, - stored_at: rfc3339_millis(100)?, - schema_id: Some("idunn.deployment_command.v1".into()), - }; - assert!( - SingleFileMessagePackBackingStore::new(&path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: "up-legacy".into(), - current: None, - }], - &[legacy], - )? - ); - assert!(ControlSnapshot::read(&path).is_err()); - Ok(()) - } - - #[test] - fn loadcredential_era_control_schemas_are_rejected_before_rehydration() -> Result<()> { - for (record_type, key, schema) in [ - ( - DeploymentTransaction::TYPE, - "tx-legacy-workload-observation", - "idunn.deployment_transaction.v1", - ), - ( - AdmittedGeneration::TYPE, - "legacy-admitted-target", - "idunn.admitted_generation.v1", - ), - ] { - let temporary = tempfile::tempdir()?; - let path = temporary.path().join("control.cc"); - let legacy = CultCacheEnvelope { - key: key.into(), - r#type: record_type.into(), - payload: Vec::new(), - stored_at: rfc3339_millis(100)?, - schema_id: Some(schema.into()), - }; - assert!( - SingleFileMessagePackBackingStore::new(&path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: record_type.into(), - key: key.into(), - current: None, - }], - &[legacy], - )? - ); - assert!(ControlSnapshot::read(&path).is_err()); - } - Ok(()) - } - - #[test] - fn source_identity_is_explicit_and_atomic() { - let partial = parse( - ["serve", "--source-uid", "1001"] - .into_iter() - .map(str::to_owned), - ) - .unwrap_err() - .to_string(); - assert!(partial.contains("supplied together")); - let Command::Serve(options) = parse( - ["serve", "--source-uid", "1001", "--source-gid", "1002"] - .into_iter() - .map(str::to_owned), - ) - .unwrap() else { - panic!("expected serve") - }; - assert_eq!( - options.source_identity, - Some(ProcessIdentity { - uid: 1001, - gid: 1002 - }) - ); - } - - // ---- v2/v3 control records lifted to the v3 generation / v4 transaction ---- - // - // The bytes under tests/fixtures/idunn-control-legacy were written by the - // encoder at Idunn 9001b58 (see the README there). Nothing below encodes a - // legacy record: the legacy layout is decoded from those bytes, and the - // lifted record is compared with what the decoded legacy record says. - - const FIXTURE_GENERATION: &str = - include_str!("../tests/fixtures/idunn-control-legacy/generation-with-receipts.hex"); - const FIXTURE_GENERATION_REPAIRING: &str = include_str!( - "../tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex" - ); - const FIXTURE_GENERATION_V3_ODIN: &str = - include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-odin.hex"); - const FIXTURE_GENERATION_V3_ROUTE_PROOF: &str = - include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex"); - const FIXTURE_TRANSACTIONS: [(&str, &str); 6] = [ - ( - "fencing", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-fencing.hex"), - ), - ( - "leasing", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-leasing.hex"), - ), - ( - "awaiting-ready", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex"), - ), - ( - "routing", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-routing.hex"), - ), - ( - "committing", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing.hex"), - ), - ( - "failed-after-fencing", - include_str!( - "../tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex" - ), - ), - ]; - const FIXTURE_TRANSACTION_V2: &str = - include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex"); - - fn fixture_envelope(text: &str, record_type: &str) -> Result { - let mut lines = text.lines(); - let schema = lines.next().context("fixture has no schema line")?; - let hex = lines.next().context("fixture has no payload line")?.trim(); - let payload = (0..hex.len()) - .step_by(2) - .map(|index| u8::from_str_radix(&hex[index..index + 2], 16)) - .collect::, _>>()?; - let key = if record_type == AdmittedGeneration::TYPE && schema == ADMITTED_GENERATION_SCHEMA_V3 { - rmp_serde::from_slice::(&payload)?.target - } else if record_type == AdmittedGeneration::TYPE { - rmp_serde::from_slice::(&payload)?.target - } else { - rmp_serde::from_slice::(&payload)?.transaction_id - }; - Ok(CultCacheEnvelope { - key, - r#type: record_type.into(), - payload, - stored_at: rfc3339_millis(1_700_000_123_456)?, - schema_id: Some(schema.into()), - }) - } - - fn fixture_generation(text: &str) -> Result<(LegacyAdmittedGeneration, AdmittedGeneration)> { - let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; - let legacy = rmp_serde::from_slice(&envelope.payload)?; - Ok((legacy, read_generation_record(&envelope)?)) - } - - fn fixture_transaction( - text: &str, - ) -> Result<(LegacyDeploymentTransaction, DeploymentTransaction)> { - let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; - let legacy = rmp_serde::from_slice(&envelope.payload)?; - Ok((legacy, read_transaction_record(&envelope)?)) - } - - #[test] - fn a_v2_generation_lifts_to_the_decisions_it_encoded() -> Result<()> { - for text in [FIXTURE_GENERATION, FIXTURE_GENERATION_REPAIRING] { - let (legacy, lifted) = fixture_generation(text)?; - // Every receipt the old decisions read is the same receipt, now - // tagged Odin-correlated. - let odin = lifted.odin_receipts()?; - assert_eq!(odin.ready, &legacy.ready); - assert_eq!(odin.latest, &legacy.latest_odin_observation); - assert_eq!(lifted.odin_authority.as_ref(), Some(&legacy.odin_authority)); - assert_eq!( - lifted.odin_publisher_sequence_cursor, - legacy.odin_publisher_sequence_cursor - ); - assert_eq!(lifted.ready.voucher(), Voucher::Odin); - assert_eq!(lifted.routing, legacy.routing); - assert_eq!(lifted.leasing, legacy.leasing); - assert_eq!(lifted.plan, legacy.plan); - assert_eq!(lifted.expected, legacy.expected); - assert_eq!(lifted.activation, legacy.activation); - assert_eq!(lifted.workload, legacy.workload); - assert_eq!(lifted.generation_id, legacy.generation_id); - // The new state starts empty and the plan is still a v2 plan. - assert_eq!(lifted.route_supervision, Some(RouteSupervisionState::default())); - assert_eq!(lifted.last_error, None); - assert_eq!(lifted.plan.schema, crate::deployment_plan::COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2); - assert_eq!( - lifted.plan.phase_deadlines(), - crate::deployment_plan::PhaseDeadlines::IDUNN_DEFAULTS - ); - } - Ok(()) - } - - #[test] - fn v2_and_v3_transactions_lift_to_the_decisions_they_encoded() -> Result<()> { - let mut phases = Vec::new(); - for (name, text) in FIXTURE_TRANSACTIONS { - let (legacy, lifted) = fixture_transaction(text)?; - phases.push((name, lifted.phase)); - assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); - assert_eq!(lifted.phase, legacy.phase); - assert_eq!(lifted.completion, legacy.completion); - assert_eq!(lifted.plan, legacy.plan); - assert_eq!(lifted.expected, legacy.expected); - assert_eq!(lifted.leasing, legacy.leasing); - assert_eq!(lifted.fencing, legacy.fencing); - assert_eq!(lifted.routing, legacy.routing); - assert_eq!(lifted.warming, legacy.warming); - assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); - assert_eq!( - lifted.odin_publisher_sequence_cursor, - legacy.odin_publisher_sequence_cursor - ); - assert_eq!( - lifted.ready.as_ref().and_then(ReadinessEvidence::odin), - legacy.ready.as_ref() - ); - assert_eq!(lifted.phase_deadline, None); - assert_eq!(lifted.lease_adoption, None); - assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); - } - assert_eq!( - phases.iter().map(|(_, phase)| *phase).collect::>(), - [ - DeploymentPhase::Fencing, - DeploymentPhase::Leasing, - DeploymentPhase::AwaitingReady, - DeploymentPhase::Routing, - DeploymentPhase::Committing, - DeploymentPhase::Complete, - ] - ); - - // The genuine v2 layout lifts to the same record as the v3 one. - let (_, from_v2) = fixture_transaction(FIXTURE_TRANSACTION_V2)?; - let (_, from_v3) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; - assert_eq!(from_v2, from_v3); - - // A terminal failure predating recovery names the recovery it always did. - let (_, failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; - assert!(matches!( - failed.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - )); - assert!(failed.is_terminal()); - Ok(()) - } - - #[test] - fn the_boot_migration_rewrites_every_legacy_record_once() -> Result<()> { - // The fixtures were cut from one build, so their records share keys - // and each migrates in a store of its own. - let mut fixtures = vec![ - fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?, - fixture_envelope(FIXTURE_GENERATION_REPAIRING, AdmittedGeneration::TYPE)?, - fixture_envelope(FIXTURE_TRANSACTION_V2, DeploymentTransaction::TYPE)?, - ]; - for (_, text) in FIXTURE_TRANSACTIONS { - fixtures.push(fixture_envelope(text, DeploymentTransaction::TYPE)?); - } - for envelope in fixtures { - let temporary = tempfile::tempdir()?; - let path = temporary.path().join("control.cc"); - let store = SingleFileMessagePackBackingStore::new(&path); - assert!(store.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: envelope.r#type.clone(), - key: envelope.key.clone(), - current: None, - }], - std::slice::from_ref(&envelope), - )?); - let debug_of = |envelope: &CultCacheEnvelope| -> Result { - Ok(if envelope.r#type == AdmittedGeneration::TYPE { - format!("{:?}", read_generation_record(envelope)?) - } else { - format!("{:?}", read_transaction_record(envelope)?) - }) - }; - let before = debug_of(&envelope)?; - - assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); - assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); - - let after = store.pull_all_read_only_snapshot()?; - assert_eq!(after.len(), 1); - let expected_schema = if envelope.r#type == AdmittedGeneration::TYPE { - ADMITTED_GENERATION_SCHEMA - } else { - DEPLOYMENT_TRANSACTION_SCHEMA - }; - assert_eq!(after[0].schema_id.as_deref(), Some(expected_schema)); - assert_eq!(after[0].stored_at, envelope.stored_at); - // The rewritten record is canonical and means what the legacy one meant. - assert_eq!(debug_of(&after[0])?, before); - } - Ok(()) - } - - #[test] - fn history_still_reads_legacy_transactions() -> Result<()> { - let temporary = tempfile::tempdir()?; - let state_store = temporary.path().join("control.cc"); - let envelope = fixture_envelope(FIXTURE_TRANSACTIONS[5].1, DeploymentTransaction::TYPE)?; - SingleFileMessagePackBackingStore::new(&history_store_path(&state_store)) - .compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: envelope.key.clone(), - current: None, - }], - std::slice::from_ref(&envelope), - )?; - let archived = read_history_transactions(&state_store); - assert_eq!(archived.len(), 1); - assert_eq!(archived[0].transaction_id, envelope.key); - assert_eq!(archived[0].schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); - Ok(()) - } - - #[test] - fn readiness_class_follows_the_expected_odin_dependency() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - let mut expected = generation.expected.clone(); - assert!(expected.dependencies.iter().any(|dependency| { - dependency.kind == "shared-infrastructure" - && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY - })); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); - for dependency in &mut expected.dependencies { - // Same capability, other kind: only the shared-infrastructure - // declaration makes a target Odin-correlated. - dependency.kind = "required".into(); - } - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); - expected.dependencies.clear(); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); - // Nothing to challenge and no Odin dependency: the recipe has declared - // no way to prove readiness, and Idunn does not pick one. - expected.route = None; - assert_eq!( - ReadinessClass::of(&expected), - Err(UndeclaredReadiness { - target: expected.target.clone() - }) - ); - // Declaring the Odin dependency is the declaration, routed or not: a - // routed target that only publishes to Odin is Odin-correlated. - let odin_dependency = generation.expected.dependencies.clone(); - expected.dependencies = odin_dependency.clone(); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); - expected.route = generation.expected.route.clone(); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); - // Providing the rendezvous is being Odin, whatever else is declared. - expected.dependencies.clear(); - expected.route = None; - provide_odin(&mut expected); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); - Ok(()) - } - - #[test] - fn warming_evidence_is_shape_checked_whatever_its_source() { - let good = |evidence: &ReadinessEvidence| match evidence { - ReadinessEvidence::RouteProof { evidence } => evidence.clone(), - ReadinessEvidence::OdinCorrelated { .. } => unreachable!("a route-proof fixture"), - }; - let presence = good(&route_proof_evidence()); - for warming in [ - WarmingEvidence::FirstOdinDirect { evidence: presence.clone() }, - WarmingEvidence::RouteProofDirect { evidence: presence.clone() }, - WarmingEvidence::OdinTopology { evidence: topology(3, 1) }, - ] { - warming.validate_shape().expect("well-formed evidence"); - } - let mut damaged = presence; - damaged.canonical_bytes.push(1); - for warming in [ - WarmingEvidence::FirstOdinDirect { evidence: damaged.clone() }, - WarmingEvidence::RouteProofDirect { evidence: damaged }, - WarmingEvidence::OdinTopology { - evidence: TopologyEvidence { - canonical_sha256: "sha256:not-the-digest".into(), - ..topology(3, 1) - }, - }, - ] { - assert!(warming.validate_shape().is_err(), "{warming:?}"); - } - } - - #[test] - fn a_presence_disagreement_names_each_disagreement() { - let shortfall = PresenceDisagrees { - disagreements: vec![ - OdinTopologyDisagreement { - code: "capacity-below-minimum".into(), - expected: Some("2".into()), - observed: Some("1".into()), - }, - OdinTopologyDisagreement { - code: "state".into(), - expected: None, - observed: None, - }, - ], - }; - assert_eq!( - shortfall.to_string(), - "route proof answered with a runtime that disagrees with current authority: \ - capacity-below-minimum (expected 2, observed 1); state (expected none, observed none)" - ); - } - - fn route_proof_evidence() -> ReadinessEvidence { - let canonical_bytes = vec![7, 7, 7]; - ReadinessEvidence::RouteProof { - evidence: RuntimePresenceEvidence { - canonical_sha256: sha256_id(&canonical_bytes), - canonical_bytes, - message_id: "challenge-1".into(), - challenged_at_unix_millis: 10, - admitted_at_unix_millis: 11, - }, - } - } - - #[test] - fn route_proof_readiness_is_refused_for_a_target_that_declares_odin() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - let mut proof = generation.clone(); - proof.ready = route_proof_evidence(); - proof.latest_odin_observation = None; - proof.odin_authority = None; - // The fixture's Expected declares Odin, so route proof is not its class. - assert!(proof.validate().is_err()); - - // Mixed shapes are refused whatever the class. - let mut mixed = generation.clone(); - mixed.latest_odin_observation = None; - assert!(mixed.validate().is_err()); - let mut mixed = generation; - mixed.odin_authority = None; - assert!(mixed.validate().is_err()); - - let mut transaction = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?.1; - transaction.validate()?; - transaction.ready = Some(route_proof_evidence()); - assert!(transaction.validate().is_err()); - Ok(()) - } - - #[test] - fn route_supervision_state_exists_exactly_for_a_promoted_route() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - let mut missing = generation.clone(); - missing.route_supervision = None; - assert!(missing.validate().is_err()); - - let mut unrouted = generation; - unrouted.route_supervision = None; - unrouted.routing = RoutingEvidence::SkippedUnrouted; - // Unrouted while Expected still names a route: refused for that reason - // too, so the test above pins the supervision rule on its own. - assert!(unrouted.validate().is_err()); - Ok(()) - } - - #[test] - fn a_new_generation_starts_unobserved_and_undegraded() -> Result<()> { - let committing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1; - let (_, incumbent) = fixture_generation(FIXTURE_GENERATION)?; - let authority = incumbent - .odin_authority - .clone() - .context("fixture generation has no Odin authority")?; - let next = AdmittedGeneration::from_transaction( - &committing, - Some(authority), - 1_700_000_200_000, - )?; - assert_eq!(next.route_supervision, Some(RouteSupervisionState::default())); - assert_eq!(next.last_error, None); - Ok(()) - } - - #[test] - fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - let plan = &generation.plan; - let defaults = plan.phase_deadlines(); - for (phase, seconds) in [ - (DeploymentPhase::Fencing, defaults.fencing_seconds), - (DeploymentPhase::Leasing, defaults.leasing_seconds), - (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), - (DeploymentPhase::Routing, defaults.routing_seconds), - (DeploymentPhase::Committing, defaults.committing_seconds), - ] { - let deadline = PhaseDeadline::entering(phase, plan, 1_000).context("no deadline")?; - assert_eq!(deadline.phase, phase); - assert_eq!(deadline.entered_at_unix_millis, 1_000); - assert_eq!(deadline.deadline_at_unix_millis, 1_000 + u64::from(seconds) * 1000); - } - for phase in [ - DeploymentPhase::Sealing, - DeploymentPhase::Starting, - DeploymentPhase::Warming, - DeploymentPhase::Complete, - ] { - assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); - } - Ok(()) - } - - #[test] - fn transition_stamps_the_deadline_of_the_phase_it_enters() -> Result<()> { - let world = EngineFixture::new()?; - // Routing, already holding the route receipt Committing requires. - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?; - assert_eq!(transaction.phase, DeploymentPhase::Routing); - transaction.routing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1.routing; - let envelope = transaction_envelope(&transaction, transaction.updated_at_unix_millis)?; - let command = command_envelope(&command(CommandKind::Continuity), 100)?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.key.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - ], - &[command, envelope], - )? - ); - let stored = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .next() - .context("stored transaction vanished")?; - world.engine.transition(&stored, DeploymentPhase::Committing)?; - let advanced = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .next() - .context("advanced transaction vanished")?; - let deadline = advanced.value.phase_deadline.context("no deadline stamped")?; - assert_eq!(deadline.phase, DeploymentPhase::Committing); - assert_eq!( - deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, - u64::from(advanced.value.plan.as_ref().unwrap().phase_deadlines().committing_seconds) - * 1000 - ); - assert_eq!(deadline.entered_at_unix_millis, advanced.value.updated_at_unix_millis); - Ok(()) - } - - #[test] - fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; - let plan = transaction.plan.clone().unwrap(); - let now = transaction.updated_at_unix_millis; - transaction.phase_deadline = - PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now); - transaction.validate()?; - let mut wrong_phase = transaction.clone(); - wrong_phase.phase_deadline = - PhaseDeadline::entering(DeploymentPhase::Routing, &plan, now); - assert!(wrong_phase.validate().is_err()); - let mut backwards = transaction.clone(); - backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; - assert!(backwards.validate().is_err()); - // Entry is not ordered against the wall-clock stamp of later writes. - let mut entered_after_the_stamp = transaction; - entered_after_the_stamp.phase_deadline = - PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); - entered_after_the_stamp.validate()?; - Ok(()) - } - - // --------------------------------------------------------------------- - // The phase machine, driven by the real Engine past Fencing. - // --------------------------------------------------------------------- - - /// A workload port with no systemd behind it: observing a workload - /// returns what was observed, stopping and discarding succeed, and - /// anything that would launch a process refuses. - struct StillWorkload; - - impl WorkloadPort for StillWorkload { - fn install( - &self, - _: &CompiledDeploymentPlan, - _: &crate::drivers::MaterializedRelease, - ) -> Result { - bail!("StillWorkload launches nothing") - } - fn prepare_activation( - &self, - _: &CompiledDeploymentPlan, - _: &IdunnExpectedIncarnationRecord, - _: IdunnRuntimeActivationLaunch, - ) -> Result { - bail!("StillWorkload launches nothing") - } - fn start_prepared( - &self, - _: &CompiledDeploymentPlan, - _: &SealedRelease, - _: &crate::drivers::InstalledReleaseObservation, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - ) -> Result { - bail!("StillWorkload launches nothing") - } - fn discard_prepared( - &self, - _: &CompiledDeploymentPlan, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - Ok(()) - } - fn observe( - &self, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - prior: &WorkloadObservation, - ) -> Result { - Ok(prior.clone()) - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - Ok(()) - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(false) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(false) - } - } - - /// A Continuity transaction for an unrouted, stateless target with no - /// incumbent: the shape most Engine tests need. - fn transaction_at( - world: &EngineFixture, - phase: DeploymentPhase, - ) -> Result { - seeded_transaction(world, phase, CommandKind::Continuity, None) - } - - /// The Odin publisher sequence a transaction seeded over `incumbent` - /// warms at; its Ready is the next one. Above anything an earlier - /// generation of the world admitted. - fn seeded_sequence(incumbent: Option<&AdmittedGeneration>) -> u64 { - if incumbent.is_some() { 14 } else { 4 } - } - - /// A released deployment brake receipt, signed by an operator identity - /// enrolled in the world: what a Deploy's `deployment_authorization` holds. - fn deployment_authorization( - world: &EngineFixture, - expected: &IdunnExpectedIncarnationRecord, - transaction_id: &str, - now: u64, - ) -> Result { - use cultnet_rs::{ - IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, - IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, - IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, - }; - let signer = enroll_service_identity_at::( - &world.root.join("identities/brake-operator.cc"), - )?; - export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; - let mut record = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: expected.runtime_id.clone(), - status: "released".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: "test rollout".into(), - observed_at_unix_millis: now, - expires_at_unix_millis: Some(now + 600_000), - authorization_id: Some("authorization-test".into()), - authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), - authorized_release_id: Some(expected.sealed_release_id.clone()), - authorized_deployment_id: Some(transaction_id.to_owned()), - authorized_by: Some(signer.trust_anchor()?.identity_id), - authorization_issued_at_unix_millis: Some(now), - authorization_expires_at_unix_millis: Some(now + 600_000), - signature_algorithm: Some("ed25519".into()), - signature: None, - private_state_exposed: false, - updated_by: "operator/test".into(), - }; - record.signature = Some( - signer - .sign::(&rmp_serde::to_vec(&record)?) - .signature, - ); - record.validate()?; - let canonical_brake_bytes = rmp_serde::to_vec(&record)?; - Ok(DeploymentAuthorization { - authorization_id: "authorization-test".into(), - brake_sha256: sha256_id(&canonical_brake_bytes), - canonical_brake_bytes, - authorized_at_unix_millis: now, - }) - } - - /// What makes a target Odin: it provides the rendezvous. - fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { - expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { - capability: ODIN_RENDEZVOUS_CAPABILITY.into(), - schema: "odin.verse-topology.v1".into(), - compatibility: "v1".into(), - minimum_capacity: 1, - }); - } - - /// The declaration that makes an unrouted fixture target Odin-correlated. - /// Idunn infers no way to prove readiness, and a target with no route has - /// nothing for Idunn to challenge, so the world says outright that Odin - /// reports on it. The plan selects no provider: these worlds never plan - /// against Odin, they only read its correlation, which names the provider - /// Expected does. - fn declare_odin(expected: &mut IdunnExpectedIncarnationRecord) { - expected.dependencies.push(cultnet_rs::IdunnExpectedDependency { - kind: "shared-infrastructure".into(), - capability: ODIN_RENDEZVOUS_CAPABILITY.into(), - schema: "odin.verse-topology.v1".into(), - compatibility: "v1".into(), - minimum_capacity: 1, - startup: "before-promotion".into(), - provider_id: Some("odin".into()), - provider_authority: Some("managed-incarnation".into()), - provider_expected_projection_sha256: Some(sha256_id(b"odin expected projection")), - provider_endpoint: None, - }); - } - - /// A transaction for an unrouted, stateless target, sitting at `phase` - /// (Warming or later) with every earlier phase's evidence in place. The - /// plan, sealed release, Expected, activation and Warming evidence are the - /// real constructions; only the workload and isolation observations are - /// borrowed from a recorded transaction. - /// - /// `incumbent` is an admitted generation already in the world's store. A - /// Continuity over it restarts that generation's own release, so it shares - /// the incumbent's Expected key. A Deploy over it builds a second - /// incarnation. A Deploy carries what phases past Fencing never read but - /// validation requires: a three-field frozen-source receipt and a - /// well-formed release receipt from the brake. - fn seeded_transaction( - world: &EngineFixture, - phase: DeploymentPhase, - kind: CommandKind, - incumbent: Option<&AdmittedGeneration>, - ) -> Result { - use crate::deployment_plan::tests::{ - BINDING, RECIPE, artifact_receipt, external_input_receipt, source, - }; - let provider_path = world.root.join("identities/provider.cc"); - let provider_anchor = world.root.join("identities/provider-anchor.cc"); - let provider = if provider_path.exists() { - open_service_identity_at::(&provider_path)? - } else { - let provider = - enroll_service_identity_at::(&provider_path)?; - export_service_identity_trust_anchor(&provider, &provider_anchor)?; - provider - }; - - let (binding_head, binding_tail) = BINDING - .split_once("[route]") - .context("binding has no route table")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail - .split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ) - .replace( - "/etc/gamecult/trust/service.cc", - &provider_anchor.display().to_string(), - ) - .replace("service-runtime-signer", &provider.entry().identity_id); - let recipe = RECIPE - .split("[[dependencies]]") - .next() - .context("recipe is empty")? - .replace("route_required = true", "route_required = false") - .replace( - r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, - r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, - ); - let (plan, release) = match (kind, incumbent) { - (CommandKind::Continuity, Some(incumbent)) => ( - incumbent.plan.clone(), - incumbent.sealed_release.clone(), - ), - _ => { - let incarnation = if incumbent.is_some() { - "service-incarnation-2" - } else { - "service-incarnation-1" - }; - let plan = compile_deployment_plan( - recipe.as_bytes(), - binding.as_bytes(), - source(&recipe), - incarnation, - None, - 110, - &[], - )?; - let release = SealedRelease::new( - &plan, - vec![artifact_receipt()], - vec![external_input_receipt()], - 120, - )?; - (plan, release) - } - }; - let mut expected = release.expected_projection(&plan)?; - assert!(!expected.write_lease_required && expected.route.is_none()); - declare_odin(&mut expected); - - let now = now_millis()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!( - "{}-service{}", - match kind { - CommandKind::Deploy => "up", - CommandKind::Continuity => "continuity", - }, - if incumbent.is_some() { "-again" } else { "" } - ), - kind, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let mut transaction = match (kind, incumbent) { - (CommandKind::Continuity, Some(incumbent)) => { - DeploymentTransaction::from_continuity(&command, incumbent, now)? - } - _ => DeploymentTransaction::new(&command, "service".into(), 0, incumbent, now)?, - }; - let activation = IdunnRuntimeActivationLaunch::issue( - &expected, - runtime_instance_id(&transaction.transaction_id)?, - now, - &world.engine.idunn_signer, - )? - .activation() - .clone(); - let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? - .1 - .workload - .context("recorded transaction has no workload")?; - match kind { - CommandKind::Continuity => { - transaction.lifecycle_authorized_at_unix_millis = Some(now); - } - CommandKind::Deploy => { - transaction.frozen_source = Some(FrozenSourceReceipt { - transaction_id: transaction.transaction_id.clone(), - plan_id: plan.plan_id.clone(), - snapshot_sha256: sha256_id(b"frozen source snapshot"), - }); - transaction.deployment_authorization = Some(deployment_authorization( - world, - &expected, - &transaction.transaction_id, - now, - )?); - } - } - transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); - transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); - match &mut workload { - WorkloadObservation::Systemd(observed) => { - observed.runtime_instance_id = activation.runtime_instance_id.clone(); - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), - } - let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; - transaction.isolation = recorded.isolation; - transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { - sealed_release_id: release.sealed_release_id.clone(), - root: PathBuf::from("/srv/service/releases/test"), - }); - transaction.sealed_release = Some(release); - transaction.expected = Some(expected); - transaction.activation = Some(activation); - transaction.workload = Some(workload); - transaction.plan = Some(plan); - // Odin's first word about the candidate, before it was Ready. - let sequence = seeded_sequence(incumbent); - let warming = signed_correlation(world, &transaction, sequence, false)?; - let authenticated = world.engine.authenticate_topology_bytes( - &ControlSnapshot::read(&world.state_store)?, - &transaction, - &warming, - None, - now, - )?; - transaction.warming = Some(WarmingEvidence::OdinTopology { - evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, - }); - transaction.odin_publisher_sequence_cursor = sequence; - transaction.enter_phase(phase, now); - transaction.validate()?; - - let store = SingleFileMessagePackBackingStore::new(&world.state_store); - assert!(store.compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - ], - &[ - command_envelope(&command, command.requested_at_unix_millis)?, - transaction_envelope(&transaction, now)?, - ], - )?); - Ok(transaction) - } - - /// Odin's correlation for the transaction's incarnation, signed by the key - /// the Engine trusts and stamped now. - fn signed_correlation( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ready: bool, - ) -> Result> { - signed_correlation_as(world, transaction, sequence, ready, "active", None, b"presence") - } - - /// Odin's correlation about the transaction's incarnation, observing a - /// presence in `state` that holds `lease`, identified by `presence`. - fn signed_correlation_as( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ready: bool, - state: &str, - lease: Option<&str>, - presence: &[u8], - ) -> Result> { - let expected = transaction.expected.as_ref().context("no Expected")?; - let activation = transaction.activation.as_ref().context("no activation")?; - let mut record = OdinRuntimeTopologyCorrelationRecord { - schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: expected.canonical_sha256()?, - expected: true, - current_activation_sha256: Some(activation.canonical_sha256()?), - signed_presence_sha256: Some(sha256_id(presence)), - observed_presence_state: Some(state.into()), - observed_presence_publisher_sequence: Some(sequence), - observed_write_lease_sha256: lease.map(str::to_owned), - observed_capabilities: expected - .capabilities - .iter() - .map(|capability| cultnet_rs::GameCultRuntimeCapability { - capability: capability.capability.clone(), - schema: capability.schema.clone(), - compatibility: capability.compatibility.clone(), - capacity: 1, - }) - .collect(), - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: Some(activation.runtime_instance_id.clone()), - present: true, - ready, - dependencies: expected - .dependencies - .iter() - .map(|dependency| cultnet_rs::OdinRuntimeDependencyEvidence { - kind: dependency.kind.clone(), - capability: dependency.capability.clone(), - schema: dependency.schema.clone(), - compatibility: dependency.compatibility.clone(), - provider_id: dependency.provider_id.clone(), - provider_authority: dependency.provider_authority.clone(), - provider_expected_projection_sha256: dependency - .provider_expected_projection_sha256 - .clone(), - provider_endpoint: dependency.provider_endpoint.clone(), - observed_capacity: Some(dependency.minimum_capacity), - provider_evidence_sha256: Some(sha256_id(b"odin provider evidence")), - ready: true, - }) - .collect(), - disagreements: Vec::new(), - signer_identity_id: world.odin_signer.entry().identity_id.clone(), - publisher_sequence: sequence, - observed_at_unix_millis: now_millis()?, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - }; - record.signature = world - .odin_signer - .sign::(&record.unsigned_signature_payload()?) - .signature; - record.canonical_bytes() - } - - /// Odin publishes its Ready correlation for the transaction incarnation. - fn odin_reports_ready( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ) -> Result<()> { - let expected = transaction.expected.as_ref().context("no Expected")?; - SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store) - .insert_entry_if_absent(CultCacheEnvelope { - key: crate::drivers::incarnation_key_of( - &expected.target, - &expected.canonical_sha256()?, - ), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: signed_correlation(world, transaction, sequence, true)?, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), - })?; - Ok(()) - } - - fn resident(world: &EngineFixture) -> Result> { - ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .next() - .context("the transaction left the live set") - } - - /// The transaction wherever it now lives: live, or archived once terminal. - fn latest(world: &EngineFixture) -> Result { - match resident(world) { - Ok(stored) => Ok(stored.value), - Err(_) => read_history_transactions(&world.state_store) - .into_iter() - .next() - .context("the transaction is in neither the live set nor history"), - } - } - - /// One engine step at a time until `stop` holds, recording each phase - /// entered and checking after every step that the deadline on the record - /// is exactly the current post-fencing phase's, and absent outside one. - fn drive( - world: &EngineFixture, - stop: impl Fn(&DeploymentTransaction) -> bool, - ) -> Result> { - let mut phases = Vec::new(); - for _ in 0..40 { - let Ok(current) = resident(world) else { - return Ok(phases); - }; - if stop(¤t.value) { - return Ok(phases); - } - world.engine.advance_transaction(¤t)?; - let after = latest(world)?; - match &after.phase_deadline { - Some(deadline) => assert_eq!(deadline.phase, after.phase), - None => assert!( - !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&after.phase), - "a post-fencing record lost its deadline" - ), - } - if phases.last() != Some(&after.phase) { - phases.push(after.phase); - } - } - bail!("the transaction did not reach the stop condition in 40 steps") - } - - #[test] - fn an_admission_runs_from_fencing_to_complete_and_commits() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - - let phases = drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Complete - })?; - assert_eq!( - phases, - [ - DeploymentPhase::Fencing, - DeploymentPhase::Leasing, - DeploymentPhase::AwaitingReady, - DeploymentPhase::Routing, - DeploymentPhase::Committing, - DeploymentPhase::Complete, - ] - ); - let finished = latest(&world)?; - assert_eq!(finished.phase_deadline, None); - assert!(matches!( - finished.completion, - Some(TransactionCompletion::Admitted { .. }) - )); - let snapshot = ControlSnapshot::read(&world.state_store)?; - let admitted = snapshot - .admitted_for("service") - .context("commit wrote no admitted generation")?; - assert_eq!(admitted.value.transaction_id, finished.transaction_id); - assert_eq!(Some(&admitted.value.expected), seeded.expected.as_ref()); - Ok(()) - } - - #[test] - fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - transaction_at(&world, DeploymentPhase::Warming)?; - let warming = resident(&world)?; - assert_eq!(warming.value.phase_deadline, None); - - world - .engine - .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - let finished = latest(&world)?; - assert_eq!(finished.phase, DeploymentPhase::Complete); - assert_eq!(finished.phase_deadline, None); - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); - assert!(finished.is_terminal()); - Ok(()) - } - - #[test] - fn a_post_fence_abort_runs_to_complete() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Routing - })?; - let routing = resident(&world)?; - assert!(routing.value.phase_deadline.is_some()); - - world - .engine - .begin_post_fencing_abort(&routing, anyhow!("candidate refused"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - let finished = latest(&world)?; - assert_eq!(finished.phase, DeploymentPhase::Complete); - assert_eq!(finished.phase_deadline, None); - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { .. }) - )); - assert!( - ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .is_none(), - "an aborted candidate was admitted" - ); - Ok(()) - } - - #[test] - fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; - let plan = transaction.plan.clone().unwrap(); - let base = transaction.created_at_unix_millis; - transaction.enter_phase(DeploymentPhase::Routing, base + 1_000); - let deadline = transaction.phase_deadline.context("no deadline on entry")?; - assert_eq!(deadline.phase, DeploymentPhase::Routing); - assert_eq!( - deadline.deadline_at_unix_millis, - base + 1_000 + u64::from(plan.phase_deadlines().routing_seconds) * 1000 - ); - // A same-phase write after the wall clock stepped backwards is still a - // valid record: the deadline is fixed at entry, not ordered against - // later stamps, and neither is the creation time. - transaction.updated_at_unix_millis = deadline.entered_at_unix_millis - 1; - transaction.validate()?; - transaction.updated_at_unix_millis = base - 1; - transaction.validate()?; - transaction.enter_phase(DeploymentPhase::Committing, base + 2_000); - assert_eq!( - transaction.phase_deadline.unwrap().phase, - DeploymentPhase::Committing - ); - transaction.enter_phase(DeploymentPhase::Complete, base + 3_000); - assert_eq!(transaction.phase_deadline, None); - Ok(()) - } - - #[test] - fn an_undecodable_history_record_is_counted_and_never_dropped_silently() -> Result<()> { - let (finished, _) = terminal_transaction_with_command("ghostlight")?; - let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; - let mut torn = good.clone(); - torn.key = "tx-torn".into(); - // 0xc1 is msgpack's never-used byte: decodable by nothing. - torn.payload = vec![0xc1]; - let mut foreign = good.clone(); - foreign.key = "tx-foreign".into(); - foreign.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA_V3.into()); - foreign.payload = vec![0xc1]; - - let (decoded, undecodable) = - decode_history_transactions(vec![good, torn, foreign]); - assert_eq!(decoded, [finished]); - assert_eq!( - undecodable.iter().map(|(key, _)| key.as_str()).collect::>(), - ["tx-torn", "tx-foreign"] - ); - assert!(undecodable.iter().all(|(_, error)| !error.is_empty())); - Ok(()) - } - - /// The layout `IsolationEvidence` had before c848459 turned it into an - /// untagged Linux/Host enum. That commit claimed old records decode - /// unchanged; this is the check with the old shape itself. - #[test] - fn isolation_evidence_from_before_the_host_variant_still_decodes() -> Result<()> { - #[derive(Serialize)] - struct BeforeTheHostVariant { - candidate_uid: u32, - candidate_pid_namespace_id: u64, - candidate_mount_namespace_id: u64, - incumbent_uid: Option, - incumbent_pid_namespace_id: Option, - incumbent_mount_namespace_id: Option, - } - for incumbent in [None, Some(7)] { - let old = BeforeTheHostVariant { - candidate_uid: 61_000, - candidate_pid_namespace_id: 4_026_531_836, - candidate_mount_namespace_id: 4_026_531_841, - incumbent_uid: incumbent, - incumbent_pid_namespace_id: incumbent.map(|_| 8), - incumbent_mount_namespace_id: incumbent.map(|_| 9), - }; - for bytes in [rmp_serde::to_vec(&old)?, rmp_serde::to_vec_named(&old)?] { - let decoded: IsolationEvidence = rmp_serde::from_slice(&bytes)?; - let IsolationEvidence::Linux(linux) = decoded else { - bail!("an old Linux record decoded as a host record"); - }; - assert_eq!(linux.candidate_uid, 61_000); - assert_eq!(linux.incumbent_uid, incumbent); - } - } - Ok(()) - } - - #[test] - fn an_admitted_odin_without_odin_authority_is_refused_not_replaced_by_the_bootstrap_key() -> Result<()> { - let world = EngineFixture::new()?; - let empty = ControlSnapshot::read(&world.state_store)?; - assert_eq!( - world.engine.current_odin_authority(&empty)?, - world.engine.bootstrap_odin_authority - ); - - let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; - odin.target = "odin".into(); - // Odin is the target that provides the rendezvous, whatever it is named. - odin.expected.target = "odin".into(); - odin.expected.dependencies.clear(); - provide_odin(&mut odin.expected); - odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( - &world.odin_signer.trust_anchor()?, - )?); - let mut with_authority = ControlSnapshot::default(); - with_authority.admitted.push(Stored { - value: odin.clone(), - envelope: CultCacheEnvelope { - key: "odin".into(), - r#type: AdmittedGeneration::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), - }, - }); - assert_eq!( - world.engine.current_odin_authority(&with_authority)?, - odin.odin_authority.clone().unwrap() - ); - - odin.odin_authority = None; - let mut without = ControlSnapshot::default(); - without.admitted.push(Stored { - value: odin.clone(), - envelope: CultCacheEnvelope { - key: "odin".into(), - r#type: AdmittedGeneration::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), - }, - }); - let error = world - .engine - .current_odin_authority(&without) - .expect_err("a route-proof Odin must not fall back to the bootstrap key"); - assert!(format!("{error:#}").contains("no Odin authority"), "{error:#}"); - Ok(()) - } - - fn error_text(result: Result<()>) -> String { - format!("{:#}", result.expect_err("the record must be refused")) - } - - /// A route-proof generation from a real admission: the world's Expected - /// declares no Odin dependency, so route proof is its class. - fn route_proof_generation() -> Result { - route_proof::committed_generation() - } - - #[test] - fn a_route_proof_generation_is_admitted_only_in_its_exact_shape() -> Result<()> { - let generation = route_proof_generation()?; - assert_eq!(generation.ready.voucher(), Voucher::Candidate); - assert!(generation.odin_authority.is_none()); - assert!(generation.latest_odin_observation.is_none()); - assert!(generation.odin_receipts().is_err()); - - // Route proof carries no Odin receipt and no Odin authority. - let mut with_receipt = generation.clone(); - with_receipt.latest_odin_observation = Some(topology(1, 1)); - assert!(with_receipt.validate().is_err()); - let mut with_authority = generation.clone(); - with_authority.odin_authority = Some(AdmittedOdinAuthority { - signer_identity_id: "odin-signer".into(), - signer_public_key: vec![1; 32], - }); - assert!(with_authority.validate().is_err()); - - // Its readiness evidence is checked by shape. - let ReadinessEvidence::RouteProof { evidence } = &generation.ready else { - bail!("not route proof"); - }; - let mut tampered = generation.clone(); - tampered.ready = ReadinessEvidence::RouteProof { - evidence: RuntimePresenceEvidence { - canonical_sha256: sha256_id(b"another"), - ..evidence.clone() - }, - }; - assert!(error_text(tampered.validate()).contains("runtime presence evidence")); - let mut untimed = generation.clone(); - untimed.ready = ReadinessEvidence::RouteProof { - evidence: RuntimePresenceEvidence { - admitted_at_unix_millis: evidence.challenged_at_unix_millis - 1, - ..evidence.clone() - }, - }; - assert!(error_text(untimed.validate()).contains("timeline")); - Ok(()) - } - - #[test] - fn odin_correlated_readiness_evidence_is_checked_by_shape() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.validate()?; - let ReadinessEvidence::OdinCorrelated { evidence } = &generation.ready else { - bail!("fixture is not Odin-correlated"); - }; - for broken in [ - TopologyEvidence { - canonical_sha256: sha256_id(b"another"), - ..evidence.clone() - }, - TopologyEvidence { - publisher_sequence: 0, - ..evidence.clone() - }, - TopologyEvidence { - admitted_at_unix_millis: 0, - ..evidence.clone() - }, - ] { - let mut tampered = generation.clone(); - tampered.ready = ReadinessEvidence::OdinCorrelated { evidence: broken }; - assert!(tampered.validate().is_err()); - } - Ok(()) - } - - #[test] - fn lease_adoption_evidence_is_checked_by_shape_on_the_transaction() -> Result<()> { - let (_, transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; - let adoption = LeaseAdoptionEvidence { - write_lease_sha256: sha256_id(b"lease"), - signed_presence_sha256: sha256_id(b"presence"), - source: AdoptionSource::Direct, - observed_at_unix_millis: 100, - }; - let refused = |change: fn(&mut LeaseAdoptionEvidence)| { - let mut adopting = transaction.clone(); - let mut evidence = adoption.clone(); - change(&mut evidence); - adopting.lease_adoption = Some(evidence); - error_text(adopting.validate()) - }; - assert!(refused(|evidence| evidence.write_lease_sha256.clear()) - .contains("adopted lease digest")); - assert!(refused(|evidence| evidence.signed_presence_sha256 = "no spaces allowed".into()) - .contains("presence digest")); - assert!(refused(|evidence| evidence.observed_at_unix_millis = 0) - .contains("no observation time")); - // Well-formed but naming no granted lease: the record is refused for - // that reason and no other. - assert!(refused(|_| {}).contains("does not name the granted lease")); - Ok(()) - } - - #[test] - fn route_supervision_is_checked_on_the_generation_and_the_meters_on_their_own_record() -> Result<()> { - let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.validate()?; - let supervised = |change: fn(&mut RouteSupervisionState)| { - let mut tampered = generation.clone(); - change(tampered.route_supervision.as_mut().unwrap()); - error_text(tampered.validate()) - }; - assert!(supervised(|state| state.degraded_since_unix_millis = Some(0)) - .contains("no start time")); - - let meters = |change: fn(&mut TargetSupervision)| { - let mut meters = TargetSupervision::new("service"); - change(&mut meters); - error_text(meters.validate()) - }; - TargetSupervision::new("service").validate()?; - assert!(meters(|m| m.route_actuations = vec![0]).contains("ascending")); - assert!(meters(|m| m.route_actuations = vec![9, 5]).contains("ascending")); - assert!( - meters(|m| m.route_actuations = (1..=13).collect()).contains("exceeds its bound") - ); - assert!(meters(|m| m.continuity_restarts = (1..=7).collect()).contains("exceeds its bound")); - assert!(meters(|m| m.route_survivals = (1..=13).collect()).contains("exceeds its bound")); - assert!(meters(|m| m.route_survivals = vec![9, 5]).contains("ascending")); - assert!(meters(|m| m.continuity_deferred_until = Some(5)).contains("time or a reason")); - assert!(meters(|m| m.continuity_deferral_reason = Some("x".into())).contains("time or a reason")); - assert!(meters(|m| m.schema_version = "idunn.target_supervision.v0".into()) - .contains("unsupported")); - Ok(()) - } - - #[test] - fn max_odin_sequence_counts_admitted_generations_by_target_and_signer() -> Result<()> { - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - let authority = generation.odin_authority.clone().context("no authority")?; - generation.odin_publisher_sequence_cursor = 88; - let target = generation.target.clone(); - let snapshot = ControlSnapshot { - expiry_requests: Vec::new(), - commands: Vec::new(), - transactions: Vec::new(), - admitted: vec![Stored { - envelope: CultCacheEnvelope { - key: target.clone(), - r#type: AdmittedGeneration::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), - }, - value: generation, - }], - targets: Vec::new(), - }; - assert_eq!(snapshot.max_odin_sequence(&target, &authority.signer_identity_id), 88); - assert_eq!(snapshot.max_odin_sequence(&target, "another-signer"), 0); - assert_eq!( - snapshot.max_odin_sequence("another-target", &authority.signer_identity_id), - 0 - ); - Ok(()) - } - - #[test] - fn a_store_holding_a_lifted_legacy_generation_reads_and_keeps_its_key_honest() -> Result<()> { - let temp = TempDir::new()?; - let store = temp.path().join("control.cc"); - let envelope = fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?; - let backing = SingleFileMessagePackBackingStore::new(&store); - backing.insert_entry_if_absent(envelope.clone())?; - - let snapshot = ControlSnapshot::read(&store)?; - assert_eq!(snapshot.admitted.len(), 1); - let lifted = &snapshot.admitted[0].value; - assert_eq!(lifted.target, envelope.key); - assert_eq!(lifted.ready.voucher(), Voucher::Odin); - assert!(snapshot.admitted_for(&envelope.key).is_some()); - - // The lifted record is held to the same key rule as a current one. - let other = temp.path().join("misfiled.cc"); - let mut misfiled = envelope; - misfiled.key = "not-its-target".into(); - SingleFileMessagePackBackingStore::new(&other).insert_entry_if_absent(misfiled)?; - assert!(error_text(ControlSnapshot::read(&other).map(|_| ())) - .contains("admitted generation key is not its target")); - Ok(()) - } - - /// Replace Odin's one correlation for the incarnation with a newer one. - fn odin_republishes( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ) -> Result<()> { - let path = &world.engine.options.odin_correlation_store; - let store = SingleFileMessagePackBackingStore::new(path); - let existing = store.pull_all_read_only_snapshot()?; - assert!(store.delete_batch_if_unchanged(&existing)?); - odin_reports_ready(world, transaction, sequence) - } - - fn ready_sequence(transaction: &DeploymentTransaction) -> Option { - transaction - .ready - .as_ref() - .and_then(ReadinessEvidence::odin) - .map(|evidence| evidence.publisher_sequence) - } - - #[test] - fn a_newer_odin_reading_refreshes_the_receipt_before_awaiting_ready_or_routing_advance() - -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - - // AwaitingReady: the receipt is written, then confirmed, then the - // phase advances only on a receipt equal to the latest reading. - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::AwaitingReady - && ready_sequence(transaction) == Some(5) - })?; - odin_republishes(&world, &seeded, 6)?; - world.engine.advance_transaction(&resident(&world)?)?; // admits 6 - let admitted = resident(&world)?.value; - assert_eq!(admitted.phase, DeploymentPhase::AwaitingReady); - assert_eq!(admitted.latest_odin_observation.as_ref().unwrap().publisher_sequence, 6); - assert_eq!(ready_sequence(&admitted), Some(5)); - world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 6 - let refreshed = resident(&world)?.value; - assert_eq!(refreshed.phase, DeploymentPhase::AwaitingReady); - assert_eq!(ready_sequence(&refreshed), Some(6)); - world.engine.advance_transaction(&resident(&world)?)?; - assert_eq!(resident(&world)?.value.phase, DeploymentPhase::Routing); - - // Routing: route admission demands a receipt equal to the latest - // reading, so a newer one is adopted before anything is routed. - odin_republishes(&world, &seeded, 7)?; - world.engine.advance_transaction(&resident(&world)?)?; // admits 7 - assert_eq!(ready_sequence(&resident(&world)?.value), Some(6)); - world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 7 - let routing = resident(&world)?.value; - assert_eq!(routing.phase, DeploymentPhase::Routing); - assert_eq!(ready_sequence(&routing), Some(7)); - assert!(routing.routing.is_none()); - world.engine.advance_transaction(&resident(&world)?)?; - assert!(resident(&world)?.value.routing.is_some()); - Ok(()) - } - - #[test] - fn lease_adoption_names_only_the_granted_lease() -> Result<()> { - let lease = write_lease(); - let lease_sha256 = lease.canonical_sha256()?; - let adoption = LeaseAdoptionEvidence { - write_lease_sha256: lease_sha256.clone(), - signed_presence_sha256: sha256_id(b"adopting-presence"), - source: AdoptionSource::Direct, - observed_at_unix_millis: 100, - }; - adoption.validate_shape()?; - let granted = LeasingEvidence::Granted { - lease: lease.clone(), - lease_sha256: lease_sha256.clone(), - }; - assert!(adoption.names(&granted)); - // Prepared is not granted, stateless has nothing to adopt, and another - // lease's digest is not this lease. - assert!(!adoption.names(&LeasingEvidence::Prepared { - lease, - lease_sha256, - })); - assert!(!adoption.names(&LeasingEvidence::SkippedStateless)); - let mut other = adoption.clone(); - other.write_lease_sha256 = sha256_id(b"another-lease"); - assert!(!other.names(&granted)); - - // On a transaction, adoption without a granted lease is refused. - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; - transaction.lease_adoption = Some(adoption); - assert!(transaction.validate().is_err()); - Ok(()) - } - - /// Q3 as a table: which ended phases abort, which name the unadopted - /// lease, which are left to the operator, and which recovery the terminal - /// record names. - #[test] - fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { - let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; - let lease = write_lease(); - let sha256 = lease.canonical_sha256()?; - let granted = LeasingEvidence::Granted { - lease: lease.clone(), - lease_sha256: sha256.clone(), - }; - let prepared = LeasingEvidence::Prepared { - lease, - lease_sha256: sha256.clone(), - }; - let adoption = LeaseAdoptionEvidence::with_ready( - Some(&sha256), - &sha256_id(b"adopting-presence"), - AdoptionSource::Direct, - 1, - ); - assert!(adoption.is_some()); - assert_eq!( - LeaseAdoptionEvidence::with_ready(None, &sha256_id(b"p"), AdoptionSource::Direct, 1), - None - ); - let abort = |lease_not_adopted| PhaseEnd::Abort { lease_not_adopted }; - use CommandKind::{Continuity, Deploy}; - let cases = [ - (Deploy, None, None, abort(false)), - (Deploy, Some(LeasingEvidence::SkippedStateless), None, abort(false)), - (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), - (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), - (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), - (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), - (Continuity, Some(prepared), None, abort(false)), - (Continuity, Some(granted.clone()), None, abort(true)), - (Continuity, Some(granted), adoption.clone(), abort(false)), - ]; - for (kind, leasing, lease_adoption, expected) in cases { - transaction.command_kind = kind; - transaction.leasing = leasing.clone(); - transaction.lease_adoption = lease_adoption.clone(); - assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); - let recovery = if kind == Continuity && lease_adoption.is_some() { - TerminalRecovery::RestartAdmitted - } else { - TerminalRecovery::RestoreIncumbent - }; - assert_eq!(transaction.terminal_recovery(), recovery, "{kind:?} {lease_adoption:?}"); - } - Ok(()) - } - - #[test] - fn an_operator_required_recovery_must_say_why() -> Result<()> { - let (_, mut failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; - let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = - failed.completion.as_mut() - else { - panic!("fixture is not a post-fencing failure") - }; - *recovery = TerminalRecovery::OperatorRequired { - reason: "adopted-lease".into(), - }; - failed.validate()?; - assert!(failed.is_terminal()); - let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = - failed.completion.as_mut() - else { - unreachable!() - }; - *recovery = TerminalRecovery::OperatorRequired { - reason: String::new(), - }; - assert!(failed.validate().is_err()); - Ok(()) - } - /// The persisted projection a continuity restart leaves behind, read at - /// the file the daemon writes. - struct ContinuityProjection { - _temp: TempDir, - topology: CultCacheTopologyDriver, - provider_anchor: ServiceIdentityTrustAnchor, - expected: IdunnExpectedIncarnationRecord, - admitted_activation: IdunnRuntimeActivationRecord, - candidate_activation: IdunnRuntimeActivationRecord, - } - - impl ContinuityProjection { - /// The production sequence: the admitted incarnation dies and is - /// demoted to Expected-only, then the continuity candidate publishes - /// its own activation under the same key once it is observed. - fn after_candidate_observed() -> Result { - let temp = TempDir::new()?; - let root = temp.path(); - let idunn = enroll_service_identity_at::(&root.join("idunn.cc"))?; - let provider = enroll_service_identity_at::( - &root.join("provider.cc"), - )?; - let provider_anchor = provider.trust_anchor()?; - let mut expected = TopologyFixture::new("ghostlight")?.expected; - expected.expected_signer_identity_id = provider.entry().identity_id.clone(); - expected.artifact_sha256 = sha256_id(&[1]); - expected.validate()?; - let topology = CultCacheTopologyDriver { - projection_store: root.join("topology.cc"), - correlation_store: root.join("correlation.cc"), - }; - let admitted = workload(1, 40, 50); - let candidate = workload(2, 41, 51); - let issue = |observation: &WorkloadObservation| -> Result<_> { - Ok(IdunnRuntimeActivationLaunch::issue( - &expected, - observation.runtime_instance_id().to_owned(), - NOW, - &idunn, - )? - .activation() - .clone()) - }; - let admitted_activation = issue(&admitted)?; - let candidate_activation = issue(&candidate)?; - assert_ne!(admitted_activation, candidate_activation); - - topology.publish_expected(&expected, &provider_anchor)?; - topology.publish_observed_activation(&expected, &admitted_activation, &admitted)?; - topology.demote_to_expected_only( - &expected, - &provider_anchor, - &admitted_activation, - None, - )?; - topology.publish_expected(&expected, &provider_anchor)?; - topology.publish_observed_activation(&expected, &candidate_activation, &candidate)?; - Ok(Self { - _temp: temp, - topology, - provider_anchor, - expected, - admitted_activation, - candidate_activation, - }) - } - - fn continuity_transaction(&self) -> Result { - let mut transaction = DeploymentTransaction::new( - &command(CommandKind::Continuity), - "ghostlight".into(), - 0, - None, - 100, - )?; - transaction.expected = Some(self.expected.clone()); - transaction.activation = Some(self.candidate_activation.clone()); - Ok(transaction) - } - - /// Types of the records projected under the shared incarnation key. - fn projected_types(&self) -> Result> { - let key = incarnation_key(&self.expected)?; - Ok( - SingleFileMessagePackBackingStore::new(&self.topology.projection_store) - .pull_all_read_only_snapshot()? - .into_iter() - .filter(|entry| entry.key == key) - .map(|entry| entry.r#type) - .collect(), - ) - } - } - - #[test] - fn a_failed_continuity_demotes_its_own_activation_and_keeps_the_expected() -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - assert!( - world - .projected_types()? - .contains(&IdunnRuntimeActivationRecord::TYPE.to_owned()) - ); - let transaction = world.continuity_transaction()?; - assert_eq!( - transaction.abort_topology_reconciliation(), - CleanupEvidence::Pending - ); - assert_eq!( - post_fencing_abort_intent(&transaction, "candidate died").topology_reconciliation, - CleanupEvidence::Pending - ); - - reconcile_failed_candidate_projection( - &world.topology, - &transaction, - &world.provider_anchor, - )?; - - assert_eq!( - world.projected_types()?, - vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()], - "the shared key must keep its Expected and nothing else" - ); - Ok(()) - } - - #[test] - fn a_continuity_that_issued_no_activation_owes_no_projection_change() -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - let mut transaction = world.continuity_transaction()?; - transaction.activation = None; - assert_eq!( - transaction.abort_topology_reconciliation(), - CleanupEvidence::Skipped - ); - assert_eq!( - post_fencing_abort_intent(&transaction, "candidate never started") - .topology_reconciliation, - CleanupEvidence::Skipped - ); - Ok(()) - } - - #[test] - fn a_deploy_fencing_after_a_failed_continuity_resolves_with_the_admitted_activation() - -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - reconcile_failed_candidate_projection( - &world.topology, - &world.continuity_transaction()?, - &world.provider_anchor, - )?; - let settled = std::fs::read(&world.topology.projection_store)?; - - // The deploy's post-fence rollback demotes the admitted generation's - // exact activation. The failed continuity left nothing standing, so it - // resolves instead of refusing as substituted. - world.topology.demote_to_expected_only( - &world.expected, - &world.provider_anchor, - &world.admitted_activation, - None, - )?; - assert_eq!(std::fs::read(&world.topology.projection_store)?, settled); - Ok(()) - } - - #[test] - fn no_path_adopts_an_activation_the_transaction_did_not_issue() -> Result<()> { - let world = ContinuityProjection::after_candidate_observed()?; - let before = std::fs::read(&world.topology.projection_store)?; - - // The candidate's activation is still projected. The admitted - // generation's exact activation is not what stands there, so its - // demotion refuses rather than taking the newer one. - assert!( - world - .topology - .demote_to_expected_only( - &world.expected, - &world.provider_anchor, - &world.admitted_activation, - None, - ) - .is_err() - ); - - // Nor may a continuity transaction demote an activation it did not - // issue. - let mut foreign = world.continuity_transaction()?; - foreign.activation = Some(world.admitted_activation.clone()); - assert!( - reconcile_failed_candidate_projection( - &world.topology, - &foreign, - &world.provider_anchor - ) - .is_err() - ); - assert_eq!(std::fs::read(&world.topology.projection_store)?, before); - Ok(()) - } - - // --------------------------------------------------------------------- - // B1 fix batch: the transition, the precondition, the boot reconciliation, - // history that fails closed, and the abort paths through the Engine. - // --------------------------------------------------------------------- - - /// A workload port whose process can be killed: observing it succeeds - /// until `kill`, then fails the way a dead unit does. Everything else is - /// `StillWorkload`'s. - struct SwitchWorkload { - alive: std::sync::atomic::AtomicBool, - rebooted: std::sync::atomic::AtomicBool, - } - - impl SwitchWorkload { - fn new() -> Arc { - Arc::new(Self { - alive: std::sync::atomic::AtomicBool::new(true), - rebooted: std::sync::atomic::AtomicBool::new(false), - }) - } - - fn revive(&self) { - self.alive - .store(true, std::sync::atomic::Ordering::SeqCst); - self.rebooted - .store(false, std::sync::atomic::Ordering::SeqCst); - } - - /// The host rebooted under the unit: it is gone and the driver can tell. - fn reboot(&self) { - self.rebooted - .store(true, std::sync::atomic::Ordering::SeqCst); - self.kill(); - } - - fn kill(&self) { - self.alive - .store(false, std::sync::atomic::Ordering::SeqCst); - } - } - - impl WorkloadPort for SwitchWorkload { - fn install( - &self, - plan: &CompiledDeploymentPlan, - release: &crate::drivers::MaterializedRelease, - ) -> Result { - StillWorkload.install(plan, release) - } - fn prepare_activation( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - launch: IdunnRuntimeActivationLaunch, - ) -> Result { - StillWorkload.prepare_activation(plan, expected, launch) - } - fn start_prepared( - &self, - plan: &CompiledDeploymentPlan, - release: &SealedRelease, - installed: &crate::drivers::InstalledReleaseObservation, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) - } - fn discard_prepared( - &self, - _: &CompiledDeploymentPlan, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - Ok(()) - } - fn observe( - &self, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - prior: &WorkloadObservation, - ) -> Result { - ensure!( - self.alive.load(std::sync::atomic::Ordering::SeqCst), - "the workload is gone" - ); - Ok(prior.clone()) - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - Ok(()) - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(false) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) - } - } - - /// Admit a generation for "service" through the real phase machine and - /// retire its transaction, leaving the world with an incumbent. - fn admit_incumbent(world: &EngineFixture) -> Result { - let seeded = transaction_at(world, DeploymentPhase::Fencing)?; - odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; - drive(world, |transaction| { - transaction.phase == DeploymentPhase::Complete - })?; - assert!(world.engine.retire_one_terminal_transaction()?); - Ok(ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("commit wrote no admitted generation")? - .value - .clone()) - } - - #[test] - fn an_admitted_odin_correlated_generation_takes_a_newer_odin_reading_once() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - let first = seeded_sequence(None) + 1; - odin_reports_ready(&world, &seeded, first)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - assert!(world.engine.retire_one_terminal_transaction()?); - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("commit wrote no admitted generation")? - .value - .clone(); - assert_eq!(admitted.readiness(), Ok(ReadinessClass::OdinCorrelated)); - assert_eq!(admitted.odin_publisher_sequence_cursor, first); - - // Nothing newer than what is held: nothing is written. - let snapshot = ControlSnapshot::read(&world.state_store)?; - let current = snapshot.admitted_for("service").context("no generation")?; - let envelope = current.envelope.clone(); - assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); - assert_eq!(incumbent_envelope(&world)?, envelope); - - // A newer reading is adopted, once. - odin_republishes(&world, &seeded, first + 4)?; - let snapshot = ControlSnapshot::read(&world.state_store)?; - let current = snapshot.admitted_for("service").context("no generation")?; - assert!(world.engine.refresh_admitted_topology(&snapshot, current)?); - let refreshed = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no generation")? - .value - .clone(); - assert_eq!(refreshed.odin_publisher_sequence_cursor, first + 4); - assert_eq!( - refreshed.latest_odin_observation.as_ref().map(|latest| latest.publisher_sequence), - Some(first + 4) - ); - assert_eq!(refreshed.ready.odin().map(|ready| ready.publisher_sequence), Some(first + 4)); - let snapshot = ControlSnapshot::read(&world.state_store)?; - let current = snapshot.admitted_for("service").context("no generation")?; - assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); - Ok(()) - } - - /// Change the incumbent in the store, compare-and-swap. - fn edit_incumbent( - world: &EngineFixture, - edit: impl FnOnce(&mut AdmittedGeneration), - ) -> Result { - let snapshot = ControlSnapshot::read(&world.state_store)?; - let stored = snapshot - .admitted_for("service") - .context("no admitted generation")?; - let mut next = stored.value.clone(); - edit(&mut next); - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: "service".into(), - current: Some(stored.envelope.clone()), - }], - &[admitted_envelope(&next, now_millis()?)?], - )? - ); - Ok(next) - } - - /// Replace or create a target's meters in the store. - fn set_meters(world: &EngineFixture, meters: &TargetSupervision) -> Result<()> { - let snapshot = ControlSnapshot::read(&world.state_store)?; - world.engine.write_target_supervision( - snapshot.supervision_for(&meters.target), - meters, - now_millis()?, - ) - } - - fn meters_of(world: &EngineFixture, target: &str) -> Result { - Ok(ControlSnapshot::read(&world.state_store)?.supervision_or_new(target)) - } - - /// Meters for "service" whose restarts happened these many milliseconds ago. - fn restarted_ago(ages: &[u64]) -> Result { - let now = now_millis()?; - let mut meters = TargetSupervision::new("service"); - meters.continuity_restarts = ages.iter().map(|age| now - age).collect(); - meters.continuity_restarts.sort_unstable(); - // Every logged attempt belongs to one streak. - meters.continuity_attempt_streak = ages.len() as u32; - Ok(meters) - } - - /// Drop every live transaction and command: what a finished restart leaves. - fn clear_live_transactions(world: &EngineFixture) -> Result<()> { - let snapshot = ControlSnapshot::read(&world.state_store)?; - let envelopes = snapshot - .transactions - .iter() - .map(|stored| stored.envelope.clone()) - .chain(snapshot.commands.iter().map(|stored| stored.envelope.clone())) - .collect::>(); - if envelopes.is_empty() { - return Ok(()); - } - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store) - .delete_batch_if_unchanged(&envelopes)? - ); - Ok(()) - } - - fn incumbent_envelope(world: &EngineFixture) -> Result { - Ok(ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no admitted generation")? - .envelope - .clone()) - } - - /// Publish what a transaction's candidate projects: its Expected and, - /// when asked, its observed activation. - fn project_candidate( - world: &EngineFixture, - transaction: &DeploymentTransaction, - with_activation: bool, - ) -> Result<()> { - let expected = transaction.expected.as_ref().context("no Expected")?; - let anchor = world - .engine - .provider_anchor_for_plan(transaction.plan.as_ref().context("no plan")?)?; - let topology = world.engine.topology(); - topology.publish_expected(expected, &anchor)?; - if with_activation { - // The observation must name the release the Expected declares; - // the borrowed workload is patched to. - let mut workload = transaction.workload.clone().context("no workload")?; - if let WorkloadObservation::Systemd(observed) = &mut workload { - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - topology.publish_observed_activation( - expected, - transaction.activation.as_ref().context("no activation")?, - &workload, - )?; - } - Ok(()) - } - - /// Publish what an admitted generation projects while it runs. - fn project_admitted(world: &EngineFixture, generation: &AdmittedGeneration) -> Result<()> { - let anchor = world.engine.provider_anchor_for_plan(&generation.plan)?; - let topology = world.engine.topology(); - topology.publish_expected(&generation.expected, &anchor)?; - topology.publish_observed_activation( - &generation.expected, - &generation.activation, - &generation.workload, - )?; - Ok(()) - } - - /// The types of the records projected under one incarnation key, sorted. - fn projected_under( - world: &EngineFixture, - expected: &IdunnExpectedIncarnationRecord, - ) -> Result> { - let path = &world.engine.options.topology_store; - if !path.exists() { - return Ok(Vec::new()); - } - let key = incarnation_key(expected)?; - let mut types = SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot()? - .into_iter() - .filter(|entry| entry.key == key) - .map(|entry| entry.r#type) - .collect::>(); - types.sort(); - Ok(types) - } - - /// One transaction by id, wherever it now lives. A failed transaction is - /// archived the moment it finishes, so it is in history, not resident. - fn record_of(world: &EngineFixture, transaction_id: &str) -> Result { - if let Some(stored) = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .find(|stored| stored.value.transaction_id == transaction_id) - { - return Ok(stored.value); - } - read_history_transactions(&world.state_store) - .into_iter() - .find(|transaction| transaction.transaction_id == transaction_id) - .context("the transaction is in neither the live set nor history") - } - - fn expected_only() -> Vec { - vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()] - } - - // ---- the Engine's abort paths, Deploy and incumbent ---- - - #[test] - fn a_deploy_that_fences_commits_and_retires_the_incumbent_it_replaced() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - admit_incumbent(&world)?; - let meters = restarted_ago(&[600_000, 60_000])?; - set_meters(&world, &meters)?; - let incumbent = edit_incumbent(&world, |_| {})?; - - let candidate = seeded_transaction( - &world, - DeploymentPhase::Fencing, - CommandKind::Deploy, - Some(&incumbent), - )?; - odin_reports_ready(&world, &candidate, seeded_sequence(Some(&incumbent)) + 1)?; - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Complete - })?; - - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("the deploy admitted nothing")? - .value - .clone(); - assert_eq!(admitted.transaction_id, candidate.transaction_id); - assert_ne!(admitted.generation_id, incumbent.generation_id); - // The restart log belongs to the target: a commit must not reset it. - assert_eq!(meters_of(&world, "service")?, meters); - assert_eq!(admitted.route_supervision, incumbent.route_supervision); - - // The commit did not stop the incumbent: it is retired after. - let committed = record_of(&world, &candidate.transaction_id)?; - let cleanup = committed.post_commit_cleanup.clone().context("no cleanup")?; - assert_eq!( - cleanup.incumbent, - IncumbentCleanupEvidence::Pending { - generation_id: incumbent.generation_id.clone(), - workload: incumbent.workload.clone(), - } - ); - assert_eq!(cleanup.source, SourceCleanupEvidence::Pending); - let frozen = world - .engine - .options - .staging_root - .join("frozen-sources") - .join(&candidate.transaction_id); - std::fs::create_dir_all(&frozen)?; - std::fs::write(frozen.join("tree"), b"frozen")?; - - drive(&world, |transaction| transaction.is_terminal())?; - let retired = record_of(&world, &candidate.transaction_id)? - .post_commit_cleanup - .context("no cleanup")?; - assert_eq!( - retired.incumbent, - IncumbentCleanupEvidence::Complete { - generation_id: incumbent.generation_id - } - ); - assert_eq!(retired.source, SourceCleanupEvidence::Complete); - assert!(!frozen.exists(), "the frozen source outlived its deploy"); - Ok(()) - } - - #[test] - fn a_deploy_post_fence_abort_cleans_up_and_restores_its_incumbent_exactly() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - project_admitted(&world, &incumbent)?; - let candidate = seeded_transaction( - &world, - DeploymentPhase::Fencing, - CommandKind::Deploy, - Some(&incumbent), - )?; - project_candidate(&world, &candidate, true)?; - let candidate_expected = candidate.expected.clone().context("no Expected")?; - assert_eq!( - projected_under(&world, &candidate_expected)?.len(), - 2, - "the candidate did not project" - ); - let frozen = world - .engine - .options - .staging_root - .join("frozen-sources") - .join(&candidate.transaction_id); - std::fs::create_dir_all(&frozen)?; - std::fs::write(frozen.join("tree"), b"frozen")?; - let before = incumbent_envelope(&world)?; - - world - .engine - .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - assert!(resident(&world)?.value.post_fencing_abort.is_some()); - drive(&world, |transaction| transaction.completion.is_some())?; - - let finished = record_of(&world, &candidate.transaction_id)?; - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - )); - let abort = finished.post_fencing_abort.context("no abort evidence")?; - assert_eq!(abort.topology_reconciliation, CleanupEvidence::Complete); - assert_eq!(abort.source_cleanup, CleanupEvidence::Complete); - assert!(!frozen.exists(), "the abandoned source was not cleaned"); - assert!( - projected_under(&world, &candidate_expected)?.is_empty(), - "the abandoned candidate is still projected" - ); - // Fencing stopped the incumbent: it stands as Expected-only, with the - // exact activation it ran under withdrawn, ready for continuity. - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - assert_eq!(incumbent_envelope(&world)?, before); - Ok(()) - } - - #[test] - fn a_deploy_pre_fence_abort_cleans_up_and_leaves_its_incumbent_running() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - project_admitted(&world, &incumbent)?; - let candidate = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Deploy, - Some(&incumbent), - )?; - project_candidate(&world, &candidate, true)?; - let candidate_expected = candidate.expected.clone().context("no Expected")?; - let frozen = world - .engine - .options - .staging_root - .join("frozen-sources") - .join(&candidate.transaction_id); - std::fs::create_dir_all(&frozen)?; - std::fs::write(frozen.join("tree"), b"frozen")?; - let before = incumbent_envelope(&world)?; - - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let intent = resident(&world)?.value.pre_fencing_abort.context("no intent")?; - assert_eq!(intent.topology_reconciliation, CleanupEvidence::Pending); - assert_eq!(intent.source_cleanup, CleanupEvidence::Pending); - drive(&world, |transaction| transaction.completion.is_some())?; - - let finished = record_of(&world, &candidate.transaction_id)?; - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); - assert!(finished.is_terminal()); - assert!(!frozen.exists(), "the abandoned source was not cleaned"); - assert!(projected_under(&world, &candidate_expected)?.is_empty()); - // Before fencing a deployment never touched its incumbent. - assert_eq!( - projected_under(&world, &incumbent.expected)?, - { - let mut types = vec![ - IdunnExpectedIncarnationRecord::TYPE.to_owned(), - IdunnRuntimeActivationRecord::TYPE.to_owned(), - ]; - types.sort(); - types - } - ); - assert_eq!(incumbent_envelope(&world)?, before); - Ok(()) - } - - #[test] - fn a_continuity_abort_over_its_incumbent_demotes_only_the_activation_it_issued() -> Result<()> { - for post_fence in [false, true] { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - let candidate = seeded_transaction( - &world, - if post_fence { - DeploymentPhase::Fencing - } else { - DeploymentPhase::Warming - }, - CommandKind::Continuity, - Some(&incumbent), - )?; - // The candidate shares the incumbent's key: same Expected, its own - // activation. - assert_eq!(candidate.expected.as_ref(), Some(&incumbent.expected)); - assert_ne!(candidate.activation.as_ref(), Some(&incumbent.activation)); - project_candidate(&world, &candidate, true)?; - let before = incumbent_envelope(&world)?; - - let stored = resident(&world)?; - if post_fence { - world - .engine - .begin_post_fencing_abort(&stored, anyhow!("candidate died"))?; - assert_eq!( - resident(&world)? - .value - .post_fencing_abort - .context("no intent")? - .topology_reconciliation, - CleanupEvidence::Pending - ); - } else { - world - .engine - .begin_pre_fencing_abort(&stored, anyhow!("candidate died"))?; - assert_eq!( - resident(&world)? - .value - .pre_fencing_abort - .context("no intent")? - .topology_reconciliation, - CleanupEvidence::Pending - ); - } - drive(&world, |transaction| transaction.completion.is_some())?; - - assert_eq!( - projected_under(&world, &incumbent.expected)?, - expected_only(), - "post_fence={post_fence}: the shared key must keep its Expected and nothing else" - ); - let finished = record_of(&world, &candidate.transaction_id)?; - assert!(finished.is_terminal()); - assert!(matches!( - finished.completion, - Some( - TransactionCompletion::FailedBeforeFencing { .. } - | TransactionCompletion::FailedAfterFencing { .. } - ) - )); - assert_eq!(incumbent_envelope(&world)?, before); - } - Ok(()) - } - - // ---- fix 1: the legacy lift owns the transition ---- - - const FIXTURE_PRE_B1_ABORTS: [(&str, &str); 3] = [ - ( - "pre-fence abort in flight", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex"), - ), - ( - "pre-fence abort already terminal", - include_str!( - "../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex" - ), - ), - ( - "post-fence abort in flight", - include_str!("../tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex"), - ), - ]; - const FIXTURE_PROVIDER_ANCHOR: &str = - include_str!("../tests/fixtures/idunn-control-legacy/provider-anchor.hex"); - /// Where the fixtures' plan binding names the provider trust anchor. - const FIXTURE_ANCHOR_PATH: &str = "/tmp/idunn-b1-legacy/provider-anchor.cc"; - - fn hex_bytes(text: &str) -> Vec { - let hex = text.trim(); - (0..hex.len()) - .step_by(2) - .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("hex fixture")) - .collect() - } - - fn write_fixture_anchor() -> Result<()> { - let path = Path::new(FIXTURE_ANCHOR_PATH); - std::fs::create_dir_all(path.parent().context("anchor path has no parent")?)?; - std::fs::write(path, hex_bytes(FIXTURE_PROVIDER_ANCHOR))?; - Ok(()) - } - - fn abort_owed(transaction: &DeploymentTransaction) -> Option { - transaction - .pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - .or_else(|| { - transaction - .post_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - }) - } - - /// Put a continuity's record and its command in the control store as they - /// stood: `envelope` is written as given, in whatever schema it is in. - fn make_resident( - world: &EngineFixture, - transaction: &DeploymentTransaction, - envelope: CultCacheEnvelope, - ) -> Result<()> { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: transaction.command_id.clone(), - kind: CommandKind::Continuity, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: envelope.key.clone(), - current: None, - }, - ], - &[command_envelope(&command, 100)?, envelope], - )? - ); - Ok(()) - } - - #[test] - fn a_pre_b1_continuity_abort_lifts_without_reopening_and_boot_resolves_its_residue() - -> Result<()> { - write_fixture_anchor()?; - for (name, text) in FIXTURE_PRE_B1_ABORTS { - let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; - - // The record is what the old rule wrote: an activation was issued - // and the abort says it owes the projection nothing. - let legacy: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload)?; - assert_eq!(legacy.command_kind, CommandKind::Continuity, "{name}"); - assert!(legacy.activation.is_some(), "{name}"); - - // History describes what happened, so it lifts the record as written. - let archived = lift_legacy_transaction(&envelope)?; - assert_eq!(abort_owed(&archived), Some(CleanupEvidence::Skipped), "{name}"); - - // The control store lifts it without reopening anything. - let lifted = read_transaction_record(&envelope)?; - let terminal = archived.completion.is_some(); - if terminal { - assert!( - abort_owed(&lifted).is_some_and(CleanupEvidence::is_legacy_marker), - "{name}" - ); - assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); - assert_eq!(lifted.phase, archived.phase, "{name}"); - assert_eq!(lifted.completion, archived.completion, "{name}"); - assert!(!lifted.owns_target_authority(), "{name}"); - } else { - assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); - assert!(!lifted.is_terminal(), "{name}"); - assert_eq!(lifted.completion, None, "{name}"); - } - - // Boot: the store reads. - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - make_resident(&world, &lifted, envelope.clone())?; - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert_eq!(snapshot.transactions.len(), 1, "{name}"); - project_candidate(&world, &lifted, true)?; - let expected = lifted.expected.clone().context("no Expected")?; - assert_eq!(projected_under(&world, &expected)?.len(), 2, "{name}"); - - if terminal { - // A finished record owes nothing and is not touched: boot - // reconciliation demotes the residue by its exact activation. - let before = resident(&world)?.envelope; - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!(outcome.demoted, vec![lifted.transaction_id.clone()], "{name}"); - assert_eq!(resident(&world)?.envelope, before, "{name}"); - } else { - // A record in flight resolves through its own abort. - for _ in 0..12 { - let Ok(current) = resident(&world) else { break }; - if current.value.completion.is_some() { - break; - } - world.engine.advance_transaction(¤t)?; - } - assert!( - record_of(&world, &envelope.key)?.completion.is_some(), - "{name}: the abort never resolved" - ); - } - assert_eq!( - projected_under(&world, &expected)?, - expected_only(), - "{name}: the projection was not demoted to Expected-only" - ); - } - Ok(()) - } - - #[test] - fn resident_terminal_pre_b1_aborts_never_claim_target_authority() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; - let first = read_transaction_record(&envelope)?; - assert_eq!(first.target, live.target); - make_resident(&world, &first, envelope)?; - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - - // A second resident abort of the same shape for the same target, - // already written as the marker the migration persists. - let mut second = first.clone(); - second.transaction_id = "tx-second-legacy".into(); - second.command_id = "continuity-second-legacy".into(); - make_resident( - &world, - &second, - cleanup_evidence::last_error_envelope(&second, "second")?, - )?; - - // The store reads: three records, one of them live, none of the - // finished ones claiming the target. - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert_eq!(snapshot.transactions.len(), 3); - let live_before = snapshot - .transactions - .iter() - .find(|stored| stored.value.transaction_id == live.transaction_id) - .context("live transaction")? - .envelope - .clone(); - - // Reconciliation cleans the residue once, by exact activation. - let expected = first.expected.clone().context("no Expected")?; - project_candidate(&world, &first, true)?; - assert_eq!(projected_under(&world, &expected)?.len(), 2); - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!(outcome.demoted.len(), 1); - assert_eq!(projected_under(&world, &expected)?, expected_only()); - let after = ControlSnapshot::read(&world.state_store)?; - assert_eq!( - after - .transactions - .iter() - .find(|stored| stored.value.transaction_id == live.transaction_id) - .context("live transaction")? - .envelope, - live_before - ); - Ok(()) - } - - #[test] - fn boot_never_republishes_a_withdrawn_incarnation_and_history_keeps_one_copy() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let lifted = read_transaction_record(&envelope)?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - make_resident(&world, &lifted, envelope.clone())?; - // A crash between the two archive steps: history already holds it. - assert!( - SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) - .insert_entry_if_absent(envelope)? - ); - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - - // The incarnation was withdrawn: nothing is projected for it. - let expected = lifted.expected.clone().context("no Expected")?; - assert!(projected_under(&world, &expected)?.is_empty()); - for _ in 0..3 { - world.engine.run_scheduler_tick()?; - } - assert!( - world - .engine - .reconcile_failed_continuity_projections()? - .demoted - .is_empty() - ); - assert!( - projected_under(&world, &expected)?.is_empty(), - "a withdrawn incarnation's Expected was published again" - ); - - // One copy in history, finished, and nothing resident. - let history = read_history(&world.state_store)?; - assert_eq!(history.transactions.len(), 1); - assert!(history.transactions[0].is_terminal()); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - Ok(()) - } - - #[test] - fn boot_runs_the_projection_reconciliation_serve_starts_from() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - let failed = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - project_candidate(&world, &failed, true)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - // A pre-B1 abort's residue: the failed candidate's activation stands. - project_candidate(&world, &failed, true)?; - assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); - - let (_lock, _engine) = boot(world.engine.options.clone())?; - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - Ok(()) - } - - /// A workload whose candidate can neither be stopped nor recover: an abort - /// that cannot finish its first step. - struct WedgedWorkload; - - impl WorkloadPort for WedgedWorkload { - fn install( - &self, - plan: &CompiledDeploymentPlan, - release: &crate::drivers::MaterializedRelease, - ) -> Result { - StillWorkload.install(plan, release) - } - fn prepare_activation( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - launch: IdunnRuntimeActivationLaunch, - ) -> Result { - StillWorkload.prepare_activation(plan, expected, launch) - } - fn start_prepared( - &self, - plan: &CompiledDeploymentPlan, - release: &SealedRelease, - installed: &crate::drivers::InstalledReleaseObservation, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) - } - fn discard_prepared( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - StillWorkload.discard_prepared(plan, expected, activation) - } - fn observe( - &self, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - prior: &WorkloadObservation, - ) -> Result { - StillWorkload.observe(expected, activation, prior) - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - bail!("the unit will not stop") - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(true) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(false) - } - } - - #[test] - fn one_wedged_pre_fence_abort_does_not_stop_the_scheduler() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; - let wedged = transaction_at(&world, DeploymentPhase::Warming)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-nowhere".into(), - kind: CommandKind::Deploy, - selector: "nowhere".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, 100)?], - )? - ); - - // Recording the error is not progress, and neither is repeating it. - assert!(!world.engine.resume_one_transaction()?); - assert!(!world.engine.resume_one_transaction()?); - // Every tick answers: the wedged abort records its error and waits, - // and the rest of the scheduler goes on to freeze the queued command. - write_service_binding(&world)?; - for _ in 0..6 { - world.engine.run_scheduler_tick()?; - } - let stuck = record_of(&world, &wedged.transaction_id)?; - assert!(!stuck.is_terminal()); - assert!(stuck.post_fencing_abort.is_none()); - assert!(stuck.last_error.is_some()); - assert!( - read_history_transactions(&world.state_store) - .iter() - .any(|transaction| transaction.command_id == command.command_id), - "the queued command was never frozen" - ); - Ok(()) - } - - #[test] - fn a_wedged_post_fence_abort_only_waits() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; - let dead = transaction_at(&world, DeploymentPhase::Fencing)?; - world - .engine - .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; - - // The abort cannot stop the unit. The abort is already durable, so - // that is a resumable error, and the tick answers every time. - for _ in 0..3 { - world.engine.run_scheduler_tick()?; - } - let stuck = record_of(&world, &dead.transaction_id)?; - assert!(!stuck.is_terminal()); - assert!(stuck.last_error.is_some()); - Ok(()) - } - - /// A wedged abort whose `stop()` error carries a changing counter, as - /// systemctl output with a PID or timestamp would. - struct ChangingErrorWorkload(std::sync::atomic::AtomicU32, std::sync::atomic::AtomicBool); - - impl WorkloadPort for ChangingErrorWorkload { - fn install( - &self, - plan: &CompiledDeploymentPlan, - release: &crate::drivers::MaterializedRelease, - ) -> Result { - WedgedWorkload.install(plan, release) - } - fn prepare_activation( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - launch: IdunnRuntimeActivationLaunch, - ) -> Result { - WedgedWorkload.prepare_activation(plan, expected, launch) - } - fn start_prepared( - &self, - plan: &CompiledDeploymentPlan, - release: &SealedRelease, - installed: &crate::drivers::InstalledReleaseObservation, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - WedgedWorkload.start_prepared(plan, release, installed, expected, activation) - } - fn discard_prepared( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - WedgedWorkload.discard_prepared(plan, expected, activation) - } - fn observe( - &self, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - prior: &WorkloadObservation, - ) -> Result { - WedgedWorkload.observe(expected, activation, prior) - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - let n = self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - if self.1.load(std::sync::atomic::Ordering::SeqCst) { - return Ok(()); - } - bail!("the unit will not stop (main pid {n})") - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(true) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(false) - } - } - - fn queue_deploy_for_nowhere(world: &EngineFixture) -> Result { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-nowhere".into(), - kind: CommandKind::Deploy, - selector: "nowhere".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, 100)?], - )? - ); - Ok(command) - } - - #[test] - fn a_wedge_whose_error_text_changes_every_tick_does_not_starve_other_targets() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(ChangingErrorWorkload( - std::sync::atomic::AtomicU32::new(0), - std::sync::atomic::AtomicBool::new(false), - )))?; - transaction_at(&world, DeploymentPhase::Warming)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let command = queue_deploy_for_nowhere(&world)?; - write_service_binding(&world)?; - for _ in 0..20 { - world.engine.run_scheduler_tick()?; - } - assert!( - read_history_transactions(&world.state_store) - .iter() - .any(|transaction| transaction.command_id == command.command_id), - "the queued command was starved by another target's wedge" - ); - Ok(()) - } - - #[test] - fn rewriting_last_error_is_not_progress() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let before = transaction_at(&world, DeploymentPhase::Warming)?; - let mut noted = before.clone(); - noted.last_error = Some("the unit will not stop (main pid 7)".into()); - noted.updated_at_unix_millis += 1; - assert!(!state_advanced(&before, ¬ed)); - let mut moved = noted.clone(); - moved.phase = DeploymentPhase::AwaitingReady; - assert!(state_advanced(&before, &moved)); - Ok(()) - } - - #[test] - fn a_failing_step_is_retried_after_a_backoff_not_every_tick() -> Result<()> { - let workload = Arc::new(ChangingErrorWorkload( - std::sync::atomic::AtomicU32::new(0), - std::sync::atomic::AtomicBool::new(false), - )); - let world = EngineFixture::with_workload(workload.clone())?; - transaction_at(&world, DeploymentPhase::Warming)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let attempts = || workload.0.load(std::sync::atomic::Ordering::SeqCst); - let before = now_millis()?; - world.engine.run_scheduler_tick()?; - let after = now_millis()?; - assert_eq!(attempts(), 1, "the step did not run"); - // The first failure waits one poll interval: the wait back_off wrote. - let poll = world.engine.options.poll_millis; - let written = world - .engine - .resume_backoff - .lock() - .unwrap() - .get(&resident(&world)?.value.transaction_id) - .map(|backoff| backoff.not_before_unix_millis) - .expect("the failure set no backoff"); - assert!( - (before + poll..=after + poll).contains(&written), - "back_off wrote {written}, not one poll ({poll} ms) after the failure" - ); - // The wait is set from the wall clock, and a loaded host can spend it - // inside a few ticks. So the test owns the wait: held open, no tick - // retries; once due, the next tick does. - let id = resident(&world)?.value.transaction_id; - let set_due = |due: u64| { - world - .engine - .resume_backoff - .lock() - .unwrap() - .get_mut(&id) - .expect("the failure set no backoff") - .not_before_unix_millis = due; - }; - set_due(now_millis()? + RESUME_BACKOFF_CEILING_MILLIS / 2); - for _ in 0..10 { - world.engine.run_scheduler_tick()?; - } - assert_eq!(attempts(), 1, "the wedged step was retried inside its backoff"); - // Each further failure doubles the wait back_off writes. - for (attempt, polls) in [(2, 2), (3, 4)] { - set_due(1); - let before = now_millis()?; - world.engine.run_scheduler_tick()?; - let after = now_millis()?; - assert_eq!(attempts(), attempt, "the step was not retried once its backoff was due"); - let written = world - .engine - .resume_backoff - .lock() - .unwrap() - .get(&id) - .map(|backoff| backoff.not_before_unix_millis) - .expect("the failure set no backoff"); - assert!( - (before + polls * poll..=after + polls * poll).contains(&written), - "failure {attempt} waits {written}, not {polls} polls ({poll} ms each) after it" - ); - } - Ok(()) - } - - /// A candidate that fails to advance and is dead for good. - struct DeadCandidateWorkload; - - impl WorkloadPort for DeadCandidateWorkload { - fn install( - &self, - plan: &CompiledDeploymentPlan, - release: &crate::drivers::MaterializedRelease, - ) -> Result { - StillWorkload.install(plan, release) - } - fn prepare_activation( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - launch: IdunnRuntimeActivationLaunch, - ) -> Result { - StillWorkload.prepare_activation(plan, expected, launch) - } - fn start_prepared( - &self, - plan: &CompiledDeploymentPlan, - release: &SealedRelease, - installed: &crate::drivers::InstalledReleaseObservation, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) - } - fn discard_prepared( - &self, - plan: &CompiledDeploymentPlan, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - ) -> Result<()> { - StillWorkload.discard_prepared(plan, expected, activation) - } - fn observe( - &self, - _: &IdunnExpectedIncarnationRecord, - _: &IdunnRuntimeActivationRecord, - _: &WorkloadObservation, - ) -> Result { - bail!("the candidate is gone") - } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { - Ok(()) - } - fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(true) - } - fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { - Ok(false) - } - } - - #[test] - fn a_permanently_dead_candidate_past_the_fence_begins_its_post_fence_abort() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(DeadCandidateWorkload))?; - let dead = transaction_at(&world, DeploymentPhase::Fencing)?; - assert!(record_of(&world, &dead.transaction_id)?.post_fencing_abort.is_none()); - for _ in 0..8 { - world.engine.resume_one_transaction()?; - } - assert!( - record_of(&world, &dead.transaction_id)? - .post_fencing_abort - .is_some(), - "a dead candidate past the fence was retried instead of aborted" - ); - Ok(()) - } - - #[test] - fn a_resume_fault_is_reported_once_and_left_in_the_record() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let live = transaction_at(&world, DeploymentPhase::Warming)?; - let error = anyhow!("store hiccup 1"); - world - .engine - .note_fault("could not resume transaction", &live.transaction_id, &error); - assert_eq!( - record_of(&world, &live.transaction_id)?.last_error.as_deref(), - Some("store hiccup 1") - ); - let reports = world.engine.fault_reports.lock().unwrap(); - assert_eq!(reports.len(), 1); - assert!(reports[&live.transaction_id].offer(Some("store hiccup 1".into())).is_none()); - Ok(()) - } - - #[test] - fn a_new_record_cannot_carry_the_legacy_marker() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let lifted = read_transaction_record(&envelope)?; - assert!(lifted.carries_legacy_marker()); - let error = error_text(transaction_envelope(&lifted, 1).map(|_| ())); - assert!(error.contains("legacy lift"), "{error}"); - // The migration's own path still writes it, and it reads back. - let (_, migrated) = cleanup_evidence::migrate_transaction_record(&envelope)?; - assert!(read_transaction_record(&migrated)?.carries_legacy_marker()); - Ok(()) - } - - #[test] - fn unreadable_history_does_not_skip_resident_residue_at_boot() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let first = read_transaction_record(&envelope)?; - make_resident(&world, &first, envelope)?; - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - let expected = first.expected.clone().context("no Expected")?; - project_candidate(&world, &first, true)?; - assert_eq!(projected_under(&world, &expected)?.len(), 2); - corrupt_history(&world)?; - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!(outcome.demoted.len(), 1); - assert_eq!(projected_under(&world, &expected)?, expected_only()); - Ok(()) - } - - #[test] - fn backoff_doubles_caps_and_waits_exactly() { - let ceiling = RESUME_BACKOFF_CEILING_MILLIS; - assert_eq!(backoff_wait(500, 0, ceiling), 500); - assert_eq!(backoff_wait(500, 1, ceiling), 1000); - assert_eq!(backoff_wait(500, 2, ceiling), 2000); - assert_eq!(backoff_wait(500, 3, ceiling), 4000); - assert_eq!(backoff_wait(500, 30, ceiling), ceiling); - assert_eq!(backoff_wait(u64::MAX, 5, ceiling), ceiling); - // The ceiling is the caller's: continuity's is lower than the resume path's. - assert_eq!(backoff_wait(5_000, 5, 300_000), 160_000); - assert_eq!(backoff_wait(5_000, 6, 300_000), 300_000); - assert_eq!(backoff_wait(5_000, 20, 300_000), 300_000); - assert!(is_waiting(9, 10, 5)); - assert!(!is_waiting(10, 10, 5)); - assert!(!is_waiting(11, 10, 5)); - // Due further ahead than the longest wait the owner can set: the clock - // stepped back, so the attempt is due, not stalled for the step. - assert!(is_waiting(5, 10, 5)); - assert!(!is_waiting(4, 10, 5)); - } - - #[test] - fn a_resume_backoff_set_by_a_stepped_back_clock_is_due_and_a_real_one_waits() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let id = transaction_at(&world, DeploymentPhase::Warming)?.transaction_id; - let not_before = |world: &EngineFixture| { - world - .engine - .resume_backoff - .lock() - .unwrap() - .get(&id) - .map(|backoff| backoff.not_before_unix_millis) - }; - let set = |world: &EngineFixture, not_before_unix_millis: u64| { - world.engine.resume_backoff.lock().unwrap().insert( - id.clone(), - ResumeBackoff { - failures: 3, - not_before_unix_millis, - }, - ); - }; - - // Half a minute ahead is a wait the backoff can set: the step is left alone. - let waiting = now_millis()? + 30_000; - set(&world, waiting); - let _ = world.engine.resume_candidate(&resident(&world)?); - assert_eq!(not_before(&world), Some(waiting)); - - // A day ahead is a clock that stepped back: the step is attempted, and - // whatever it does, it replaces the backoff. - let ahead = now_millis()? + 86_400_000; - set(&world, ahead); - let _ = world.engine.resume_candidate(&resident(&world)?); - assert!(not_before(&world).is_none_or(|due| due < ahead)); - Ok(()) - } - - #[test] - fn a_persistent_fault_is_reported_once_and_again_only_after_recovery() -> Result<()> { - let workload = Arc::new(ChangingErrorWorkload( - std::sync::atomic::AtomicU32::new(0), - std::sync::atomic::AtomicBool::new(false), - )); - let world = EngineFixture::with_workload(workload.clone())?; - let wedged = transaction_at(&world, DeploymentPhase::Warming)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; - let id = wedged.transaction_id; - let expire = || { - world.engine.resume_backoff.lock().unwrap().clear(); - }; - let reported = || world.engine.fault_reports.lock().unwrap().contains_key(&id); - // A fault already reported for this transaction (an Err from the - // resume itself); a handled, still-failing step must not re-arm it. - world - .engine - .note_fault("could not resume transaction", &id, &anyhow!("store hiccup")); - assert!(reported()); - world.engine.resume_one_transaction()?; - assert!(reported(), "a failing handled step re-armed the report"); - // Skipped inside the backoff: the fault is still the same fault. - world.engine.resume_one_transaction()?; - assert!(reported(), "a backoff skip re-armed the report"); - // Still failing after the backoff expires: still the same fault. - expire(); - world.engine.resume_one_transaction()?; - assert!(reported()); - // A real recovery clears it. - workload.1.store(true, std::sync::atomic::Ordering::SeqCst); - expire(); - world.engine.resume_one_transaction()?; - assert!(!reported(), "a step that ran and succeeded left the fault armed"); - Ok(()) - } - - #[test] - fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { - write_fixture_anchor()?; - let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let first = read_transaction_record(&envelope)?; - make_resident(&world, &first, envelope)?; - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - assert!(record_of(&world, &first.transaction_id)?.carries_legacy_marker()); - corrupt_history(&world)?; - assert!(!world.engine.retire_one_terminal_transaction()?); - let noted = record_of(&world, &first.transaction_id)?; - assert!( - noted.last_error.is_some() && noted.last_error != first.last_error, - "the retire fault never reached the record" - ); - assert!(noted.carries_legacy_marker()); - Ok(()) - } - - #[test] - fn a_tick_that_only_freezes_or_only_advances_reports_progress() -> Result<()> { - // Only a freeze. - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - queue_deploy_for_nowhere(&world)?; - write_service_binding(&world)?; - assert!(world.engine.run_scheduler_tick()?, "a freeze-only tick reported no progress"); - - // Only a phase advance. - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - transaction_at(&world, DeploymentPhase::Fencing)?; - assert!(world.engine.resume_one_transaction()?, "a real advance reported none"); - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - transaction_at(&world, DeploymentPhase::Fencing)?; - assert!(world.engine.run_scheduler_tick()?); - Ok(()) - } - - #[test] - fn a_terminal_transaction_that_cannot_be_archived_does_not_stop_the_scheduler() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let live = transaction_at(&world, DeploymentPhase::Warming)?; - odin_reports_ready(&world, &live, seeded_sequence(None) + 1)?; - let (finished, command) = terminal_transaction_with_command("ghostlight")?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: finished.transaction_id.clone(), - current: None, - }, - ], - &[ - command_envelope(&command, 100)?, - transaction_envelope(&finished, finished.updated_at_unix_millis)?, - ], - )? - ); - corrupt_history(&world)?; - let before = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .find(|stored| stored.value.transaction_id == live.transaction_id) - .context("live transaction")? - .envelope; - - // The finished record cannot be archived, and the live one still moves. - world.engine.run_scheduler_tick()?; - let after = ControlSnapshot::read(&world.state_store)?; - assert!( - after - .transactions - .iter() - .any(|stored| stored.value.transaction_id == finished.transaction_id) - ); - assert_ne!( - after - .transactions - .iter() - .find(|stored| stored.value.transaction_id == live.transaction_id) - .context("live transaction")? - .envelope, - before, - "the live transaction was not resumed" - ); - Ok(()) - } - - // ---- fix 2: supervision owns the Expected-only precondition ---- - - #[test] - fn continuity_mints_no_restart_over_a_projection_it_cannot_demote() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - - // The projection names an activation that is not the incumbent's. - let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; - let topology = world.engine.topology(); - topology.publish_expected(&incumbent.expected, &anchor)?; - let stranger = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - topology.publish_observed_activation( - &incumbent.expected, - stranger.activation.as_ref().context("no activation")?, - stranger.workload.as_ref().context("no workload")?, - )?; - // Remove the seeded stranger transaction: only the projection matters. - let seeded = resident(&world)?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged( - &[ - seeded.envelope.clone(), - ControlSnapshot::read(&world.state_store)? - .commands - .into_iter() - .find(|stored| stored.value.command_id == seeded.value.command_id) - .context("seeded command")? - .envelope, - ] - )? - ); - workload.kill(); - - let before = now_millis()?; - assert!(world.engine.supervise_one_admitted_generation()?); - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert!( - snapshot.transactions.is_empty(), - "continuity minted a restart over a projection it could not demote" - ); - let recorded = snapshot - .supervision_for("service") - .context("the failed demotion was not recorded")? - .value - .clone(); - let deferred = recorded.continuity_deferred_until.context("no deferral time")?; - assert!(recorded.continuity_deferral_reason.is_some()); - assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); - let after = now_millis()?; - assert!( - (before + CONTINUITY_DEFERRAL_MILLIS..=after + CONTINUITY_DEFERRAL_MILLIS) - .contains(&deferred) - ); - - // Deferred: the next tick neither retries nor writes. - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(meters_of(&world, "service")?, recorded); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - - // The projection is repaired and the deferral has run out: the restart - // proceeds, from an Expected-only projection. - topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; - project_admitted(&world, &incumbent)?; - let mut ran_out = recorded.clone(); - ran_out.continuity_deferred_until = Some(1); - set_meters(&world, &ran_out)?; - assert!(world.engine.supervise_one_admitted_generation()?); - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert_eq!(snapshot.transactions.len(), 1); - let minted = snapshot.supervision_for("service").context("no meters")?; - assert_eq!(minted.value.continuity_restarts.len(), 1); - assert_eq!(minted.value.continuity_deferred_until, None); - assert_eq!(minted.value.continuity_deferral_reason, None); - assert_eq!(snapshot.transactions[0].value.command_kind, CommandKind::Continuity); - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - Ok(()) - } - - // ---- fix 3: one-time boot reconciliation, by exact issuer ---- - - #[test] - fn boot_demotes_the_activation_a_failed_continuity_left_and_only_that() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - let failed = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - project_candidate(&world, &failed, true)?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - // A failed transaction is archived the moment it finishes. - let aborted = record_of(&world, &failed.transaction_id)?; - assert!(aborted.is_terminal()); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - - // Drift a pre-B1 abort left: the failed candidate's activation stands. - project_candidate(&world, &failed, true)?; - assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); - - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!( - outcome, - ProjectionReconciliation { - demoted: vec![failed.transaction_id.clone()], - unexplained: Vec::new(), - } - ); - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - - // Once: nothing is left to demote. - assert_eq!( - world.engine.reconcile_failed_continuity_projections()?, - ProjectionReconciliation::default() - ); - Ok(()) - } - - #[test] - fn boot_never_adopts_an_activation_no_failed_transaction_issued() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let incumbent = admit_incumbent(&world)?; - - // The incumbent's own activation is accounted for. - project_admitted(&world, &incumbent)?; - assert_eq!( - world.engine.reconcile_failed_continuity_projections()?, - ProjectionReconciliation::default() - ); - - // Some other activation, issued by nothing in history, is reported and - // left exactly as it is. - let stranger = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; - let topology = world.engine.topology(); - topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; - project_candidate(&world, &stranger, true)?; - // A live transaction accounts for its own activation... - assert_eq!( - world.engine.reconcile_failed_continuity_projections()?, - ProjectionReconciliation::default() - ); - // ...and for nobody else's. - topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; - let mut other = stranger.clone(); - let other_activation = IdunnRuntimeActivationLaunch::issue( - &incumbent.expected, - runtime_instance_id("tx-someone-else")?, - now_millis()?, - &world.engine.idunn_signer, - )? - .activation() - .clone(); - if let Some(WorkloadObservation::Systemd(observed)) = other.workload.as_mut() { - observed.runtime_instance_id = other_activation.runtime_instance_id.clone(); - } - other.activation = Some(other_activation); - project_candidate(&world, &other, true)?; - assert_eq!( - world.engine.reconcile_failed_continuity_projections()?.unexplained, - vec!["service".to_owned()] - ); - topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; - project_candidate(&world, &stranger, true)?; - // ...and once it is gone, nothing does. - let seeded = resident(&world)?; - let command = ControlSnapshot::read(&world.state_store)? - .commands - .into_iter() - .find(|stored| stored.value.command_id == seeded.value.command_id) - .context("seeded command")?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store) - .delete_batch_if_unchanged(&[seeded.envelope.clone(), command.envelope])? - ); - let projection = std::fs::read(&world.engine.options.topology_store)?; - let outcome = world.engine.reconcile_failed_continuity_projections()?; - assert_eq!( - outcome, - ProjectionReconciliation { - demoted: Vec::new(), - unexplained: vec!["service".into()], - } - ); - assert_eq!(std::fs::read(&world.engine.options.topology_store)?, projection); - Ok(()) - } - - // ---- fix 4: history that cannot be read stops actuation ---- - - /// An operator binding for the target "service", so the command queue can - /// resolve selectors. - fn write_service_binding(world: &EngineFixture) -> Result<()> { - use crate::deployment_plan::tests::BINDING; - let (head, tail) = BINDING - .split_once("[route]") - .context("binding has no route table")?; - let binding = format!( - "{head}[brakes]{}", - tail.split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ); - std::fs::create_dir_all(&world.engine.options.bindings_dir)?; - std::fs::write(world.engine.options.bindings_dir.join("service.toml"), binding)?; - Ok(()) - } - - fn corrupt_history(world: &EngineFixture) -> Result<()> { - std::fs::write(history_store_path(&world.state_store), b"\xc1\xc1 torn")?; - Ok(()) - } - - #[test] - fn history_that_cannot_be_read_is_an_error_and_missing_history_is_empty() -> Result<()> { - let world = EngineFixture::new()?; - let history = read_history(&world.state_store)?; - assert!(history.transactions.is_empty() && history.report().is_none()); - - corrupt_history(&world)?; - let error = read_history(&world.state_store).err().context("torn history read")?; - assert!(format!("{error:#}").contains("history"), "{error:#}"); - assert!(world.engine.history_for_decision().is_none()); - // The display path still answers, with nothing. - assert!(read_history_transactions(&world.state_store).is_empty()); - Ok(()) - } - - #[test] - fn undecodable_history_names_what_it_is_missing() -> Result<()> { - let (finished, _) = terminal_transaction_with_command("ghostlight")?; - let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; - let mut torn = good.clone(); - torn.key = "tx-torn".into(); - torn.payload = vec![0xc1]; - let (transactions, undecodable) = decode_history_transactions(vec![good, torn]); - let history = HistoryRead { - transactions, - undecodable, - }; - let report = history.report().context("nothing reported")?; - assert!(report.contains("1 archived transaction"), "{report}"); - assert!(report.contains("tx-torn"), "{report}"); - - let complete = HistoryRead { - transactions: history.transactions, - undecodable: Vec::new(), - }; - assert_eq!(complete.report(), None); - Ok(()) - } - - #[test] - fn a_fault_is_said_once_while_it_lasts() { - let once = ReportOnce::default(); - assert_eq!(once.offer(None), None); - assert_eq!(once.offer(Some("torn".into())), Some("torn".into())); - assert_eq!(once.offer(Some("torn".into())), None); - assert_eq!(once.offer(Some("torn".into())), None); - assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); - assert_eq!(once.offer(None), None); - assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); - } - - #[test] - fn unreadable_history_does_not_stop_continuity() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - corrupt_history(&world)?; - - // Crash recovery reads the target's own log, never history: the dead - // workload is restarted, and the restart is counted there. - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); - assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); - Ok(()) - } - - #[test] - fn unreadable_history_stops_command_retirement_and_freezing() -> Result<()> { - let world = EngineFixture::new()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-nowhere".into(), - kind: CommandKind::Deploy, - selector: "nowhere".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, 100)?], - )? - ); - write_service_binding(&world)?; - corrupt_history(&world)?; - - // Whether the command was consumed is unknown, so it is not acted on. - assert!(!world.engine.freeze_one_queued_command()?); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - - // Readable, it is acted on: the unknown selector is refused. - std::fs::remove_file(history_store_path(&world.state_store))?; - assert!(world.engine.freeze_one_queued_command()?); - assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); - Ok(()) - } - - #[test] - fn the_legacy_lift_owes_only_a_continuity_that_issued_an_activation() -> Result<()> { - let envelope = fixture_envelope( - FIXTURE_PRE_B1_ABORTS[0].1, - DeploymentTransaction::TYPE, - )?; - let written = lift_legacy_transaction(&envelope)?; - let owed = |transaction: &DeploymentTransaction| { - transaction - .pre_fencing_abort - .as_ref() - .map(|abort| abort.topology_reconciliation) - }; - assert_eq!(owed(&written), Some(CleanupEvidence::Skipped)); - - let mut lifted = written.clone(); - cleanup_evidence::owe_legacy_continuity_projection(&mut lifted); - assert_eq!(owed(&lifted), Some(CleanupEvidence::Pending)); - - // A deployment published its own key and recorded what it owed. - let mut deploy = written.clone(); - deploy.command_kind = CommandKind::Deploy; - cleanup_evidence::owe_legacy_continuity_projection(&mut deploy); - assert_eq!(owed(&deploy), Some(CleanupEvidence::Skipped)); - - // A continuity that issued nothing owes nothing. - let mut unissued = written; - unissued.activation = None; - cleanup_evidence::owe_legacy_continuity_projection(&mut unissued); - assert_eq!(owed(&unissued), Some(CleanupEvidence::Skipped)); - Ok(()) - } - - // ---- operator incidents ---------------------------------------------- - - fn incident_path(world: &EngineFixture) -> PathBuf { - world.root.join("incidents.cc") - } - - fn incidents_of(world: &EngineFixture) -> Result> { - IncidentStore::new(&incident_path(world)).read() - } - - /// The control store never holds an incident: it would stop every older - /// binary that reads it. - fn assert_control_store_holds_no_incident(world: &EngineFixture) -> Result<()> { - ControlSnapshot::read(&world.state_store)?; - for envelope in SingleFileMessagePackBackingStore::new(&world.state_store) - .pull_all_read_only_snapshot()? - { - assert_ne!(envelope.r#type, "idunn.operator_incident"); - } - Ok(()) - } - - /// Six restarts inside the window, the last long enough ago that its wait is - /// over: only the ceiling holds. - fn spend_the_window(world: &EngineFixture) -> Result<()> { - set_meters( - world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - ) - } - - /// One entry has left the window, and the newest restart is recent enough - /// that continuity is still waiting: exhaustion is over, nothing restarts. - fn slide_the_window(world: &EngineFixture) -> Result<()> { - set_meters( - world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 10_000])?, - ) - } - - #[test] - fn an_exhausted_target_opens_one_incident_across_ticks_and_restarts() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - assert!(!incident_path(&world).exists()); - - // The first pass over the dead target starts its wait: progress, no mint. - assert!(world.engine.supervise_one_admitted_generation()?); - let opened = incidents_of(&world)?; - assert_eq!(opened.len(), 1); - assert_eq!(opened[0].subject, "service"); - assert_eq!(opened[0].condition, IncidentCondition::ContinuityExhausted); - assert!(opened[0].is_open()); - let written = std::fs::read(incident_path(&world))?; - - for _ in 0..3 { - assert!(!world.engine.supervise_one_admitted_generation()?); - } - // An Idunn restart forgets its in-memory trace; the durable record - // keeps the decision from being written again. - let restarted = - Engine::open_with_systemd_workload(world.engine.options.clone(), workload.clone())?; - assert!(restarted.fault_reports.lock().unwrap().is_empty()); - assert!(!restarted.supervise_one_admitted_generation()?); - assert_eq!(std::fs::read(incident_path(&world))?, written); - assert_eq!(incidents_of(&world)?, opened); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - assert_control_store_holds_no_incident(&world) - } - - #[test] - fn an_exhaustion_incident_closes_on_observed_health_not_on_the_window() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - let first = incidents_of(&world)?; - assert_eq!(first.len(), 1); - - // The window slides while the workload is still dead: exhaustion is - // over, the target is not healthy, and the incident stays open. - slide_the_window(&world)?; - let _ = world.engine.supervise_one_admitted_generation()?; - assert_eq!(incidents_of(&world)?, first); - - // Seen running, it closes as recovered. - workload.revive(); - assert!(!world.engine.supervise_one_admitted_generation()?); - let closed = incidents_of(&world)?; - assert_eq!(closed.len(), 1); - assert_eq!(closed[0].incident_key, first[0].incident_key); - assert!( - closed[0] - .closed_at_unix_millis - .is_some_and(|at| at >= closed[0].opened_at_unix_millis) - ); - assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); - - // A later exhaustion is a new record with a new opening time. - std::thread::sleep(Duration::from_millis(5)); - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - let again = incidents_of(&world)?; - assert_eq!(again.len(), 2); - let reopened = again - .iter() - .find(|record| record.is_open()) - .context("no new incident")?; - assert!(reopened.opened_at_unix_millis > first[0].opened_at_unix_millis); - assert_ne!(reopened.incident_key, first[0].incident_key); - assert_control_store_holds_no_incident(&world) - } - - #[test] - fn an_incident_whose_target_is_no_longer_admitted_closes() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - assert!(incidents_of(&world)?[0].is_open()); - - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no admitted generation")? - .envelope - .clone(); - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store) - .delete_batch_if_unchanged(&[admitted])? - ); - world.engine.supervise_one_admitted_generation()?; - let records = incidents_of(&world)?; - assert_eq!(records.len(), 1); - assert_eq!(records[0].close_reason, Some(CloseReason::NoLongerAdmitted)); - assert_control_store_holds_no_incident(&world) - } - - #[test] - fn a_tick_retires_incidents_closed_past_retention() -> Result<()> { - let world = EngineFixture::with_workload(SwitchWorkload::new())?; - let store = IncidentStore::new(&incident_path(&world)); - let condition = IncidentCondition::ContinuityExhausted; - let long_ago = now_millis()? - INCIDENT_RETENTION_MILLIS - 10_000; - assert!(store.open(condition, "gone", long_ago)?); - assert!(store.close(condition, "gone", CloseReason::Recovered, long_ago + 1)?); - assert!(store.open(condition, "recent", now_millis()?)?); - assert!(store.close(condition, "recent", CloseReason::Recovered, now_millis()?)?); - - world.engine.supervise_one_admitted_generation()?; - let live = store.read()?; - assert_eq!(live.len(), 1); - assert_eq!(live[0].subject, "recent"); - assert!(world.root.join("incident-history.cc").exists()); - Ok(()) - } - - /// The same exhaustion and recovery sequence, reduced to what continuity - /// decided: each tick's return, and the transactions and restart log it left. - fn continuity_decisions( - world: &EngineFixture, - engine: &Engine, - workload: &SwitchWorkload, - ) -> Result> { - let mut decided = Vec::new(); - let tick = |decided: &mut Vec<(bool, usize, usize)>| -> Result<()> { - let moved = engine.supervise_one_admitted_generation()?; - let snapshot = ControlSnapshot::read(&world.state_store)?; - decided.push(( - moved, - snapshot.transactions.len(), - snapshot.supervision_or_new("service").continuity_restarts.len(), - )); - Ok(()) - }; - workload.kill(); - spend_the_window(world)?; - tick(&mut decided)?; - tick(&mut decided)?; - slide_the_window(world)?; - tick(&mut decided)?; - workload.revive(); - tick(&mut decided)?; - workload.kill(); - spend_the_window(world)?; - tick(&mut decided)?; - Ok(decided) - } - - #[test] - fn an_unwritable_incident_store_changes_nothing_continuity_decides() -> Result<()> { - const CANARY: &str = "CANARY-6b1d-secret-segment"; - let healthy_workload = SwitchWorkload::new(); - let healthy = EngineFixture::with_workload(healthy_workload.clone())?; - admit_incumbent(&healthy)?; - let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; - assert!(incidents_of(&healthy)?.iter().any(|record| record.is_open())); - - // Not permission-dependent: a store path beneath a regular file cannot - // be created for any uid, root included. - let broken_workload = SwitchWorkload::new(); - let mut broken = EngineFixture::with_workload(broken_workload.clone())?; - admit_incumbent(&broken)?; - let blocker = broken.root.join(CANARY); - std::fs::write(&blocker, b"a file, not a directory")?; - let mut options = broken.engine.options.clone(); - options.incident_store = blocker.join("incidents.cc"); - broken.engine = Engine::open_with_systemd_workload(options, broken_workload.clone())?; - - let actual = continuity_decisions(&broken, &broken.engine, &broken_workload)?; - assert_eq!(actual, expected, "a broken incident store changed a continuity decision"); - assert!(!blocker.join("incidents.cc").exists()); - - // The fault is traced once, under its operation, and echoes no path. - let reports = broken.engine.fault_reports.lock().unwrap(); - let faults = reports - .iter() - .filter(|(key, _)| key.starts_with("incident-store:")) - .collect::>(); - assert_eq!( - faults.len(), - 1, - "{:?}", - faults.iter().map(|(key, _)| key).collect::>() - ); - let message = faults[0].1.last.lock().unwrap().clone().context("fault cleared")?; - assert!(message.contains("open an operator incident"), "{message}"); - assert!(!message.contains(CANARY), "{message}"); - drop(reports); - assert_control_store_holds_no_incident(&broken) - } - - #[test] - fn status_renders_incidents_and_survives_an_unreadable_store() -> Result<()> { - const CANARY: &str = "CANARY-90c2-secret-segment"; - let temp = TempDir::new()?; - let now = now_millis()?; - let mut snapshot = ControlSnapshot::default(); - let meters = TargetSupervision::new("service"); - snapshot.targets.push(Stored { - envelope: target_supervision_envelope(&meters, now)?, - value: meters, - }); - - let readable = temp.path().join("incidents.cc"); - IncidentStore::new(&readable).open(IncidentCondition::ContinuityExhausted, "service", now)?; - let lines = render_targets(&snapshot, &readable, now); - assert!(lines[0].starts_with("target service "), "{lines:?}"); - assert_eq!( - lines.last().map(String::as_str), - Some(format!(" incident continuity-exhausted opened-at {now}").as_str()) - ); - - let unreadable = temp.path().join(CANARY); - std::fs::write(&unreadable, b"not a cultcache file")?; - let lines = render_targets(&snapshot, &unreadable, now); - assert!(lines[0].starts_with("target service "), "{lines:?}"); - let last = lines.last().context("no lines")?; - assert!(last.starts_with("incidents unreadable: "), "{last}"); - assert!(!lines.iter().any(|line| line.contains(CANARY)), "{lines:?}"); - Ok(()) - } - - #[test] - fn the_incident_store_is_named_by_serve_and_status() { - let Command::Serve(options) = - parse(["serve", "--incident-store", "/tmp/i.cc"].into_iter().map(str::to_owned)) - .unwrap() - else { - panic!("expected serve") - }; - assert_eq!(options.incident_store, PathBuf::from("/tmp/i.cc")); - assert!(usage().contains("idunn status [--state-store PATH] [--incident-store PATH]")); - let Command::Status { incident_store, .. } = - parse(["status", "--incident-store", "/tmp/j.cc"].into_iter().map(str::to_owned)) - .unwrap() - else { - panic!("expected status") - }; - assert_eq!(incident_store, PathBuf::from("/tmp/j.cc")); - } - - /// A standing fault is traced once. A key is cleared only by an operation - /// that succeeded, so another site's short-circuit must not end it: a - /// target that is running but still exhausted closes nothing, and that - /// used to clear the reconcile pass's fault and print it again every pass. - #[test] - fn a_failing_reconcile_is_traced_once_while_a_running_target_stays_exhausted() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - std::fs::write(incident_path(&world), b"not an incident store")?; - spend_the_window(&world)?; - - for pass in 0..6 { - world.engine.supervise_one_admitted_generation()?; - let reports = world.engine.fault_reports.lock().unwrap(); - assert!( - reports.contains_key("incident-store:close:*"), - "pass {pass}: the standing reconcile fault was cleared, so it prints again" - ); - } - Ok(()) - } - - #[test] - fn a_release_that_runs_one_pass_after_each_restart_keeps_one_incident() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - - // Each cycle: the restarts are spent, the release dies, is seen running - // for one pass (its last restart took), and dies again. - for _ in 0..4 { - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - workload.revive(); - world.engine.supervise_one_admitted_generation()?; - let records = incidents_of(&world)?; - assert_eq!(records.len(), 1, "a flapping release opened another incident"); - assert!(records[0].is_open(), "running while still exhausted closed the incident"); - } - - // Real recovery: running, and the window no longer holds the restarts. - slide_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - let records = incidents_of(&world)?; - assert_eq!(records.len(), 1); - assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); - assert_control_store_holds_no_incident(&world) - } - - #[test] - fn a_target_leaving_admission_closes_only_its_own_incident() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let service = admit_incumbent(&world)?; - let store = IncidentStore::new(&incident_path(&world)); - let condition = IncidentCondition::ContinuityExhausted; - let now = now_millis()?; - for subject in ["service", "neighbour", "gone"] { - assert!(store.open(condition, subject, now)?); - } - - // Two targets are admitted: service, and a neighbour. Only the target - // that is not admitted has its incident closed. - let mut snapshot = ControlSnapshot::read(&world.state_store)?; - let envelope = snapshot - .admitted_for("service") - .context("no admitted generation")? - .envelope - .clone(); - snapshot.admitted.push(Stored { - envelope, - value: AdmittedGeneration { - target: "neighbour".into(), - ..service - }, - }); - world.engine.reconcile_incidents(&snapshot); - - let closed = |subject: &str| -> Result> { - Ok(incidents_of(&world)? - .into_iter() - .find(|record| record.subject == subject) - .context("no such incident")? - .close_reason) - }; - assert_eq!(closed("service")?, None); - assert_eq!(closed("neighbour")?, None); - assert_eq!(closed("gone")?, Some(CloseReason::NoLongerAdmitted)); - Ok(()) - } - - #[cfg(target_os = "linux")] - #[test] - fn a_held_incident_lock_changes_nothing_continuity_decides() -> Result<()> { - use crate::incident::tests::{STUCK_AFTER, hold_lock_of, run_while_held}; - - let healthy_workload = SwitchWorkload::new(); - let healthy = EngineFixture::with_workload(healthy_workload.clone())?; - admit_incumbent(&healthy)?; - let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; - - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - let path = incident_path(&world); - // Three ticks reach the open of an exhausted target, one exchange each. - let actual = run_while_held(hold_lock_of(&path), 3, || { - continuity_decisions(&world, &world.engine, &workload) - })?; - assert_eq!(actual, expected, "a held incident lock changed a continuity decision"); - assert!(!path.exists(), "a contended write landed"); - - // Every refused write is a traced fault naming no path. - { - let reports = world.engine.fault_reports.lock().unwrap(); - let faults = reports - .iter() - .filter(|(key, _)| key.starts_with("incident-store:")) - .map(|(_, report)| report.last.lock().unwrap().clone().unwrap_or_default()) - .collect::>(); - assert!(!faults.is_empty()); - let root = world.root.display().to_string(); - assert!(faults.iter().all(|fault| !fault.contains(&root)), "{faults:?}"); - } - - // The lock is free again: the next pass retries and the incident lands. - // (A child another test forked while it was held can keep it a moment.) - let deadline = std::time::Instant::now() + STUCK_AFTER; - while !incidents_of(&world)?.iter().any(|record| record.is_open()) { - assert!(std::time::Instant::now() < deadline, "the incident never landed"); - world.engine.supervise_one_admitted_generation()?; - std::thread::sleep(Duration::from_millis(20)); - } - assert_control_store_holds_no_incident(&world) - } - - /// A publish that could not land is not done, at the tick: the demotion - /// continuity needs before a restart is one exchange on the topology lock, - /// and another holder of it defers the restart instead of waiting for it. - #[cfg(target_os = "linux")] - #[test] - fn a_held_topology_lock_defers_continuity_and_mints_nothing() -> Result<()> { - use crate::incident::tests::{hold_lock_of, run_while_held}; - - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - project_admitted(&world, &incumbent)?; - let store = world.engine.options.topology_store.clone(); - let projected = projected_under(&world, &incumbent.expected)?; - assert!(projected.contains(&IdunnRuntimeActivationRecord::TYPE.to_owned())); - let before = std::fs::read(&store)?; - workload.kill(); - - // One exchange: the demotion. The window is not spent, so continuity - // would restart if the projection could be demoted. - let moved = run_while_held(hold_lock_of(&store), 1, || { - world.engine.supervise_one_admitted_generation() - })?; - assert!(moved, "the deferral is a write the tick made"); - assert_eq!(std::fs::read(&store)?, before, "a contended publish changed the projection"); - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert!(snapshot.transactions.is_empty(), "continuity minted over a projection it could not demote"); - let recorded = snapshot.supervision_for("service").context("no deferral recorded")?.value.clone(); - assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); - let reason = recorded.continuity_deferral_reason.clone().context("no deferral reason")?; - assert!(reason.contains("locked by another holder"), "{reason}"); - assert!(!reason.contains(&world.root.display().to_string()), "{reason}"); - - // The holder is gone and the deferral has run out: the demotion lands - // and the restart proceeds. - let mut ran_out = recorded; - ran_out.continuity_deferred_until = Some(1); - set_meters(&world, &ran_out)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); - assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); - Ok(()) - } - - /// A deployment phase records a publication digest only after the publish - /// returned: the digest is the claim that it landed. - #[cfg(target_os = "linux")] - #[test] - fn a_contended_expected_publish_is_not_recorded_as_published() -> Result<()> { - use crate::incident::tests::{hold_lock_of, run_while_held}; - - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Warming)?; - let stored = resident(&world)?; - let mut starting = seeded.clone(); - starting.phase = DeploymentPhase::Starting; - starting.expected_publication_sha256 = None; - starting.activation = None; - starting.workload = None; - starting.activation_publication_sha256 = None; - starting.warming = None; - replace_transaction(&world.state_store, &stored, &starting)?; - let store = world.engine.options.topology_store.clone(); - let stored = resident(&world)?; - - let error = run_while_held(hold_lock_of(&store), 1, || { - world.engine.advance_transaction(&stored) - }) - .expect_err("a contended publish must not advance the phase"); - assert!(format!("{error:#}").contains("locked by another holder"), "{error:#}"); - assert!(!store.exists(), "a contended publish wrote the projection"); - let after = resident(&world)?.value; - assert_eq!(after.phase, DeploymentPhase::Starting); - assert_eq!(after.expected_publication_sha256, None); - - // Free again: the same step publishes and records what it published. - world.engine.advance_transaction(&resident(&world)?)?; - let expected = seeded.expected.context("seeded transaction has no Expected")?; - assert_eq!( - resident(&world)?.value.expected_publication_sha256, - Some(expected.canonical_sha256()?) - ); - Ok(()) - } - - /// A Continuity transaction parked in Starting with nothing published yet, - /// plus the stored record the tick will see. - #[cfg(target_os = "linux")] - fn parked_in_starting(world: &EngineFixture) -> Result { - let seeded = transaction_at(world, DeploymentPhase::Warming)?; - let stored = resident(world)?; - let mut starting = seeded.clone(); - starting.phase = DeploymentPhase::Starting; - starting.expected_publication_sha256 = None; - starting.activation = None; - starting.workload = None; - starting.activation_publication_sha256 = None; - starting.warming = None; - replace_transaction(&world.state_store, &stored, &starting)?; - Ok(seeded) - } - - /// A publish that lost to a held lock is transient: before the fence the - /// transaction stays in its phase with the error recorded, no abort is - /// begun, and the next tick after the release lands the publish. A - /// continuity restart is a Continuity transaction on this same path, so - /// its attempt is minted once and a contended publish spends no second one. - #[cfg(target_os = "linux")] - #[test] - fn a_contended_starting_publish_is_retried_on_a_later_tick_not_aborted() -> Result<()> { - use crate::incident::tests::{hold_lock_of, run_while_held}; - - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = parked_in_starting(&world)?; - let mut meters = TargetSupervision::new("service"); - meters.continuity_restarts = vec![now_millis()?]; - set_meters(&world, &meters)?; - let store = world.engine.options.topology_store.clone(); - - for tick in 0..2 { - run_while_held(hold_lock_of(&store), 1, || world.engine.resume_one_transaction())?; - world.engine.resume_backoff.lock().unwrap().clear(); - let held = resident(&world)?.value; - assert_eq!(held.phase, DeploymentPhase::Starting, "tick {tick}"); - assert!(held.pre_fencing_abort.is_none(), "tick {tick}: a held lock aborted the transaction"); - assert!(!held.is_terminal(), "tick {tick}"); - assert_eq!(held.expected_publication_sha256, None, "tick {tick}"); - let error = held.last_error.context("a contended publish left no last_error")?; - assert!(error.contains("locked by another holder"), "{error}"); - assert!(!store.exists(), "a contended publish wrote the projection"); - } - - // Released: the next tick lands the publish and records its digest. - world.engine.resume_one_transaction()?; - let landed = resident(&world)?.value; - assert_eq!(landed.phase, DeploymentPhase::Starting); - assert!(landed.pre_fencing_abort.is_none()); - assert_eq!( - landed.expected_publication_sha256, - Some(seeded.expected.context("seeded transaction has no Expected")?.canonical_sha256()?) - ); - assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); - Ok(()) - } - - /// Only contention is transient: a publish that fails for any other reason - /// still ends the transaction before the fence, as it always did. - #[cfg(target_os = "linux")] - #[test] - fn a_non_contention_publish_error_still_aborts_before_the_fence() -> Result<()> { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - parked_in_starting(&world)?; - // A directory where the store should be: the snapshot cannot be read. - std::fs::create_dir_all(&world.engine.options.topology_store)?; - world.engine.resume_one_transaction()?; - let after = resident(&world)?.value; - assert!(after.pre_fencing_abort.is_some(), "a non-contention error did not abort"); - assert!( - !after.last_error.as_deref().unwrap_or_default().contains("locked by another holder"), - "{:?}", - after.last_error - ); - Ok(()) - } - - #[test] - fn status_prints_an_open_incident_under_its_target() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - world.engine.supervise_one_admitted_generation()?; - let opened = incidents_of(&world)?[0].opened_at_unix_millis; - - let mut printed = Vec::new(); - status(&mut printed, &world.state_store, &incident_path(&world), None)?; - let printed = String::from_utf8(printed)?; - let target = printed.find("target service ").context("no target line")?; - let incident = printed - .find(&format!(" incident continuity-exhausted opened-at {opened}")) - .context("status printed no incident")?; - assert!(target < incident, "{printed}"); - Ok(()) - } - - #[test] - fn serve_ticks_until_told_to_stop() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - workload.kill(); - spend_the_window(&world)?; - let mut options = world.engine.options.clone(); - options.poll_millis = 1; - let engine = Engine::open_with_systemd_workload(options, workload.clone())?; - - let mut asked = 0; - serve_while(&engine, || { - asked += 1; - asked <= 3 - })?; - assert_eq!(asked, 4); - assert!(incidents_of(&world)?.iter().any(|record| record.is_open()), "no tick ran"); - Ok(()) - } - - #[test] - fn run_dispatches_serve_and_status() { - let temp = TempDir::new().unwrap(); - let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); - // A control store beneath a regular file: serve's boot cannot make it, - // and a serve that returned at once would say Ok instead. - let file = temp.path().join("not-a-directory"); - std::fs::write(&file, b"a file").unwrap(); - let store = file.join("control.cc"); - assert!( - run(args(&["serve", "--state-store", store.to_str().unwrap()]).into_iter()).is_err() - ); - // A control store that is not one: status reads it and refuses. - let junk = temp.path().join("junk.cc"); - std::fs::write(&junk, b"not a cultcache file").unwrap(); - assert!( - run(args(&["status", "--state-store", junk.to_str().unwrap()]).into_iter()).is_err() - ); - } - - /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment - /// waiting before Fencing. Continuity never stops and never yields a - /// deployment (budget-never-stop, handover-phase-split), so the window - /// sliding must not turn the deployment into a casualty. - #[test] - fn a_deployment_before_fencing_over_a_dead_incumbent_that_paged_is_never_yielded_and_lands() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - workload.kill(); - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - world.engine.supervise_one_admitted_generation()?; - // The oldest leaves the window: base would now yield the deployment. - set_meters( - &world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - for _ in 0..3 { - let _ = world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!( - after.pre_fencing_abort.is_none(), - "the deployment was yielded: {:?}", - after.pre_fencing_abort - ); - assert_eq!( - continuity_mints(&world)?, - 0, - "continuity minted beside a pre-fencing deployment" - ); - } - workload.revive(); - odin_reports_ready(&world, &redeploy, seeded_sequence(Some(&incumbent)) + 1)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - let landed = latest(&world)?; - assert_eq!(landed.transaction_id, redeploy.transaction_id); - assert_eq!(landed.phase, DeploymentPhase::Complete); - assert!(landed.pre_fencing_abort.is_none()); - Ok(()) - } - - /// The incumbent was admitted this long ago, so an anchor written "in the - /// past" is not older than its generation. - fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { - edit_incumbent(world, |generation| generation.admitted_at_unix_millis -= millis)?; - Ok(()) - } - - /// Put the target's backoff anchor `ago` milliseconds back. - fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { - let mut meters = meters_of(world, "service")?; - meters.continuity_backoff_from_unix_millis = Some(now_millis()? - ago); - set_meters(world, &meters) - } - - /// Continuity transactions minted over an incumbent (the one that admitted - /// it was seeded over nothing). - fn continuity_mints(world: &EngineFixture) -> Result { - Ok(every_transaction(world)? - .iter() - .filter(|transaction| { - transaction.command_kind == CommandKind::Continuity - && transaction.incumbent_generation_id.is_some() - }) - .count()) - } - - #[test] - fn continuity_keeps_restarting_past_the_paging_threshold() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - // Six charged attempts in the hour: the incident pages, nothing stops. - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); - let opened = incidents_of(&world)?; - assert_eq!(opened.len(), 1); - assert!(opened[0].is_open()); - - // The streak-6 wait is 160 s. - anchor_ago(&world, 159_000)?; - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - anchor_ago(&world, 160_001)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 1); - let after = meters_of(&world, "service")?; - assert_eq!(after.continuity_restarts.len(), 6); - assert!(after.continuity_restarts.last().is_some_and(|&last| now_millis().unwrap() - last < 60_000)); - assert_eq!(after.continuity_attempt_streak, 7); - assert_eq!(after.continuity_backoff_from_unix_millis, None); - assert_eq!(incidents_of(&world)?, opened); - Ok(()) - } - - #[test] - fn continuity_backoff_is_a_capped_doubling_of_the_attempt_streak() { - let from = 10_000_000_u64; - for (streak, seconds) in [ - (1_u32, 5_u64), - (2, 10), - (3, 20), - (4, 40), - (5, 80), - (6, 160), - (7, 300), - (8, 300), - (9, 300), - ] { - let mut meters = TargetSupervision::new("service"); - meters.continuity_attempt_streak = streak; - meters.continuity_backoff_from_unix_millis = Some(from); - // The paging log disagrees with the streak on purpose: the wait - // is a function of the streak, not of the window's count. - meters.continuity_restarts = vec![from - 10; if streak == 1 { 6 } else { 2 }]; - let due = from + seconds * 1_000; - assert_eq!(meters.backoff_due(0), Some(due), "streak {streak}"); - assert_eq!( - meters.continuity_backoff(due - 1, 0), - ContinuityBackoff::Waiting, - "streak {streak}" - ); - assert_eq!( - meters.continuity_backoff(due, 0), - ContinuityBackoff::Due { streak }, - "streak {streak}" - ); - } - // No streak, no wait. - let mut fresh = TargetSupervision::new("service"); - fresh.continuity_backoff_from_unix_millis = Some(from); - assert_eq!(fresh.continuity_backoff(from, 0), ContinuityBackoff::Due { streak: 0 }); - } - - #[test] - fn a_sustained_run_resets_the_streak_and_only_a_sustained_run() { - let now = 100_000_000_u64; - let mut meters = TargetSupervision::new("service"); - meters.continuity_attempt_streak = 7; - meters.continuity_restarts = vec![1_000]; - // Lived the sustained time, and no attempt since admission. - assert_eq!( - meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), - ContinuityBackoff::Due { streak: 0 } - ); - // A moment short of it. - assert_eq!( - meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + 10_000), - ContinuityBackoff::Begin - ); - // Attempts after admission failed on this generation: it has not run well. - meters.continuity_restarts = vec![now - 100_000]; - assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); - // Never attempted over this log, long lived: reset. - meters.continuity_restarts = Vec::new(); - assert_eq!( - meters.continuity_backoff(now, now - 700_000), - ContinuityBackoff::Due { streak: 0 } - ); - assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); - // An anchor older than its generation decides nothing. - meters.continuity_backoff_from_unix_millis = Some(now - 800_000); - assert_eq!(meters.backoff_due(now - 700_000), None); - assert_eq!( - meters.continuity_backoff(now, now - 700_000), - ContinuityBackoff::Due { streak: 0 } - ); - assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); - // Streak zero with no history is simply due. - let fresh = TargetSupervision::new("service"); - assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); - } - - #[test] - fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - let mut meters = restarted_ago(&[1_000_000])?; - meters.continuity_attempt_streak = 3; - set_meters(&world, &meters)?; - - // The last attempt was long ago, but the wait starts now: a wait - // anchored on the mint would already be over. - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - - // Streak 3 waits 20 s. - anchor_ago(&world, 19_000)?; - assert!(!world.engine.supervise_one_admitted_generation()?); - anchor_ago(&world, 20_001)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 1); - assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 4); - Ok(()) - } - - #[test] - fn a_queued_deployment_freezes_in_the_gap_after_a_continuity_attempt() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - let mut meters = restarted_ago(&[3_000_000])?; - meters.continuity_attempt_streak = 2; - set_meters(&world, &meters)?; - queue_declaring_redeploy(&world, "up-service-queued")?; - - for _ in 0..4 { - let _ = world.engine.run_scheduler_tick(); - } - let all = every_transaction(&world)?; - assert!( - all.iter().any(|transaction| transaction.command_id == "up-service-queued"), - "the queued deployment never froze" - ); - assert_eq!(continuity_mints(&world)?, 0); - for _ in 0..3 { - let _ = world.engine.run_scheduler_tick(); - assert_eq!(continuity_mints(&world)?, 0); - let aborts = every_transaction(&world)? - .into_iter() - .filter_map(|transaction| transaction.pre_fencing_abort) - .collect::>(); - // The deployment may fail on its own account (a root test runner - // cannot resolve its source); continuity must never be the reason. - assert!( - aborts.iter().all(|abort| !abort.error.contains("continuity")), - "the deployment was yielded: {aborts:?}" - ); - } - Ok(()) - } - - #[test] - fn a_unit_lost_to_a_host_reboot_is_rematerialized_once_with_no_backoff_step() -> Result<()> { - let fresh_world = |workload: &Arc| -> Result { - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - let mut meters = restarted_ago(&[1_000_000])?; - meters.continuity_attempt_streak = 3; - meters.continuity_backoff_from_unix_millis = Some(now_millis()?); - set_meters(&world, &meters)?; - Ok(world) - }; - let workload = SwitchWorkload::new(); - let world = fresh_world(&workload)?; - let before = meters_of(&world, "service")?; - - // Control: dead but not rebooted, the ordinary 20 s wait holds. - workload.kill(); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - - // The host rebooted under the unit: one mint, charged to nothing. - workload.reboot(); - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 1); - let after = meters_of(&world, "service")?; - assert_eq!(after.continuity_restarts, before.continuity_restarts); - assert_eq!(after.continuity_attempt_streak, 3); - assert!(after.continuity_rematerialized_at_unix_millis.is_some()); - assert_eq!(after.continuity_backoff_from_unix_millis, None); - - // The bound is spent: the next death of this generation waits like any other. - clear_live_transactions(&world)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - - // Control: a deferred projection spaces a re-materialization too. - let workload = SwitchWorkload::new(); - let world = fresh_world(&workload)?; - let mut deferred = meters_of(&world, "service")?; - deferred.defer_continuity(now_millis()?, "demotion failed"); - set_meters(&world, &deferred)?; - workload.reboot(); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(continuity_mints(&world)?, 0); - Ok(()) - } - - fn supervision_envelope_raw(payload: Vec, schema: &str) -> Result { - Ok(CultCacheEnvelope { - key: "service".into(), - r#type: TargetSupervision::TYPE.into(), - payload, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(schema.into()), - }) - } - - #[test] - fn a_v1_target_supervision_record_is_lifted_and_migrated_once() -> Result<()> { - let world = EngineFixture::new()?; - let meters = restarted_ago(&[200_000])?; - // A record as every one on a running host was written: seven keys, v1. - let payload = rmp_serde::to_vec(&( - TARGET_SUPERVISION_SCHEMA_V1, - &meters.target, - &meters.route_actuations, - &meters.continuity_restarts, - &meters.continuity_deferred_until, - &meters.continuity_deferral_reason, - &meters.route_survivals, - ))?; - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( - supervision_envelope_raw(payload, TARGET_SUPERVISION_SCHEMA_V1)?, - )?; - let read = meters_of(&world, "service")?; - assert_eq!(read.continuity_restarts, meters.continuity_restarts); - assert_eq!(read.continuity_attempt_streak, 0); - assert_eq!(read.continuity_backoff_from_unix_millis, None); - assert_eq!(read.continuity_rematerialized_at_unix_millis, None); - - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); - assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 0); - let stored = ControlSnapshot::read(&world.state_store)?; - let envelope = &stored.supervision_for("service").unwrap().envelope; - assert_eq!(envelope.schema_id.as_deref(), Some(TARGET_SUPERVISION_SCHEMA)); - assert_eq!(meters_of(&world, "service")?.continuity_restarts, meters.continuity_restarts); - - let mut anchored = TargetSupervision::new("service"); - anchored.continuity_backoff_from_unix_millis = Some(0); - assert!(anchored.validate().is_err()); - anchored.continuity_backoff_from_unix_millis = None; - anchored.continuity_rematerialized_at_unix_millis = Some(0); - assert!(anchored.validate().is_err()); - Ok(()) - } - - #[test] - fn one_target_supervision_schema_id_names_one_shape() -> Result<()> { - let meters = restarted_ago(&[200_000])?; - let ten = rmp_serde::to_vec(&meters)?; - let seven = rmp_serde::to_vec(&( - TARGET_SUPERVISION_SCHEMA, - &meters.target, - &meters.route_actuations, - &meters.continuity_restarts, - &meters.continuity_deferred_until, - &meters.continuity_deferral_reason, - &meters.route_survivals, - ))?; - // The current id refuses the old shape and the old id refuses the new one. - assert!( - read_supervision_record(&supervision_envelope_raw(seven, TARGET_SUPERVISION_SCHEMA)?) - .is_err() - ); - assert!( - read_supervision_record(&supervision_envelope_raw( - ten.clone(), - TARGET_SUPERVISION_SCHEMA_V1 - )?) - .is_err() - ); - assert_eq!( - read_supervision_record(&supervision_envelope_raw(ten, TARGET_SUPERVISION_SCHEMA)?)?, - meters - ); - Ok(()) - } - - #[test] - fn continuity_keeps_minting_at_every_streak_and_log_length() -> Result<()> { - // No stop at any streak, however long, with the paging log full or not. - let full = [3_000_000_u64, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000]; - for streak in 1..=12_u32 { - for log in [&[1_000_000_u64][..], &full[..]] { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - let mut meters = restarted_ago(log)?; - meters.continuity_attempt_streak = streak; - set_meters(&world, &meters)?; - world.engine.supervise_one_admitted_generation()?; - assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the wait begins"); - anchor_ago(&world, 300_001)?; - world.engine.supervise_one_admitted_generation()?; - assert_eq!(continuity_mints(&world)?, 1, "streak {streak} log {} stopped", log.len()); - assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, streak + 1); - } - } - Ok(()) - } - - #[test] - fn a_mint_after_a_sustained_run_stores_the_reset_streak() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 700_000)?; - workload.kill(); - let mut meters = restarted_ago(&[3_000_000])?; - meters.continuity_attempt_streak = 7; - set_meters(&world, &meters)?; - // Ran 700 s since the last attempt: no wait, and the streak restarts at 1. - world.engine.supervise_one_admitted_generation()?; - assert_eq!(continuity_mints(&world)?, 1); - assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); - Ok(()) - } - - #[test] - fn a_re_materialization_counts_as_the_last_attempt_for_the_sustained_reset() { - let admitted_at = 10_000_000_u64; - let now = admitted_at + 700_000; - let mut meters = TargetSupervision::new("service"); - meters.continuity_attempt_streak = 5; - meters.continuity_restarts = vec![admitted_at - 1_000]; - // Before admission: a sustained run, reset. - meters.continuity_rematerialized_at_unix_millis = Some(admitted_at - 1); - assert_eq!( - meters.continuity_backoff(now, admitted_at), - ContinuityBackoff::Due { streak: 0 } - ); - // After admission: that attempt failed on this generation. - meters.continuity_rematerialized_at_unix_millis = Some(admitted_at + 1); - assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); - // Even with no streak to step: the re-materialization is an attempt, - // and the next one waits the base step from its anchor. - meters.continuity_attempt_streak = 0; - meters.continuity_restarts = Vec::new(); - assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); - meters.continuity_backoff_from_unix_millis = Some(now); - assert_eq!(meters.backoff_due(admitted_at), Some(now + CONTINUITY_RESTART_BACKOFF_MILLIS)); - assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Waiting); - assert_eq!( - meters.continuity_backoff(now + CONTINUITY_RESTART_BACKOFF_MILLIS, admitted_at), - ContinuityBackoff::Due { streak: 0 } - ); - meters.continuity_rematerialized_at_unix_millis = None; - assert_eq!(meters.backoff_due(admitted_at), Some(now)); - } - - mod after_a_reboot { - use super::*; - - fn resident_continuity(world: &EngineFixture) -> Result { - Ok(ControlSnapshot::read(&world.state_store)? - .transactions - .iter() - .filter(|s| s.value.command_kind == CommandKind::Continuity) - .count()) - } - - /// The attempt failed and was retired: drop continuity transactions - /// and their commands only. - fn drop_continuity(world: &EngineFixture) -> Result<()> { - let snapshot = ControlSnapshot::read(&world.state_store)?; - let ids = snapshot - .transactions - .iter() - .filter(|s| s.value.command_kind == CommandKind::Continuity) - .map(|s| s.value.command_id.clone()) - .collect::>(); - let envelopes = snapshot - .transactions - .iter() - .filter(|s| s.value.command_kind == CommandKind::Continuity) - .map(|s| s.envelope.clone()) - .chain( - snapshot - .commands - .iter() - .filter(|s| ids.contains(&s.value.command_id)) - .map(|s| s.envelope.clone()), - ) - .collect::>(); - if !envelopes.is_empty() { - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store) - .delete_batch_if_unchanged(&envelopes)? - ); - } - Ok(()) - } - - /// (attempts minted on tick 1, on tick 2, whether the queued deploy froze by then). - fn gap_run(reboot: bool) -> Result<(usize, usize, bool)> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 700_000)?; - if reboot { - workload.reboot(); - } else { - workload.kill(); - } - queue_declaring_redeploy(&world, "up-gap")?; - // One tick, in the tick's order: supervise, then freeze. - world.engine.supervise_one_admitted_generation()?; - let first = resident_continuity(&world)?; - world.engine.freeze_one_queued_command()?; - // That attempt failed and was retired. - drop_continuity(&world)?; - world.engine.supervise_one_admitted_generation()?; - let second = resident_continuity(&world)?; - world.engine.freeze_one_queued_command()?; - let frozen = every_transaction(&world)?.iter().any(|t| t.command_id == "up-gap"); - Ok((first, second, frozen)) - } - - #[test] - fn a_failed_re_materialization_is_followed_by_a_gap_a_queued_deployment_can_take( - ) -> Result<()> { - // Control: an ordinary attempt after a sustained run leaves a gap. - assert_eq!(gap_run(false)?, (1, 0, true)); - // The re-materialization is an attempt too: the next pass waits. - assert_eq!(gap_run(true)?, (1, 0, true)); - Ok(()) - } - - #[test] - fn a_running_pass_ends_the_wait_so_a_later_death_after_a_sustained_run_resets( - ) -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - age_incumbent(&world, 500_000)?; - workload.kill(); - let mut meters = restarted_ago(&[1_000_000])?; - meters.continuity_attempt_streak = 3; - set_meters(&world, &meters)?; - world.engine.supervise_one_admitted_generation()?; - assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); - workload.revive(); - world.engine.supervise_one_admitted_generation()?; - assert_eq!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis, None); - - // It runs on for days, then dies: no wait, and the streak restarts. - age_incumbent(&world, 200_000)?; - workload.kill(); - world.engine.supervise_one_admitted_generation()?; - assert_eq!(continuity_mints(&world)?, 1); - assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); - Ok(()) - } - } - - /// A source that resolves one fixed recipe and never freezes anything. - struct FixedSource { - recipe: String, - } - - impl SourcePort for FixedSource { - fn resolve( - &self, - _binding: &OperatorBinding, - _resolution_id: &str, - _selected_at_unix_millis: u64, - ) -> Result { - Ok(crate::drivers::ResolvedSource { - facts: crate::deployment_plan::tests::source(&self.recipe), - recipe_bytes: self.recipe.clone().into_bytes(), - }) - } - - fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { - bail!("this source freezes nothing") - } - - fn observe_frozen( - &self, - _: &CompiledDeploymentPlan, - _: &FrozenSourceReceipt, - ) -> Result { - bail!("this source freezes nothing") - } - - fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { - Ok(()) - } - } - - /// A Deploy transaction for an unrouted target, still Sealing, whose - /// source resolves `recipe`. One scheduler tick admits its plan or refuses it. - fn sealing_world(recipe: &str) -> Result { - use crate::deployment_plan::tests::BINDING; - let mut world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let (binding_head, binding_tail) = BINDING - .split_once("[route]") - .context("binding has no route table")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail - .split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ); - std::fs::create_dir_all(world.root.join("bindings"))?; - std::fs::write(world.root.join("bindings/service.toml"), binding)?; - world.engine.source = Arc::new(FixedSource { - recipe: recipe.to_owned(), - }); - let now = now_millis()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-service".into(), - kind: CommandKind::Deploy, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; - assert!( - SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - ], - &[ - command_envelope(&command, command.requested_at_unix_millis)?, - transaction_envelope(&transaction, now)?, - ], - )? - ); - Ok(world) - } - - fn unrouted_recipe() -> String { - crate::deployment_plan::tests::RECIPE - .split("[[dependencies]]") - .next() - .expect("the recipe has a body") - .replace("route_required = true", "route_required = false") - .replace( - r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, - r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, - ) - } - - #[test] - fn admission_refuses_a_recipe_that_declares_no_way_to_prove_readiness() -> Result<()> { - let recipe = unrouted_recipe(); - let world = sealing_world(&recipe)?; - world.engine.run_scheduler_tick()?; - let refused = latest(&world)?; - assert!(refused.plan.is_none(), "the refused recipe was planned"); - assert!(refused.frozen_source.is_none(), "the refused recipe was frozen"); - let abort = refused - .pre_fencing_abort - .context("a recipe with no declared proof of readiness must be refused")?; - assert!( - abort.error.contains("target service declares no way to prove readiness"), - "{}", - abort.error - ); - Ok(()) - } - - #[test] - fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { - let recipe = unrouted_recipe().replace( - "capability = \"service.runtime\"", - &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), - ); - let world = sealing_world(&recipe)?; - world.engine.run_scheduler_tick()?; - let planned = latest(&world)?; - assert!(planned.pre_fencing_abort.is_none(), "{:?}", planned.pre_fencing_abort); - assert_eq!( - planned.plan.as_ref().map(CompiledDeploymentPlan::readiness_class).transpose()?, - Some(ReadinessClass::OdinSelf) - ); - Ok(()) - } - - // --------------------------------------------------------------------- - // A held record is reported and never decided for. Supervision mints no - // continuity over a held generation, and nothing yields to a held record. - // --------------------------------------------------------------------- - - /// Rewrite the world's admitted generation into what a pre-B3 binary left - /// behind: no Odin declaration, admitted on Odin receipts. - fn undeclare_incumbent(world: &EngineFixture) -> Result { - let now = now_millis()?; - let mut failure = None; - let next = edit_incumbent(world, |generation| { - generation.expected.dependencies.clear(); - match IdunnRuntimeActivationLaunch::issue( - &generation.expected, - generation.activation.runtime_instance_id.clone(), - now, - &world.engine.idunn_signer, - ) { - Ok(launch) => generation.activation = launch.activation().clone(), - Err(error) => failure = Some(error), - } - })?; - if let Some(error) = failure { - return Err(error); - } - ControlSnapshot::read(&world.state_store).context("store with the undeclared generation")?; - assert!(matches!(next.readiness(), Err(ReadinessDisagreement::Undeclared(_)))); - Ok(next) - } - - /// Queue the operator's declaring redeploy of `service`. - fn queue_declaring_redeploy(world: &EngineFixture, command_id: &str) -> Result<()> { - use crate::deployment_plan::tests::BINDING; - let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail.split_once("[brakes]").context("brakes")?.1 - ); - std::fs::create_dir_all(world.root.join("bindings"))?; - std::fs::write(world.root.join("bindings/service.toml"), binding)?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: command_id.into(), - kind: CommandKind::Deploy, - selector: "service".into(), - requested_by: "operator".into(), - requested_at_unix_millis: now_millis()?, - }; - assert!(SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }], - &[command_envelope(&command, command.requested_at_unix_millis)?], - )?); - Ok(()) - } - - /// Every transaction the store knows: live, then archived. - fn every_transaction(world: &EngineFixture) -> Result> { - let mut all = ControlSnapshot::read(&world.state_store)? - .transactions - .into_iter() - .map(|stored| stored.value) - .collect::>(); - all.extend(read_history_transactions(&world.state_store)); - Ok(all) - } - - fn is_continuity_over(transaction: &DeploymentTransaction, held: &AdmittedGeneration) -> bool { - transaction.command_kind == CommandKind::Continuity - && transaction.incumbent_generation_id.as_deref() == Some(held.generation_id.as_str()) - } - - #[test] - fn a_held_generation_that_dies_mints_no_continuity_and_leaves_its_target_free() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - let held = undeclare_incumbent(&world)?; - world.engine.validate_durable_authority(&ControlSnapshot::read(&world.state_store)?)?; - - workload.kill(); - for _ in 0..4 { - assert!(!world.engine.supervise_one_admitted_generation()?); - } - assert!( - ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), - "supervision minted a transaction over a held generation" - ); - // Reported, once per state change: the same text is not offered again. - let reports = world.engine.fault_reports.lock().unwrap(); - let down = reports.get("generation-down:service").context("the dead hold is not reported")?; - let last = down.last.lock().unwrap().clone(); - assert!(last.is_some()); - assert!(down.offer(last).is_none()); - drop(reports); - - // The target is free: the declaring redeploy freezes and is not - // displaced by a continuity. - queue_declaring_redeploy(&world, "up-service-declared")?; - for _ in 0..8 { - let _ = world.engine.run_scheduler_tick(); - } - let all = every_transaction(&world)?; - assert!( - all.iter().all(|transaction| !is_continuity_over(transaction, &held)), - "a continuity was minted over the held generation" - ); - assert!( - all.iter().any(|transaction| transaction.command_id == "up-service-declared"), - "the declaring redeploy never froze" - ); - Ok(()) - } - - #[test] - fn a_declaring_redeploy_is_not_yielded_when_the_held_incumbent_dies() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - admit_incumbent(&world)?; - let held = undeclare_incumbent(&world)?; - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; - assert_eq!(redeploy.held_disagreement(), None); - workload.kill(); - for _ in 0..4 { - let _ = world.engine.supervise_one_admitted_generation()?; - } - let all = every_transaction(&world)?; - assert!( - all.iter().all(|transaction| !is_continuity_over(transaction, &held)), - "a continuity was minted over the held incumbent" - ); - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!( - after.pre_fencing_abort.is_none(), - "the redeploy yielded: {:?}", - after.pre_fencing_abort - ); - Ok(()) - } - - /// `seeded` with its own Expected undeclared, so Idunn holds it. - fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { - let mut bare = seeded.clone(); - // Undeclare the Expected and re-derive what is bound to its digest. - let expected = bare.expected.as_mut().context("no expected")?; - expected.dependencies.clear(); - bare.expected_publication_sha256 = Some(expected.canonical_sha256()?); - let activation = IdunnRuntimeActivationLaunch::issue( - expected, - seeded.activation.as_ref().context("no activation")?.runtime_instance_id.clone(), - now_millis()?, - &world.engine.idunn_signer, - )? - .activation() - .clone(); - bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); - bare.activation = Some(activation); - bare.validate()?; - assert!(bare.held_disagreement().is_some()); - Ok(bare) - } - - /// A pre-fence Deploy whose own Expected is undeclared, stored in place. - fn held_predeploy( - world: &EngineFixture, - incumbent: &AdmittedGeneration, - ) -> Result { - let seeded = - seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(incumbent))?; - let bare = undeclared(world, &seeded)?; - replace_transaction(&world.state_store, &resident(world)?, &bare)?; - Ok(bare) - } - - #[test] - fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - // The incumbent is declared; the Deploy in front of it is the held one. - let held = held_predeploy(&world, &incumbent)?; - - workload.kill(); - for _ in 0..4 { - let _ = world.engine.supervise_one_admitted_generation()?; - } - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, held.transaction_id); - assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); - assert!(after.completion.is_none()); - Ok(()) - } - - #[test] - fn cancel_withdraws_a_held_predeploy_and_leaves_the_incumbent_untouched() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let held = held_predeploy(&world, &incumbent)?; - let before = incumbent_envelope(&world)?; - - cancel(&world.state_store, &held.command_id, "operator")?; - let cancelling = resident(&world)?.value; - assert_eq!( - cancelling.pre_fencing_abort.as_ref().map(|abort| abort.error.as_str()), - Some("cancelled by operator") - ); - assert!(cancelling.post_fencing_abort.is_none()); - - drive(&world, |transaction| transaction.completion.is_some())?; - // History also holds the incumbent's own admission: find ours by id. - let finished = every_transaction(&world)? - .into_iter() - .find(|transaction| transaction.transaction_id == held.transaction_id) - .context("the cancelled redeploy is nowhere")?; - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); - assert!(finished.is_terminal()); - // The target is released and nothing of the incumbent was touched. - assert!(!finished.blocks_new_target_mutation()); - assert_eq!(incumbent_envelope(&world)?, before); - assert!( - every_transaction(&world)? - .iter() - .all(|transaction| !is_continuity_over(transaction, &incumbent)), - "cancelling the redeploy minted a continuity" - ); - Ok(()) - } - - #[test] - fn cancel_refuses_a_predeploy_that_is_not_held_and_a_held_one_past_the_fence() -> Result<()> { - let world = EngineFixture::with_workload(SwitchWorkload::new())?; - let incumbent = admit_incumbent(&world)?; - let seeded = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - assert!(seeded.held_disagreement().is_none()); - assert!(cancel(&world.state_store, &seeded.command_id, "operator").is_err()); - assert!(resident(&world)?.value.pre_fencing_abort.is_none()); - - let mut past_the_fence = undeclared(&world, &seeded)?; - past_the_fence.enter_phase(DeploymentPhase::Fencing, now_millis()?); - assert!(past_the_fence.held_disagreement().is_some()); - assert!(!cancel_is_safe_for_held_prefence(&past_the_fence)); - Ok(()) - } - - // --------------------------------------------------------------------- - // B2: the meters, the v4 lift, and what `status` shows. - // --------------------------------------------------------------------- - - #[test] - fn the_actuation_log_slides() -> Result<()> { - let window = ROUTE_ACTUATION_WINDOW_MILLIS; - let t = 1_000_000_000_u64; - let mut meters = TargetSupervision::new("service"); - for offset in 0..12 { - meters.charge_route(t + offset, RouteActuation::Forward)?; - } - let refused = meters.charge_route(t + window - 1, RouteActuation::Forward).unwrap_err(); - assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + window)); - // Only the first entry has left the window: exactly one slot reopens. - meters.charge_route(t + window, RouteActuation::Forward)?; - let refused = meters.charge_route(t + window, RouteActuation::Forward).unwrap_err(); - assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + 1 + window)); - meters.validate()?; - Ok(()) - } - - #[test] - fn a_survival_actuation_is_recorded_but_never_counted_or_refused() -> Result<()> { - let t = 1_000_000_000_u64; - let mut meters = TargetSupervision::new("service"); - // Survivals first, past the ceiling: every one is admitted and - // recorded, and none takes a Forward slot. - for offset in 0..20 { - meters.charge_route(t + offset, RouteActuation::Survival)?; - } - assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); - assert_eq!(meters.route_survivals.last(), Some(&(t + 19))); - assert!(meters.route_actuations.is_empty()); - // Twelve Forward changes interleaved with more Survivals: the twelve - // are admitted, and the thirteenth is refused with a reopen time that - // only the oldest Forward entry decides. - for offset in 0..12 { - meters.charge_route(t + 100 + 2 * offset, RouteActuation::Forward)?; - meters.charge_route(t + 101 + 2 * offset, RouteActuation::Survival)?; - } - let refused = meters.charge_route(t + 200, RouteActuation::Forward).unwrap_err(); - assert_eq!( - (refused.used, refused.reopens_at_unix_millis), - (ROUTE_ACTUATION_CEILING, t + 100 + ROUTE_ACTUATION_WINDOW_MILLIS) - ); - // A full Forward ceiling still admits and records a Survival. - meters.charge_route(t + 201, RouteActuation::Survival)?; - assert_eq!(meters.route_survivals.last(), Some(&(t + 201))); - meters.validate()?; - Ok(()) - } - - #[test] - fn a_clock_stepped_back_stalls_nothing_past_one_window() -> Result<()> { - let t = 5_000_000_000_u64; - let now = t - 86_400_000; - let mut meters = TargetSupervision::new("service"); - meters.route_actuations = vec![t; ROUTE_ACTUATION_CEILING]; - meters.continuity_restarts = vec![t; CONTINUITY_PAGING_RESTARTS]; - meters.continuity_deferred_until = Some(t); - meters.continuity_deferral_reason = Some("demotion failed".into()); - meters.validate()?; - - // Entries from the future count as now: the meters hold for one window. - let read_only = meters.clone(); - assert!(read_only.restarts_page(now)); - let refused = meters.charge_route(now, RouteActuation::Forward).unwrap_err(); - assert!(refused.reopens_at_unix_millis <= now + ROUTE_ACTUATION_WINDOW_MILLIS); - // The clamp is written, so the window runs from the step and no longer. - assert_eq!(meters.route_actuations, vec![now; ROUTE_ACTUATION_CEILING]); - assert_eq!(meters.continuity_restarts, vec![now; CONTINUITY_PAGING_RESTARTS]); - assert!(!meters.restarts_page(now + CONTINUITY_RESTART_WINDOW_MILLIS)); - meters.charge_route(now + ROUTE_ACTUATION_WINDOW_MILLIS, RouteActuation::Forward)?; - - // Waits set for the future are due, not stalled for the length of the step. - assert!(!read_only.continuity_deferred(now)); - let mut anchored = TargetSupervision::new("service"); - anchored.continuity_attempt_streak = 7; - anchored.continuity_backoff_from_unix_millis = Some(t); - assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Due { streak: 7 }); - anchored.continuity_attempt_streak = 1; - anchored.continuity_backoff_from_unix_millis = Some(now + 10_000); - assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Waiting); - let route = RouteSupervisionState { - next_challenge_at_unix_millis: Some(t), - ..RouteSupervisionState::default() - }; - assert!(!route.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); - assert!(!is_waiting(now, t, RESUME_BACKOFF_CEILING_MILLIS)); - // The same waits, set for a moment ahead, still wait. - let ahead = RouteSupervisionState { - next_challenge_at_unix_millis: Some(now + 5_000), - ..RouteSupervisionState::default() - }; - assert!(ahead.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); - Ok(()) - } - - #[test] - fn a_route_challenge_wait_widens_to_a_cap_and_a_proof_clears_it() { - let age = DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS; - let mut state = RouteSupervisionState::default(); - let mut waits = Vec::new(); - for _ in 0..8 { - state.record_failed_challenge(1_000, age); - waits.push(state.next_challenge_at_unix_millis.unwrap() - 1_000); - } - assert_eq!(&waits[..3], &[age, age * 2, age * 4]); - assert_eq!(waits[7], ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); - assert_eq!(state.degraded_since_unix_millis, Some(1_000)); - state.record_proved_challenge(9_000); - assert_eq!(state.consecutive_failures, 0); - assert_eq!(state.next_challenge_at_unix_millis, None); - assert_eq!(state.degraded_since_unix_millis, None); - } - - #[test] - fn a_v3_generation_lifts_to_v4_once_and_reencodes_canonically() -> Result<()> { - for text in [FIXTURE_GENERATION_V3_ODIN, FIXTURE_GENERATION_V3_ROUTE_PROOF] { - let temp = TempDir::new()?; - let store = temp.path().join("control.cc"); - let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; - let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; - SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(envelope.clone())?; - // The v3 bytes are not v4 bytes: only the typed lift may read them. - assert!(decode_record::(&envelope).is_err()); - - assert_eq!(migrate_control_store_to_current_schema(&store)?, 1); - assert_eq!(migrate_control_store_to_current_schema(&store)?, 0); - let snapshot = ControlSnapshot::read(&store)?; - let migrated = snapshot.admitted.first().context("the lift lost the record")?; - assert_eq!(migrated.envelope.schema_id.as_deref(), Some(ADMITTED_GENERATION_SCHEMA)); - // After the lift the canonical re-encode accepts its own bytes. - let lifted: AdmittedGeneration = decode_record(&migrated.envelope)?; - assert_eq!(lifted, migrated.value); - - // The decisions are the ones the v3 record encoded; only the retired - // fields are gone, and no meters were invented. - assert_eq!(lifted.target, legacy.target); - assert_eq!(lifted.generation_id, legacy.generation_id); - assert_eq!(lifted.plan, legacy.plan); - assert_eq!(lifted.expected, legacy.expected); - assert_eq!(lifted.activation, legacy.activation); - assert_eq!(lifted.workload, legacy.workload); - assert_eq!(lifted.leasing, legacy.leasing); - assert_eq!(lifted.ready, legacy.ready); - assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); - assert_eq!(lifted.routing, legacy.routing); - assert_eq!(lifted.odin_authority, legacy.odin_authority); - assert_eq!( - lifted.odin_publisher_sequence_cursor, - legacy.odin_publisher_sequence_cursor - ); - assert_eq!( - lifted.route_supervision, - legacy.route_supervision.map(|state| RouteSupervisionState { - last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, - consecutive_failures: state.consecutive_failures, - next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, - degraded_since_unix_millis: state.degraded_since_unix_millis, - }) - ); - assert_eq!(lifted.last_error, None); - assert!(snapshot.targets.is_empty()); - } - Ok(()) - } - - #[test] - fn a_v3_generation_that_shows_written_meters_is_refused_not_dropped() -> Result<()> { - let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; - let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; - assert!(legacy.route_supervision.is_some(), "the fixture is a routed generation"); - let refused = |change: fn(&mut LegacyAdmittedGenerationV3)| -> Result { - let mut edited = legacy.clone(); - change(&mut edited); - let mut tampered = envelope.clone(); - tampered.payload = rmp_serde::to_vec(&edited)?; - Ok(error_text(read_generation_record(&tampered).map(|_| ()))) - }; - let restarts = refused(|edited| edited.continuity_backoff.attempts = 2)?; - assert!(restarts.contains("refusing to drop"), "{restarts}"); - assert!(restarts.contains(&legacy.target), "the error names the record: {restarts}"); - let actuations = - refused(|edited| edited.route_supervision.as_mut().unwrap().actuations.count = 1)?; - assert!(actuations.contains("refusing to drop"), "{actuations}"); - - // A whole store carrying one refuses to boot, naming the record. - let temp = TempDir::new()?; - let store = temp.path().join("control.cc"); - let mut edited = legacy.clone(); - edited.continuity_backoff.attempts = 2; - let mut tampered = envelope.clone(); - tampered.payload = rmp_serde::to_vec(&edited)?; - SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(tampered)?; - assert!(migrate_control_store_to_current_schema(&store).is_err()); - Ok(()) - } - - #[test] - fn a_v3_generation_lifts_each_route_supervision_field_to_its_own_place() -> Result<()> { - let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; - let mut legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; - // Four distinct non-default values, so a swap cannot land on itself. - legacy.route_supervision = Some(LegacyRouteSupervisionStateV3 { - last_challenge_at_unix_millis: Some(1_111), - consecutive_failures: 3, - next_challenge_at_unix_millis: Some(2_222), - degraded_since_unix_millis: Some(3_333), - actuations: LegacyActuationWindowV3::default(), - }); - let mut edited = envelope.clone(); - edited.payload = rmp_serde::to_vec(&legacy)?; - let lifted = read_generation_record(&edited)?; - assert_eq!( - lifted.route_supervision, - Some(RouteSupervisionState { - last_challenge_at_unix_millis: Some(1_111), - consecutive_failures: 3, - next_challenge_at_unix_millis: Some(2_222), - degraded_since_unix_millis: Some(3_333), - }) - ); - Ok(()) - } - - #[test] - fn status_covers_the_admitted_targets_and_the_metered_ones() -> Result<()> { - // "service" is admitted with no meters at all; "metered" has meters and - // no generation. Neither set contains the other. - let world = EngineFixture::with_workload(SwitchWorkload::new())?; - admit_incumbent(&world)?; - assert!(ControlSnapshot::read(&world.state_store)?.targets.is_empty()); - set_meters(&world, &TargetSupervision::new("metered"))?; - let snapshot = ControlSnapshot::read(&world.state_store)?; - assert_eq!( - supervised_targets(&snapshot).into_iter().collect::>(), - ["metered", "service"] - ); - Ok(()) - } - - #[test] - fn target_supervision_reads_back_and_its_key_must_be_its_target() -> Result<()> { - let world = EngineFixture::new()?; - let meters = restarted_ago(&[5_000])?; - set_meters(&world, &meters)?; - assert_eq!(meters_of(&world, "service")?, meters); - assert_eq!(ControlSnapshot::read(&world.state_store)?.targets.len(), 1); - - let mut elsewhere = target_supervision_envelope(&meters, 5)?; - elsewhere.key = "other".into(); - SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent(elsewhere)?; - let error = error_text(ControlSnapshot::read(&world.state_store).map(|_| ())); - assert!(error.contains("not its target"), "{error}"); - Ok(()) - } - - #[test] - fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { - let now = 10_000_000_u64; - let mut meters = TargetSupervision::new("service"); - meters.route_actuations = (0..12).map(|offset| now - 5_000 + offset).collect(); - meters.continuity_restarts = vec![now - 9_000, now - 4_000]; - meters.continuity_deferred_until = Some(now + 500); - meters.continuity_deferral_reason = Some("projection would not demote".into()); - meters.continuity_attempt_streak = 2; - meters.continuity_backoff_from_unix_millis = Some(now - 4_000); - - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.last_error = Some("held".into()); - generation.admitted_at_unix_millis = 1; - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); - assert!(lines.contains(&format!("target service {}", generation.generation_id)), "{lines}"); - assert!(lines.contains("held: held"), "{lines}"); - assert!(lines.contains(&format!( - "continuity attempts-in-hour 2 pages-at 6 streak 2 next-attempt-at {}", - now - 4_000 + 10_000 - )), "{lines}"); - assert!(!lines.contains("rematerialized-at"), "{lines}"); - meters.continuity_rematerialized_at_unix_millis = Some(now - 1_000); - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" -"); - assert!(lines.contains(&format!("continuity rematerialized-at {}", now - 1_000)), "{lines}"); - // An anchor older than the generation is not shown as a pending wait. - generation.admitted_at_unix_millis = now; - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" -"); - assert!(lines.contains("streak 2 next-attempt-at none"), "{lines}"); - generation.admitted_at_unix_millis = 1; - assert!(lines.contains("continuity deferred until 10000500: projection would not demote"), "{lines}"); - assert!(lines.contains(&format!( - "route actuations 12/12 in window, reopens-at {}", - now - 5_000 + ROUTE_ACTUATION_WINDOW_MILLIS - )), "{lines}"); - assert!(lines.contains("route healthy consecutive-failures 0"), "{lines}"); - assert!( - lines.contains("route survival actuations 0 in window (uncounted), last-at none"), - "{lines}" - ); - meters.route_survivals = vec![now - 7_000, now - 3_000]; - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); - assert!(lines.contains(&format!( - "route survival actuations 2 in window (uncounted), last-at {}", - now - 3_000 - )), "{lines}"); - meters.route_survivals = (0..ROUTE_SURVIVAL_RECORD as u64).map(|n| now - 100 + n).collect(); - let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); - assert!(lines.contains("route survival actuations at least 12 in window"), "{lines}"); - // Survivals never move the ceiling's count. - assert!(lines.contains("route actuations 12/12 in window"), "{lines}"); - - // A first deployment has meters and no generation: it shows them alone. - let mut first = TargetSupervision::new("fresh"); - first.route_actuations = vec![now - 1]; - let lines = render_supervision("fresh", None, Some(&first), now).join("\n"); - assert!(lines.contains("target fresh no-admitted-generation"), "{lines}"); - assert!(lines.contains("route actuations 1/12 in window, reopens-at none"), "{lines}"); - // An unrouted, unmetered target shows no route line at all. - let lines = render_supervision("plain", Some(&unrouted_generation()?), None, now).join("\n"); - assert!(!lines.contains("route actuations"), "{lines}"); - // A generation that no longer routes, whose meters hold only survival - // actuations, still shows the record. - let mut survived = TargetSupervision::new("plain"); - survived.route_survivals = vec![now - 1]; - let lines = - render_supervision("plain", Some(&unrouted_generation()?), Some(&survived), now).join("\n"); - assert!(lines.contains("route survival actuations 1 in window"), "{lines}"); - Ok(()) - } - - fn unrouted_generation() -> Result { - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.route_supervision = None; - Ok(generation) - } - - // --------------------------------------------------------------------- - // Route-proof readiness: a routed target that declares no Odin dependency - // is admitted by Idunn's own challenges. Odin is never read. - // --------------------------------------------------------------------- - mod route_proof { - use super::*; - use std::io::{Read, Write}; - use std::net::{TcpListener, TcpStream}; - use std::sync::Mutex; - use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; - use std::time::Duration; - - /// The runtime behind both endpoints, answering Idunn's snapshot - /// challenge the way the TypeScript signer does: locally, in the - /// reported health, with `route-observation:` as the detail. - struct RuntimeStub { - expected: IdunnExpectedIncarnationRecord, - activation: IdunnRuntimeActivationRecord, - provider: cultnet_rs::ServiceIdentitySigner, - activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, - state: Mutex<&'static str>, - capacity: AtomicU32, - sequence: AtomicU64, - candidate_hits: AtomicUsize, - stable_hits: AtomicUsize, - stop: AtomicBool, - /// Accept the connection and close it without a word: a process - /// that is up but not answering. Unlike a closed port, no other - /// test's listener can be handed the address. - hang_up: AtomicBool, - /// The write lease its presence claims to hold: what a candidate - /// that picked up Idunn's grant reports. - lease: Mutex>, - reply: Mutex, - /// Runs when the stable listener accepts, before it answers. - on_stable_hit: Mutex>>, - } - - /// How the runtime answers a well-formed challenge. - #[derive(Clone, Copy, PartialEq, Eq)] - enum Reply { - Honest, - /// An answer to some other challenge. - ForeignChallengeId, - /// An HTTP error where the snapshot should be. - HttpFailure, - /// A first Odin candidate answering its Warming challenge: the - /// detail names the challenge as `idunn-warming:`. - FirstOdinWarming, - } - - impl RuntimeStub { - fn answer(&self, message_id: &str) -> Result> { - let expected = &self.expected; - let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { - schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: self.activation.expected_projection_sha256.clone(), - plan_id: expected.plan_id.clone(), - incarnation_id: expected.incarnation_id.clone(), - sealed_release_id: expected.sealed_release_id.clone(), - activation_witness_sha256: self.activation.canonical_sha256()?, - state_schema_generation: expected.state_schema_generation.clone(), - state_contract_sha256: expected.state_contract_sha256.clone(), - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: self.activation.runtime_instance_id.clone(), - bound_endpoint: expected - .route - .as_ref() - .map(|route| route.candidate_endpoint.clone()), - capabilities: expected - .capabilities - .iter() - .map(|capability| cultnet_rs::GameCultRuntimeCapability { - capability: capability.capability.clone(), - schema: capability.schema.clone(), - compatibility: capability.compatibility.clone(), - capacity: self.capacity.load(Ordering::SeqCst), - }) - .collect(), - health_contract: expected.health_contract.clone(), - state: (*self.state.lock().unwrap()).into(), - detail: match *self.reply.lock().unwrap() { - Reply::FirstOdinWarming => format!("idunn-warming:{message_id}"), - _ => format!("route-observation:{message_id}"), - }, - write_lease_sha256: self.lease.lock().unwrap().clone(), - signer_identity_id: self.provider.entry().identity_id.clone(), - publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, - observed_at_unix_millis: now_millis()?, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - activation_signer_identity_id: self.activation_signer.identity_id(), - activation_signature: Vec::new(), - }; - let proof = presence.canonical_proof_payload()?; - presence.signature = self - .provider - .sign::(&proof) - .signature; - presence.activation_signature = - self.activation_signer.sign_presence_proof(&presence)?; - Ok(rmp_serde::to_vec(&presence)?) - } - - fn set_state(&self, state: &'static str) { - *self.state.lock().unwrap() = state; - } - - fn serve_one(&self, mut stream: TcpStream) -> Result<()> { - stream.set_read_timeout(Some(Duration::from_secs(3)))?; - let mut request = Vec::new(); - let mut chunk = [0_u8; 4096]; - let (header_end, length) = loop { - let read = stream.read(&mut chunk)?; - ensure!(read > 0, "stub client closed early"); - request.extend_from_slice(&chunk[..read]); - if let Some(index) = request.windows(4).position(|w| w == b"\r\n\r\n") { - let head = String::from_utf8_lossy(&request[..index]).to_lowercase(); - let length = head - .lines() - .find_map(|line| line.strip_prefix("content-length:")) - .context("stub request has no Content-Length")? - .trim() - .parse::()?; - if request.len() >= index + 4 + length { - break (index + 4, length); - } - } - }; - let message = cultnet_rs::decode_cultnet_message_from_slice( - &request[header_end..header_end + length], - cultnet_rs::CultNetWireContract::CultNetSchemaV0, - )?; - let cultnet_rs::CultNetMessage::SnapshotRequest { message_id, .. } = message - else { - bail!("stub received something other than a snapshot request"); - }; - let reply = *self.reply.lock().unwrap(); - if reply == Reply::HttpFailure { - stream.write_all( - b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - )?; - return Ok(()); - } - let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { - message_id: if reply == Reply::ForeignChallengeId { - "candidate-foreign".to_owned() - } else { - message_id.clone() - }, - documents: vec![cultnet_rs::CultNetRawDocumentRecord { - schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), - record_key: self.expected.target.clone(), - stored_at: rfc3339_millis(now_millis()?)?, - payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, - payload: self.answer(&message_id)?, - source_runtime_id: None, - source_agent_id: None, - source_role: None, - tags: None, - }], - }; - let body = cultnet_rs::encode_cultnet_message_to_vec( - &response, - cultnet_rs::CultNetWireContract::CultNetSchemaV0, - )?; - stream.write_all( - format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ) - .as_bytes(), - )?; - stream.write_all(&body)?; - Ok(()) - } - - fn listen(self: &Arc, listener: TcpListener, candidate: bool) { - listener.set_nonblocking(true).expect("nonblocking listener"); - let stub = Arc::clone(self); - std::thread::spawn(move || { - while !stub.stop.load(Ordering::SeqCst) { - match listener.accept() { - Ok((stream, _)) => { - stream.set_nonblocking(false).ok(); - let hits = if candidate { - &stub.candidate_hits - } else { - &stub.stable_hits - }; - hits.fetch_add(1, Ordering::SeqCst); - if !candidate - && let Some(hook) = stub.on_stable_hit.lock().unwrap().as_ref() - { - hook(); - } - if !stub.hang_up.load(Ordering::SeqCst) { - let _ = stub.serve_one(stream); - } - } - Err(_) => std::thread::sleep(Duration::from_millis(5)), - } - } - }); - } - } - - impl Drop for RuntimeStub { - fn drop(&mut self) { - self.stop.store(true, Ordering::SeqCst); - } - } - - /// Odin, as the world it runs in sees it. - #[derive(Clone, Copy, PartialEq, Eq)] - enum Odin { - /// Every read of the correlation store fails. - Unreachable, - /// Odin publishes a correlation that says the target is not Ready. - NotReady, - } - - struct RoutedWorld { - world: EngineFixture, - workload: Arc, - stub: Arc, - odin: Odin, - poisoned: Vec, - } - - /// A stateless, routed target that declares no Odin dependency, over - /// `minimum_capacity`, seeded at `phase` (Warming or Fencing): its - /// Warming evidence, when the phase needs it, is the candidate's own - /// answer to a real challenge. Listeners stand at both endpoints. - fn routed_world( - odin: Odin, - minimum_capacity: u32, - phase: DeploymentPhase, - provides_odin: bool, - ) -> Result { - build_routed_world( - odin, - minimum_capacity, - phase, - provides_odin, - false, - CommandKind::Continuity, - ) - } - - /// The same world owned by a deployment command instead of a - /// continuity: its route actuations are `Forward` and can be refused. - fn routed_deploy_world( - odin: Odin, - minimum_capacity: u32, - phase: DeploymentPhase, - ) -> Result { - build_routed_world(odin, minimum_capacity, phase, false, false, CommandKind::Deploy) - } - - /// The same world for a target with process-writable state: its - /// candidate is granted a write lease and Ready must be the candidate - /// reporting exactly that lease. - fn stateful_routed_world(odin: Odin, phase: DeploymentPhase) -> Result { - build_routed_world(odin, 1, phase, false, true, CommandKind::Continuity) - } - - fn build_routed_world( - odin: Odin, - minimum_capacity: u32, - phase: DeploymentPhase, - provides_odin: bool, - stateful: bool, - kind: CommandKind, - ) -> Result { - use crate::deployment_plan::tests::{ - BINDING, RECIPE, artifact_receipt, external_input_receipt, source, - }; - let switch = SwitchWorkload::new(); - let world = EngineFixture::with_workload(switch.clone())?; - let candidate_listener = TcpListener::bind("127.0.0.1:0")?; - let stable_listener = TcpListener::bind("127.0.0.1:0")?; - let candidate_port = candidate_listener.local_addr()?.port(); - let stable_port = stable_listener.local_addr()?.port(); - - let provider_path = world.root.join("identities/provider.cc"); - let provider_anchor = world.root.join("identities/provider-anchor.cc"); - let provider = - enroll_service_identity_at::(&provider_path)?; - export_service_identity_trust_anchor(&provider, &provider_anchor)?; - let config = world.root.join("service.conf"); - let binding = BINDING - .replace( - "/etc/gamecult/trust/service.cc", - &provider_anchor.display().to_string(), - ) - .replace("service-runtime-signer", &provider.entry().identity_id) - .replace("127.0.0.1:17999", &format!("127.0.0.1:{stable_port}")) - .replace( - "private_port_start = 18000", - &format!("private_port_start = {candidate_port}"), - ) - .replace( - "private_port_end = 18009", - &format!("private_port_end = {}", candidate_port + 1), - ) - .replace( - "/etc/nginx/idunn-stream-routes/service.conf", - &config.display().to_string().replace('\\', "/"), - ); - let (binding, recipe_source) = if stateful { - std::fs::create_dir_all(world.root.join("lease"))?; - ( - binding.replace( - "[brakes]", - &format!( - "[process_write_lease]\nrecord_path = \"{}\"\n\n[brakes]", - world.root.join("lease/process-write-lease.cc").display() - ), - ), - RECIPE - .replace( - "writer = \"none\"\nrecovery = \"rebuildable\"\nstartup = \"open-at-start\"", - "writer = \"process-bound-single-writer\"\nrecovery = \"preserve\"\nstartup = \"create-or-open-after-write-lease\"", - ) - .replace( - "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", - "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_PROCESS_WRITE_LEASE\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", - ), - ) - } else { - (binding, RECIPE.to_owned()) - }; - let recipe = recipe_source - .split("[[dependencies]]") - .next() - .context("recipe is empty")? - .replace( - "capability = \"service.runtime\"", - &format!( - "capability = \"{}\"\ncapacity = {minimum_capacity}", - if provides_odin { - ODIN_RENDEZVOUS_CAPABILITY - } else { - "service.runtime" - } - ), - ); - let plan = compile_deployment_plan( - recipe.as_bytes(), - binding.as_bytes(), - source(&recipe), - "service-incarnation-1", - Some(candidate_port), - 110, - &[], - )?; - let release = SealedRelease::new( - &plan, - vec![artifact_receipt()], - vec![external_input_receipt()], - 120, - )?; - let expected = release.expected_projection(&plan)?; - assert!(expected.route.is_some() && expected.dependencies.is_empty()); - assert_eq!(expected.write_lease_required, stateful); - assert_eq!( - ReadinessClass::of(&expected), - Ok(if provides_odin { - ReadinessClass::OdinSelf - } else { - ReadinessClass::RouteProof - }) - ); - - let now = now_millis()?; - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: match kind { - CommandKind::Deploy => "up-service", - CommandKind::Continuity => "continuity-service", - } - .into(), - kind, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let mut transaction = - DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; - match kind { - CommandKind::Continuity => { - transaction.lifecycle_authorized_at_unix_millis = Some(now); - } - CommandKind::Deploy => { - transaction.frozen_source = Some(FrozenSourceReceipt { - transaction_id: transaction.transaction_id.clone(), - plan_id: plan.plan_id.clone(), - snapshot_sha256: sha256_id(b"frozen source snapshot"), - }); - transaction.deployment_authorization = Some(deployment_authorization( - &world, - &expected, - &transaction.transaction_id, - now, - )?); - } - } - let launch = IdunnRuntimeActivationLaunch::issue( - &expected, - runtime_instance_id(&transaction.transaction_id)?, - now, - &world.engine.idunn_signer, - )?; - let mut seed = Vec::new(); - let activation = launch.write_credential(&mut seed)?; - let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; - let mut workload = recorded.workload.clone().context("no recorded workload")?; - match &mut workload { - WorkloadObservation::Systemd(observed) => { - observed.runtime_instance_id = activation.runtime_instance_id.clone(); - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), - } - transaction.isolation = recorded.isolation; - transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { - sealed_release_id: release.sealed_release_id.clone(), - root: PathBuf::from("/srv/service/releases/test"), - }); - transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); - transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); - transaction.sealed_release = Some(release); - transaction.expected = Some(expected.clone()); - transaction.activation = Some(activation.clone()); - transaction.workload = Some(workload); - transaction.plan = Some(plan); - - let stub = Arc::new(RuntimeStub { - expected, - activation, - provider: open_service_identity_at::( - &provider_path, - )?, - activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( - &seed[..], - )?, - state: Mutex::new("warming"), - capacity: AtomicU32::new(minimum_capacity), - sequence: AtomicU64::new(0), - candidate_hits: AtomicUsize::new(0), - stable_hits: AtomicUsize::new(0), - stop: AtomicBool::new(false), - hang_up: AtomicBool::new(false), - lease: Mutex::new(None), - reply: Mutex::new(Reply::Honest), - on_stable_hit: Mutex::new(None), - }); - stub.listen(candidate_listener, true); - stub.listen(stable_listener, false); - - if phase >= DeploymentPhase::Fencing && provides_odin { - // Odin's own first word about the candidate, as for any - // Odin-correlated target. - // A stateful candidate is warming, not active, until it holds - // the lease Idunn grants after the fence. - let warming = signed_correlation_as( - &world, - &transaction, - 4, - false, - if stateful { "warming" } else { "active" }, - None, - b"presence", - )?; - let authenticated = world.engine.authenticate_topology_bytes( - &ControlSnapshot::read(&world.state_store)?, - &transaction, - &warming, - None, - now, - )?; - transaction.warming = Some(WarmingEvidence::OdinTopology { - evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, - }); - transaction.odin_publisher_sequence_cursor = 4; - } else if phase >= DeploymentPhase::Fencing { - let CandidateAnswer::Answered { evidence, .. } = - world - .engine - .challenge_candidate(&transaction, &["warming"], None)? - else { - bail!("the stub candidate did not answer"); - }; - transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); - } - if phase >= DeploymentPhase::Fencing { - transaction.route_preflight = Some(candidate_preflight(&transaction)?); - } - if stateful { - // The lease driver reads the Expected and the observed - // activation from the topology store, as Starting left them. - let plan = transaction.plan.as_ref().unwrap(); - let anchor = world.engine.provider_anchor_for_plan(plan)?; - let topology = world.engine.topology(); - topology.publish_expected(transaction.expected.as_ref().unwrap(), &anchor)?; - topology.publish_observed_activation( - transaction.expected.as_ref().unwrap(), - transaction.activation.as_ref().unwrap(), - transaction.workload.as_ref().unwrap(), - )?; - } - transaction.enter_phase(phase, now); - transaction.validate()?; - let store = SingleFileMessagePackBackingStore::new(&world.state_store); - assert!(store.compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: command.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: transaction.transaction_id.clone(), - current: None, - }, - ], - &[ - command_envelope(&command, command.requested_at_unix_millis)?, - transaction_envelope(&transaction, now)?, - ], - )?); - - // Odin's world. Unreachable: the correlation store is bytes no - // reader can decode, so any read of it is an error. NotReady: a - // valid correlation that says the target is not Ready. - let store = &world.engine.options.odin_correlation_store; - let poisoned = match odin { - _ if provides_odin && stateful => { - // A new warming word after the fence: what the lease - // grant waits for. - SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( - CultCacheEnvelope { - key: crate::drivers::incarnation_key_of( - &transaction.target, - &transaction.expected.as_ref().unwrap().canonical_sha256()?, - ), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: signed_correlation_as( - &world, - &transaction, - 5, - false, - "warming", - None, - b"presence-5", - )?, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some( - cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - ), - }, - )?; - Vec::new() - } - _ if provides_odin => { - odin_reports_ready(&world, &transaction, 5)?; - Vec::new() - } - Odin::Unreachable => { - let garbage = b"odin is not a cache".to_vec(); - std::fs::write(store, &garbage)?; - garbage - } - Odin::NotReady => { - let wrongly = signed_correlation(&world, &transaction, 9, false)?; - SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( - CultCacheEnvelope { - key: crate::drivers::incarnation_key_of( - &transaction.target, - &transaction.expected.as_ref().unwrap().canonical_sha256()?, - ), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: wrongly, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some( - cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), - ), - }, - )?; - Vec::new() - } - }; - Ok(RoutedWorld { - world, - workload: switch, - stub, - odin, - poisoned, - }) - } - - /// The preflight receipt of a candidate whose route was never - /// installed: what Warming records before Fencing, without running the - /// host's nginx. - fn candidate_preflight( - transaction: &DeploymentTransaction, - ) -> Result { - let rendered = NginxRouteDriver::new( - transaction - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1 - .route - .context("no route binding")?, - ) - .render(transaction.expected.as_ref().unwrap())?; - Ok(crate::drivers::RoutePreflightReceipt { - route_id: "service".into(), - candidate_runtime_instance_id: transaction - .activation - .as_ref() - .unwrap() - .runtime_instance_id - .clone(), - candidate_membership_sha256: sha256_id(&rendered), - incumbent_runtime_instance_id: None, - incumbent_membership_sha256: None, - incumbent_configuration: None, - }) - } - - impl RoutedWorld { - fn step(&self) -> Result<()> { - self.world.engine.advance_transaction(&resident(&self.world)?) - } - - fn run_to_routing(&self) -> Result<()> { - for _ in 0..12 { - if self.transaction()?.phase == DeploymentPhase::Routing { - return Ok(()); - } - self.step()?; - } - bail!("the transaction never reached Routing") - } - - fn transaction(&self) -> Result { - Ok(resident(&self.world)?.value) - } - - /// Odin was never read: the poisoned store is byte for byte what - /// it was, and no step failed on it. - fn assert_odin_untouched(&self) -> Result<()> { - if self.odin == Odin::Unreachable { - assert_eq!( - std::fs::read(&self.world.engine.options.odin_correlation_store)?, - self.poisoned - ); - } - Ok(()) - } - - /// Routing runs for real: it installs the fragment through the - /// world's stub host programs and proves the stable route against - /// the stub runtime. The world only drives the steps until the - /// transaction enters Committing. - fn promote(&self) -> Result<()> { - assert_eq!(self.transaction()?.phase, DeploymentPhase::Routing); - for _ in 0..4 { - if self.transaction()?.phase == DeploymentPhase::Committing { - return Ok(()); - } - self.step()?; - } - bail!("the transaction never reached Committing") - } - } - - /// A route-proof generation from a real admission. - pub(super) fn committed_generation() -> Result { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - routed.step()?; - admitted(&routed) - } - - fn admitted(world: &RoutedWorld) -> Result { - Ok(ControlSnapshot::read(&world.world.state_store)? - .admitted - .into_iter() - .next() - .context("nothing was admitted")? - .value) - } - - #[test] - fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - // Fencing, Leasing, AwaitingReady: the candidate is still warming. - for _ in 0..12 { - routed.step()?; - if routed.transaction()?.last_error.is_some() { - break; - } - } - let waiting = routed.transaction()?; - assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); - assert!(waiting.ready.is_none()); - assert_eq!( - waiting.last_error.as_deref(), - Some("candidate is still warming") - ); - // The candidate finishes warming: Ready is its own Active answer. - routed.stub.set_state("active"); - routed.step()?; - assert!(matches!( - routed.transaction()?.ready, - Some(ReadinessEvidence::RouteProof { .. }) - )); - routed.step()?; - assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); - routed.promote()?; - routed.step()?; - - let generation = admitted(&routed)?; - generation.validate()?; - assert_eq!(generation.ready.voucher(), Voucher::Candidate); - assert!(generation.odin_authority.is_none()); - assert!(generation.latest_odin_observation.is_none()); - assert!(generation.route_supervision.is_some()); - // Idunn's own challenges carried every proof: the seed's warming, - // two Ready polls; the stable endpoint proved the route twice. - assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 3); - assert!(routed.stub.stable_hits.load(Ordering::SeqCst) >= 2); - routed.assert_odin_untouched()?; - - // Boot and supervision read no Odin either: the topology refresh - // returns before it opens Odin's store, and durable authority - // skips a route-proof generation's receipts. - let stored = ControlSnapshot::read(&routed.world.state_store)?; - let current = stored.admitted.first().context("no admitted generation")?; - assert!(!routed.world.engine.refresh_admitted_topology(&stored, current)?); - routed.world.engine.validate_durable_authority(&stored)?; - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { - let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - routed.step()?; - let generation = admitted(&routed)?; - assert_eq!(generation.ready.voucher(), Voucher::Candidate); - // The valid, not-Ready correlation was neither admitted as a - // receipt nor consulted: the record carries no Odin cursor. - assert_eq!(generation.odin_publisher_sequence_cursor, 0); - Ok(()) - } - - #[test] - fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - routed.step()?; - let warmed = routed.transaction()?; - assert_eq!(warmed.phase, DeploymentPhase::Warming); - let Some(WarmingEvidence::RouteProofDirect { evidence }) = &warmed.warming else { - bail!("Warming evidence is not the candidate's own presence"); - }; - assert!(evidence.message_id.starts_with("candidate-")); - assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); - routed.assert_odin_untouched()?; - - // Nothing answers: the same step waits, and says why. - let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - silent.stub.hang_up.store(true, Ordering::SeqCst); - silent.step()?; - let waiting = silent.transaction()?; - assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); - assert!( - waiting - .last_error - .as_deref() - .is_some_and(|error| error.contains("candidate endpoint did not answer")), - "{:?}", - waiting.last_error - ); - Ok(()) - } - - #[test] - fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - routed.stub.capacity.store(1, Ordering::SeqCst); - let failure = routed.step().expect_err("a shortfall must be refused"); - // The shortfall is a typed value, not a sentence to search. - let disagrees = failure - .downcast_ref::() - .with_context(|| format!("the refusal is not typed: {failure:#}"))?; - let shortfall = disagrees - .disagreements - .iter() - .find(|disagreement| disagreement.code == "expected-capability-000-capacity") - .context("the capacity shortfall is not named")?; - assert!( - shortfall.expected.as_deref().is_some_and(|text| text.ends_with("capacity>=2")), - "{shortfall:?}" - ); - assert!(shortfall.observed.is_some(), "{shortfall:?}"); - assert!(admitted(&routed).is_err()); - Ok(()) - } - - #[test] - fn odin_itself_is_never_route_proof() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let mut expected = routed.stub.expected.clone(); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); - // Odin provides the rendezvous capability and declares no dependency. - expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); - assert!(expected.dependencies.is_empty()); - assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); - - // Route-proof evidence for it is refused: its admitted generation - // is what names the Odin authority, so it must carry one. - let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; - odin.target = "odin".into(); - odin.expected = expected; - odin.ready = route_proof_evidence(); - odin.latest_odin_observation = None; - odin.odin_authority = None; - assert!(error_text(odin.validate()).contains("not route-proof")); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn route_admission_installs_through_the_host_programs_without_reading_odin() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - // One real Routing step: the fragment goes in through the world's - // stub nginx, ufw and systemctl, and the stable listener proves it. - routed.step()?; - assert!(matches!( - routed.transaction()?.routing, - Some(RoutingEvidence::Promoted { .. }) - )); - assert_eq!(routed.world.route_stubs.count("systemctl reload"), 1); - assert_eq!(routed.world.route_stubs.count("ufw allow"), 1); - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - routed.stub.set_state("active"); - routed.step()?; - assert!(matches!( - routed.transaction()?.warming, - Some(WarmingEvidence::RouteProofDirect { .. }) - )); - Ok(()) - } - - #[test] - fn the_lease_refresh_asks_the_candidate_for_a_new_warming_presence() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let current = resident(&routed.world)?; - let prior = routed - .world - .engine - .rehydrate_warming_token(¤t.value, now_millis()?, false)?; - let (_, evidence, token) = routed - .world - .engine - .fresh_warming_for_lease(¤t, now_millis()?)? - .context("a warming candidate must refresh")?; - assert!(matches!(evidence, WarmingEvidence::RouteProofDirect { .. })); - assert_ne!(token.signed_presence_sha256(), prior.signed_presence_sha256()); - // A candidate that went Active is not warming: nothing to bind. - routed.stub.set_state("active"); - let failure = routed - .world - .engine - .fresh_warming_for_lease(¤t, now_millis()?) - .map(|_| ()) - .expect_err("an active candidate is not a warming one"); - assert!(format!("{failure:#}").contains("not warming"), "{failure:#}"); - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn boot_reauthenticates_a_live_route_proof_transactions_evidence() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.validate_durable_authority(&snapshot)?; - - // Warming evidence that does not answer the challenge it names. - let current = resident(&routed.world)?; - let mut forged = current.value.clone(); - let Some(WarmingEvidence::RouteProofDirect { evidence }) = &mut forged.warming else { - bail!("no route-proof Warming evidence"); - }; - evidence.message_id = "candidate-another".into(); - replace_transaction(&routed.world.state_store, ¤t, &forged)?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - let failure = routed - .world - .engine - .validate_durable_authority(&snapshot) - .expect_err("Warming evidence for another challenge"); - assert!(format!("{failure:#}").contains("exact challenge"), "{failure:#}"); - - // A Ready label carried by a presence that was only warming. - let current = resident(&routed.world)?; - let CandidateAnswer::Answered { evidence, .. } = routed - .world - .engine - .challenge_candidate(¤t.value, &["warming"], None)? - else { - bail!("the stub candidate did not answer"); - }; - let mut forged = current.value.clone(); - forged.warming = Some(WarmingEvidence::RouteProofDirect { - evidence: evidence.clone(), - }); - forged.ready = Some(ReadinessEvidence::RouteProof { evidence }); - replace_transaction(&routed.world.state_store, ¤t, &forged)?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - let failure = routed - .world - .engine - .validate_durable_authority(&snapshot) - .expect_err("Ready carried by a warming presence"); - assert!(format!("{failure:#}").contains("not active"), "{failure:#}"); - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() - -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let ready = routed.transaction()?; - assert!(ready.ready.as_ref().and_then(ReadinessEvidence::odin).is_some()); - routed.promote()?; - routed.step()?; - let generation = admitted(&routed)?; - generation.validate()?; - assert_eq!(generation.ready.voucher(), Voucher::Odin); - assert_eq!( - generation.odin_authority, - Some(routed.world.engine.bootstrap_odin_authority.clone()) - ); - assert!(generation.latest_odin_observation.is_some()); - // Boot re-authenticates every Odin receipt this generation carries. - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.validate_durable_authority(&snapshot)?; - Ok(()) - } - - #[test] - fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() - -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - routed.step()?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.validate_durable_authority(&snapshot)?; - assert_eq!( - routed.world.engine.current_odin_authority(&snapshot)?, - routed.world.engine.bootstrap_odin_authority - ); - Ok(()) - } - - impl RoutedWorld { - /// Boot would accept this store, and Odin was never read. - fn assert_boots(&self) -> Result<()> { - let snapshot = ControlSnapshot::read(&self.world.state_store)?; - self.world.engine.validate_durable_authority(&snapshot)?; - self.assert_odin_untouched() - } - - /// Step until `done`, checking after every step that boot would - /// still accept the store. - fn drive_until(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { - for _ in 0..20 { - if done(&self.transaction()?) { - return Ok(()); - } - self.step()?; - self.assert_boots()?; - } - bail!("the transaction never reached the awaited state") - } - - fn answer_now(&self) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { - match self.world.engine.challenge_candidate( - &self.transaction()?, - &["warming", "active"], - None, - )? { - CandidateAnswer::Answered { evidence, present } => Ok((evidence, present)), - CandidateAnswer::Silent(reason) => bail!("the candidate was silent: {reason}"), - } - } - } - - // ----------------------------------------------------------------- - // The stateful path, end to end through the Engine. - // ----------------------------------------------------------------- - - #[test] - fn a_stateful_route_proof_target_holds_its_lease_from_fencing_to_commit() -> Result<()> { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - routed.assert_boots()?; - - routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Prepared { .. })))?; - assert_eq!(routed.transaction()?.phase, DeploymentPhase::Leasing); - routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; - routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; - let lease_sha256 = routed - .transaction()? - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("no granted lease")? - .to_owned(); - - // The candidate has not picked the lease up: still warming. - routed.step()?; - assert_eq!( - routed.transaction()?.last_error.as_deref(), - Some("candidate is still warming") - ); - assert!(routed.transaction()?.ready.is_none()); - // Active without the lease Idunn granted is not Ready. - routed.stub.set_state("active"); - let refusal = routed - .step() - .expect_err("an active candidate without the granted lease is refused"); - assert!( - format!("{refusal:#}").contains("exact current process write lease"), - "{refusal:#}" - ); - assert!(routed.transaction()?.ready.is_none()); - // Active holding exactly that lease is. - *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); - routed.step()?; - assert!(matches!( - routed.transaction()?.ready, - Some(ReadinessEvidence::RouteProof { .. }) - )); - routed.assert_boots()?; - - routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; - routed.promote()?; - routed.assert_boots()?; - routed.step()?; - - let generation = admitted(&routed)?; - generation.validate()?; - assert_eq!(generation.ready.voucher(), Voucher::Candidate); - assert!(generation.expected.write_lease_required); - assert_eq!(generation.leasing.lease_sha256(), Some(lease_sha256.as_str())); - assert!(generation.odin_authority.is_none()); - routed.assert_boots()?; - Ok(()) - } - - // ----------------------------------------------------------------- - // Warming is one challenge, then the phase moves. - // ----------------------------------------------------------------- - - #[test] - fn warming_challenges_once_and_then_advances_on_every_tick() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - routed.step()?; - let recorded = routed - .transaction()? - .warming - .context("the first step recorded no Warming evidence")?; - assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); - - // The route preflight needs the host's nginx; its receipt is recorded - // here so the phase can move on. - let current = resident(&routed.world)?; - let mut next = current.value.clone(); - next.route_preflight = Some(candidate_preflight(&next)?); - replace_transaction(&routed.world.state_store, ¤t, &next)?; - - routed.drive_until(|t| t.phase == DeploymentPhase::Fencing)?; - assert_eq!( - routed.stub.candidate_hits.load(Ordering::SeqCst), - 1, - "a recorded Warming answer is never asked for again" - ); - assert_eq!(routed.transaction()?.warming, Some(recorded)); - Ok(()) - } - - #[test] - fn a_stateful_candidate_may_not_answer_its_warming_challenge_active() { - let mut expected = fixture_generation(FIXTURE_GENERATION).unwrap().1.expected; - expected.write_lease_required = true; - assert_eq!(route_proof_warming_states(&expected), ["warming"]); - expected.write_lease_required = false; - assert_eq!(route_proof_warming_states(&expected), ["warming", "active"]); - } - - #[test] - fn a_stateful_candidate_that_answers_warming_active_is_refused() -> Result<()> { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; - routed.stub.set_state("active"); - let refusal = routed - .step() - .expect_err("a stateful candidate cannot be active before it holds a lease"); - assert!(format!("{refusal:#}").contains("not warming"), "{refusal:#}"); - assert!(routed.transaction()?.warming.is_none()); - Ok(()) - } - - // ----------------------------------------------------------------- - // What a challenged presence must prove. - // ----------------------------------------------------------------- - - #[test] - fn a_presence_holds_the_write_lease_exactly_when_it_is_not_warming() -> Result<()> { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; - let authority = routed - .world - .engine - .runtime_authority(&routed.transaction()?)?; - let lease = sha256_id(b"the granted lease"); - let other = sha256_id(b"some other lease"); - let judge = |state: &'static str, - held: Option<&str>, - current: Option<&str>, - states: &[&str]| - -> Result<()> { - let challenged = now_millis()?; - routed.stub.set_state(state); - *routed.stub.lease.lock().unwrap() = held.map(str::to_owned); - let bytes = routed.stub.answer("candidate-probe")?; - routed - .world - .engine - .authenticate_challenged_presence( - &authority, - states, - current, - "candidate-probe", - challenged, - now_millis()?, - &bytes, - ) - .map(|_| ()) - }; - let either = ["warming", "active"]; - judge("warming", None, None, &either)?; - judge("active", Some(&lease), Some(&lease), &either)?; - for (state, held, current, named) in [ - // Warming holds nothing, whatever Idunn has granted. - ("warming", Some(lease.as_str()), None, "write lease"), - ("warming", Some(lease.as_str()), Some(lease.as_str()), "write lease"), - // Anything else holds exactly the current grant. - ("active", None, Some(lease.as_str()), "exact current process write lease"), - ("active", Some(lease.as_str()), None, "exact current process write lease"), - ( - "active", - Some(other.as_str()), - Some(lease.as_str()), - "exact current process write lease", - ), - ] { - let error = judge(state, held, current, &either) - .expect_err("a presence with the wrong lease is refused"); - assert!( - format!("{error:#}").contains(named), - "{state} {held:?} {current:?}: {error:#}" - ); - } - Ok(()) - } - - #[test] - fn a_presence_minted_before_its_challenge_is_refused() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let authority = routed - .world - .engine - .runtime_authority(&routed.transaction()?)?; - let bytes = routed.stub.answer("candidate-probe")?; - let minted = now_millis()?; - // Answered before the challenge that claims it: an old presence - // replayed for a new question. - let replay = routed - .world - .engine - .authenticate_challenged_presence( - &authority, - &["warming"], - None, - "candidate-probe", - minted + 1_000, - minted + 1_000, - &bytes, - ) - .expect_err("a presence minted before its challenge"); - assert!( - format!("{replay:#}").contains("minted before its challenge"), - "{replay:#}" - ); - let early = routed - .world - .engine - .authenticate_challenged_presence( - &authority, - &["warming"], - None, - "candidate-probe", - minted + 1_000, - minted, - &bytes, - ) - .expect_err("a receipt before its challenge"); - assert!(format!("{early:#}").contains("predates its challenge"), "{early:#}"); - Ok(()) - } - - #[test] - fn direct_warming_evidence_is_bound_to_the_bytes_it_authenticated() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let id = routed.transaction()?.transaction_id; - let direct = |evidence: RuntimePresenceEvidence| WarmingEvidence::RouteProofDirect { evidence }; - - let (evidence, present) = routed.answer_now()?; - SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present)?; - - // The digest is the authenticated presence's, but the bytes are not. - let (mut evidence, present) = routed.answer_now()?; - evidence.canonical_bytes.push(0); - let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) - .expect_err("bytes that are not the authenticated presence"); - assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); - - // The bytes are the authenticated presence's, but the digest is not. - let (mut evidence, present) = routed.answer_now()?; - evidence.canonical_sha256 = sha256_id(b"another presence"); - let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) - .expect_err("a digest that is not the authenticated presence's"); - assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); - - // Evidence about another answer altogether. - let (other, _) = routed.answer_now()?; - let (_, present) = routed.answer_now()?; - SequenceAdmittedWarming::from_direct_presence(id, direct(other), present) - .expect_err("evidence about another answer"); - Ok(()) - } - - // ----------------------------------------------------------------- - // Validation: candidate-vouched evidence belongs to route-proof targets. - // ----------------------------------------------------------------- - - #[test] - fn candidate_vouched_evidence_is_refused_on_a_target_odin_reports_on() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let (evidence, _) = routed.answer_now()?; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let odin_reported = transaction_at(&world, DeploymentPhase::Warming)?; - assert_eq!( - ReadinessClass::of(odin_reported.expected.as_ref().unwrap()), - Ok(ReadinessClass::OdinCorrelated) - ); - - let mut warming = odin_reported.clone(); - warming.warming = Some(WarmingEvidence::RouteProofDirect { - evidence: evidence.clone(), - }); - assert!(error_text(warming.validate()).contains("not route-proof")); - - let mut ready = odin_reported.clone(); - ready.ready = Some(ReadinessEvidence::RouteProof { - evidence: evidence.clone(), - }); - assert!(error_text(ready.validate()).contains("not route-proof")); - - // Direct Odin warming is Odin's alone, by class: a route-proof - // target may not claim it, and a target that provides the - // rendezvous may, whatever it is named. - let mut direct = routed.transaction()?; - direct.warming = Some(WarmingEvidence::FirstOdinDirect { - evidence: evidence.clone(), - }); - assert!(error_text(direct.validate()).contains("reserved for Odin")); - let odin = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; - let mut direct = odin.transaction()?; - assert_eq!(direct.target, "service"); - direct.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); - direct.validate()?; - Ok(()) - } - - #[test] - fn a_route_proof_transaction_past_awaiting_ready_needs_the_candidates_ready_proof() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let routing = routed.transaction()?; - routing.validate()?; - let mut bare = routing.clone(); - bare.ready = None; - assert!(error_text(bare.validate()).contains("lacks Ready evidence")); - routed.promote()?; - let mut bare = routed.transaction()?; - assert_eq!(bare.phase, DeploymentPhase::Committing); - bare.ready = None; - assert!(error_text(bare.validate()).contains("lacks Ready evidence")); - Ok(()) - } - - // ----------------------------------------------------------------- - // The class is the Expected's alone. Evidence collected under another - // class is held and reported, never stepped, never repaired. - // ----------------------------------------------------------------- - - /// Odin's Ready for the world's target, as pre-B3 code recorded it for - /// every routed target. - fn odin_receipt_for( - routed: &RoutedWorld, - transaction: &DeploymentTransaction, - sequence: u64, - ) -> Result { - odin_receipt_with(routed, transaction, sequence, true) - } - - fn odin_receipt_with( - routed: &RoutedWorld, - transaction: &DeploymentTransaction, - sequence: u64, - ready: bool, - ) -> Result { - let now = now_millis()?; - let bytes = signed_correlation(&routed.world, transaction, sequence, ready)?; - let authenticated = routed.world.engine.authenticate_topology_bytes( - &ControlSnapshot::read(&routed.world.state_store)?, - transaction, - &bytes, - transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256), - now, - )?; - TopologyEvidence::from_authenticated(&authenticated, now) - } - - fn stored(generation: &AdmittedGeneration) -> Stored { - Stored { - value: generation.clone(), - envelope: CultCacheEnvelope { - key: generation.target.clone(), - r#type: AdmittedGeneration::TYPE.into(), - payload: Vec::new(), - stored_at: "1970-01-01T00:00:00.100Z".into(), - schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), - }, - } - } - - #[test] - fn a_pre_b3_transaction_over_a_route_proof_target_is_held_and_reported_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - - // Ready as pre-B3 Idunn wrote it: an Odin receipt. - let current = resident(&routed.world)?; - let mut legacy = current.value.clone(); - let receipt = odin_receipt_for(&routed, &legacy, 9)?; - legacy.latest_odin_observation = Some(receipt.clone()); - legacy.odin_publisher_sequence_cursor = 9; - legacy.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: receipt }); - replace_transaction(&routed.world.state_store, ¤t, &legacy)?; - - let disagreement = ReadinessDisagreement::WrongVoucher { - target: "service".into(), - required: ReadinessClass::RouteProof, - collected: Voucher::Odin, - }; - assert_eq!(legacy.readiness_disagreement(), Some(disagreement.clone())); - // The record is readable and boot accepts it: held, not refused. - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.validate_durable_authority(&snapshot)?; - - let hits = ( - routed.stub.candidate_hits.load(Ordering::SeqCst), - routed.stub.stable_hits.load(Ordering::SeqCst), - ); - for _ in 0..4 { - assert!(!routed.world.engine.run_scheduler_tick()?); - } - let held = resident(&routed.world)?; - assert_eq!(held.value.phase, DeploymentPhase::Routing); - assert!(held.value.pre_fencing_abort.is_none()); - assert!(held.value.post_fencing_abort.is_none()); - assert_eq!(held.value.ready, legacy.ready, "nothing re-derived the evidence"); - assert_eq!( - held.value.last_error.as_deref(), - Some(disagreement.to_string().as_str()) - ); - // Reported once: later ticks leave the record alone. - routed.world.engine.run_scheduler_tick()?; - assert_eq!(resident(&routed.world)?.envelope, held.envelope); - assert_eq!( - hits, - ( - routed.stub.candidate_hits.load(Ordering::SeqCst), - routed.stub.stable_hits.load(Ordering::SeqCst) - ), - "a held transaction is asked nothing" - ); - routed.assert_odin_untouched()?; - - // An abort already under way is not held: it never reads the class. - let mut aborting = held.value.clone(); - aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); - replace_transaction(&routed.world.state_store, &held, &aborting)?; - let after = resident(&routed.world)?; - let _ = routed.world.engine.resume_candidate(&after); - assert_ne!( - resident(&routed.world)?.value.last_error.as_deref(), - Some(disagreement.to_string().as_str()), - "an abort in progress runs instead of being held" - ); - Ok(()) - } - - #[test] - fn a_pre_b3_generation_over_a_route_proof_target_boots_and_is_reported_not_refreshed() - -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let at_routing = routed.transaction()?; - routed.promote()?; - routed.step()?; - - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - let current = snapshot.admitted.first().context("nothing was admitted")?; - // Receipts that would not survive being re-proved: authentic, but - // not Ready. Boot re-proves against the current Odin authority and - // requires exact semantic Ready, never what a generation stores, so - // this record is only bootable if boot does not re-prove it. - let receipt = odin_receipt_with(&routed, &at_routing, 9, false)?; - let mut legacy = current.value.clone(); - legacy.ready = ReadinessEvidence::OdinCorrelated { - evidence: receipt.clone(), - }; - legacy.latest_odin_observation = Some(receipt); - // The stored authority is present because an Odin-tagged record - // carries one; it is not what boot reads. - let stranger = cultnet_rs::enroll_service_identity_at::( - &routed.world.root.join("identities/stranger.cc"), - )?; - legacy.odin_authority = Some(AdmittedOdinAuthority::from_anchor(&stranger.trust_anchor()?)?); - legacy.odin_publisher_sequence_cursor = 9; - assert!( - SingleFileMessagePackBackingStore::new(&routed.world.state_store) - .compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: AdmittedGeneration::TYPE.into(), - key: legacy.target.clone(), - current: Some(current.envelope.clone()), - }], - &[admitted_envelope(&legacy, now_millis()?)?], - )? - ); - - assert_eq!( - legacy.readiness(), - Err(ReadinessDisagreement::WrongVoucher { - target: "service".into(), - required: ReadinessClass::RouteProof, - collected: Voucher::Odin, - }) - ); - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - let current = snapshot.admitted.first().context("the generation is gone")?; - // The class chooses the path: no Odin receipt is re-proved at boot - // and none is refreshed, though the evidence is an Odin receipt. - routed.world.engine.validate_durable_authority(&snapshot)?; - assert!(!routed.world.engine.refresh_admitted_topology(&snapshot, current)?); - routed.assert_odin_untouched()?; - - routed.world.engine.supervise_one_admitted_generation()?; - assert!( - routed - .world - .engine - .fault_reports - .lock() - .unwrap() - .contains_key("generation:service"), - "supervision reports the disagreement" - ); - routed.assert_odin_untouched()?; - Ok(()) - } - - #[test] - fn only_a_transaction_that_still_reads_its_class_is_held_for_its_evidence() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let at_routing = routed.transaction()?; - // What pre-B3 Idunn would have collected: an Odin receipt. - let mut legacy = at_routing.clone(); - legacy.ready = Some(ReadinessEvidence::OdinCorrelated { - evidence: odin_receipt_for(&routed, &at_routing, 9)?, - }); - let disagreement = legacy.readiness_disagreement(); - assert!(disagreement.is_some()); - assert_eq!(legacy.held_disagreement(), disagreement); - - // Finished work, and an abort under way, never read the class. - let mut completed = legacy.clone(); - completed.completion = Some(TransactionCompletion::Admitted { - generation_id: "generation-1".into(), - }); - assert_eq!(completed.held_disagreement(), None); - let mut aborting = legacy.clone(); - aborting.pre_fencing_abort = Some(PreFencingAbort { - error: "operator abort".into(), - candidate_cleanup: CleanupEvidence::Pending, - topology_reconciliation: CleanupEvidence::Pending, - source_cleanup: CleanupEvidence::Pending, - }); - assert_eq!(aborting.held_disagreement(), None); - let mut aborting = legacy; - aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); - assert_eq!(aborting.held_disagreement(), None); - Ok(()) - } - - #[test] - fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { - let world = EngineFixture::new()?; - // Odin's store is unreadable: any read of it is an error. - std::fs::write(&world.engine.options.odin_correlation_store, b"odin is not a cache")?; - - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - let target = generation.expected.target.clone(); - assert_eq!(generation.readiness(), Ok(ReadinessClass::OdinCorrelated)); - generation.expected.dependencies.clear(); - generation.expected.route = None; - let undeclared = UndeclaredReadiness { target: target.clone() }; - assert_eq!( - generation.readiness(), - Err(ReadinessDisagreement::Undeclared(undeclared.clone())) - ); - assert!( - !world - .engine - .refresh_admitted_topology(&ControlSnapshot::default(), &stored(&generation))? - ); - - // The mirror: Odin declared, but the receipt is the candidate's. - let (_, mut odin_class) = fixture_generation(FIXTURE_GENERATION)?; - odin_class.ready = route_proof_evidence(); - assert_eq!( - odin_class.readiness(), - Err(ReadinessDisagreement::WrongVoucher { - target: target.clone(), - required: ReadinessClass::OdinCorrelated, - collected: Voucher::Candidate, - }) - ); - - // A transaction says the same about what it has collected. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let (evidence, _) = routed.answer_now()?; - let odin_world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&odin_world, DeploymentPhase::Warming)?; - assert_eq!(seeded.readiness_disagreement(), None); - let mut collected = seeded.clone(); - collected.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); - assert_eq!( - collected.readiness_disagreement(), - Some(ReadinessDisagreement::WrongVoucher { - target: "service".into(), - required: ReadinessClass::OdinCorrelated, - collected: Voucher::Candidate, - }) - ); - let mut bare = seeded.clone(); - bare.expected.as_mut().unwrap().dependencies.clear(); - assert_eq!( - bare.readiness_disagreement(), - Some(ReadinessDisagreement::Undeclared(UndeclaredReadiness { - target: "service".into() - })) - ); - let mut no_expected = seeded; - no_expected.expected = None; - assert_eq!(no_expected.readiness_disagreement(), None); - Ok(()) - } - - // ----------------------------------------------------------------- - // Odin is whichever target provides the rendezvous. - // ----------------------------------------------------------------- - - #[test] - fn odin_is_the_target_that_provides_the_rendezvous_not_the_target_named_odin() -> Result<()> { - let world = EngineFixture::new()?; - let (_, fixture) = fixture_generation(FIXTURE_GENERATION)?; - - // A target named "odin" that merely depends on Odin is not Odin. - let mut named = fixture.clone(); - named.target = "odin".into(); - named.expected.target = "odin".into(); - let mut snapshot = ControlSnapshot::default(); - snapshot.admitted.push(stored(&named)); - assert!(snapshot.admitted_odin().is_none()); - assert_eq!( - world.engine.current_odin_authority(&snapshot)?, - world.engine.bootstrap_odin_authority - ); - - // The target that provides the rendezvous is, whatever it is called. - let mut provider = fixture.clone(); - provider.target = "verse".into(); - provider.expected.target = "verse".into(); - provider.expected.dependencies.clear(); - provide_odin(&mut provider.expected); - let authority = AdmittedOdinAuthority::from_anchor(&world.odin_signer.trust_anchor()?)?; - provider.odin_authority = Some(authority.clone()); - snapshot.admitted.push(stored(&provider)); - assert_eq!( - snapshot.admitted_odin().map(|found| found.value.target.as_str()), - Some("verse") - ); - assert_eq!( - snapshot.admitted_odin().and_then(|found| found.value.odin_authority.clone()), - Some(authority) - ); - Ok(()) - } - - #[test] - fn a_target_that_provides_the_rendezvous_warms_as_odin_whatever_it_is_named() -> Result<()> { - // Nothing else has been admitted that could observe it, so Odin - // observes itself directly -- and only a stateful incarnation may. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; - assert_eq!(routed.transaction()?.target, "service"); - let refusal = routed - .step() - .expect_err("a stateless Odin cannot be observed directly"); - assert!( - format!("{refusal:#}").contains("direct Odin warming must be a stateful incarnation"), - "{refusal:#}" - ); - Ok(()) - } - - #[test] - fn direct_odin_warming_is_reserved_to_odin_by_class_not_by_name() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let mut named = routed.transaction()?; - named.target = "odin".into(); - let error = routed - .world - .engine - .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) - .expect_err("a target named odin that is not Odin"); - assert!( - format!("{error:#}").contains("reserved for a stateful Odin incarnation"), - "{error:#}" - ); - Ok(()) - } - - #[test] - fn a_stateful_target_that_provides_the_rendezvous_is_observed_directly_whatever_it_is_named() - -> Result<()> { - let provider = build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true, true, CommandKind::Continuity)?; - let transaction = provider.transaction()?; - assert_eq!(transaction.target, "service"); - assert!(transaction.expected.as_ref().unwrap().write_lease_required); - provider.stub.set_state("warming"); - *provider.stub.reply.lock().unwrap() = Reply::FirstOdinWarming; - let challenged_at = now_millis()?; - let answer = provider.stub.answer("warming-probe")?; - provider.world.engine.authenticate_first_odin_warming_presence( - &transaction, - "warming-probe", - challenged_at, - now_millis()?, - &answer, - )?; - - // The reverse: named odin, stateful, but nothing provides the - // rendezvous. - let stateful = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; - let mut named = stateful.transaction()?; - assert!(named.expected.as_ref().unwrap().write_lease_required); - named.target = "odin".into(); - let error = stateful - .world - .engine - .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) - .expect_err("a stateful target named odin that does not provide the rendezvous"); - assert!( - format!("{error:#}").contains("reserved for a stateful Odin incarnation"), - "{error:#}" - ); - Ok(()) - } - - // ----------------------------------------------------------------- - // A bad answer is an error. Only silence waits. - // ----------------------------------------------------------------- - - #[test] - fn a_bad_answer_to_a_challenge_is_an_error_and_only_silence_waits() -> Result<()> { - for reply in [Reply::ForeignChallengeId, Reply::HttpFailure] { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - *routed.stub.reply.lock().unwrap() = reply; - let error = match routed.world.engine.challenge_candidate( - &routed.transaction()?, - &["warming"], - None, - ) { - Err(error) => error, - Ok(_) => bail!("a bad answer was taken for silence"), - }; - assert!( - matches!( - error.downcast_ref::(), - Some(ChallengeFailure::Refused(_)) - ), - "{error:#}" - ); - // Before the fence a candidate that answers wrongly is aborted, - // not waited on. - routed.world.engine.run_scheduler_tick()?; - let after = routed.transaction()?; - assert!(after.pre_fencing_abort.is_some(), "the bad answer aborted it"); - assert!(after.warming.is_none()); - } - - let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - silent.stub.hang_up.store(true, Ordering::SeqCst); - assert!(matches!( - silent.world.engine.challenge_candidate(&silent.transaction()?, &["warming"], None)?, - CandidateAnswer::Silent(_) - )); - Ok(()) - } - - // ----------------------------------------------------------------- - // B2: the route gate, the meters and the supervision pass, through - // the Engine, over the world's stub host programs. - // ----------------------------------------------------------------- - - impl RoutedWorld { - /// Run a fresh world to an admitted generation whose transaction - /// is retired: what admitted-route supervision looks at. - fn admit(&self) -> Result<()> { - self.stub.set_state("active"); - self.run_to_routing()?; - self.promote()?; - self.drive_until(|transaction| transaction.is_terminal())?; - assert!(self.world.engine.retire_one_terminal_transaction()?); - Ok(()) - } - - fn fragment_path(&self) -> PathBuf { - self.world.root.join("service.conf") - } - - /// The target's route ledger holds `entries` recent actuations. - fn ledger(&self, entries: usize) -> Result<()> { - let now = now_millis()?; - let mut meters = TargetSupervision::new("service"); - meters.route_actuations = (0..entries as u64).map(|offset| now - 1_000 + offset).collect(); - set_meters(&self.world, &meters) - } - - #[cfg(unix)] - fn reloads(&self) -> usize { - self.world.route_stubs.count("systemctl reload") - } - - /// Something else rewrote the fragment: the admitted membership drifted. - fn drift(&self) -> Result<()> { - std::fs::write(self.fragment_path(), b"drifted bytes\n")?; - Ok(()) - } - - fn tick(&self) -> Result { - self.world.engine.supervise_one_admitted_generation() - } - } - - // A store that reads but cannot be written (ENOSPC on the state volume) - // is simulated with RLIMIT_FSIZE, which is process-wide: the test - // re-runs itself in a child process so no sibling test loses its writes. - // Returns true in the child, which runs the body; the parent returns - // false after asserting the child passed and hands back its stderr. - #[cfg(unix)] - fn in_unwritable_store_child(test: &str) -> Result> { - if std::env::var_os("IDUNN_TEST_UNWRITABLE_STORE_CHILD").is_some() { - return Ok(None); - } - let path = module_path!().split_once("::").map_or("", |(_, rest)| rest); - let output = std::process::Command::new(std::env::current_exe()?) - .args(["--exact", &format!("{path}::{test}"), "--nocapture", "--test-threads=1"]) - .env("IDUNN_TEST_UNWRITABLE_STORE_CHILD", "1") - .output()?; - let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); - assert!( - output.status.success(), - "child failed:\n{}\n{stderr}", - String::from_utf8_lossy(&output.stdout) - ); - assert!( - String::from_utf8_lossy(&output.stdout).contains("1 passed"), - "the child ran no test: {}", - String::from_utf8_lossy(&output.stdout) - ); - Ok(Some(stderr)) - } - - /// Cap file growth one byte under the store's size: reads work, every - /// write fails. Child process only. - #[cfg(unix)] - fn break_store_writes(world: &EngineFixture) { - let size = std::fs::metadata(&world.state_store).unwrap().len(); - unsafe { - libc::signal(libc::SIGXFSZ, libc::SIG_IGN); - let limit = libc::rlimit { rlim_cur: size - 1, rlim_max: libc::RLIM_INFINITY }; - assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0); - } - assert!( - set_meters(world, &TargetSupervision::new("probe")).is_err(), - "the store is still writable" - ); - assert!(ControlSnapshot::read(&world.state_store).is_ok(), "the store is unreadable"); - } - - #[cfg(unix)] - fn challenge_due(world: &EngineFixture) -> Option { - world.engine.challenge_wait.lock().unwrap().get("service").map(|wait| wait.due_unix_millis) - } - - #[cfg(unix)] - #[test] - fn an_unwritable_store_neither_spins_a_failing_challenge_nor_reports_progress() -> Result<()> { - let name = "an_unwritable_store_neither_spins_a_failing_challenge_nor_reports_progress"; - let Some(stderr) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.route_stubs.refuse_reloads()?; - routed.drift()?; - let before = routed.reloads(); - let began = now_millis()?; - break_store_writes(&routed.world); - let stored_wait = |routed: &RoutedWorld| -> Result> { - Ok(ControlSnapshot::read(&routed.world.state_store)? - .admitted - .iter() - .find_map(|stored| stored.value.route_supervision.as_ref()) - .and_then(|state| state.next_challenge_at_unix_millis)) - }; - let stored = stored_wait(&routed)?; - - // Fifty laps: one attempt, nothing reported as progress. - for _ in 0..50 { - assert!(!routed.tick()?, "a write the store refused was reported as progress"); - } - assert_eq!(routed.reloads() - before, 1, "the failing reload spun"); - assert_eq!(stored_wait(&routed)?, stored, "the record changed though writes fail"); - let due = challenge_due(&routed.world).expect("no in-memory wait after a failed record"); - let age = routed.world.engine.options.topology_maximum_age_millis; - assert!(due >= began + age, "the wait is shorter than one observation age: {due}"); - - // The wait expiring is the only thing that admits another - // attempt, and each one is again a single reload with no - // progress. The failed record is said once, not per attempt. - for attempt in 2..=4 { - routed.world.engine.challenge_wait.lock().unwrap().clear(); - assert!(!routed.tick()?); - assert!(!routed.tick()?); - assert_eq!(routed.reloads() - before, attempt); - } - return Ok(()); - }; - assert_eq!( - stderr.matches("could not record the failed route challenge of service").count(), - 1, - "{stderr}" - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_unwritable_store_challenges_a_stale_route_once_per_wait() -> Result<()> { - let name = "an_unwritable_store_challenges_a_stale_route_once_per_wait"; - let Some(stderr) = in_unwritable_store_child(name)? else { - let mut routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.engine.options.topology_maximum_age_millis = 8_000; - std::thread::sleep(Duration::from_millis(8_300)); - break_store_writes(&routed.world); - assert!(!routed.tick()?, "a write the store refused was reported as progress"); - let first = challenge_due(&routed.world).expect("the proved challenge left no wait"); - for _ in 0..5 { - std::thread::sleep(Duration::from_millis(3)); - assert!(!routed.tick()?); - } - assert_eq!(challenge_due(&routed.world), Some(first), "the route was challenged again"); - return Ok(()); - }; - assert_eq!( - stderr.matches("could not record the proved route challenge of service").count(), - 1, - "{stderr}" - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_landed_challenge_record_is_progress_and_leaves_no_in_memory_wait() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.route_stubs.refuse_reloads()?; - routed.drift()?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.challenge_wait.lock().unwrap().insert( - "service".into(), - ChallengeWait { due_unix_millis: 0, stamp: snapshot.target_stamp("service") }, - ); - assert!(routed.tick()?, "a landed record is progress"); - assert_eq!(challenge_due(&routed.world), None); - Ok(()) - } - - /// A fragment that re-drifts after every restore, with the store - /// unwritable, is reloaded once per wait, however many laps run. - #[cfg(unix)] - #[test] - fn a_re_drifting_fragment_is_reloaded_once_per_wait_when_the_store_is_unwritable() -> Result<()> { - let name = "a_re_drifting_fragment_is_reloaded_once_per_wait_when_the_store_is_unwritable"; - let Some(_) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let before = routed.reloads(); - break_store_writes(&routed.world); - for _ in 0..30 { - routed.drift()?; - assert!(!routed.tick()?, "a refused write was reported as progress"); - } - assert_eq!(routed.reloads() - before, 1, "a re-drifting fragment reloaded every lap"); - for attempt in 2..=4 { - routed.world.engine.challenge_wait.lock().unwrap().clear(); - routed.drift()?; - assert!(!routed.tick()?); - routed.drift()?; - assert!(!routed.tick()?); - assert_eq!(routed.reloads() - before, attempt); - } - return Ok(()); - }; - Ok(()) - } - - /// The wait of a proved challenge whose record failed is one - /// observation age from the attempt. - #[cfg(unix)] - #[test] - fn a_proved_challenge_that_cannot_be_recorded_waits_one_observation_age() -> Result<()> { - let name = "a_proved_challenge_that_cannot_be_recorded_waits_one_observation_age"; - let Some(_) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let before = routed.reloads(); - break_store_writes(&routed.world); - let began = now_millis()?; - routed.drift()?; - assert!(!routed.tick()?); - assert_eq!(routed.reloads() - before, 1, "the repair did not run"); - let due = challenge_due(&routed.world).expect("no wait"); - let age = routed.world.engine.options.topology_maximum_age_millis; - assert!(due >= began + age, "the wait is shorter than one age: {}", due - began); - return Ok(()); - }; - Ok(()) - } - - /// The stored schedule sets the in-memory wait: with three failures - /// recorded, the fourth failed challenge waits eight ages (capped), and - /// no repair runs inside that wait. - #[cfg(unix)] - #[test] - fn a_failed_challenge_that_cannot_be_recorded_waits_as_long_as_the_stored_schedule() -> Result<()> { - let name = "a_failed_challenge_that_cannot_be_recorded_waits_as_long_as_the_stored_schedule"; - let Some(_) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - edit_incumbent(&routed.world, |generation| { - let state = generation.route_supervision.as_mut().unwrap(); - state.consecutive_failures = 3; - state.next_challenge_at_unix_millis = Some(1); - state.degraded_since_unix_millis = Some(1); - })?; - break_store_writes(&routed.world); - let began = now_millis()?; - routed.drift()?; - assert!(!routed.tick()?); - let due = challenge_due(&routed.world).expect("no wait"); - let age = routed.world.engine.options.topology_maximum_age_millis; - let want = (age * 8).min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); - assert!(due >= began + want, "the wait ignores the stored doubling: {}", due - began); - let reloads = routed.reloads(); - for _ in 0..10 { - routed.drift()?; - assert!(!routed.tick()?); - } - assert_eq!(routed.reloads(), reloads, "the route was repaired inside its wait"); - return Ok(()); - }; - Ok(()) - } - - /// A lost race is progress, and the next pass is governed by the - /// winner's record, not by an in-memory wait of the loser. - #[cfg(unix)] - #[test] - fn a_lost_race_leaves_no_in_memory_wait() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.route_stubs.refuse_reloads()?; - let old = ControlSnapshot::read(&routed.world.state_store)?; - let stale = old.admitted_for("service").context("no generation")?; - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().last_challenge_at_unix_millis = Some(5); - })?; - routed.drift()?; - assert!(routed.world.engine.supervise_admitted_route(&old, stale)?); - assert_eq!(challenge_due(&routed.world), None, "a lost race left an in-memory wait"); - Ok(()) - } - - /// The wait left by a full disk does not outlive it: once any write for - /// the target lands, the stored schedule owns the target again and a - /// drift is repaired without waiting. - #[cfg(unix)] - #[test] - fn a_wait_left_by_a_full_disk_ends_when_any_write_for_the_target_lands() -> Result<()> { - let name = "a_wait_left_by_a_full_disk_ends_when_any_write_for_the_target_lands"; - let Some(_) = in_unwritable_store_child(name)? else { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let before = routed.reloads(); - break_store_writes(&routed.world); - routed.drift()?; - assert!(!routed.tick()?); - assert!(challenge_due(&routed.world).is_some(), "no wait to outlive"); - assert_eq!(routed.reloads() - before, 1); - - // Space is freed. Nothing has landed yet, so the wait still holds. - unsafe { - let limit = libc::rlimit { rlim_cur: libc::RLIM_INFINITY, rlim_max: libc::RLIM_INFINITY }; - assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0); - } - routed.drift()?; - assert!(!routed.tick()?); - assert_eq!(routed.reloads() - before, 1, "the wait did not hold before any write landed"); - - // Something else writes for the target: the wait is over. - routed.ledger(1)?; - routed.drift()?; - assert!(routed.tick()?, "the drift was not repaired"); - assert_eq!(routed.reloads() - before, 2, "a stale wait held the repair"); - assert_eq!(challenge_due(&routed.world), None); - return Ok(()); - }; - Ok(()) - } - - /// A wait belongs to an admitted target: once the target is retired, - /// the wait is gone. - #[cfg(unix)] - #[test] - fn a_wait_ends_when_its_target_is_retired() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - routed.world.engine.challenge_wait.lock().unwrap().insert( - "service".into(), - ChallengeWait { - due_unix_millis: now_millis()? + 500_000, - stamp: snapshot.target_stamp("service"), - }, - ); - assert!(!routed.tick()?); - assert!(challenge_due(&routed.world).is_some(), "an unchanged store dropped the wait"); - assert!( - SingleFileMessagePackBackingStore::new(&routed.world.state_store) - .delete_batch_if_unchanged(&[snapshot.admitted_for("service").unwrap().envelope.clone()])? - ); - routed.tick()?; - assert_eq!(challenge_due(&routed.world), None, "a retired target kept its wait"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_full_ledger_still_restores_the_admitted_and_incumbent_route() -> Result<()> { - let full = ROUTE_ACTUATION_CEILING; - // The newest recorded Survival, or 0; the Forward ledger stays full. - let newest = |world: &EngineFixture| -> Result { - let meters = meters_of(world, "service")?; - assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); - Ok(meters.route_survivals.last().copied().unwrap_or(0)) - }; - - // (i) A post-fence abort withdraws the candidate's membership. - let aborting = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - aborting.ledger(full)?; - let before = newest(&aborting.world)?; - aborting - .world - .engine - .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; - aborting.step()?; - assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); - assert!(newest(&aborting.world)? > before, "the withdrawal was not recorded"); - - // (ii) Supervision repairs a drifted admitted fragment. - let supervised = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - supervised.admit()?; - supervised.ledger(full)?; - let (before, reloads) = (newest(&supervised.world)?, supervised.reloads()); - supervised.drift()?; - assert!(supervised.tick()?); - assert_eq!(supervised.reloads(), reloads + 1, "the repair did not run"); - assert!(newest(&supervised.world)? > before, "the repair was not recorded"); - assert!(admitted(&supervised)?.route_supervision.unwrap().degraded_since_unix_millis.is_none()); - - // (iii) A failed proof at Routing rolls the route back, though the - // install that preceded it took the last free slot. The rollback - // is recorded and leaves the Forward ledger exactly full. - let failing = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - failing.stub.set_state("active"); - failing.run_to_routing()?; - failing.ledger(full - 1)?; - failing.stub.hang_up.store(true, Ordering::SeqCst); - let before = now_millis()?; - assert!(failing.step().is_err(), "an unanswered stable route was admitted"); - assert_eq!(failing.reloads(), 2, "install then rollback"); - assert!(newest(&failing.world)? >= before, "the rollback was not recorded"); - assert!(!failing.fragment_path().exists(), "the rollback left the candidate's route"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_continuity_restart_is_never_refused_by_a_full_ledger() -> Result<()> { - // Warming: the preflight of a continuity runs and is recorded, - // and the Forward ledger is left as it was. - let warming = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - warming.step()?; - assert!(warming.transaction()?.warming.is_some()); - warming.ledger(ROUTE_ACTUATION_CEILING)?; - let ledger = meters_of(&warming.world, "service")?.route_actuations; - warming.step()?; - assert!(warming.transaction()?.route_preflight.is_some(), "the preflight was refused"); - let after = meters_of(&warming.world, "service")?; - assert_eq!(after.route_survivals.len(), 1, "the preflight was not recorded"); - assert_eq!(after.route_actuations, ledger, "a survival was counted"); - - // Routing: its install runs and is recorded, and not counted. - let routing = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routing.stub.set_state("active"); - routing.run_to_routing()?; - routing.ledger(ROUTE_ACTUATION_CEILING)?; - let ledger = meters_of(&routing.world, "service")?.route_actuations; - routing.step()?; - assert!(routing.reloads() >= 1, "the install did not run"); - assert!(routing.fragment_path().exists()); - let after = meters_of(&routing.world, "service")?; - assert_eq!(after.route_survivals.len(), 1, "the install was not recorded"); - assert_eq!(after.route_actuations, ledger, "a survival was counted"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_survival_that_cannot_be_recorded_runs_and_a_forward_is_refused() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let transaction = routed.transaction()?; - let preflight = transaction.route_preflight.clone().context("no preflight")?; - let driver = routed.world.engine.route_driver( - transaction - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1 - .route - .context("no route binding")?, - ); - // The ledger cannot be read, so nothing can be recorded. - std::fs::write(&routed.world.state_store, b"unreadable")?; - let engine = &routed.world.engine; - - let refused = engine - .route_gate("service", CommandKind::Deploy) - .admit(RouteActuation::Forward) - .unwrap_err(); - assert!(refused.downcast_ref::().is_none(), "{refused:#}"); - assert!(engine.charge_route_actuation("service", RouteActuation::Forward).is_err()); - - engine.charge_route_actuation("service", RouteActuation::Survival)?; - driver.withdraw_candidate_membership( - &preflight, - &engine.route_gate("service", CommandKind::Deploy), - )?; - assert_eq!(routed.reloads(), 1, "the unrecorded withdrawal did not run"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_second_failed_challenge_waits_twice_as_long_before_the_next_repair() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - let elapse = |routed: &RoutedWorld| { - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = - Some(1); - }) - }; - - routed.drift()?; - assert!(routed.tick()?); - elapse(&routed)?; - routed.drift()?; - assert!(routed.tick()?); - let state = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(state.consecutive_failures, 2); - assert_eq!( - state.next_challenge_at_unix_millis.unwrap() - state.last_challenge_at_unix_millis.unwrap(), - 2 * DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS - ); - - // The wait is longer than one observation age: no challenge, so no - // repair and no reload, however often the pass runs. - let reloads = routed.reloads(); - for _ in 0..20 { - routed.drift()?; - routed.tick()?; - } - assert_eq!(routed.reloads(), reloads, "the route was repaired inside its wait"); - assert_eq!( - admitted(&routed)?.route_supervision.unwrap().consecutive_failures, - 2 - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_proved_challenge_that_cannot_be_recorded_ends_the_pass_without_failing_it() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let old = ControlSnapshot::read(&routed.world.state_store)?; - let stale = old.admitted_for("service").context("no generation")?; - // Something else writes the generation after the pass read it. - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().last_challenge_at_unix_millis = Some(5); - })?; - routed.drift()?; - assert!(routed.world.engine.supervise_admitted_route(&old, stale)?); - assert_eq!( - admitted(&routed)?.route_supervision.unwrap().last_challenge_at_unix_millis, - Some(5), - "the stale write replaced the newer generation" - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_first_routed_deploy_is_metered_and_refused_at_the_ceiling() -> Result<()> { - let metered = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - metered.stub.set_state("active"); - metered.run_to_routing()?; - assert!(ControlSnapshot::read(&metered.world.state_store)?.targets.is_empty()); - metered.step()?; - assert_eq!(meters_of(&metered.world, "service")?.route_actuations.len(), 1); - - let refused = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - refused.stub.set_state("active"); - refused.run_to_routing()?; - refused.ledger(ROUTE_ACTUATION_CEILING)?; - let error = refused.step().unwrap_err(); - assert!(error.downcast_ref::().is_some(), "{error:#}"); - assert_eq!(refused.world.route_stubs.count(""), 0, "a refused install ran a program"); - assert!(!refused.fragment_path().exists()); - - // Past the fence the refusal is resumable, not an abort: the resume - // backoff spaces it and no actuation runs while it waits. - assert!(!refused.world.engine.resume_one_transaction()?); - let waiting = refused.transaction()?; - assert_eq!(waiting.phase, DeploymentPhase::Routing); - assert!(waiting.post_fencing_abort.is_none()); - assert!(waiting.last_error.as_deref().is_some_and(|text| text.contains("ceiling"))); - assert_eq!(refused.world.route_stubs.count(""), 0); - Ok(()) - } - - #[test] - fn a_stepped_back_clock_is_settled_where_the_decision_is_made() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let ahead = now_millis()? + 86_400_000; - - // A supervision pass that would decide from the restart log settles - // it first: the write ends the pass, and nothing is minted from - // entries that only look recent because the clock stepped back. - let mut restarts = TargetSupervision::new("service"); - restarts.continuity_restarts = vec![ahead; CONTINUITY_PAGING_RESTARTS]; - set_meters(&routed.world, &restarts)?; - routed.workload.kill(); - assert!(routed.tick()?); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let settled = meters_of(&routed.world, "service")?; - assert!(settled.continuity_restarts.iter().all(|&at| at < ahead)); - - // A refused forward actuation writes the clamp, so the window runs - // from now and not from the day the clock stood on. - let mut ledger = TargetSupervision::new("service"); - ledger.route_actuations = vec![ahead; ROUTE_ACTUATION_CEILING]; - set_meters(&routed.world, &ledger)?; - let refused = routed - .world - .engine - .charge_route_actuation("service", RouteActuation::Forward) - .unwrap_err(); - assert!(refused.downcast_ref::().is_some()); - let settled = meters_of(&routed.world, "service")?; - assert!(settled.route_actuations.iter().all(|&at| at < ahead)); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_refused_preflight_starts_no_private_mount_unit_and_fails_at_once_with_the_reopen_time() - -> Result<()> { - let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; - routed.step()?; - assert!(routed.transaction()?.warming.is_some()); - routed.ledger(ROUTE_ACTUATION_CEILING)?; - let reopens_at = meters_of(&routed.world, "service")? - .route_reopens_at(now_millis()?) - .context("the ledger is not full")?; - - // The refusal is an error of its own phase, before the fence: the - // deployment fails now, and its error carries the reopen time. - let id = routed.transaction()?.transaction_id; - routed.world.engine.resume_one_transaction()?; - let abort = routed - .transaction()? - .pre_fencing_abort - .context("the refusal did not abort the deployment")?; - assert!(abort.error.contains(&format!("reopens at unix ms {reopens_at}")), "{}", abort.error); - for _ in 0..8 { - if record_of(&routed.world, &id)?.completion.is_some() { - break; - } - routed.step()?; - } - let Some(TransactionCompletion::FailedBeforeFencing { error }) = - record_of(&routed.world, &id)?.completion - else { - let record = record_of(&routed.world, &id)?; - bail!( - "the deployment did not fail before the fence: phase {:?} completion {:?} abort {:?} error {:?}", - record.phase, record.completion, record.pre_fencing_abort, record.last_error - ); - }; - assert!(error.contains(&format!("reopens at unix ms {reopens_at}")), "{error}"); - assert_eq!(routed.world.route_stubs.count(""), 0, "a program ran"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.workload.kill(); - routed.stub.hang_up.store(true, Ordering::SeqCst); - routed.drift()?; - - // Tick 1: the repair charges the target's meters and the proof - // fails. Both are writes, so they end the pass: nothing else is - // decided from the snapshot they made stale. - assert!(routed.tick()?); - let state = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(state.consecutive_failures, 1); - assert!(state.degraded_since_unix_millis.is_some()); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let meters = meters_of(&routed.world, "service")?; - assert_eq!(meters.route_survivals.len(), 2); - assert!(meters.route_actuations.is_empty()); - - // Tick 2: the route waits, and the dead unit is restarted from a - // fresh read, counted in the same write that mints it. - assert!(routed.tick()?); - assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); - assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_proved_challenge_clears_the_degradation() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - routed.drift()?; - assert!(routed.tick()?); - let degraded = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(degraded.consecutive_failures, 1); - assert!(degraded.degraded_since_unix_millis.is_some()); - assert!(degraded.next_challenge_at_unix_millis.is_some()); - - routed.stub.hang_up.store(false, Ordering::SeqCst); - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); - })?; - routed.drift()?; - assert!(routed.tick()?); - let healed = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(healed.consecutive_failures, 0); - assert_eq!(healed.next_challenge_at_unix_millis, None); - assert_eq!(healed.degraded_since_unix_millis, None); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_failing_reload_that_puts_the_drift_back_backs_the_route_off() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.world.route_stubs.refuse_reloads()?; - let reloads = routed.reloads(); - routed.drift()?; - for _ in 0..200 { - routed.tick()?; - } - assert_eq!(routed.reloads() - reloads, 1, "the route was reloaded again and again"); - assert_eq!(std::fs::read(routed.fragment_path())?, b"drifted bytes\n"); - assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_candidate_whose_route_proof_fails_reloads_at_most_twice_the_ceiling() -> Result<()> { - let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - let mut last = None; - for _ in 0..40 { - last = Some(routed.step().unwrap_err()); - } - // Every install is a counted Forward and is followed by one - // recorded rollback, so the rollbacks are bounded by the installs: - // twelve installs spend the window, each reloads twice, and every - // later attempt is refused before it runs anything. - assert_eq!(routed.reloads(), 2 * ROUTE_ACTUATION_CEILING); - let meters = meters_of(&routed.world, "service")?; - assert_eq!(meters.route_used(now_millis()?), ROUTE_ACTUATION_CEILING); - assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); - let last = last.unwrap(); - assert!(last.downcast_ref::().is_some(), "{last:#}"); - Ok(()) - } - - /// The rescue (ruling 3d A): a target whose continuity spent every - /// restart, each charging its preflight and install as Survival, is - /// still deployed: its route changes are the ceiling's first Forward. - #[cfg(unix)] - #[test] - fn a_target_that_spent_its_restarts_can_still_be_deployed() -> Result<()> { - let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; - routed.stub.set_state("active"); - routed.step()?; - assert!(routed.transaction()?.warming.is_some()); - set_meters( - &routed.world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - for _ in 0..2 * CONTINUITY_PAGING_RESTARTS { - routed.world.engine.charge_route_actuation("service", RouteActuation::Survival)?; - } - let spent = meters_of(&routed.world, "service")?; - assert!(spent.restarts_page(now_millis()?)); - assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_PAGING_RESTARTS); - - // Warming's preflight and Routing's install are both admitted. - routed.step()?; - assert!(routed.transaction()?.route_preflight.is_some(), "the preflight was refused"); - routed.run_to_routing()?; - routed.promote()?; - assert!(routed.fragment_path().exists(), "the install was refused"); - assert!(routed.reloads() >= 1); - let after = meters_of(&routed.world, "service")?; - assert_eq!(after.route_actuations.len(), 2, "preflight and install are the Forward"); - Ok(()) - } - - // ----------------------------------------------------------------- - // B5: every post-fencing phase ends. - // ----------------------------------------------------------------- - - impl RoutedWorld { - /// One scheduler pass over the transaction, as the daemon runs it: - /// through the deadline resolver, with the resume backoff spent. - fn resume(&self) -> Result<()> { - self.world.engine.resume_backoff.lock().unwrap().clear(); - self.world.engine.resume_one_transaction()?; - Ok(()) - } - - fn resume_until_terminal(&self) -> Result { - for _ in 0..12 { - if latest(&self.world)?.completion.is_some() { - break; - } - self.resume()?; - } - let finished = latest(&self.world)?; - ensure!(finished.completion.is_some(), "never finished: {:?}", finished.last_error); - Ok(finished) - } - } - - /// The live transaction's phase deadline is already behind it. - fn pass_deadline(world: &EngineFixture) -> Result<()> { - let current = resident(world)?; - let mut next = current.value.clone(); - let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; - deadline.entered_at_unix_millis = 1; - deadline.deadline_at_unix_millis = 2; - replace_transaction(&world.state_store, ¤t, &next) - } - - /// Soul's F1 probe, committed. A continuity's route changes are - /// Survival, so no ceiling refuses its install and rollback while its - /// route proof keeps failing. The Routing deadline ends it: no attempt - /// runs after the deadline. - #[cfg(unix)] - #[test] - fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - for _ in 0..3 { - routed.resume()?; - } - // Before the deadline each attempt installs and rolls back, and no - // resume moves the deadline. - assert_eq!(routed.reloads(), 6); - let waiting = routed.transaction()?; - assert_eq!(waiting.phase, DeploymentPhase::Routing); - assert!(waiting.post_fencing_abort.is_none(), "aborted before its deadline"); - assert!(waiting.last_error.is_some()); - assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); - - pass_deadline(&routed.world)?; - routed.resume()?; - let abort = routed - .transaction()? - .post_fencing_abort - .context("the passed deadline did not end the phase")?; - assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); - let finished = routed.resume_until_terminal()?; - assert!( - matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - ), - "{:?}", - finished.completion - ); - // After the deadline no attempt runs; the withdrawal writes and - // reloads once, whatever the rolled-back disk shows. - assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); - Ok(()) - } - - /// The live shape of F1. A continuity reuses the admitted Expected, so - /// its membership is the admitted fragment already on disk: while its - /// proof fails, the install and the rollback write nothing and reload - /// nothing, and neither does the withdrawal after its deadline. - #[cfg(unix)] - #[test] - fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() - -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let current = resident(&routed.world)?; - let mut next = current.value.clone(); - let binding = next.plan.as_ref().unwrap().parsed_inputs()?.1; - let admitted = NginxRouteDriver::new(binding.route.context("no route binding")?) - .render(next.expected.as_ref().context("no Expected")?)?; - let preflight = next.route_preflight.as_mut().context("no preflight")?; - preflight.incumbent_runtime_instance_id = Some(sha256_id(b"admitted-instance")); - preflight.incumbent_membership_sha256 = Some(sha256_id(&admitted)); - preflight.incumbent_configuration = Some(admitted.clone()); - replace_transaction(&routed.world.state_store, ¤t, &next)?; - std::fs::write(routed.fragment_path(), &admitted)?; - - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - for _ in 0..5 { - routed.resume()?; - } - assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); - assert!(routed.transaction()?.last_error.is_some(), "the proof did not fail"); - assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); - - pass_deadline(&routed.world)?; - routed.resume_until_terminal()?; - assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); - assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); - Ok(()) - } - - /// The abort's withdrawal wrote the baseline back and Idunn died before - /// the reload: the disk already shows the baseline, and the retry still - /// reloads, so nginx does not keep proxying to the stopped candidate. - #[cfg(unix)] - #[test] - fn an_abort_whose_reload_a_crash_cut_off_reloads_on_retry() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - let installed = routed.reloads(); - routed - .world - .engine - .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; - std::fs::remove_file(routed.fragment_path())?; - routed.resume()?; - assert_eq!( - routed.transaction()?.post_fencing_abort.context("no abort")?.route_restoration, - CleanupEvidence::Complete - ); - assert_eq!(routed.reloads(), installed + 1, "the withdrawal trusted unloaded bytes"); - Ok(()) - } - - /// Soul's F2 probe, committed: a post-fence withdrawal whose - /// `ufw delete` keeps failing withdraws the firewall allow first, so its - /// retries reload nginx not at all. - #[cfg(unix)] - #[test] - fn a_withdrawal_whose_firewall_step_keeps_failing_never_reloads() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - let installed = routed.reloads(); - routed.world.route_stubs.refuse_ufw_deletes()?; - routed - .world - .engine - .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; - for _ in 0..20 { - routed.resume()?; - } - assert_eq!(routed.reloads(), installed, "a failing withdrawal reloaded nginx"); - assert_eq!(routed.world.route_stubs.count("ufw delete"), 20); - assert!(routed.fragment_path().exists(), "the route went before its firewall rule"); - Ok(()) - } - - /// A record lifted from before deadlines existed is timed once, from - /// the first time the resolver sees it, and never again. - #[test] - fn a_post_fence_record_without_a_deadline_is_timed_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let current = resident(&routed.world)?; - let mut lifted = current.value.clone(); - lifted.phase_deadline = None; - replace_transaction(&routed.world.state_store, ¤t, &lifted)?; - let engine = &routed.world.engine; - - let before = now_millis()?; - assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, Some(true)); - let stamped = routed.transaction()?.phase_deadline.context("the record was not timed")?; - assert_eq!(stamped.phase, DeploymentPhase::Fencing); - assert!(stamped.entered_at_unix_millis >= before); - assert_eq!( - stamped.deadline_at_unix_millis - stamped.entered_at_unix_millis, - u64::from(lifted.plan.as_ref().unwrap().phase_deadlines().fencing_seconds) * 1000 - ); - assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, None); - assert_eq!(routed.transaction()?.phase_deadline, Some(stamped)); - Ok(()) - } - - /// `idunn expire` writes a request and nothing else; the resolver ends - /// the phase from it, and the spent request is deleted. - #[test] - fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { - let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - let refused = expire(&early.world.state_store, "continuity-service", "operator") - .expect_err("a transaction before the fence was expired"); - assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); - - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let untouched = resident(&routed.world)?.envelope; - expire(&routed.world.state_store, "continuity-service", "operator")?; - assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - assert_eq!(snapshot.expiry_requests.len(), 1); - assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); - - routed.resume()?; - let abort = routed - .transaction()? - .post_fencing_abort - .context("the expiry did not end the phase")?; - assert!(abort.error.contains("Fencing phase was expired by the operator"), "{}", abort.error); - routed.resume()?; - assert!( - ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), - "the spent request stayed" - ); - Ok(()) - } - - /// The `expire` verb as the CLI parses and runs it: the options land - /// where they say, and it asks only about its own command's live - /// transaction. - #[test] - fn idunn_expire_parses_its_options_and_asks_only_about_its_own_live_transaction() - -> Result<()> { - let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); - assert_eq!( - parse(args(&["expire", "up-service", "--requested-by", "alice", "--state-store", "/s/c.cc"]).into_iter())?, - Command::Expire { - state_store: PathBuf::from("/s/c.cc"), - command_id: "up-service".into(), - requested_by: "alice".into(), - } - ); - let help = parse(args(&["expire", "up-service", "--help"]).into_iter()).unwrap_err(); - assert!(format!("{help}").contains("idunn expire "), "{help}"); - assert!(parse(args(&["expire", "up-service", "--force"]).into_iter()).is_err()); - assert!(parse(args(&["expire"]).into_iter()).is_err()); - - // The verb runs: a store with nothing live refuses. - let empty = TempDir::new()?; - let store = empty.path().join("control.cc").display().to_string(); - let refused = run(args(&["expire", "up-service", "--state-store", &store]).into_iter()) - .expect_err("expired a command with no transaction"); - assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); - - // Another target's deployment in flight is not this command's. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let now = now_millis()?; - let other = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-other".into(), - kind: CommandKind::Deploy, - selector: "other".into(), - requested_by: "test".into(), - requested_at_unix_millis: now, - }; - let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; - assert!(SingleFileMessagePackBackingStore::new(&routed.world.state_store).compare_exchange( - &[ - CultCacheExpectedEnvelope { - r#type: DeploymentCommand::TYPE.into(), - key: other.command_id.clone(), - current: None, - }, - CultCacheExpectedEnvelope { - r#type: DeploymentTransaction::TYPE.into(), - key: sealing.transaction_id.clone(), - current: None, - }, - ], - &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], - )?); - expire(&routed.world.state_store, "continuity-service", "operator")?; - let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].value.command_id, "continuity-service"); - Ok(()) - } - - /// The resolver reads a request as its own transaction's only: with two - /// live transactions, a request for the other leaves this one running. - #[test] - fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let now = now_millis()?; - let other = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-other".into(), - kind: CommandKind::Deploy, - selector: "other".into(), - requested_by: "test".into(), - requested_at_unix_millis: now, - }; - let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; - // The request names a command whose id has this transaction's - // length and first bytes, and differs only in its last. - let near = "continuity-servicx"; - assert_eq!(near.len(), "continuity-service".len()); - let request = ExpiryRequest { - schema_version: EXPIRY_REQUEST_SCHEMA.into(), - command_id: near.into(), - requested_by: "operator".into(), - requested_at_unix_millis: now, - }; - // And commands whose ids are a strict prefix of this one's, and - // extend it: a match is the whole id, in neither direction a part. - let prefix = ExpiryRequest { - command_id: "continuity-servic".into(), - ..request.clone() - }; - let extension = ExpiryRequest { - command_id: "continuity-service-2".into(), - ..request.clone() - }; - let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); - let absent = |r#type: &str, key: &str| CultCacheExpectedEnvelope { - r#type: r#type.into(), - key: key.into(), - current: None, - }; - assert!(store.compare_exchange( - &[ - absent(DeploymentCommand::TYPE, &other.command_id), - absent(DeploymentTransaction::TYPE, &sealing.transaction_id), - absent(ExpiryRequest::TYPE, &request.command_id), - absent(ExpiryRequest::TYPE, &prefix.command_id), - absent(ExpiryRequest::TYPE, &extension.command_id), - ], - &[ - command_envelope(&other, now)?, - transaction_envelope(&sealing, now)?, - expiry_request_envelope(&request, now)?, - expiry_request_envelope(&prefix, now)?, - expiry_request_envelope(&extension, now)?, - ], - )?); - assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.len(), 3); - let snapshot = ControlSnapshot::read(&routed.world.state_store)?; - assert_eq!(snapshot.transactions.iter().filter(|t| !t.value.is_terminal()).count(), 2); - let mine = snapshot - .transactions - .iter() - .find(|t| t.value.command_id == "continuity-service") - .context("no continuity")?; - assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); - let after = ControlSnapshot::read(&routed.world.state_store)?; - let unchanged = after - .transactions - .iter() - .find(|t| t.value.command_id == "continuity-service") - .context("no continuity")?; - assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); - Ok(()) - } - - /// Q3 at the deadline, through the Engine. A continuity whose candidate - /// was granted a lease and never proved it held it fails as - /// `lease-not-adopted`; one whose Ready proved it restarts the admitted - /// release. A deployment whose candidate holds a lease is left to the - /// operator: nothing is aborted and its step still runs. - #[test] - fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { - let granted = |routed: &RoutedWorld| -> Result { - routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; - Ok(routed - .transaction()? - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("no granted lease")? - .to_owned()) - }; - - let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - granted(&unadopted)?; - pass_deadline(&unadopted.world)?; - unadopted.resume()?; - let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); - assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); - let finished = unadopted.resume_until_terminal()?; - assert!(matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - )); - - let adopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - let lease = granted(&adopted)?; - adopted.stub.set_state("active"); - *adopted.stub.lease.lock().unwrap() = Some(lease.clone()); - adopted.step()?; - let ready = adopted.transaction()?; - let Some(ReadinessEvidence::RouteProof { evidence }) = &ready.ready else { - bail!("the candidate was not Ready"); - }; - let adoption = ready.lease_adoption.clone().context("Ready recorded no adoption")?; - assert_eq!(adoption.write_lease_sha256, lease); - assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); - assert_eq!(adoption.source, AdoptionSource::Direct); - adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; - pass_deadline(&adopted.world)?; - adopted.resume()?; - let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); - let finished = adopted.resume_until_terminal()?; - assert!( - matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestartAdmitted, - .. - }) - ), - "{:?}", - finished.completion - ); - - let deploy = - build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; - granted(&deploy)?; - pass_deadline(&deploy.world)?; - for _ in 0..3 { - deploy.resume()?; - } - let held = deploy.transaction()?; - assert!(held.post_fencing_abort.is_none(), "a leased deployment was aborted"); - assert_eq!(held.phase, DeploymentPhase::AwaitingReady); - assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); - Ok(()) - } - - /// B5 Soul A. The physical grant lands before `Granted` is durable. A - /// continuity that dies in that window and passes its deadline is - /// aborted from `Prepared`; the abort still withdraws the lease on - /// disk, so the next fencing of the target is not refused. - #[test] - fn an_abort_from_a_prepared_lease_withdraws_a_grant_already_on_disk() -> Result<()> { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; - let current = resident(&routed.world)?; - let mut crashed = current.value.clone(); - let Some(LeasingEvidence::Granted { lease, lease_sha256 }) = crashed.leasing.clone() else { - bail!("no granted lease"); - }; - crashed.leasing = Some(LeasingEvidence::Prepared { lease: lease.clone(), lease_sha256 }); - replace_transaction(&routed.world.state_store, ¤t, &crashed)?; - let driver = CultCacheWriteLeaseDriver::new( - "service", - routed.world.root.join("lease/process-write-lease.cc"), - ); - assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); - - pass_deadline(&routed.world)?; - routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); - routed.resume_until_terminal()?; - assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); - // What the next fencing does to a candidate's lease path. - driver.revoke_exact(None)?; - Ok(()) - } - - /// B5 Soul F. A stateful candidate Odin reports on: its Ready word - /// carries the granted lease, and that Ready records the adoption, - /// sourced from Odin's topology and naming the presence Odin observed. - #[test] - fn an_odin_ready_word_records_a_stateful_candidates_lease_adoption() -> Result<()> { - let routed = build_routed_world( - Odin::Unreachable, - 1, - DeploymentPhase::Fencing, - true, - true, - CommandKind::Continuity, - )?; - for _ in 0..8 { - if routed.transaction()?.phase == DeploymentPhase::AwaitingReady { - break; - } - routed.step()?; - } - let waiting = routed.transaction()?; - assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady, "{:?}", waiting.last_error); - let lease = waiting - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("no granted lease")? - .to_owned(); - assert_eq!(waiting.lease_adoption, None); - - let ready = CultCacheEnvelope { - key: crate::drivers::incarnation_key_of( - &waiting.target, - &waiting.expected.as_ref().unwrap().canonical_sha256()?, - ), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: signed_correlation_as( - &routed.world, - &waiting, - waiting.odin_publisher_sequence_cursor + 1, - true, - "active", - Some(&lease), - b"presence-ready", - )?, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), - }; - replace_odin_entry(&routed.world.engine.options.odin_correlation_store, ready)?; - // One step admits the new sequence, the next records Ready from it. - for _ in 0..3 { - if routed.transaction()?.ready.is_some() { - break; - } - routed.step()?; - } - let adopted = routed.transaction()?; - assert!( - matches!(adopted.ready, Some(ReadinessEvidence::OdinCorrelated { .. })), - "{:?}", - adopted.last_error - ); - let adoption = adopted.lease_adoption.context("Odin's Ready recorded no adoption")?; - assert_eq!(adoption.source, AdoptionSource::OdinTopology); - assert_eq!(adoption.write_lease_sha256, lease); - assert_eq!(adoption.signed_presence_sha256, sha256_id(b"presence-ready")); - Ok(()) - } - - /// F16: Commit goes on from the topology it admitted, so Odin - /// re-stamping before the step and again during the final proof does - /// not send it round again. - #[test] - fn commit_completes_while_odin_restamps_every_tick() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - routed.promote()?; - let transaction = routed.transaction()?; - let cursor = transaction.odin_publisher_sequence_cursor; - let store = routed.world.engine.options.odin_correlation_store.clone(); - replace_odin_entry(&store, odin_ready_envelope(&routed.world, &transaction, cursor + 1)?)?; - let later = odin_ready_envelope(&routed.world, &transaction, cursor + 2)?; - *routed.stub.on_stable_hit.lock().unwrap() = Some(Box::new(move || { - replace_odin_entry(&store, later.clone()).expect("Odin re-stamps"); - })); - routed.step()?; - let generation = admitted(&routed)?; - assert_eq!( - generation.latest_odin_observation.context("no Odin receipt")?.publisher_sequence, - cursor + 2, - "commit did not go on from the re-stamped topology" - ); - Ok(()) - } - - fn odin_ready_envelope( - world: &EngineFixture, - transaction: &DeploymentTransaction, - sequence: u64, - ) -> Result { - let expected = transaction.expected.as_ref().context("no Expected")?; - Ok(CultCacheEnvelope { - key: crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?), - r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), - payload: signed_correlation(world, transaction, sequence, true)?, - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), - }) - } - - fn replace_odin_entry(store: &Path, envelope: CultCacheEnvelope) -> Result<()> { - let backing = SingleFileMessagePackBackingStore::new(store); - let current = backing - .pull_all_read_only_snapshot()? - .into_iter() - .find(|entry| entry.key == envelope.key && entry.r#type == envelope.r#type); - ensure!( - backing.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: envelope.r#type.clone(), - key: envelope.key.clone(), - current, - }], - &[envelope], - )?, - "the Odin store changed under the re-stamp" - ); - Ok(()) - } - } -} +use std::collections::{BTreeMap, BTreeSet}; +use std::env; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use cultnet_rs::{ + AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, + IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, + IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, + IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, + IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, + OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, + ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, + authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, + correlate_runtime_presence_claim, derive_service_identity_id, + evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, + verify_idunn_deployment_brake_authorization, verify_runtime_authority, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::deployment::{ + DependencyKind, OperatorBinding, RolloutStrategy, RouteBinding, WorkloadBinding, + capability_compatible, +}; +use crate::deployment_plan::{ + CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, +}; +use crate::drivers::{ + ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, + GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, + LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, + RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, + SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, + WriteLeasePort, +}; +use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; +use crate::host_actuator::{ + HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, + HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, +}; + +const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2"; +const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v4"; +const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3"; +const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2"; +const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v4"; +const ADMITTED_GENERATION_SCHEMA_V3: &str = "idunn.admitted_generation.v3"; +const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; +/// The capability whose declaration makes a target Odin-correlated. +pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; +const TARGET_SUPERVISION_SCHEMA: &str = "idunn.target_supervision.v2"; +/// The seven-key layout written before continuity's streak, anchor and +/// re-materialization bound. Read and migrated by `read_supervision_record` +/// and `migrate_control_store_to_current_schema`; delete it, `LegacyTargetSupervisionV1` +/// and those two arms once every store has opened under a build that +/// migrates (the key-34 precedent's deletion line). +const TARGET_SUPERVISION_SCHEMA_V1: &str = "idunn.target_supervision.v1"; +const EXPIRY_REQUEST_SCHEMA: &str = "idunn.expiry_request.v1"; +/// The number of charged continuity attempts inside +/// `CONTINUITY_RESTART_WINDOW_MILLIS` at which the ContinuityExhausted incident +/// opens and is held. It gates no actuation: continuity never stops +/// (ruling budget-never-stop). +const CONTINUITY_PAGING_RESTARTS: usize = 6; +const CONTINUITY_RESTART_WINDOW_MILLIS: u64 = 3_600_000; +/// The base of the streak backoff: the wait after the first charged attempt; +/// it doubles with each further one in the streak. +const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; +/// The longest streak backoff between charged continuity attempts (ruling +/// continuity-backoff-defaults). +const CONTINUITY_BACKOFF_CEILING_MILLIS: u64 = 300_000; +/// How long an admitted generation must have run, with no attempt failing on +/// it, before the attempt streak resets (ruling continuity-backoff-defaults). +const CONTINUITY_SUSTAINED_RUN_MILLIS: u64 = 600_000; + +/// Forward route actuations (fragment write, firewall, `nginx -t`, reload, +/// private-mount validation) one target may perform per sliding window. A +/// healthy target performs none; a legitimate deployment performs one or two. +/// Only deployments count: a Survival actuation is recorded beside this +/// ceiling, never counted against it and never refused by it, so a target +/// that is being restarted can still be rescued by a deploy. +/// Survival is bounded elsewhere: a continuity's actuations by the streak +/// backoff (at most one attempt per `CONTINUITY_BACKOFF_CEILING_MILLIS` once +/// the streak is long) times what one transaction can do before its phase +/// deadline ends it (none, over its admitted route), and an admitted-route +/// repair by the challenge backoff. A withdrawal that keeps failing retries, +/// reloading each time, under the resume backoff alone: the abort has no +/// deadline. Nothing bounds +/// reloads across the host. +const ROUTE_ACTUATION_CEILING: usize = 12; +const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; +/// How many Survival route actuation times a target keeps, for `status` +/// only: they are a record, not a meter, and no decision reads them. +const ROUTE_SURVIVAL_RECORD: usize = 12; +/// The longest wait between route challenges while proofs keep failing. The +/// shortest is the observation max age. +const ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS: u64 = 600_000; + +/// How long continuity waits after it could not prepare a restart (the +/// projection would not demote) before trying again. +const CONTINUITY_DEFERRAL_MILLIS: u64 = 30_000; + +const DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS: u64 = 30_000; +const DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS: u64 = 2_000; + +/// An immutable request. Execution state deliberately does not fit in this +/// record; status is derived from the transactions created for it. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_command", + schema = "idunn.deployment_command.v2" +)] +struct DeploymentCommand { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + command_id: String, + #[cultcache(key = 2)] + kind: CommandKind, + #[cultcache(key = 3)] + selector: String, + #[cultcache(key = 4)] + requested_by: String, + #[cultcache(key = 5)] + requested_at_unix_millis: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum CommandKind { + Deploy, + Continuity, +} + +impl DeploymentCommand { + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == DEPLOYMENT_COMMAND_SCHEMA, + "deployment command schema is unsupported" + ); + require_id(&self.command_id, "deployment command id")?; + require_selector(&self.selector)?; + require_value(&self.requested_by, "deployment requester")?; + ensure!( + self.requested_at_unix_millis > 0, + "deployment command has no request time" + ); + match self.kind { + CommandKind::Deploy => ensure!( + !self.command_id.starts_with("continuity-"), + "operator deployment uses the continuity command namespace" + ), + CommandKind::Continuity => ensure!( + self.command_id.starts_with("continuity-") + && !self.selector.starts_with("profile:"), + "continuity command identity or selector is invalid" + ), + } + Ok(()) + } +} + +/// An operator's request that a live transaction's post-fencing phase end now +/// (`idunn expire`). The CLI only writes this record. The daemon's deadline +/// resolver reads it as a deadline that has passed, and the record is deleted +/// once its transaction no longer has a phase to end. No CLI writes a +/// transaction field. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.expiry_request", schema = "idunn.expiry_request.v1")] +struct ExpiryRequest { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + command_id: String, + #[cultcache(key = 2)] + requested_by: String, + #[cultcache(key = 3)] + requested_at_unix_millis: u64, +} + +impl ExpiryRequest { + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == EXPIRY_REQUEST_SCHEMA, + "expiry request schema is unsupported" + ); + require_id(&self.command_id, "expiry request command id")?; + require_value(&self.requested_by, "expiry requester")?; + ensure!( + self.requested_at_unix_millis > 0, + "expiry request has no request time" + ); + Ok(()) + } +} + +fn expiry_request_envelope(value: &ExpiryRequest, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.command_id, + ExpiryRequest::TYPE, + EXPIRY_REQUEST_SCHEMA, + value, + now, + ) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +#[repr(u8)] +enum DeploymentPhase { + Sealing, + Starting, + Warming, + Fencing, + Leasing, + AwaitingReady, + Routing, + Committing, + Complete, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct DeploymentAuthorization { + authorization_id: String, + brake_sha256: String, + canonical_brake_bytes: Vec, + authorized_at_unix_millis: u64, +} + +impl DeploymentAuthorization { + fn validate_shape(&self) -> Result<()> { + require_id(&self.authorization_id, "deployment authorization id")?; + ensure!( + !self.canonical_brake_bytes.is_empty() + && self.brake_sha256 == sha256_id(&self.canonical_brake_bytes) + && self.authorized_at_unix_millis > 0, + "deployment authorization receipt is incomplete" + ); + let record: IdunnDeploymentBrakeRecord = + rmp_serde::from_slice(&self.canonical_brake_bytes)?; + record.validate()?; + ensure!( + rmp_serde::to_vec(&record)? == self.canonical_brake_bytes + && record.authorization_id.as_deref() == Some(&self.authorization_id), + "deployment authorization receipt is noncanonical or mismatched" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct TopologyEvidence { + canonical_bytes: Vec, + canonical_sha256: String, + signer_identity_id: String, + publisher_sequence: u64, + admitted_at_unix_millis: u64, +} + +impl TopologyEvidence { + fn from_authenticated( + topology: &AuthenticatedOdinRuntimeTopologyCorrelation, + admitted_at_unix_millis: u64, + ) -> Result { + let record = topology.record(); + let evidence = Self { + canonical_bytes: topology.canonical_bytes().to_vec(), + canonical_sha256: sha256_id(topology.canonical_bytes()), + signer_identity_id: record.signer_identity_id.clone(), + publisher_sequence: record.publisher_sequence, + admitted_at_unix_millis, + }; + evidence.validate_shape()?; + Ok(evidence) + } + + fn validate_shape(&self) -> Result<()> { + ensure!( + !self.canonical_bytes.is_empty() + && self.canonical_sha256 == sha256_id(&self.canonical_bytes), + "topology evidence bytes or digest are invalid" + ); + require_id(&self.signer_identity_id, "Odin topology signer")?; + ensure!( + self.publisher_sequence > 0 && self.admitted_at_unix_millis > 0, + "topology evidence sequence or admission time is invalid" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RuntimePresenceEvidence { + canonical_bytes: Vec, + canonical_sha256: String, + message_id: String, + challenged_at_unix_millis: u64, + admitted_at_unix_millis: u64, +} + +impl RuntimePresenceEvidence { + fn from_present( + present: &cultnet_rs::VerifiedRuntimePresence, + message_id: String, + challenged_at_unix_millis: u64, + admitted_at_unix_millis: u64, + ) -> Result { + let evidence = Self { + canonical_bytes: present.canonical_bytes().to_vec(), + canonical_sha256: present.signed_presence_sha256().to_owned(), + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + }; + evidence.validate_shape()?; + Ok(evidence) + } + + fn validate_shape(&self) -> Result<()> { + ensure!( + !self.canonical_bytes.is_empty() + && self.canonical_sha256 == sha256_id(&self.canonical_bytes), + "runtime presence evidence bytes or digest are invalid" + ); + require_id(&self.message_id, "runtime presence challenge")?; + ensure!( + self.challenged_at_unix_millis > 0 + && self.admitted_at_unix_millis >= self.challenged_at_unix_millis, + "runtime presence evidence timeline is invalid" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "source", rename_all = "kebab-case", deny_unknown_fields)] +enum WarmingEvidence { + OdinTopology { evidence: TopologyEvidence }, + FirstOdinDirect { evidence: RuntimePresenceEvidence }, + /// A route-proof target's candidate answered Idunn's challenge itself. + RouteProofDirect { evidence: RuntimePresenceEvidence }, +} + +impl WarmingEvidence { + fn validate_shape(&self) -> Result<()> { + match self { + Self::OdinTopology { evidence } => evidence.validate_shape(), + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + evidence.validate_shape() + } + } + } + + fn voucher(&self) -> Voucher { + match self { + Self::OdinTopology { .. } | Self::FirstOdinDirect { .. } => Voucher::Odin, + Self::RouteProofDirect { .. } => Voucher::Candidate, + } + } + + /// The presence Idunn challenged for itself, when no Odin carried it. + fn direct(&self) -> Option<&RuntimePresenceEvidence> { + match self { + Self::OdinTopology { .. } => None, + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + Some(evidence) + } + } + } +} + +/// Opaque capability minted only after the transaction CAS stores exact +/// Warming evidence. Adjacent drivers receive only the candidate identity and +/// signed-presence digest needed to bind the write lease; they cannot inspect +/// or construct semantic admission. +#[derive(Clone, Debug)] +pub struct SequenceAdmittedWarming { + transaction_id: String, + signed_presence_sha256: String, + runtime_instance_id: String, +} + +impl SequenceAdmittedWarming { + fn from_topology( + transaction_id: String, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + ) -> Result { + let record = authenticated.record(); + let signed_presence_sha256 = record + .signed_presence_sha256 + .clone() + .context("Warming topology has no signed presence")?; + let runtime_instance_id = record + .runtime_instance_id + .clone() + .context("Warming topology has no runtime instance")?; + Ok(Self { + transaction_id, + signed_presence_sha256, + runtime_instance_id, + }) + } + + fn from_direct_presence( + transaction_id: String, + warming: WarmingEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + ) -> Result { + let evidence = warming + .direct() + .context("direct Warming evidence is an Odin topology receipt")?; + ensure!( + present.canonical_bytes() == evidence.canonical_bytes + && present.signed_presence_sha256() == evidence.canonical_sha256, + "direct Warming evidence differs from its authenticated presence" + ); + let runtime_instance_id = present.record().runtime_instance_id.clone(); + Ok(Self { + transaction_id, + signed_presence_sha256: evidence.canonical_sha256.clone(), + runtime_instance_id, + }) + } + + #[cfg(test)] + pub(crate) fn for_test( + transaction_id: impl Into, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + ) -> Result { + let record = authenticated.record(); + ensure!( + record.present + && !record.ready + && record.observed_presence_state.as_deref() == Some("warming") + && record.observed_write_lease_sha256.is_none() + && record.disagreements.is_empty(), + "test receipt is not exact Warming evidence" + ); + let transaction_id = transaction_id.into(); + require_id(&transaction_id, "test transaction id")?; + let signed_presence_sha256 = record + .signed_presence_sha256 + .clone() + .context("test Warming receipt has no signed presence")?; + let runtime_instance_id = record + .runtime_instance_id + .clone() + .context("test Warming receipt has no runtime instance")?; + let token = Self::from_topology(transaction_id, authenticated)?; + ensure!( + token.signed_presence_sha256 == signed_presence_sha256 + && token.runtime_instance_id == runtime_instance_id, + "test Warming token extraction changed" + ); + Ok(token) + } + + pub(crate) fn transaction_id(&self) -> &str { + &self.transaction_id + } + + pub(crate) fn signed_presence_sha256(&self) -> &str { + &self.signed_presence_sha256 + } + + pub(crate) fn runtime_instance_id(&self) -> &str { + &self.runtime_instance_id + } +} + +/// Ready proof with the same construction boundary. Dependency planning and +/// promotion receive this token, never caller-supplied digest/sequence tuples. +#[derive(Clone, Debug)] +pub(crate) struct SequenceAdmittedReady { + transaction_id: String, + evidence: TopologyEvidence, + expected: IdunnExpectedIncarnationRecord, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, +} + +impl SequenceAdmittedReady { + #[cfg(test)] + pub(crate) fn for_test( + expected: &IdunnExpectedIncarnationRecord, + authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, + admitted_at_unix_millis: u64, + ) -> Result { + expected.validate()?; + let record = authenticated.record(); + ensure!( + record.expected_projection_sha256 == expected.canonical_sha256()? + && record.target == expected.target + && record.runtime_id == expected.runtime_id + && record.present + && record.ready + && record.observed_presence_state.as_deref() == Some("active") + && record.disagreements.is_empty(), + "test receipt is not exact Ready evidence" + ); + let evidence = + TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; + Ok(Self { + transaction_id: "test-sequence-admission".into(), + evidence, + expected: expected.clone(), + authenticated, + }) + } + + pub(crate) fn authenticated(&self) -> &AuthenticatedOdinRuntimeTopologyCorrelation { + &self.authenticated + } + + pub(crate) fn expected(&self) -> &IdunnExpectedIncarnationRecord { + &self.expected + } + + pub(crate) fn transaction_id(&self) -> &str { + &self.transaction_id + } + + pub(crate) fn evidence_sha256(&self) -> &str { + &self.evidence.canonical_sha256 + } + + pub(crate) fn publisher_sequence(&self) -> u64 { + self.evidence.publisher_sequence + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +enum FencingEvidence { + SkippedStateless, + Revoked { + incumbent_lease_sha256: Option, + candidate_lease_path_verified_empty: bool, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +enum LeasingEvidence { + SkippedStateless, + Prepared { + lease: IdunnProcessWriteLeaseRecord, + lease_sha256: String, + }, + Granted { + lease: IdunnProcessWriteLeaseRecord, + lease_sha256: String, + }, +} + +impl LeasingEvidence { + fn lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { + match self { + Self::SkippedStateless | Self::Prepared { .. } => None, + Self::Granted { lease, .. } => Some(lease), + } + } + + /// The lease Idunn issued, granted or only prepared. A prepared lease may + /// already be on disk: the physical grant runs before `Granted` is durable. + fn issued_lease(&self) -> Option<&IdunnProcessWriteLeaseRecord> { + match self { + Self::SkippedStateless => None, + Self::Prepared { lease, .. } | Self::Granted { lease, .. } => Some(lease), + } + } + + fn prepared_lease(&self) -> Option<(&IdunnProcessWriteLeaseRecord, &str)> { + match self { + Self::Prepared { + lease, + lease_sha256, + } => Some((lease, lease_sha256)), + Self::SkippedStateless | Self::Granted { .. } => None, + } + } + + fn lease_sha256(&self) -> Option<&str> { + match self { + Self::SkippedStateless | Self::Prepared { .. } => None, + Self::Granted { lease_sha256, .. } => Some(lease_sha256), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +enum RoutingEvidence { + SkippedUnrouted, + Promoted { + observation: RouteObservation, + promoted_at_unix_millis: u64, + }, +} + +impl RoutingEvidence { + fn observation(&self) -> Option<&RouteObservation> { + match self { + Self::SkippedUnrouted => None, + Self::Promoted { observation, .. } => Some(observation), + } + } + + fn promoted_at_unix_millis(&self) -> Option { + match self { + Self::SkippedUnrouted => None, + Self::Promoted { + promoted_at_unix_millis, + .. + } => Some(*promoted_at_unix_millis), + } + } +} + +/// The states a route-proof candidate may answer its Warming challenge in. A +/// stateful one is warming until it holds a lease it cannot hold yet, and the +/// lease binds that warming presence. A stateless one may have finished +/// warming before the first challenge landed. +fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'static [&'static str] { + if expected.write_lease_required { + &["warming"] + } else { + &["warming", "active"] + } +} + +/// A challenged runtime answered as something that disagrees with current +/// authority. Each disagreement is named, so a capacity below the Expected +/// minimum reads as that and not as a generic refusal. It stays the error's +/// typed value; callers read `disagreements`, and the text is only its +/// rendering. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PresenceDisagrees { + pub(crate) disagreements: Vec, +} + +impl std::fmt::Display for PresenceDisagrees { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let named = self + .disagreements + .iter() + .map(|disagreement| { + format!( + "{} (expected {}, observed {})", + disagreement.code, + disagreement.expected.as_deref().unwrap_or("none"), + disagreement.observed.as_deref().unwrap_or("none"), + ) + }) + .collect::>() + .join("; "); + write!( + formatter, + "route proof answered with a runtime that disagrees with current authority: {named}" + ) + } +} + +impl std::error::Error for PresenceDisagrees {} + +/// What a direct challenge to a candidate endpoint produced. +enum CandidateAnswer { + /// No answer: the reason it is still being waited on. + Silent(String), + Answered { + evidence: RuntimePresenceEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + }, +} + +//// Which kind of proof admits a target as Ready, derived from what the target's +/// recipe declares and from nothing else. Idunn infers no way to prove +/// readiness: a target that provides `odin.verse-rendezvous` is Odin itself, one +/// that declares a `shared-infrastructure odin.verse-rendezvous` dependency +/// advertises into the Verse and is Odin-correlated, and a routed one that +/// declares neither answers Idunn's own challenge and is route-proof. A target +/// with no route and no Odin dependency declares no proof at all, and is +/// refused. Odin collects the same receipts an Odin-correlated target does, and +/// its admitted generation is what names the Odin authority, so it is never +/// route-proof. +/// +/// `of` is the only place a class is decided, and Expected is digest-bound, so +/// the class cannot drift from the incarnation it describes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ReadinessClass { + OdinSelf, + OdinCorrelated, + RouteProof, +} + +/// A recipe that declares neither a route nor an Odin dependency, and is not +/// Odin: it has said nothing about how it proves readiness. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct UndeclaredReadiness { + pub(crate) target: String, +} + +impl std::fmt::Display for UndeclaredReadiness { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "target {} declares no way to prove readiness: its recipe must declare a route \ + (Idunn challenges it) or a shared-infrastructure {ODIN_RENDEZVOUS_CAPABILITY} \ + dependency (Odin correlates it)", + self.target + ) + } +} + +impl std::error::Error for UndeclaredReadiness {} + +impl ReadinessClass { + /// The class of a recipe's declarations. Plan compilation and `of` both + /// call it, so admission refuses exactly what `of` would refuse. + pub(crate) fn declared( + target: &str, + provides_odin: bool, + declares_odin: bool, + routed: bool, + ) -> Result { + if provides_odin { + Ok(Self::OdinSelf) + } else if declares_odin { + Ok(Self::OdinCorrelated) + } else if routed { + Ok(Self::RouteProof) + } else { + Err(UndeclaredReadiness { + target: target.to_owned(), + }) + } + } + + pub(crate) fn of( + expected: &IdunnExpectedIncarnationRecord, + ) -> Result { + Self::declared( + &expected.target, + expected + .capabilities + .iter() + .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY), + expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }), + expected.route.is_some(), + ) + } + + /// The class of `expected`, provided every receipt collected under it was + /// vouched for by that class's voucher. The one way a stored record is + /// asked what class it is. + fn confirmed( + expected: &IdunnExpectedIncarnationRecord, + collected: impl IntoIterator, + ) -> Result { + let required = Self::of(expected).map_err(ReadinessDisagreement::Undeclared)?; + match collected + .into_iter() + .find(|voucher| *voucher != required.voucher()) + { + None => Ok(required), + Some(collected) => Err(ReadinessDisagreement::WrongVoucher { + target: expected.target.clone(), + required, + collected, + }), + } + } + + /// Who vouches for a target of this class: Odin's correlation, or the + /// candidate answering Idunn. + fn voucher(self) -> Voucher { + match self { + Self::OdinSelf | Self::OdinCorrelated => Voucher::Odin, + Self::RouteProof => Voucher::Candidate, + } + } +} + +/// Who vouched for a piece of stored readiness evidence. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Voucher { + Odin, + Candidate, +} + +/// A stored record whose readiness evidence is not what its own Expected +/// requires. A record admitted before the target's recipe declared how it +/// proves readiness looks like this. It is reported and held, never repaired: +/// re-deriving the evidence around it would make the evidence tag a second +/// authority for the class. +#[derive(Clone, Debug, PartialEq, Eq)] +enum ReadinessDisagreement { + Undeclared(UndeclaredReadiness), + WrongVoucher { + target: String, + required: ReadinessClass, + collected: Voucher, + }, +} + +impl std::fmt::Display for ReadinessDisagreement { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Undeclared(undeclared) => write!(formatter, "{undeclared}"), + Self::WrongVoucher { + target, + required, + collected, + } => write!( + formatter, + "target {target} is {required:?} but its stored readiness evidence was vouched \ + for by {collected:?}: it was collected under a different class, and is held \ + until the target is redeployed" + ), + } + } +} + +impl std::error::Error for ReadinessDisagreement {} + +/// The receipt that admitted a candidate as Ready, tagged by who vouched for +/// it. The tag records what was collected; what a target must collect is +/// `ReadinessClass::of(expected)`, and a tag that differs is a +/// `ReadinessDisagreement`, never a second opinion about the class. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)] +enum ReadinessEvidence { + OdinCorrelated { evidence: TopologyEvidence }, + RouteProof { evidence: RuntimePresenceEvidence }, +} + +impl ReadinessEvidence { + fn voucher(&self) -> Voucher { + match self { + Self::OdinCorrelated { .. } => Voucher::Odin, + Self::RouteProof { .. } => Voucher::Candidate, + } + } + + fn validate_shape(&self) -> Result<()> { + match self { + Self::OdinCorrelated { evidence } => evidence.validate_shape(), + Self::RouteProof { evidence } => evidence.validate_shape(), + } + } + + fn odin(&self) -> Option<&TopologyEvidence> { + match self { + Self::OdinCorrelated { evidence } => Some(evidence), + Self::RouteProof { .. } => None, + } + } +} + +// Signed evidence that the candidate process picked up the lease Idunn +/// granted: its presence carries `write_lease_sha256` equal to the grant. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LeaseAdoptionEvidence { + write_lease_sha256: String, + signed_presence_sha256: String, + source: AdoptionSource, + observed_at_unix_millis: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum AdoptionSource { + Direct, + OdinTopology, +} + +impl LeaseAdoptionEvidence { + /// The adoption a stateful candidate's Ready proves: its signed presence + /// held exactly the granted lease. `None` for a stateless candidate. + fn with_ready( + granted_lease_sha256: Option<&str>, + signed_presence_sha256: &str, + source: AdoptionSource, + observed_at_unix_millis: u64, + ) -> Option { + granted_lease_sha256.map(|lease| Self { + write_lease_sha256: lease.to_owned(), + signed_presence_sha256: signed_presence_sha256.to_owned(), + source, + observed_at_unix_millis, + }) + } + + /// Whether this evidence is about exactly the lease that was granted. + fn names(&self, leasing: &LeasingEvidence) -> bool { + leasing.lease_sha256() == Some(self.write_lease_sha256.as_str()) + } + + fn validate_shape(&self) -> Result<()> { + require_id(&self.write_lease_sha256, "adopted lease digest")?; + require_id(&self.signed_presence_sha256, "lease adoption presence digest")?; + ensure!( + self.observed_at_unix_millis > 0, + "lease adoption has no observation time" + ); + Ok(()) + } +} + +/// When the transaction entered its current post-fencing phase and when that +/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PhaseDeadline { + phase: DeploymentPhase, + entered_at_unix_millis: u64, + deadline_at_unix_millis: u64, +} + +impl PhaseDeadline { + /// `None` for a phase that has no deadline (pre-fencing and Complete). + fn entering( + phase: DeploymentPhase, + plan: &CompiledDeploymentPlan, + now: u64, + ) -> Option { + let deadlines = plan.phase_deadlines(); + let seconds = match phase { + DeploymentPhase::Fencing => deadlines.fencing_seconds, + DeploymentPhase::Leasing => deadlines.leasing_seconds, + DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, + DeploymentPhase::Routing => deadlines.routing_seconds, + DeploymentPhase::Committing => deadlines.committing_seconds, + _ => return None, + }; + Some(Self { + phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: now.saturating_add(u64::from(seconds) * 1000), + }) + } +} + +/// How the deadline resolver ends a post-fencing phase. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PhaseEnd { + /// Abort through the post-fencing path. `lease_not_adopted` names a + /// granted lease with no adoption evidence. + Abort { lease_not_adopted: bool }, + /// A stateful deployment whose candidate holds an issued lease: the phase + /// is reported and not resolved. + OperatorRequired, +} + +/// How a post-fencing failure recovers. `RestoreIncumbent` is what a +/// post-fencing abort has always done and is the default for records that +/// predate the field. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "action", rename_all = "kebab-case", deny_unknown_fields)] +enum TerminalRecovery { + RestartAdmitted, + #[default] + RestoreIncumbent, + OperatorRequired { + reason: String, + }, +} + +/// Route supervision's durable memory for one admitted routed incarnation: +/// challenge pacing and the degradation mark. It belongs to the incarnation, so +/// a new generation starts from `default()`. What a target may actuate outlives +/// every generation and lives in `TargetSupervision`. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RouteSupervisionState { + last_challenge_at_unix_millis: Option, + consecutive_failures: u32, + next_challenge_at_unix_millis: Option, + /// Set while route proof keeps failing. Marks the route degraded so + /// dependents stop selecting it; it never authorizes a restart. + degraded_since_unix_millis: Option, +} + +impl RouteSupervisionState { + /// The next challenge is not due yet. + fn is_waiting(&self, now: u64, maximum_age_millis: u64) -> bool { + self.next_challenge_at_unix_millis.is_some_and(|next| { + is_waiting( + now, + next, + ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(maximum_age_millis), + ) + }) + } + + /// A failed proof, or a failed repair of the fragment it needs. Observation + /// state only: the route is marked degraded and the next challenge waits, + /// twice as long each time, from the observation max age up to a cap. + fn record_failed_challenge(&mut self, now: u64, maximum_age_millis: u64) { + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + let doublings = self.consecutive_failures.saturating_sub(1).min(16); + let wait = maximum_age_millis + .saturating_mul(1 << doublings) + .min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS) + .max(maximum_age_millis); + self.last_challenge_at_unix_millis = Some(now); + self.next_challenge_at_unix_millis = Some(now.saturating_add(wait)); + self.degraded_since_unix_millis.get_or_insert(now); + } + + fn record_proved_challenge(&mut self, now: u64) { + self.consecutive_failures = 0; + self.last_challenge_at_unix_millis = Some(now); + self.next_challenge_at_unix_millis = None; + self.degraded_since_unix_millis = None; + } + + fn validate(&self) -> Result<()> { + if let Some(since) = self.degraded_since_unix_millis { + ensure!(since > 0, "route degradation has no start time"); + } + Ok(()) + } +} + +/// What one target has been allowed to do, kept across every generation and +/// present before the first: Forward route actuations and continuity restarts, +/// each a sliding log of the times they happened, and a record of Survival +/// route actuations. Owned by the meter primitives below; +/// the route driver only asks, and no generation write carries it. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.target_supervision", + schema = "idunn.target_supervision.v2" +)] +struct TargetSupervision { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + /// The newest `ROUTE_ACTUATION_CEILING` Forward actuation times, + /// ascending: the only entries the ceiling counts. + #[cultcache(key = 2)] + route_actuations: Vec, + /// The newest `CONTINUITY_PAGING_RESTARTS` restart times, ascending: the + /// paging threshold's log. + #[cultcache(key = 3)] + continuity_restarts: Vec, + /// A restart whose projection could not be prepared waits until this time. + #[cultcache(key = 4)] + continuity_deferred_until: Option, + #[cultcache(key = 5)] + continuity_deferral_reason: Option, + /// The newest `ROUTE_SURVIVAL_RECORD` Survival actuation times, ascending. + /// Visibility only: never counted, never refused. + #[cultcache(key = 6)] + route_survivals: Vec, + // Keys 7-9 are why this is v2: a v1 record is lifted by + // `LegacyTargetSupervisionV1` with no streak, no anchor and no + // re-materialization. + /// Charged attempts since the last sustained run. Owned by continuity + /// supervision; it decides the next attempt's backoff step. + #[cultcache(key = 7)] + continuity_attempt_streak: u32, + /// When a dead-and-free pass began waiting out the next attempt. Owned by + /// continuity supervision; it decides when the wait is over. Older than + /// the admitted generation, it decides nothing. + #[cultcache(key = 8)] + continuity_backoff_from_unix_millis: Option, + /// When the admitted generation's one uncharged re-materialization was + /// minted. Owned by continuity supervision; it can only withhold a free + /// re-materialization, never grant one. + #[cultcache(key = 9)] + continuity_rematerialized_at_unix_millis: Option, +} + +/// What continuity supervision does about the next attempt for a dead target. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ContinuityBackoff { + /// Mint now, carrying this streak into the attempt. + Due { streak: u32 }, + /// Start the wait: write the anchor and mint nothing this pass. + Begin, + /// The wait has not run out. + Waiting, +} + +/// The entries of a sliding log still inside `window` at `now`. An entry from +/// the future means the clock stepped back; it counts as `now`, so a backwards +/// step holds a meter for at most one window however far the clock moved. +fn live_entries(log: &[u64], now: u64, window: u64) -> Vec { + log.iter() + .map(|&at| at.min(now)) + .filter(|&at| now - at < window) + .collect() +} + +/// Record `now`, keeping the newest `limit` entries: all a meter needs to +/// decide "at most `limit` in any window". +fn record_entry(log: &mut Vec, now: u64, limit: usize) { + clamp_to(log, now); + log.push(now); + if log.len() > limit { + log.drain(..log.len() - limit); + } +} + +/// Make a stepped-back clock's future entries the present. `live_entries` +/// only reads them that way, so a log that is never written would keep +/// looking recent until the clock caught up; writing the clamp is what makes +/// a backwards step cost one window and no more. +fn clamp_to(log: &mut [u64], now: u64) { + for at in log.iter_mut() { + *at = (*at).min(now); + } +} + +impl TargetSupervision { + fn new(target: &str) -> Self { + Self { + schema_version: TARGET_SUPERVISION_SCHEMA.into(), + target: target.into(), + route_actuations: Vec::new(), + continuity_restarts: Vec::new(), + continuity_deferred_until: None, + continuity_deferral_reason: None, + route_survivals: Vec::new(), + continuity_attempt_streak: 0, + continuity_backoff_from_unix_millis: None, + continuity_rematerialized_at_unix_millis: None, + } + } + + /// Clamp every log to `now`. Callers that decide from a log write the + /// result back when it changed. + fn settle(&mut self, now: u64) { + clamp_to(&mut self.route_actuations, now); + clamp_to(&mut self.continuity_restarts, now); + clamp_to(&mut self.route_survivals, now); + } + + fn route_used(&self, now: u64) -> usize { + live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() + } + + /// When the oldest counted actuation leaves the window, if the ceiling is reached. + fn route_reopens_at(&self, now: u64) -> Option { + let live = live_entries(&self.route_actuations, now, ROUTE_ACTUATION_WINDOW_MILLIS); + (live.len() >= ROUTE_ACTUATION_CEILING) + .then(|| live[live.len() - ROUTE_ACTUATION_CEILING] + ROUTE_ACTUATION_WINDOW_MILLIS) + } + + /// Charge one route actuation. A `Forward` change is counted against the + /// ceiling and refused at it; a `Survival` is only recorded, never + /// counted and never refused. + fn charge_route(&mut self, now: u64, kind: RouteActuation) -> Result<(), RouteActuationRefused> { + self.settle(now); + match kind { + RouteActuation::Forward => { + if let Some(reopens_at) = self.route_reopens_at(now) { + return Err(RouteActuationRefused { + target: self.target.clone(), + used: self.route_used(now), + reopens_at_unix_millis: reopens_at, + }); + } + record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING); + } + RouteActuation::Survival => { + record_entry(&mut self.route_survivals, now, ROUTE_SURVIVAL_RECORD); + } + } + Ok(()) + } + + fn survivals_in_window(&self, now: u64) -> usize { + live_entries(&self.route_survivals, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() + } + + fn restarts_used(&self, now: u64) -> usize { + live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len() + } + + /// The paging threshold of the ContinuityExhausted incident; it gates no actuation. + fn restarts_page(&self, now: u64) -> bool { + self.restarts_used(now) >= CONTINUITY_PAGING_RESTARTS + } + + /// Whether continuity must still wait out the deferral after a projection + /// that would not demote. + fn continuity_deferred(&self, now: u64) -> bool { + self.continuity_deferred_until + .is_some_and(|due| is_waiting(now, due, CONTINUITY_DEFERRAL_MILLIS)) + } + + /// The doubling steps the next attempt waits, if it waits. A streak of `n` + /// waits `n - 1` steps. The generation's one re-materialization is an + /// attempt that failed on it though it charged no streak, so the attempt + /// after it waits the base step: every attempt leaves a gap. + fn owed_steps(&self, admitted_at: u64) -> Option { + match self.continuity_attempt_streak.checked_sub(1) { + None if self.rematerialization_spent(admitted_at) => Some(0), + steps => steps, + } + } + + /// When the wait for the next charged attempt ends, from the anchor and + /// the streak; the one spelling `continuity_backoff` and `status` share. + /// An anchor older than the admitted generation decides nothing. + fn backoff_due(&self, admitted_at: u64) -> Option { + let from = self + .continuity_backoff_from_unix_millis + .filter(|&from| from >= admitted_at)?; + let wait = self.owed_steps(admitted_at).map_or(0, |steps| { + backoff_wait( + CONTINUITY_RESTART_BACKOFF_MILLIS, + steps, + CONTINUITY_BACKOFF_CEILING_MILLIS, + ) + }); + Some(from.saturating_add(wait)) + } + + /// What to do about the next attempt for the dead admitted generation + /// admitted at `admitted_at`. The streak resets only after a sustained + /// run: the generation has lived the sustained time and no attempt, charged + /// or the one re-materialization, has been minted since it was admitted. + /// A standing anchor belongs to a death the supervision pass has seen + /// through; an observed-running pass clears it, so it never hides a reset. + fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { + let streak = self.continuity_attempt_streak; + if let Some(due) = self.backoff_due(admitted_at) { + return if self.owed_steps(admitted_at).is_some() + && is_waiting(now, due, CONTINUITY_BACKOFF_CEILING_MILLIS) + { + ContinuityBackoff::Waiting + } else { + ContinuityBackoff::Due { streak } + }; + } + let sustained = self + .continuity_restarts + .last() + .into_iter() + .chain(&self.continuity_rematerialized_at_unix_millis) + .all(|&last| admitted_at >= last) + && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; + if sustained || self.owed_steps(admitted_at).is_none() { + ContinuityBackoff::Due { streak: 0 } + } else { + ContinuityBackoff::Begin + } + } + + /// One charged attempt scheduled at `streak`. It ends the wait and any + /// deferral: the projection was prepared. + fn record_restart(&mut self, now: u64, streak: u32) { + record_entry(&mut self.continuity_restarts, now, CONTINUITY_PAGING_RESTARTS); + self.continuity_attempt_streak = streak.saturating_add(1); + self.continuity_backoff_from_unix_millis = None; + self.continuity_deferred_until = None; + self.continuity_deferral_reason = None; + } + + /// The target was found dead and free: the wait for its next attempt runs from now. + fn begin_backoff(&mut self, now: u64) { + self.continuity_backoff_from_unix_millis = Some(now); + } + + /// The admitted workload was seen running: the death the anchor waited + /// out is over, and the next death waits from its own discovery. + fn end_backoff(&mut self) { + self.continuity_backoff_from_unix_millis = None; + } + + /// The one uncharged re-materialization scheduled. The log and the streak + /// are left alone. + fn record_rematerialization(&mut self, now: u64) { + self.continuity_rematerialized_at_unix_millis = Some(now); + self.continuity_backoff_from_unix_millis = None; + self.continuity_deferred_until = None; + self.continuity_deferral_reason = None; + } + + /// Whether the generation admitted at `admitted_at` has had its free re-materialization. + fn rematerialization_spent(&self, admitted_at: u64) -> bool { + self.continuity_rematerialized_at_unix_millis + .is_some_and(|at| at >= admitted_at) + } + + fn defer_continuity(&mut self, now: u64, reason: &str) { + self.continuity_deferred_until = Some(now.saturating_add(CONTINUITY_DEFERRAL_MILLIS)); + self.continuity_deferral_reason = Some(reason.to_owned()); + } + + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == TARGET_SUPERVISION_SCHEMA, + "target supervision schema is unsupported" + ); + require_id(&self.target, "supervised target")?; + for (log, limit, what) in [ + (&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"), + (&self.continuity_restarts, CONTINUITY_PAGING_RESTARTS, "continuity restart"), + (&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"), + ] { + ensure!(log.len() <= limit, "{what} log exceeds its bound"); + ensure!( + log.iter().all(|&at| at > 0) && log.windows(2).all(|pair| pair[0] <= pair[1]), + "{what} log is not an ascending list of times" + ); + } + ensure!( + self.continuity_backoff_from_unix_millis != Some(0) + && self.continuity_rematerialized_at_unix_millis != Some(0), + "continuity backoff anchor or re-materialization has no time" + ); + ensure!( + self.continuity_deferred_until.is_some() == self.continuity_deferral_reason.is_some(), + "continuity deferral has a time or a reason but not both" + ); + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "kebab-case", deny_unknown_fields)] +enum TransactionCompletion { + Admitted { generation_id: String }, + FailedBeforeFencing { error: String }, + FailedAfterFencing { + error: String, + #[serde(default)] + recovery: TerminalRecovery, + }, +} + +use cleanup_evidence::CleanupEvidence; + +/// Where the legacy marker lives, so that only the lift can make one. +/// +/// `LegacyLift` has a private field: outside this module no expression can +/// construct it, so no other code can write `CleanupEvidence::Legacy(..)`. +/// Decoding a stored record is the one other way in, and it stays here too. +/// `transaction_envelope` refuses a record carrying the marker; the schema +/// migration, which persists what the lift produced, writes through +/// `migrate_transaction_record`, and a fault note on an existing record through +/// `last_error_envelope`; both are defined here beside the only constructor, +/// and the encoder they share is private to this module. +mod cleanup_evidence { + use super::*; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(super) struct LegacyLift(()); + + const LEGACY_MARKER: &str = "legacy-skipped-before-b1"; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(super) enum CleanupEvidence { + Pending, + Skipped, + Complete, + /// Set only by the legacy lift, on a terminal continuity abort written + /// before the single resolution rule: it issued an activation and + /// recorded `Skipped`. Boot reconciliation, not the record, owes that + /// residue's demotion. The marker is what lets validation accept the + /// old shape without reopening a finished transaction. + Legacy(LegacyLift), + } + + impl CleanupEvidence { + pub(super) fn is_complete(self) -> bool { + !matches!(self, Self::Pending) + } + + pub(super) fn is_legacy_marker(self) -> bool { + matches!(self, Self::Legacy(_)) + } + } + + impl Serialize for CleanupEvidence { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(match self { + Self::Pending => "pending", + Self::Skipped => "skipped", + Self::Complete => "complete", + Self::Legacy(_) => LEGACY_MARKER, + }) + } + } + + impl<'de> Deserialize<'de> for CleanupEvidence { + fn deserialize>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + match text.as_str() { + "pending" => Ok(Self::Pending), + "skipped" => Ok(Self::Skipped), + "complete" => Ok(Self::Complete), + LEGACY_MARKER => Ok(Self::Legacy(LegacyLift(()))), + other => Err(serde::de::Error::unknown_variant( + other, + &["pending", "skipped", "complete", LEGACY_MARKER], + )), + } + } + } + + impl DeploymentTransaction { + pub(super) fn carries_legacy_marker(&self) -> bool { + self.pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .into_iter() + .chain( + self.post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation), + ) + .any(CleanupEvidence::is_legacy_marker) + } + } + + /// The schema migration's one entry point: read a stored v2, v3 or v4 + /// transaction (lifting older ones) and return its key and current-shape + /// envelope. The record may carry the marker the lift just set. + pub(super) fn migrate_transaction_record( + envelope: &CultCacheEnvelope, + ) -> Result<(String, CultCacheEnvelope)> { + let value = read_transaction_record(envelope)?; + let next = encode_transaction(&value, value.updated_at_unix_millis)?; + Ok((value.transaction_id, next)) + } + + /// The envelope for an existing record with only its `last_error` set. The + /// record may already carry the marker; this cannot add one. + pub(super) fn last_error_envelope( + value: &DeploymentTransaction, + detail: &str, + ) -> Result { + let mut next = value.clone(); + next.last_error = Some(detail.to_string()); + encode_transaction(&next, next.updated_at_unix_millis) + } + +/// A continuity abort written before the single resolution rule recorded no +/// projection cleanup although its transaction had issued an activation, and +/// so left that activation standing. +/// +/// Nothing is reopened. A record already terminal keeps its terminal phase and +/// completion and is marked the legacy marker, the one shape validation +/// accepts for a terminal abort that owes nothing yet issued an activation; +/// boot reconciliation, the single owner of that residue, demotes it by exact +/// activation. A record still in flight lifts to `Pending`: it already holds +/// its target's authority (a live record claims it before and after the lift), +/// so the lift adds no claimant, and its own abort demotes the activation. +/// Only the control store's read applies this: history describes what happened +/// and is lifted unchanged. +pub(super) fn owe_legacy_continuity_projection(transaction: &mut DeploymentTransaction) { + if transaction.command_kind != CommandKind::Continuity || transaction.activation.is_none() { + return; + } + let terminal = transaction.completion.is_some(); + if let Some(abort) = transaction.pre_fencing_abort.as_mut() + && abort.topology_reconciliation == CleanupEvidence::Skipped + { + abort.topology_reconciliation = if terminal { + CleanupEvidence::Legacy(LegacyLift(())) + } else { + CleanupEvidence::Pending + }; + } + if let Some(abort) = transaction.post_fencing_abort.as_mut() + && abort.topology_reconciliation == CleanupEvidence::Skipped + && !terminal + { + abort.topology_reconciliation = CleanupEvidence::Pending; + } +} + +} + +/// Rollback of a transaction that has already fenced the incumbent. +/// +/// Fencing stops the admitted incarnation and revokes its write lease, so a +/// candidate that then dies for good leaves the target with nothing serving it. +/// Retrying forever cannot fix that -- a transient unit with `Restart=no` will +/// not come back -- and it holds the target, leaving every later command queued +/// behind a transaction that can never finish. +/// +/// So the candidate's own artifacts are withdrawn in the order that never +/// leaves two writers: route first, then the write lease, then the process, +/// then the projection. Reconciling the projection restores the incumbent's +/// admitted Expected where there is one, which is what lets continuity bring +/// the incumbent back; where there is none, the failed Expected is withdrawn. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PostFencingAbort { + error: String, + route_restoration: CleanupEvidence, + lease_withdrawal: CleanupEvidence, + candidate_cleanup: CleanupEvidence, + topology_reconciliation: CleanupEvidence, + source_cleanup: CleanupEvidence, +} + +impl PostFencingAbort { + fn is_complete(&self) -> bool { + self.route_restoration.is_complete() + && self.lease_withdrawal.is_complete() + && self.candidate_cleanup.is_complete() + && self.topology_reconciliation.is_complete() + && self.source_cleanup.is_complete() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PreFencingAbort { + error: String, + candidate_cleanup: CleanupEvidence, + topology_reconciliation: CleanupEvidence, + source_cleanup: CleanupEvidence, +} + +impl PreFencingAbort { + fn is_complete(&self) -> bool { + self.candidate_cleanup.is_complete() + && self.topology_reconciliation.is_complete() + && self.source_cleanup.is_complete() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "kebab-case", deny_unknown_fields)] +enum IncumbentCleanupEvidence { + SkippedNoIncumbent, + Pending { + generation_id: String, + workload: WorkloadObservation, + }, + Complete { + generation_id: String, + }, +} + +impl IncumbentCleanupEvidence { + fn is_complete(&self) -> bool { + matches!(self, Self::SkippedNoIncumbent | Self::Complete { .. }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum SourceCleanupEvidence { + SkippedContinuity, + Pending, + Complete, +} + +impl SourceCleanupEvidence { + fn is_complete(self) -> bool { + !matches!(self, Self::Pending) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PostCommitCleanup { + incumbent: IncumbentCleanupEvidence, + source: SourceCleanupEvidence, +} + +impl PostCommitCleanup { + fn is_complete(&self) -> bool { + self.incumbent.is_complete() && self.source.is_complete() + } +} + +/// The sole owner of an in-flight deployment decision. Every actuator result +/// is durable here before a later phase is entered. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_transaction", + schema = "idunn.deployment_transaction.v4" +)] +struct DeploymentTransaction { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + transaction_id: String, + #[cultcache(key = 2)] + command_id: String, + #[cultcache(key = 3)] + command_kind: CommandKind, + #[cultcache(key = 4)] + target: String, + #[cultcache(key = 5)] + ordinal: u32, + #[cultcache(key = 6)] + phase: DeploymentPhase, + #[cultcache(key = 7)] + created_at_unix_millis: u64, + #[cultcache(key = 8)] + updated_at_unix_millis: u64, + #[cultcache(key = 9)] + incumbent_generation_id: Option, + #[cultcache(key = 10)] + plan: Option, + #[cultcache(key = 11)] + frozen_source: Option, + #[cultcache(key = 12)] + sealed_release: Option, + #[cultcache(key = 13)] + installed_release: Option, + #[cultcache(key = 14)] + expected: Option, + #[cultcache(key = 15)] + expected_publication_sha256: Option, + #[cultcache(key = 16)] + deployment_authorization: Option, + #[cultcache(key = 17)] + lifecycle_authorized_at_unix_millis: Option, + #[cultcache(key = 18)] + activation: Option, + #[cultcache(key = 19)] + workload: Option, + #[cultcache(key = 20)] + activation_publication_sha256: Option, + #[cultcache(key = 21)] + latest_odin_observation: Option, + #[cultcache(key = 22)] + warming: Option, + #[cultcache(key = 23)] + route_preflight: Option, + #[cultcache(key = 24)] + isolation: Option, + #[cultcache(key = 25)] + fencing: Option, + #[cultcache(key = 26)] + leasing: Option, + #[cultcache(key = 27)] + ready: Option, + #[cultcache(key = 28)] + routing: Option, + #[cultcache(key = 29)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 30)] + last_error: Option, + #[cultcache(key = 31)] + completion: Option, + #[cultcache(key = 32)] + pre_fencing_abort: Option, + #[cultcache(key = 33)] + post_commit_cleanup: Option, + // because every transaction already durable predates this field: + // an absent key 34 decodes as None, which is exactly "this transaction has + // no post-fencing abort". Without it the daemon refuses to read its own + // control store and crashloops. + #[cultcache(key = 34, default)] + post_fencing_abort: Option, + /// Set by `transition` on entry to each post-fencing phase. + #[cultcache(key = 35)] + phase_deadline: Option, + #[cultcache(key = 36)] + lease_adoption: Option, +} + +impl DeploymentTransaction { + /// The one writer of `phase`. It stamps the update time and the phase + /// deadline together, so a record outside Fencing..=Committing carries no + /// deadline by construction. No other code assigns `phase`. + fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { + self.phase = phase; + self.updated_at_unix_millis = now; + self.phase_deadline = self + .plan + .as_ref() + .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); + } + + /// A live transaction past the fence that is not already being aborted: + /// the only kind whose phase a deadline, or an operator's expiry, ends. + fn resolves_by_deadline(&self) -> bool { + self.completion.is_none() + && self.pre_fencing_abort.is_none() + && self.post_fencing_abort.is_none() + && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) + } + + /// What an ended post-fencing phase resolves to (Q3). A candidate that was + /// never issued a write lease cannot have written, so the transaction + /// aborts to the incumbent through the post-fencing path, stateless or not. + /// A continuity always aborts: its candidate is the admitted release, and + /// supervision restarts that release under the continuity backoff. A + /// deployment whose candidate was issued a lease may have written under a + /// new state contract, and restoring the incumbent over that is not + /// Idunn's call: it is left for the operator. + fn phase_end(&self) -> PhaseEnd { + let leasing = self.leasing.as_ref(); + let granted = leasing.and_then(LeasingEvidence::lease).is_some(); + let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); + match self.command_kind { + CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, + _ => PhaseEnd::Abort { + lease_not_adopted: granted && self.lease_adoption.is_none(), + }, + } + } + + /// The recovery a completed post-fencing abort names. A continuity whose + /// candidate adopted its lease restarts the admitted release; everything + /// else restores the incumbent. + fn terminal_recovery(&self) -> TerminalRecovery { + if self.command_kind == CommandKind::Continuity && self.lease_adoption.is_some() { + TerminalRecovery::RestartAdmitted + } else { + TerminalRecovery::RestoreIncumbent + } + } + + /// Whether this transaction's binding declares stop-then-start. A + /// transaction without a plan yet cannot, so it reads as false. + fn rollout_stops_incumbent_first(&self) -> bool { + self.plan + .as_ref() + .and_then(|plan| plan.parsed_inputs().ok()) + .is_some_and(|(_, binding)| binding.rollout.strategy == RolloutStrategy::StopThenStart) + } + + fn new( + command: &DeploymentCommand, + target: String, + ordinal: u32, + incumbent: Option<&AdmittedGeneration>, + now: u64, + ) -> Result { + let transaction = Self { + schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), + transaction_id: format!("tx-{}", Uuid::new_v4()), + command_id: command.command_id.clone(), + command_kind: command.kind, + target, + ordinal, + phase: DeploymentPhase::Sealing, + created_at_unix_millis: now, + updated_at_unix_millis: now, + incumbent_generation_id: incumbent.map(|value| value.generation_id.clone()), + plan: None, + frozen_source: None, + sealed_release: None, + installed_release: None, + expected: None, + expected_publication_sha256: None, + deployment_authorization: None, + lifecycle_authorized_at_unix_millis: None, + activation: None, + workload: None, + activation_publication_sha256: None, + latest_odin_observation: None, + warming: None, + route_preflight: None, + isolation: None, + fencing: None, + leasing: None, + ready: None, + routing: None, + odin_publisher_sequence_cursor: incumbent + .map_or(0, |value| value.odin_publisher_sequence_cursor), + last_error: None, + completion: None, + pre_fencing_abort: None, + post_fencing_abort: None, + post_commit_cleanup: None, + phase_deadline: None, + lease_adoption: None, + }; + transaction.validate()?; + Ok(transaction) + } + + fn from_continuity( + command: &DeploymentCommand, + incumbent: &AdmittedGeneration, + now: u64, + ) -> Result { + let mut transaction = + Self::new(command, incumbent.target.clone(), 0, Some(incumbent), now)?; + transaction.plan = Some(incumbent.plan.clone()); + transaction.sealed_release = Some(incumbent.sealed_release.clone()); + transaction.installed_release = Some(incumbent.installed_release.clone()); + transaction.expected = Some(incumbent.expected.clone()); + transaction.validate()?; + Ok(transaction) + } + + /// Whether an abort owes the projection a reconciliation. A deployment + /// published its own Expected key. A continuity shares the admitted key, + /// whose Expected is never withdrawn, so it owes only the demotion of the + /// activation it issued. + fn abort_topology_reconciliation(&self) -> CleanupEvidence { + let owes = match self.command_kind { + CommandKind::Deploy => self.expected_publication_sha256.is_some(), + CommandKind::Continuity => self.activation.is_some(), + }; + if owes { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + } + } + + /// Whether an abort owes the candidate's durable cleanup: it does once an + /// activation was issued or a workload was observed, either of which leaves + /// something behind. + fn candidate_cleanup_owed(&self) -> CleanupEvidence { + if self.activation.is_some() || self.workload.is_some() { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + } + } + + /// Whether an abort's recorded candidate cleanup is what + /// `candidate_cleanup_owed` says it must be: owed cleanup is pending or + /// done, unowed cleanup is skipped. + fn candidate_cleanup_recorded(&self, recorded: CleanupEvidence) -> bool { + matches!( + (self.candidate_cleanup_owed(), recorded), + ( + CleanupEvidence::Pending, + CleanupEvidence::Pending | CleanupEvidence::Complete + ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) + ) + } + + fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { + let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; + let detail = truncate(&format!("{error:#}"), 2048); + transaction.enter_phase(DeploymentPhase::Complete, now); + transaction.last_error = Some(detail.clone()); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: detail.clone(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Skipped, + }); + transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { error: detail }); + transaction.validate()?; + Ok(transaction) + } + + fn is_terminal(&self) -> bool { + if self.phase != DeploymentPhase::Complete { + return false; + } + match &self.completion { + Some(TransactionCompletion::FailedBeforeFencing { .. }) => self + .pre_fencing_abort + .as_ref() + .is_some_and(PreFencingAbort::is_complete), + Some(TransactionCompletion::FailedAfterFencing { .. }) => self + .post_fencing_abort + .as_ref() + .is_some_and(PostFencingAbort::is_complete), + Some(TransactionCompletion::Admitted { .. }) => self + .post_commit_cleanup + .as_ref() + .is_some_and(PostCommitCleanup::is_complete), + None => false, + } + } + + /// Complete transactions retain retryable cleanup work, but admission has + /// already transferred current-incarnation authority to AdmittedGeneration. + fn owns_target_authority(&self) -> bool { + self.phase != DeploymentPhase::Complete + } + + /// Admission has moved to the new generation at Complete, but an exact + /// draining incumbent still reserves its process and candidate endpoint. + /// No later mutation for the same target may overlap that cleanup. + fn blocks_new_target_mutation(&self) -> bool { + self.owns_target_authority() + || self + .post_commit_cleanup + .as_ref() + .is_some_and(|cleanup| !cleanup.is_complete()) + } + + /// Where what this transaction has collected disagrees with what its own + /// Expected requires. There is nothing to disagree with before Expected. + fn readiness_disagreement(&self) -> Option { + let expected = self.expected.as_ref()?; + ReadinessClass::confirmed( + expected, + self.warming + .iter() + .map(WarmingEvidence::voucher) + .chain(self.ready.iter().map(ReadinessEvidence::voucher)), + ) + .err() + } + + /// The disagreement that holds this transaction: evidence collected under + /// another class is held and reported, not advanced and not repaired, since + /// no step can make it the right kind and running one anyway only fails it + /// every tick. What is finished, or being finished (a completion, or an + /// abort under way), never reads the class, so it is left to run. + fn held_disagreement(&self) -> Option { + let finishing = self.completion.is_some() + || self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some(); + self.readiness_disagreement().filter(|_| !finishing) + } + + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, + "deployment transaction schema is unsupported" + ); + require_id(&self.transaction_id, "deployment transaction id")?; + require_id(&self.command_id, "transaction command id")?; + require_id(&self.target, "transaction target")?; + ensure!( + self.created_at_unix_millis > 0 && self.updated_at_unix_millis > 0, + "deployment transaction timestamps are invalid" + ); + if let Some(generation) = &self.incumbent_generation_id { + require_id(generation, "incumbent generation id")?; + } + if let Some(plan) = &self.plan { + plan.validate()?; + ensure!( + plan.parsed_inputs()?.1.target == self.target, + "transaction plan belongs to another target" + ); + } + if let (Some(plan), Some(frozen)) = (&self.plan, &self.frozen_source) { + frozen.validate_against(plan)?; + } + if let (Some(plan), Some(release)) = (&self.plan, &self.sealed_release) { + release.validate_against(plan)?; + } + if let Some(expected) = &self.expected { + expected.validate()?; + ensure!(expected.target == self.target, "Expected target differs"); + if let Some(plan) = &self.plan { + ensure!(expected.plan_id == plan.plan_id, "Expected plan differs"); + } + if let Some(release) = &self.sealed_release { + ensure!( + expected.sealed_release_id == release.sealed_release_id, + "Expected release differs" + ); + } + if let Some(digest) = &self.expected_publication_sha256 { + ensure!( + digest == &expected.canonical_sha256()?, + "Expected publication receipt differs" + ); + } + } + if let (Some(expected), Some(activation)) = (&self.expected, &self.activation) { + activation.validate()?; + ensure!( + activation.expected_projection_sha256 == expected.canonical_sha256()?, + "activation belongs to another Expected projection" + ); + if let Some(workload) = &self.workload { + ensure!( + workload.runtime_instance_id() == activation.runtime_instance_id, + "workload belongs to another activation" + ); + } + if let Some(digest) = &self.activation_publication_sha256 { + ensure!( + digest == &activation.canonical_sha256()?, + "activation publication receipt differs" + ); + } + } + ensure!( + self.workload.is_none() || self.activation.is_some(), + "workload observation exists without its prepared activation" + ); + let ready_odin = self.ready.as_ref().and_then(ReadinessEvidence::odin); + for evidence in [self.latest_odin_observation.as_ref(), ready_odin] + .into_iter() + .flatten() + { + evidence.validate_shape()?; + ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, + "topology evidence exceeds the transaction replay cursor" + ); + } + if let Some(ready) = &self.ready { + ready.validate_shape()?; + // A candidate-vouched receipt is only ever written by this code, so + // one on a target that is not route-proof is forged. The other + // direction -- an Odin receipt on a route-proof target -- is what a + // pre-B3 record looks like, and is a ReadinessDisagreement to hold, + // not a record to refuse: refusing it would refuse to boot. + if let (Voucher::Candidate, Some(expected)) = (ready.voucher(), &self.expected) { + ensure!( + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" + ); + } + } + if let Some(warming) = &self.warming { + warming.validate_shape()?; + match warming { + WarmingEvidence::OdinTopology { evidence } => ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor, + "Warming topology evidence exceeds the transaction replay cursor" + ), + WarmingEvidence::FirstOdinDirect { .. } => ensure!( + self.expected.as_ref().is_some_and(|expected| { + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::OdinSelf) + }), + "direct Warming evidence is reserved for Odin observing itself" + ), + WarmingEvidence::RouteProofDirect { .. } => ensure!( + self.expected.as_ref().is_none_or(|expected| { + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof) + }), + "direct route-proof Warming evidence for a target that is not route-proof" + ), + } + } + if let Some(leasing) = &self.leasing { + if let Some(expected) = &self.expected { + ensure!( + matches!( + (expected.write_lease_required, leasing), + (true, LeasingEvidence::Prepared { .. }) + | (true, LeasingEvidence::Granted { .. }) + | (false, LeasingEvidence::SkippedStateless) + ), + "lease evidence differs from the Expected state contract" + ); + } + let lease_and_sha256 = match leasing { + LeasingEvidence::SkippedStateless => None, + LeasingEvidence::Prepared { + lease, + lease_sha256, + } + | LeasingEvidence::Granted { + lease, + lease_sha256, + } => Some((lease, lease_sha256)), + }; + if let Some((lease, lease_sha256)) = lease_and_sha256 { + lease.validate()?; + ensure!( + lease.canonical_sha256()? == *lease_sha256, + "lease evidence digest differs" + ); + } + } + if let (Some(expected), Some(fencing)) = (&self.expected, &self.fencing) { + match fencing { + FencingEvidence::SkippedStateless => ensure!( + !expected.write_lease_required, + "stateful candidate claims stateless fencing" + ), + FencingEvidence::Revoked { + candidate_lease_path_verified_empty, + .. + } => ensure!( + *candidate_lease_path_verified_empty == expected.write_lease_required, + "candidate lease-path fencing differs from Expected" + ), + } + } + if let Some(error) = &self.last_error { + require_detail(error, "transaction error")?; + } + if let Some(deadline) = &self.phase_deadline { + ensure!( + deadline.phase == self.phase + && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) + .contains(&deadline.phase) + && deadline.entered_at_unix_millis > 0 + && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, + "phase deadline does not describe the current post-fencing phase" + ); + } + if let Some(adoption) = &self.lease_adoption { + adoption.validate_shape()?; + ensure!( + self.leasing + .as_ref() + .is_some_and(|leasing| adoption.names(leasing)), + "lease adoption does not name the granted lease" + ); + } + if let Some(authorization) = &self.deployment_authorization { + authorization.validate_shape()?; + } + if let Some(abort) = &self.pre_fencing_abort { + require_detail(&abort.error, "pre-fencing abort error")?; + ensure!( + self.post_commit_cleanup.is_none(), + "aborted transaction also carries post-commit cleanup" + ); + ensure!( + self.candidate_cleanup_recorded(abort.candidate_cleanup), + "abort candidate cleanup differs from its prepared activation" + ); + ensure!( + matches!( + ( + self.abort_topology_reconciliation(), + abort.topology_reconciliation + ), + ( + CleanupEvidence::Pending, + CleanupEvidence::Pending | CleanupEvidence::Complete + ) | (CleanupEvidence::Skipped, CleanupEvidence::Skipped) + ) || (self.command_kind == CommandKind::Continuity + && self.phase == DeploymentPhase::Complete + && self.completion.is_some() + && abort.topology_reconciliation.is_legacy_marker()), + "abort topology cleanup differs from what the transaction projected" + ); + ensure!( + matches!( + (self.command_kind, abort.source_cleanup), + ( + CommandKind::Deploy, + CleanupEvidence::Pending | CleanupEvidence::Complete + ) | (CommandKind::Continuity, CleanupEvidence::Skipped) + ) || (self.command_kind == CommandKind::Deploy + && abort.source_cleanup == CleanupEvidence::Skipped + && self.plan.is_none() + && self.frozen_source.is_none() + && self.expected.is_none() + && self.phase == DeploymentPhase::Complete), + "abort source cleanup differs from command work" + ); + ensure!( + self.phase < DeploymentPhase::Fencing || self.phase == DeploymentPhase::Complete, + "pre-fencing abort crossed the fencing boundary" + ); + } + + if let Some(abort) = &self.post_fencing_abort { + require_detail(&abort.error, "post-fencing abort error")?; + ensure!( + self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_none(), + "post-fencing abort collides with another terminal path" + ); + ensure!( + self.phase >= DeploymentPhase::Fencing, + "post-fencing abort exists before the fence" + ); + ensure!( + self.candidate_cleanup_recorded(abort.candidate_cleanup), + "abort candidate cleanup differs from its prepared activation" + ); + } + + let failed_after_fencing = matches!( + self.completion, + Some(TransactionCompletion::FailedAfterFencing { .. }) + ); + if failed_after_fencing { + ensure!( + self.phase == DeploymentPhase::Complete, + "terminal failure is not Complete" + ); + let abort = required( + &self.post_fencing_abort, + "terminal post-fence abort evidence", + )?; + let TransactionCompletion::FailedAfterFencing { error, recovery } = + self.completion.as_ref().unwrap() + else { + unreachable!() + }; + if let TerminalRecovery::OperatorRequired { reason } = recovery { + require_detail(reason, "operator-required recovery reason")?; + } + ensure!( + abort.is_complete() && abort.error == *error, + "terminal failure lacks complete matching abort evidence" + ); + return Ok(()); + } + + let failed = matches!( + self.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + ); + if failed { + ensure!( + self.phase == DeploymentPhase::Complete, + "terminal failure is not Complete" + ); + let abort = required(&self.pre_fencing_abort, "terminal abort evidence")?; + let TransactionCompletion::FailedBeforeFencing { error } = + self.completion.as_ref().unwrap() + else { + unreachable!() + }; + ensure!( + abort.is_complete() && abort.error == *error, + "terminal failure lacks complete matching abort evidence" + ); + } else { + ensure!( + !matches!( + self.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + ), + "non-failure branch carries failure completion" + ); + if self.phase >= DeploymentPhase::Starting { + ensure!( + self.plan.is_some() + && self.sealed_release.is_some() + && self.installed_release.is_some() + && self.expected.is_some(), + "Starting transaction lacks sealed release evidence" + ); + // Expected is published as Starting's first step, after the + // brake; from the activation onward it must be there. + ensure!( + self.expected_publication_sha256.is_some() + || (self.activation.is_none() && self.phase == DeploymentPhase::Starting), + "started transaction has not published its Expected" + ); + match self.command_kind { + CommandKind::Deploy => ensure!( + self.frozen_source.is_some() && self.deployment_authorization.is_some(), + "deployment entered Starting without frozen source and brake consumption" + ), + CommandKind::Continuity => ensure!( + self.lifecycle_authorized_at_unix_millis.is_some() + && self.deployment_authorization.is_none(), + "continuity entered Starting without its lifecycle gate" + ), + } + } + if self.phase >= DeploymentPhase::Warming { + ensure!( + self.activation.is_some() + && self.workload.is_some() + && self.activation_publication_sha256.is_some(), + "Warming transaction lacks observed activation" + ); + } + if self.phase >= DeploymentPhase::Fencing { + ensure!( + self.warming.is_some() && self.isolation.is_some(), + "Fencing transaction lacks warming or isolation evidence" + ); + ensure!( + self.expected + .as_ref() + .and_then(|expected| expected.route.as_ref()) + .is_some() + == self.route_preflight.is_some(), + "route preflight does not match routed Expected" + ); + if let Some(preflight) = &self.route_preflight { + preflight.validate()?; + } + } + if self.phase >= DeploymentPhase::Leasing { + ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence"); + } + if self.phase >= DeploymentPhase::AwaitingReady { + ensure!( + matches!( + self.leasing.as_ref(), + Some(LeasingEvidence::SkippedStateless) + | Some(LeasingEvidence::Granted { .. }) + ), + "AwaitingReady lacks finalized lease evidence" + ); + } + if self.phase >= DeploymentPhase::Routing { + ensure!(self.ready.is_some(), "Routing lacks Ready evidence"); + } + if self.phase >= DeploymentPhase::Committing { + ensure!(self.routing.is_some(), "Committing lacks routing evidence"); + } + if let Some(RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + }) = &self.routing + { + observation.validate()?; + ensure!(*promoted_at_unix_millis > 0, "route promotion has no time"); + } + if self.phase == DeploymentPhase::Complete { + ensure!( + matches!( + self.completion, + Some(TransactionCompletion::Admitted { .. }) + ), + "successful Complete transaction lacks admission receipt" + ); + ensure!( + self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_some(), + "admitted transaction lacks exclusive post-commit cleanup evidence" + ); + let TransactionCompletion::Admitted { generation_id } = + self.completion.as_ref().unwrap() + else { + unreachable!() + }; + ensure!( + generation_id == &format!("generation-{}", self.transaction_id), + "admission completion names another transaction generation" + ); + } else { + ensure!( + self.completion.is_none() && self.post_commit_cleanup.is_none(), + "non-Complete transaction has completion or post-commit state" + ); + } + } + if let Some(cleanup) = &self.post_commit_cleanup { + ensure!( + self.phase == DeploymentPhase::Complete + && matches!( + self.completion, + Some(TransactionCompletion::Admitted { .. }) + ), + "post-commit cleanup exists without an admitted Complete transaction" + ); + match (&self.incumbent_generation_id, &cleanup.incumbent) { + (None, IncumbentCleanupEvidence::SkippedNoIncumbent) => {} + ( + Some(expected), + IncumbentCleanupEvidence::Pending { generation_id, .. } + | IncumbentCleanupEvidence::Complete { generation_id }, + ) if expected == generation_id => {} + _ => bail!("post-commit incumbent cleanup names another generation"), + } + ensure!( + matches!( + (self.command_kind, cleanup.source), + (CommandKind::Deploy, SourceCleanupEvidence::Pending) + | (CommandKind::Deploy, SourceCleanupEvidence::Complete) + | ( + CommandKind::Continuity, + SourceCleanupEvidence::SkippedContinuity + ) + ), + "post-commit source cleanup differs from command kind" + ); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AdmittedOdinAuthority { + signer_identity_id: String, + signer_public_key: Vec, +} + +impl AdmittedOdinAuthority { + fn from_anchor(anchor: &ServiceIdentityTrustAnchor) -> Result { + ensure!( + derive_service_identity_id::(&anchor.public_key)? + == anchor.identity_id, + "Odin topology anchor identity differs from its key" + ); + Ok(Self { + signer_identity_id: anchor.identity_id.clone(), + signer_public_key: anchor.public_key.clone(), + }) + } + + fn validate(&self) -> Result<()> { + require_id(&self.signer_identity_id, "admitted Odin signer")?; + ensure!( + derive_service_identity_id::(&self.signer_public_key)? + == self.signer_identity_id, + "admitted Odin signer identity differs from its key" + ); + Ok(()) + } +} + +/// The only current-incarnation owner. Unit state, route state, topology +/// projections, and CLI output are observations of this record, never peers. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.admitted_generation", + schema = "idunn.admitted_generation.v4" +)] +struct AdmittedGeneration { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + generation_id: String, + #[cultcache(key = 3)] + command_id: String, + #[cultcache(key = 4)] + transaction_id: String, + #[cultcache(key = 5)] + admitted_at_unix_millis: u64, + #[cultcache(key = 6)] + plan: CompiledDeploymentPlan, + #[cultcache(key = 7)] + sealed_release: SealedRelease, + #[cultcache(key = 8)] + installed_release: InstalledReleaseObservation, + #[cultcache(key = 9)] + expected: IdunnExpectedIncarnationRecord, + #[cultcache(key = 10)] + activation: IdunnRuntimeActivationRecord, + #[cultcache(key = 11)] + workload: WorkloadObservation, + #[cultcache(key = 12)] + leasing: LeasingEvidence, + /// The receipt that admitted this incarnation, tagged by proof class. + #[cultcache(key = 13)] + ready: ReadinessEvidence, + /// Present exactly when `ready` is Odin-correlated. + #[cultcache(key = 14)] + latest_odin_observation: Option, + #[cultcache(key = 15)] + routing: RoutingEvidence, + /// Present exactly when `ready` is Odin-correlated. + #[cultcache(key = 16)] + odin_authority: Option, + /// Meaningful only for an Odin-correlated generation. + #[cultcache(key = 17)] + odin_publisher_sequence_cursor: u64, + // Keys 18 (route repair start) and 20 (continuity backoff) are retired. + // The slots stay as gaps: the layout is positional, so a slot cannot be + // reused without a bump. + /// Present exactly when `routing` is Promoted. + #[cultcache(key = 19)] + route_supervision: Option, + /// Why this generation is held or down, for `status`. Cleared when the + /// generation is replaced. + #[cultcache(key = 21)] + last_error: Option, +} + +/// The Odin-side receipts of an Odin-correlated generation. Decisions that +/// consume Odin evidence read it through here, so a route-proof generation +/// cannot be mistaken for one. +struct OdinReceipts<'a> { + ready: &'a TopologyEvidence, + latest: &'a TopologyEvidence, +} + +impl AdmittedGeneration { + /// This generation's class, or where its evidence disagrees with it. The + /// class comes from its Expected, never from the tag on its Ready receipt. + fn readiness(&self) -> Result { + ReadinessClass::confirmed(&self.expected, [self.ready.voucher()]) + } + + fn odin_receipts(&self) -> Result> { + match ( + &self.ready, + &self.latest_odin_observation, + &self.odin_authority, + ) { + (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(_)) => { + Ok(OdinReceipts { + ready: evidence, + latest, + }) + } + _ => bail!("admitted generation of {} is not Odin-correlated", self.target), + } + } + + fn from_transaction( + transaction: &DeploymentTransaction, + odin_authority: Option, + now: u64, + ) -> Result { + ensure!( + transaction.phase == DeploymentPhase::Committing, + "only Committing can create an admitted generation" + ); + let ready = required(&transaction.ready, "Ready receipt")?.clone(); + let (latest_odin_observation, odin_authority) = match &ready { + ReadinessEvidence::OdinCorrelated { .. } => ( + Some( + required(&transaction.latest_odin_observation, "latest Odin receipt")? + .clone(), + ), + Some(required(&odin_authority, "Odin authority")?.clone()), + ), + ReadinessEvidence::RouteProof { .. } => (None, None), + }; + let routing = required(&transaction.routing, "route disposition")?.clone(); + let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. }) + .then(RouteSupervisionState::default); + let generation = Self { + schema_version: ADMITTED_GENERATION_SCHEMA.into(), + target: transaction.target.clone(), + generation_id: format!("generation-{}", transaction.transaction_id), + command_id: transaction.command_id.clone(), + transaction_id: transaction.transaction_id.clone(), + admitted_at_unix_millis: now, + plan: required(&transaction.plan, "transaction plan")?.clone(), + sealed_release: required(&transaction.sealed_release, "sealed release")?.clone(), + installed_release: required(&transaction.installed_release, "installed release")? + .clone(), + expected: required(&transaction.expected, "Expected projection")?.clone(), + activation: required(&transaction.activation, "activation")?.clone(), + workload: required(&transaction.workload, "workload")?.clone(), + leasing: required(&transaction.leasing, "lease disposition")?.clone(), + ready, + latest_odin_observation, + routing, + odin_authority, + odin_publisher_sequence_cursor: transaction.odin_publisher_sequence_cursor, + route_supervision, + last_error: None, + }; + generation.validate()?; + Ok(generation) + } + + fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == ADMITTED_GENERATION_SCHEMA, + "admitted generation schema is unsupported" + ); + require_id(&self.target, "admitted target")?; + require_id(&self.generation_id, "admitted generation id")?; + require_id(&self.command_id, "admitted command id")?; + require_id(&self.transaction_id, "admitted transaction id")?; + ensure!(self.admitted_at_unix_millis > 0, "admission has no time"); + self.plan.validate()?; + self.sealed_release.validate_against(&self.plan)?; + self.expected.validate()?; + self.activation.validate()?; + self.ready.validate_shape()?; + match ( + &self.ready, + &self.latest_odin_observation, + &self.odin_authority, + ) { + (ReadinessEvidence::OdinCorrelated { evidence }, Some(latest), Some(authority)) => { + latest.validate_shape()?; + authority.validate()?; + ensure!( + evidence.publisher_sequence <= self.odin_publisher_sequence_cursor + && latest.publisher_sequence == self.odin_publisher_sequence_cursor, + "admitted generation evidence does not describe one incarnation" + ); + } + (ReadinessEvidence::RouteProof { .. }, None, None) => ensure!( + ReadinessClass::of(&self.expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" + ), + _ => bail!("admitted readiness evidence and Odin receipts disagree"), + } + ensure!( + self.target == self.expected.target + && self.expected.plan_id == self.plan.plan_id + && self.expected.sealed_release_id == self.sealed_release.sealed_release_id + && self.activation.expected_projection_sha256 + == self.expected.canonical_sha256()? + && self.activation.runtime_instance_id == self.workload.runtime_instance_id(), + "admitted generation evidence does not describe one incarnation" + ); + ensure!( + self.expected.write_lease_required == self.leasing.lease().is_some(), + "admitted write-lease disposition differs from Expected" + ); + ensure!( + self.expected.route.is_some() == self.routing.observation().is_some(), + "admitted route disposition differs from Expected" + ); + if let Some(route) = self.routing.observation() { + route.validate()?; + ensure!( + route.runtime_instance_id == self.activation.runtime_instance_id, + "admitted route observation belongs to another runtime instance" + ); + } + ensure!( + self.route_supervision.is_some() + == matches!(&self.routing, RoutingEvidence::Promoted { .. }), + "route supervision state exists exactly for a promoted route" + ); + if let Some(state) = &self.route_supervision { + state.validate()?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RuntimeOptions { + state_store: PathBuf, + bindings_dir: PathBuf, + source_root: PathBuf, + staging_root: PathBuf, + topology_store: PathBuf, + /// Operator incidents, published world-readable beside the topology + /// projection. Not part of the control store: see `incident`. + incident_store: PathBuf, + odin_correlation_store: PathBuf, + odin_trust_anchor: PathBuf, + idunn_identity_store: PathBuf, + deployment_brake_operator_anchor: PathBuf, + source_identity: Option, + topology_maximum_age_millis: u64, + topology_maximum_future_skew_millis: u64, + poll_millis: u64, + /// Where host actuators dial in. `None` means no host-actuator workload + /// can be served; a binding that names one then fails at its first + /// driver call, not at startup, so an Idunn without hosts is unchanged. + host_actuator_bind: Option, + /// Where route drivers find nginx, systemctl, ufw and systemd-run, and + /// their scratch directory. Every route driver is built from this. + route_actuators: RouteActuators, +} + +impl Default for RuntimeOptions { + fn default() -> Self { + Self { + state_store: PathBuf::from("/var/lib/gamecult/idunn/control.cc"), + bindings_dir: PathBuf::from("/etc/gamecult/idunn/bindings"), + source_root: PathBuf::from("/var/lib/gamecult/idunn/sources"), + staging_root: PathBuf::from("/var/lib/gamecult/idunn/staging"), + topology_store: PathBuf::from("/var/lib/gamecult/idunn/topology.cc"), + incident_store: PathBuf::from("/var/lib/gamecult/idunn-projection/incidents.cc"), + odin_correlation_store: PathBuf::from( + "/var/lib/gamecult/odin/idunn-runtime-topology.cc", + ), + odin_trust_anchor: PathBuf::from("/etc/gamecult/idunn/odin-topology-anchor.cc"), + idunn_identity_store: PathBuf::from( + "/var/lib/gamecult/idunn/idunn-service-identity.cc", + ), + deployment_brake_operator_anchor: PathBuf::from( + "/etc/gamecult/idunn/deployment-brake-operator-anchor.cc", + ), + source_identity: None, + topology_maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, + topology_maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, + poll_millis: 500, + host_actuator_bind: None, + route_actuators: RouteActuators::default(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Command { + Serve(RuntimeOptions), + Up { + selector: String, + requested_by: String, + state_store: PathBuf, + wait: bool, + timeout_seconds: u64, + }, + Status { + state_store: PathBuf, + incident_store: PathBuf, + command_id: Option, + }, + Cancel { + state_store: PathBuf, + command_id: String, + requested_by: String, + }, + Expire { + state_store: PathBuf, + command_id: String, + requested_by: String, + }, + Validate { + recipe: PathBuf, + binding: Option, + }, +} + +pub fn run(args: impl Iterator) -> Result<()> { + match parse(args)? { + Command::Serve(options) => serve(options), + Command::Up { + selector, + requested_by, + state_store, + wait, + timeout_seconds, + } => submit( + &state_store, + &selector, + &requested_by, + wait, + timeout_seconds, + ), + Command::Status { + state_store, + incident_store, + command_id, + } => status( + &mut std::io::stdout().lock(), + &state_store, + &incident_store, + command_id.as_deref(), + ), + Command::Cancel { + state_store, + command_id, + requested_by, + } => cancel(&state_store, &command_id, &requested_by), + Command::Expire { + state_store, + command_id, + requested_by, + } => expire(&state_store, &command_id, &requested_by), + Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + } +} + +/// Offline admission check for authoring. Reads a recipe, optionally reads a +/// binding, and runs the same `admit` the daemon runs at load. It opens no +/// store, contacts no host, and actuates nothing, so it is safe to run against +/// a candidate binding before installing it. +fn validate(recipe_path: &Path, binding_path: Option<&Path>) -> Result<()> { + let recipe_text = fs::read_to_string(recipe_path) + .with_context(|| format!("reading recipe {}", recipe_path.display()))?; + let recipe = crate::deployment::TargetDeclaration::parse(&recipe_text) + .with_context(|| format!("validating recipe {}", recipe_path.display()))?; + println!( + "recipe ok: {} declares target {}", + recipe_path.display(), + recipe.target + ); + + let Some(binding_path) = binding_path else { + return Ok(()); + }; + let binding_text = fs::read_to_string(binding_path) + .with_context(|| format!("reading operator binding {}", binding_path.display()))?; + let binding = OperatorBinding::parse(&binding_text) + .with_context(|| format!("validating operator binding {}", binding_path.display()))?; + ensure!( + binding.target == recipe.target, + "operator binding targets {} but the recipe declares {}", + binding.target, + recipe.target + ); + binding + .admit(&recipe) + .with_context(|| format!("admitting {} against its recipe", binding.target))?; + println!( + "binding ok: {} admits target {}", + binding_path.display(), + binding.target + ); + Ok(()) +} + +fn parse(args: impl Iterator) -> Result { + let mut args = args.peekable(); + let command = args.next().ok_or_else(|| anyhow!(usage()))?; + match command.as_str() { + "serve" => parse_serve(args), + "up" => parse_up(args), + "status" => parse_status(args), + "cancel" => parse_cancel(args), + "expire" => parse_expire(args), + "validate" => parse_validate(args), + "--help" | "-h" | "help" => bail!(usage()), + _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), + } +} + +fn parse_serve(mut args: impl Iterator) -> Result { + let mut options = RuntimeOptions::default(); + let mut source_uid = None; + let mut source_gid = None; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, &argument)?, + "--bindings-dir" => options.bindings_dir = path_value(&mut args, &argument)?, + "--source-root" => options.source_root = path_value(&mut args, &argument)?, + "--staging-root" => options.staging_root = path_value(&mut args, &argument)?, + "--topology-store" => options.topology_store = path_value(&mut args, &argument)?, + "--incident-store" => options.incident_store = path_value(&mut args, &argument)?, + "--odin-correlation-store" => { + options.odin_correlation_store = path_value(&mut args, &argument)? + } + "--odin-trust-anchor" => options.odin_trust_anchor = path_value(&mut args, &argument)?, + "--idunn-identity-store" => { + options.idunn_identity_store = path_value(&mut args, &argument)? + } + "--deployment-brake-operator-anchor" => { + options.deployment_brake_operator_anchor = path_value(&mut args, &argument)? + } + "--topology-maximum-age-millis" => { + options.topology_maximum_age_millis = u64_value(&mut args, &argument)? + } + "--topology-maximum-future-skew-millis" => { + options.topology_maximum_future_skew_millis = u64_value(&mut args, &argument)? + } + "--source-uid" => source_uid = Some(u32_value(&mut args, &argument)?), + "--source-gid" => source_gid = Some(u32_value(&mut args, &argument)?), + "--poll-millis" => options.poll_millis = u64_value(&mut args, &argument)?, + "--host-actuator-bind" => { + let value = string_value(&mut args, &argument)?; + options.host_actuator_bind = Some( + value + .parse() + .with_context(|| format!("{argument} is not a socket address"))?, + ); + } + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn serve option {argument:?}"), + } + } + ensure!( + options.poll_millis > 0 && options.topology_maximum_age_millis > 0, + "poll and topology maximum age must be positive" + ); + options.source_identity = match (source_uid, source_gid) { + (Some(uid), Some(gid)) => { + ensure!( + uid > 0 && gid > 0, + "source UID and GID must be unprivileged" + ); + Some(ProcessIdentity { uid, gid }) + } + (None, None) => None, + _ => bail!("--source-uid and --source-gid must be supplied together"), + }; + Ok(Command::Serve(options)) +} + +fn parse_up(mut args: impl Iterator) -> Result { + let selector = args + .next() + .ok_or_else(|| anyhow!("idunn up requires a service or profile selector"))?; + require_selector(&selector)?; + let mut state_store = RuntimeOptions::default().state_store; + let mut requested_by = env::var("SUDO_USER") + .or_else(|_| env::var("USER")) + .or_else(|_| env::var("USERNAME")) + .unwrap_or_else(|_| "operator".into()); + let mut wait = true; + let mut timeout_seconds = 1800; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--requested-by" => requested_by = string_value(&mut args, &argument)?, + "--no-wait" => wait = false, + "--timeout-seconds" => timeout_seconds = u64_value(&mut args, &argument)?, + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn up option {argument:?}"), + } + } + ensure!(timeout_seconds > 0, "--timeout-seconds must be positive"); + require_value(&requested_by, "deployment requester")?; + Ok(Command::Up { + selector, + requested_by, + state_store, + wait, + timeout_seconds, + }) +} + +fn parse_status(mut args: impl Iterator) -> Result { + let defaults = RuntimeOptions::default(); + let mut state_store = defaults.state_store; + let mut incident_store = defaults.incident_store; + let mut command_id = None; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--incident-store" => incident_store = path_value(&mut args, &argument)?, + "--command" => command_id = Some(string_value(&mut args, &argument)?), + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn status option {argument:?}"), + } + } + Ok(Command::Status { + state_store, + incident_store, + command_id, + }) +} + +fn parse_expire(mut args: impl Iterator) -> Result { + let command_id = args + .next() + .ok_or_else(|| anyhow!("idunn expire requires a command id"))?; + require_id(&command_id, "command id")?; + let mut state_store = RuntimeOptions::default().state_store; + let mut requested_by = env::var("SUDO_USER") + .or_else(|_| env::var("USER")) + .or_else(|_| env::var("USERNAME")) + .unwrap_or_else(|_| "operator".into()); + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--requested-by" => requested_by = string_value(&mut args, &argument)?, + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn expire option {argument:?}"), + } + } + require_value(&requested_by, "expiry requester")?; + Ok(Command::Expire { + state_store, + command_id, + requested_by, + }) +} + +/// Ask the daemon to end a live transaction's post-fencing phase now. This +/// writes a request and nothing else: the daemon's deadline resolver ends the +/// phase exactly as it ends one whose deadline has passed. +fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { + let snapshot = ControlSnapshot::read(store_path)?; + let mut live = snapshot + .transactions + .iter() + .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); + let current = live + .next() + .context("command has no live transaction to expire")?; + ensure!( + live.next().is_none(), + "command has multiple live transactions" + ); + ensure!( + current.value.resolves_by_deadline(), + "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" + ); + let now = now_millis()?; + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: command_id.to_owned(), + requested_by: requested_by.to_owned(), + requested_at_unix_millis: now, + }; + ensure!( + SingleFileMessagePackBackingStore::new(store_path) + .insert_entry_if_absent(expiry_request_envelope(&request, now)?)?, + "an expiry request for this command is already waiting" + ); + println!("{command_id} expiry requested; Idunn ends its phase as an expired deadline"); + Ok(()) +} + +fn parse_cancel(mut args: impl Iterator) -> Result { + let command_id = args + .next() + .ok_or_else(|| anyhow!("idunn cancel requires a deployment command id"))?; + require_id(&command_id, "deployment command id")?; + let mut state_store = RuntimeOptions::default().state_store; + let mut requested_by = env::var("SUDO_USER") + .or_else(|_| env::var("USER")) + .or_else(|_| env::var("USERNAME")) + .unwrap_or_else(|_| "operator".into()); + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => state_store = path_value(&mut args, &argument)?, + "--requested-by" => requested_by = string_value(&mut args, &argument)?, + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn cancel option {argument:?}"), + } + } + require_value(&requested_by, "cancellation requester")?; + Ok(Command::Cancel { + state_store, + command_id, + requested_by, + }) +} + +/// Withdraw queued work, a held deployment that has not fenced, or ask Idunn to +/// clean up a live stateless candidate. Stateful candidates past fencing cannot be cancelled because their prior +/// writer may already be stopped and its state boundary needs explicit repair. +fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { + let snapshot = ControlSnapshot::read(store_path)?; + let command = snapshot + .commands + .iter() + .find(|stored| stored.value.command_id == command_id) + .context("deployment command is unknown or already retired")?; + ensure!( + command.value.kind == CommandKind::Deploy, + "only deployment commands can be cancelled; continuity is Idunn's own" + ); + let mut live = snapshot + .transactions + .iter() + .filter(|stored| stored.value.command_id == command_id && !stored.value.is_terminal()); + if let Some(current) = live.next() { + ensure!( + live.next().is_none(), + "deployment command has multiple live transactions" + ); + ensure!( + cancel_is_safe_for_live_stateless(¤t.value) + || cancel_is_safe_for_held_prefence(¤t.value), + "live deployment can only be cancelled before fencing when Idunn holds it, or after fencing when its candidate is stateless" + ); + let error = format!("cancelled by {requested_by}"); + let mut next = current.value.clone(); + if cancel_is_safe_for_held_prefence(&next) { + next.pre_fencing_abort = Some(pre_fencing_abort_intent(&next, &error)); + } else { + next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); + } + next.last_error = Some(error); + next.updated_at_unix_millis = now_millis()?; + replace_transaction(store_path, current, &next)?; + println!("{command_id} cancellation requested; Idunn is performing exact candidate cleanup"); + return Ok(()); + } + let now = now_millis()?; + let refusal = DeploymentTransaction::rejected( + &command.value, + anyhow!("cancelled by {requested_by} before freezing"), + now, + )?; + let envelope = transaction_envelope(&refusal, now)?; + ensure!( + SingleFileMessagePackBackingStore::new(store_path).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command_id.to_owned(), + current: Some(command.envelope.clone()), + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: refusal.transaction_id.clone(), + current: None, + }, + ], + std::slice::from_ref(&envelope), + )?, + "deployment command changed before it could be cancelled" + ); + archive_terminal_transaction(store_path, &envelope)?; + println!("{command_id} cancelled"); + Ok(()) +} + +/// A deployment Idunn holds before it fenced anything has touched nothing of +/// the incumbent, so an operator may withdraw it. Idunn never does so itself. +fn cancel_is_safe_for_held_prefence(transaction: &DeploymentTransaction) -> bool { + transaction.command_kind == CommandKind::Deploy + && transaction.phase < DeploymentPhase::Fencing + && transaction.held_disagreement().is_some() +} + +fn cancel_is_safe_for_live_stateless(transaction: &DeploymentTransaction) -> bool { + transaction.command_kind == CommandKind::Deploy + && transaction.phase >= DeploymentPhase::Fencing + && matches!( + transaction.fencing.as_ref(), + Some(FencingEvidence::SkippedStateless) + ) + && transaction.completion.is_none() + && transaction.pre_fencing_abort.is_none() + && transaction.post_fencing_abort.is_none() + && transaction.post_commit_cleanup.is_none() +} + +fn parse_validate(mut args: impl Iterator) -> Result { + let mut recipe = None; + let mut binding = None; + while let Some(argument) = args.next() { + match argument.as_str() { + "--recipe" => recipe = Some(path_value(&mut args, &argument)?), + "--binding" => binding = Some(path_value(&mut args, &argument)?), + "--help" | "-h" => bail!(usage()), + _ => bail!("unknown Idunn validate option {argument:?}"), + } + } + let recipe = recipe.ok_or_else(|| anyhow!("validate requires --recipe"))?; + Ok(Command::Validate { recipe, binding }) +} + +#[derive(Clone)] +struct Stored { + envelope: CultCacheEnvelope, + value: T, +} + +#[derive(Default)] +struct ControlSnapshot { + commands: Vec>, + transactions: Vec>, + admitted: Vec>, + targets: Vec>, + expiry_requests: Vec>, +} + +impl ControlSnapshot { + fn read(path: &Path) -> Result { + if !path.exists() { + return Ok(Self::default()); + } + let mut snapshot = Self::default(); + for envelope in SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading Idunn control snapshot")? + { + match envelope.r#type.as_str() { + DeploymentCommand::TYPE => { + ensure!( + envelope.schema_id.as_deref() == Some(DEPLOYMENT_COMMAND_SCHEMA), + "Idunn control store contains an unsupported command" + ); + let value: DeploymentCommand = decode_record(&envelope)?; + value.validate()?; + ensure!( + envelope.key == value.command_id, + "deployment command key differs from its identity" + ); + snapshot.commands.push(Stored { envelope, value }); + } + DeploymentTransaction::TYPE => { + let value = read_transaction_record(&envelope)?; + ensure!( + envelope.key == value.transaction_id, + "deployment transaction key differs from its identity" + ); + snapshot.transactions.push(Stored { envelope, value }); + } + AdmittedGeneration::TYPE => { + let value = read_generation_record(&envelope)?; + ensure!( + envelope.key == value.target, + "admitted generation key is not its target" + ); + snapshot.admitted.push(Stored { envelope, value }); + } + TargetSupervision::TYPE => { + let value = read_supervision_record(&envelope)?; + ensure!( + envelope.key == value.target, + "target supervision key is not its target" + ); + snapshot.targets.push(Stored { envelope, value }); + } + ExpiryRequest::TYPE => { + ensure!( + envelope.schema_id.as_deref() == Some(EXPIRY_REQUEST_SCHEMA), + "Idunn control store contains an unsupported expiry request" + ); + let value: ExpiryRequest = decode_record(&envelope)?; + value.validate()?; + ensure!( + envelope.key == value.command_id, + "expiry request key is not its command" + ); + snapshot.expiry_requests.push(Stored { envelope, value }); + } + _ => bail!("Idunn control store contains a foreign document"), + } + } + snapshot.validate_relations()?; + Ok(snapshot) + } + + fn validate_relations(&self) -> Result<()> { + let command_ids = self + .commands + .iter() + .map(|stored| stored.value.command_id.as_str()) + .collect::>(); + ensure!( + command_ids.len() == self.commands.len(), + "Idunn control store contains duplicate commands" + ); + let transaction_ids = self + .transactions + .iter() + .map(|stored| stored.value.transaction_id.as_str()) + .collect::>(); + ensure!( + transaction_ids.len() == self.transactions.len(), + "Idunn control store contains duplicate transactions" + ); + let admitted_targets = self + .admitted + .iter() + .map(|stored| stored.value.target.as_str()) + .collect::>(); + ensure!( + admitted_targets.len() == self.admitted.len(), + "Idunn control store contains duplicate current generations" + ); + ensure!( + self.targets + .iter() + .map(|stored| stored.value.target.as_str()) + .collect::>() + .len() + == self.targets.len(), + "Idunn control store contains duplicate target supervision records" + ); + for transaction in &self.transactions { + let command = self + .commands + .iter() + .find(|candidate| candidate.value.command_id == transaction.value.command_id) + .context("deployment transaction has no immutable command")?; + ensure!( + command.value.kind == transaction.value.command_kind, + "transaction kind differs from its immutable command" + ); + } + let mut live_targets = BTreeSet::new(); + for transaction in self + .transactions + .iter() + .filter(|stored| stored.value.owns_target_authority()) + { + ensure!( + live_targets.insert(transaction.value.target.as_str()), + "multiple transactions claim current-incarnation authority for one target" + ); + if let Some(fencing) = transaction + .value + .fencing + .as_ref() + .filter(|_| transaction.value.phase < DeploymentPhase::Complete) + { + let incumbent = self.admitted_for(&transaction.value.target); + let incumbent_matches = + match (&transaction.value.incumbent_generation_id, incumbent) { + (None, None) => true, + (Some(expected), Some(current)) => current.value.generation_id == *expected, + _ => false, + }; + ensure!( + incumbent_matches, + "fenced transaction incumbent differs from current admission" + ); + let incumbent_lease_sha256 = incumbent + .and_then(|stored| stored.value.leasing.lease()) + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose()?; + match fencing { + FencingEvidence::SkippedStateless => ensure!( + incumbent_lease_sha256.is_none(), + "stateless fencing omitted the admitted incumbent lease" + ), + FencingEvidence::Revoked { + incumbent_lease_sha256: recorded, + .. + } => { + ensure!( + recorded == &incumbent_lease_sha256, + "fencing evidence names another incumbent lease" + ); + ensure!( + incumbent_lease_sha256.is_some() + || transaction + .value + .expected + .as_ref() + .is_some_and(|expected| expected.write_lease_required), + "stateless transaction did not use its explicit fencing skip" + ); + } + } + } + } + let mut authorizations = BTreeSet::new(); + for authorization in self + .transactions + .iter() + .filter_map(|stored| stored.value.deployment_authorization.as_ref()) + { + ensure!( + authorizations.insert(authorization.authorization_id.as_str()), + "deployment authorization was consumed more than once" + ); + } + Ok(()) + } + + /// Every stored record that concerns `target`, as read: the generation, the + /// meters and its transactions. Two reads with equal stamps saw no write + /// for the target between them. + fn target_stamp(&self, target: &str) -> Vec { + let mut stamp = self + .admitted + .iter() + .filter(|stored| stored.value.target == target) + .map(|stored| stored.envelope.clone()) + .chain( + self.targets + .iter() + .filter(|stored| stored.value.target == target) + .map(|stored| stored.envelope.clone()), + ) + .chain( + self.transactions + .iter() + .filter(|stored| stored.value.target == target) + .map(|stored| stored.envelope.clone()), + ) + .collect::>(); + stamp.sort_by(|a, b| (&a.r#type, &a.key).cmp(&(&b.r#type, &b.key))); + stamp + } + + fn admitted_for(&self, target: &str) -> Option<&Stored> { + self.admitted + .iter() + .find(|stored| stored.value.target == target) + } + + fn supervision_for(&self, target: &str) -> Option<&Stored> { + self.targets + .iter() + .find(|stored| stored.value.target == target) + } + + /// The target's meters, or empty ones: a target that was never metered has + /// used nothing. + fn supervision_or_new(&self, target: &str) -> TargetSupervision { + self.supervision_for(target) + .map_or_else(|| TargetSupervision::new(target), |stored| stored.value.clone()) + } + + /// The admitted generation of the target that provides + /// `odin.verse-rendezvous`: Odin is whichever target says so, not + /// whichever is named "odin". + fn admitted_odin(&self) -> Option<&Stored> { + self.admitted.iter().find(|stored| { + ReadinessClass::of(&stored.value.expected) + .is_ok_and(|class| class == ReadinessClass::OdinSelf) + }) + } + + fn transaction_for_command(&self, command_id: &str) -> Vec<&DeploymentTransaction> { + let mut values = self + .transactions + .iter() + .filter(|stored| stored.value.command_id == command_id) + .map(|stored| &stored.value) + .collect::>(); + values.sort_by_key(|value| value.ordinal); + values + } + + fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool { + self.transactions.iter().any(|stored| { + stored.value.owns_target_authority() + && stored.value.command_id == transaction.command_id + && stored.value.ordinal < transaction.ordinal + }) + } + + /// The highest Odin publisher sequence this target has ever *acted on*. + /// + /// A transaction that failed contributes nothing. Its observations were + /// never built upon -- the deployment was abandoned and its projection + /// withdrawn -- so treating them as admitted history only raises the bar + /// for the next attempt. Odin does not reuse a sequence, so nothing is + /// weakened by forgetting the readings of an attempt that came to nothing; + /// what would be weakened is refusing every retry after a failure, which is + /// how a target ends up unable to deploy at all. + fn max_odin_sequence(&self, target: &str, signer_identity_id: &str) -> u64 { + let transaction_max = self + .transactions + .iter() + .filter_map(|stored| { + stored + .value + .latest_odin_observation + .as_ref() + .filter(|evidence| { + stored.value.target == target + && evidence.signer_identity_id == signer_identity_id + && !matches!( + stored.value.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + | Some(TransactionCompletion::FailedAfterFencing { .. }) + ) + }) + .map(|evidence| evidence.publisher_sequence) + }) + .max() + .unwrap_or(0); + let admitted_max = self + .admitted + .iter() + .filter(|stored| { + stored.value.target == target + && stored + .value + .odin_authority + .as_ref() + .is_some_and(|authority| authority.signer_identity_id == signer_identity_id) + }) + .map(|stored| stored.value.odin_publisher_sequence_cursor) + .max() + .unwrap_or(0); + transaction_max.max(admitted_max) + } +} + +pub(crate) fn decode_record(envelope: &CultCacheEnvelope) -> Result +where + T: for<'de> Deserialize<'de> + Serialize, +{ + let value: T = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&value)? == envelope.payload, + "Idunn control store contains a noncanonical record" + ); + Ok(value) +} + +fn command_envelope(value: &DeploymentCommand, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.command_id, + DeploymentCommand::TYPE, + DEPLOYMENT_COMMAND_SCHEMA, + value, + now, + ) +} + +/// The transaction layout of v2 and v3 records: `ready` is a bare Odin receipt, +/// and there are no phase deadline or lease adoption slots. It exists only to +/// decode records written before v4. `migrate_control_store_to_current_schema` +/// rewrites every one in `control.cc` at boot, but `history.cc` is never +/// migrated or pruned, so this struct dies only when history is pruned or +/// migrated. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_transaction", + schema = "idunn.deployment_transaction.v3" +)] +struct LegacyDeploymentTransaction { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + transaction_id: String, + #[cultcache(key = 2)] + command_id: String, + #[cultcache(key = 3)] + command_kind: CommandKind, + #[cultcache(key = 4)] + target: String, + #[cultcache(key = 5)] + ordinal: u32, + #[cultcache(key = 6)] + phase: DeploymentPhase, + #[cultcache(key = 7)] + created_at_unix_millis: u64, + #[cultcache(key = 8)] + updated_at_unix_millis: u64, + #[cultcache(key = 9)] + incumbent_generation_id: Option, + #[cultcache(key = 10)] + plan: Option, + #[cultcache(key = 11)] + frozen_source: Option, + #[cultcache(key = 12)] + sealed_release: Option, + #[cultcache(key = 13)] + installed_release: Option, + #[cultcache(key = 14)] + expected: Option, + #[cultcache(key = 15)] + expected_publication_sha256: Option, + #[cultcache(key = 16)] + deployment_authorization: Option, + #[cultcache(key = 17)] + lifecycle_authorized_at_unix_millis: Option, + #[cultcache(key = 18)] + activation: Option, + #[cultcache(key = 19)] + workload: Option, + #[cultcache(key = 20)] + activation_publication_sha256: Option, + #[cultcache(key = 21)] + latest_odin_observation: Option, + #[cultcache(key = 22)] + warming: Option, + #[cultcache(key = 23)] + route_preflight: Option, + #[cultcache(key = 24)] + isolation: Option, + #[cultcache(key = 25)] + fencing: Option, + #[cultcache(key = 26)] + leasing: Option, + #[cultcache(key = 27)] + ready: Option, + #[cultcache(key = 28)] + routing: Option, + #[cultcache(key = 29)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 30)] + last_error: Option, + #[cultcache(key = 31)] + completion: Option, + #[cultcache(key = 32)] + pre_fencing_abort: Option, + #[cultcache(key = 33)] + post_commit_cleanup: Option, + // A v2 record has no key 34: it predates post-fencing aborts. + #[cultcache(key = 34, default)] + post_fencing_abort: Option, +} + +impl LegacyDeploymentTransaction { + /// Every legacy Ready receipt is an Odin receipt: no route-proof class + /// existed. The new slots start empty, which is what an older transaction + /// had. + fn into_current(self) -> DeploymentTransaction { + DeploymentTransaction { + schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), + transaction_id: self.transaction_id, + command_id: self.command_id, + command_kind: self.command_kind, + target: self.target, + ordinal: self.ordinal, + phase: self.phase, + created_at_unix_millis: self.created_at_unix_millis, + updated_at_unix_millis: self.updated_at_unix_millis, + incumbent_generation_id: self.incumbent_generation_id, + plan: self.plan, + frozen_source: self.frozen_source, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + expected_publication_sha256: self.expected_publication_sha256, + deployment_authorization: self.deployment_authorization, + lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, + activation: self.activation, + workload: self.workload, + activation_publication_sha256: self.activation_publication_sha256, + latest_odin_observation: self.latest_odin_observation, + warming: self.warming, + route_preflight: self.route_preflight, + isolation: self.isolation, + fencing: self.fencing, + leasing: self.leasing, + ready: self + .ready + .map(|evidence| ReadinessEvidence::OdinCorrelated { evidence }), + routing: self.routing, + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + last_error: self.last_error, + completion: self.completion, + pre_fencing_abort: self.pre_fencing_abort, + post_commit_cleanup: self.post_commit_cleanup, + post_fencing_abort: self.post_fencing_abort, + phase_deadline: None, + lease_adoption: None, + } + } +} + +/// The v2 admitted-generation layout. Same lifetime rule as +/// `LegacyDeploymentTransaction`. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.admitted_generation", + schema = "idunn.admitted_generation.v2" +)] +struct LegacyAdmittedGeneration { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + generation_id: String, + #[cultcache(key = 3)] + command_id: String, + #[cultcache(key = 4)] + transaction_id: String, + #[cultcache(key = 5)] + admitted_at_unix_millis: u64, + #[cultcache(key = 6)] + plan: CompiledDeploymentPlan, + #[cultcache(key = 7)] + sealed_release: SealedRelease, + #[cultcache(key = 8)] + installed_release: InstalledReleaseObservation, + #[cultcache(key = 9)] + expected: IdunnExpectedIncarnationRecord, + #[cultcache(key = 10)] + activation: IdunnRuntimeActivationRecord, + #[cultcache(key = 11)] + workload: WorkloadObservation, + #[cultcache(key = 12)] + leasing: LeasingEvidence, + #[cultcache(key = 13)] + ready: TopologyEvidence, + #[cultcache(key = 14)] + latest_odin_observation: TopologyEvidence, + #[cultcache(key = 15)] + routing: RoutingEvidence, + #[cultcache(key = 16)] + odin_authority: AdmittedOdinAuthority, + #[cultcache(key = 17)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 18)] + route_repair_started_at_unix_millis: Option, +} + +impl LegacyAdmittedGeneration { + /// Every v2 generation was admitted on Odin receipts, so it migrates as + /// Odin-correlated with those receipts unchanged. Route supervision state + /// starts empty for a promoted route. The v2 repair start decided nothing + /// after S1 and is not carried. + fn into_current(self) -> AdmittedGeneration { + let route_supervision = matches!(&self.routing, RoutingEvidence::Promoted { .. }) + .then(RouteSupervisionState::default); + AdmittedGeneration { + schema_version: ADMITTED_GENERATION_SCHEMA.into(), + target: self.target, + generation_id: self.generation_id, + command_id: self.command_id, + transaction_id: self.transaction_id, + admitted_at_unix_millis: self.admitted_at_unix_millis, + plan: self.plan, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + activation: self.activation, + workload: self.workload, + leasing: self.leasing, + ready: ReadinessEvidence::OdinCorrelated { + evidence: self.ready, + }, + latest_odin_observation: Some(self.latest_odin_observation), + routing: self.routing, + odin_authority: Some(self.odin_authority), + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + route_supervision, + last_error: None, + } + } +} + +/// The v3 admitted-generation layout. Its meters (`actuations`, the continuity +/// backoff, the repair start) are retired: no released Idunn ever wrote a +/// nonzero one, and a record that shows one is refused instead of lifted. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.admitted_generation", + schema = "idunn.admitted_generation.v3" +)] +struct LegacyAdmittedGenerationV3 { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + generation_id: String, + #[cultcache(key = 3)] + command_id: String, + #[cultcache(key = 4)] + transaction_id: String, + #[cultcache(key = 5)] + admitted_at_unix_millis: u64, + #[cultcache(key = 6)] + plan: CompiledDeploymentPlan, + #[cultcache(key = 7)] + sealed_release: SealedRelease, + #[cultcache(key = 8)] + installed_release: InstalledReleaseObservation, + #[cultcache(key = 9)] + expected: IdunnExpectedIncarnationRecord, + #[cultcache(key = 10)] + activation: IdunnRuntimeActivationRecord, + #[cultcache(key = 11)] + workload: WorkloadObservation, + #[cultcache(key = 12)] + leasing: LeasingEvidence, + #[cultcache(key = 13)] + ready: ReadinessEvidence, + #[cultcache(key = 14)] + latest_odin_observation: Option, + #[cultcache(key = 15)] + routing: RoutingEvidence, + #[cultcache(key = 16)] + odin_authority: Option, + #[cultcache(key = 17)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 18)] + route_repair_started_at_unix_millis: Option, + #[cultcache(key = 19)] + route_supervision: Option, + #[cultcache(key = 20)] + continuity_backoff: LegacyContinuityBackoffV3, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyRouteSupervisionStateV3 { + last_challenge_at_unix_millis: Option, + consecutive_failures: u32, + next_challenge_at_unix_millis: Option, + degraded_since_unix_millis: Option, + actuations: LegacyActuationWindowV3, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyActuationWindowV3 { + window_started_at_unix_millis: u64, + count: u32, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyContinuityBackoffV3 { + window_started_at_unix_millis: Option, + attempts: u32, + next_restart_at_unix_millis: Option, +} + +impl LegacyAdmittedGenerationV3 { + /// Drops the retired meters. The lift creates no `TargetSupervision`: every + /// target starts unmetered, which is what no released binary ever changed. + fn into_current(self) -> Result { + ensure!( + self.continuity_backoff.attempts == 0 + && self + .route_supervision + .as_ref() + .is_none_or(|state| state.actuations.count == 0), + "admitted generation of {} records route actuations or continuity restarts that no released Idunn wrote; refusing to drop them", + self.target + ); + Ok(AdmittedGeneration { + schema_version: ADMITTED_GENERATION_SCHEMA.into(), + target: self.target, + generation_id: self.generation_id, + command_id: self.command_id, + transaction_id: self.transaction_id, + admitted_at_unix_millis: self.admitted_at_unix_millis, + plan: self.plan, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + activation: self.activation, + workload: self.workload, + leasing: self.leasing, + ready: self.ready, + latest_odin_observation: self.latest_odin_observation, + routing: self.routing, + odin_authority: self.odin_authority, + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + route_supervision: self.route_supervision.map(|state| RouteSupervisionState { + last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, + consecutive_failures: state.consecutive_failures, + next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, + degraded_since_unix_millis: state.degraded_since_unix_millis, + }), + last_error: None, + }) + } +} + +/// Decode a v2 or v3 transaction and lift it to the current shape, without +/// validating it. History reads stop here; the control store validates. +fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result { + let schema = match envelope.schema_id.as_deref() { + Some(schema @ (DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3)) => { + schema + } + _ => bail!("Idunn control store contains an unsupported transaction"), + }; + let value: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload) + .with_context(|| format!("decoding a {schema} deployment transaction"))?; + ensure!( + value.schema_version == schema, + "stored transaction schema differs from its envelope" + ); + ensure!( + schema != DEPLOYMENT_TRANSACTION_SCHEMA_V2 || value.post_fencing_abort.is_none(), + "a v2 transaction cannot carry post-fencing abort evidence" + ); + Ok(value.into_current()) +} + +/// Read one stored transaction, lifting a v2 or v3 record to the current shape. +/// +/// The control store refuses noncanonical bytes: a record must re-encode to +/// exactly what is stored. That check is what makes tampering visible, and it +/// also means adding a field is a schema change: an older record re-encodes +/// with extra keys and would be read as tampered. +/// +/// So an older record is decoded by its own layout and lifted. Byte-exactness +/// cannot apply across a version boundary -- the bytes are a different schema +/// by definition -- so the lift leans on the full semantic `validate()` +/// instead, and everything downstream sees only the current shape. +fn read_transaction_record(envelope: &CultCacheEnvelope) -> Result { + match envelope.schema_id.as_deref() { + Some(DEPLOYMENT_TRANSACTION_SCHEMA) => { + let value: DeploymentTransaction = decode_record(envelope)?; + value.validate()?; + Ok(value) + } + _ => { + let mut value = lift_legacy_transaction(envelope)?; + cleanup_evidence::owe_legacy_continuity_projection(&mut value); + value.validate()?; + Ok(value) + } + } +} + +/// Read one stored admitted generation, lifting a v2 or v3 record to the +/// current shape by the same rule as `read_transaction_record`. +fn read_generation_record(envelope: &CultCacheEnvelope) -> Result { + match envelope.schema_id.as_deref() { + Some(ADMITTED_GENERATION_SCHEMA) => { + let value: AdmittedGeneration = decode_record(envelope)?; + value.validate()?; + Ok(value) + } + Some(ADMITTED_GENERATION_SCHEMA_V3) => { + let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload) + .context("decoding a v3 admitted generation")?; + ensure!( + legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V3, + "stored generation schema differs from its envelope" + ); + let value = legacy.into_current()?; + value.validate()?; + Ok(value) + } + Some(ADMITTED_GENERATION_SCHEMA_V2) => { + let legacy: LegacyAdmittedGeneration = rmp_serde::from_slice(&envelope.payload) + .context("decoding a v2 admitted generation")?; + ensure!( + legacy.schema_version == ADMITTED_GENERATION_SCHEMA_V2, + "stored generation schema differs from its envelope" + ); + let value = legacy.into_current(); + value.validate()?; + Ok(value) + } + _ => bail!("Idunn control store contains an unsupported admitted generation"), + } +} + +/// The v1 target-supervision layout: the first seven keys of the current one. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.target_supervision", + schema = "idunn.target_supervision.v1" +)] +struct LegacyTargetSupervisionV1 { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + target: String, + #[cultcache(key = 2)] + route_actuations: Vec, + #[cultcache(key = 3)] + continuity_restarts: Vec, + #[cultcache(key = 4)] + continuity_deferred_until: Option, + #[cultcache(key = 5)] + continuity_deferral_reason: Option, + #[cultcache(key = 6)] + route_survivals: Vec, +} + +impl LegacyTargetSupervisionV1 { + /// No streak, no anchor, no re-materialization: the same defaults a fresh + /// record starts with. + fn into_current(self) -> TargetSupervision { + TargetSupervision { + schema_version: TARGET_SUPERVISION_SCHEMA.into(), + target: self.target, + route_actuations: self.route_actuations, + continuity_restarts: self.continuity_restarts, + continuity_deferred_until: self.continuity_deferred_until, + continuity_deferral_reason: self.continuity_deferral_reason, + route_survivals: self.route_survivals, + continuity_attempt_streak: 0, + continuity_backoff_from_unix_millis: None, + continuity_rematerialized_at_unix_millis: None, + } + } +} + +/// Read one stored target supervision record, lifting a v1 record to the +/// current shape by the same rule as `read_generation_record`. One schema id +/// names one shape: a v1 record that carries ten keys, or a v2 record that +/// carries seven, is refused. +fn read_supervision_record(envelope: &CultCacheEnvelope) -> Result { + let value = match envelope.schema_id.as_deref() { + Some(TARGET_SUPERVISION_SCHEMA) => decode_record::(envelope)?, + Some(TARGET_SUPERVISION_SCHEMA_V1) => { + let legacy: LegacyTargetSupervisionV1 = decode_record(envelope) + .context("decoding a v1 target supervision record")?; + ensure!( + legacy.schema_version == TARGET_SUPERVISION_SCHEMA_V1, + "stored supervision schema differs from its envelope" + ); + legacy.into_current() + } + _ => bail!("Idunn control store contains an unsupported target supervision record"), + }; + value.validate()?; + ensure!( + envelope.key == value.target, + "target supervision key is not its target" + ); + Ok(value) +} + +/// Rewrite every older record as current, once, before the engine runs. +/// +/// The read path lifts older records on its own, so this is convergence rather +/// than correctness: without it the store keeps records in several shapes for +/// as long as the oldest one survives. Each rewrite is a compare-exchange +/// against the exact stored envelope, so a record that changed underneath is +/// left alone rather than clobbered. +fn migrate_control_store_to_current_schema(store_path: &Path) -> Result { + if !store_path.exists() { + return Ok(0); + } + let store = SingleFileMessagePackBackingStore::new(store_path); + let stale = store + .pull_all_read_only_snapshot() + .context("reading Idunn control snapshot for migration")? + .into_iter() + .filter(|envelope| { + let schema = envelope.schema_id.as_deref(); + (envelope.r#type == DeploymentTransaction::TYPE + && matches!( + schema, + Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) + )) + || (envelope.r#type == AdmittedGeneration::TYPE + && matches!( + schema, + Some(ADMITTED_GENERATION_SCHEMA_V2 | ADMITTED_GENERATION_SCHEMA_V3) + )) + || (envelope.r#type == TargetSupervision::TYPE + && schema == Some(TARGET_SUPERVISION_SCHEMA_V1)) + }) + .collect::>(); + let mut migrated = 0; + for envelope in stale { + let (record_type, key, mut next) = if envelope.r#type == DeploymentTransaction::TYPE { + let (key, next) = cleanup_evidence::migrate_transaction_record(&envelope)?; + (DeploymentTransaction::TYPE, key, next) + } else if envelope.r#type == TargetSupervision::TYPE { + let value = read_supervision_record(&envelope)?; + let next = target_supervision_envelope(&value, now_millis()?)?; + (TargetSupervision::TYPE, value.target, next) + } else { + let value = read_generation_record(&envelope)?; + let next = admitted_envelope(&value, value.admitted_at_unix_millis)?; + (AdmittedGeneration::TYPE, value.target, next) + }; + next.stored_at = envelope.stored_at.clone(); + ensure!( + store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: record_type.into(), + key, + current: Some(envelope.clone()), + }], + &[next], + )?, + "control record changed during schema migration" + ); + migrated += 1; + } + Ok(migrated) +} + +fn transaction_envelope(value: &DeploymentTransaction, now: u64) -> Result { + ensure!( + !value.carries_legacy_marker(), + "only the legacy lift may produce a legacy-marked record; it cannot be written as a new one" + ); + encode_transaction(value, now) +} + +fn encode_transaction(value: &DeploymentTransaction, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.transaction_id, + DeploymentTransaction::TYPE, + DEPLOYMENT_TRANSACTION_SCHEMA, + value, + now, + ) +} + +fn admitted_envelope(value: &AdmittedGeneration, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.target, + AdmittedGeneration::TYPE, + ADMITTED_GENERATION_SCHEMA, + value, + now, + ) +} + +fn target_supervision_envelope(value: &TargetSupervision, now: u64) -> Result { + value.validate()?; + typed_envelope( + &value.target, + TargetSupervision::TYPE, + TARGET_SUPERVISION_SCHEMA, + value, + now, + ) +} + +pub(crate) fn typed_envelope( + key: &str, + record_type: &str, + schema: &str, + value: &T, + now: u64, +) -> Result { + Ok(CultCacheEnvelope { + key: key.into(), + r#type: record_type.into(), + payload: rmp_serde::to_vec(value)?, + stored_at: rfc3339_millis(now)?, + schema_id: Some(schema.into()), + }) +} + +/// `history.cc`, beside the control store. Terminal transactions live here so +/// the control store holds only what still gates a decision. +fn history_store_path(state_store: &Path) -> PathBuf { + state_store.with_file_name("history.cc") +} + +/// Move one finished transaction out of the live set. +/// +/// History first, then the live copy: a crash between the two leaves the +/// record in both, which reads as still-resident and is archived again on the +/// next attempt. The other order can lose it, and a completion nobody can +/// observe is the failure that actually costs an operator something. +/// +/// `insert_entry_if_absent` makes the repeat a no-op, and the delete is a +/// compare-and-swap on the exact envelope archived, so a transaction that +/// changed underneath us is left alone rather than dropped. +fn archive_terminal_transaction(state_store: &Path, envelope: &CultCacheEnvelope) -> Result<()> { + SingleFileMessagePackBackingStore::new(&history_store_path(state_store)) + .insert_entry_if_absent(envelope.clone()) + .context("archiving a finished transaction to history")?; + SingleFileMessagePackBackingStore::new(state_store) + .delete_batch_if_unchanged(std::slice::from_ref(envelope)) + .context("retiring a finished transaction from the control store")?; + // The command goes with its last transaction. A command is consumed by + // being frozen once; left resident after its transaction retired, it read + // as queued again and was frozen again on the next tick, which is how one + // `idunn up` became an unbounded series of deployment attempts. + let transaction = read_transaction_record(envelope)?; + let snapshot = ControlSnapshot::read(state_store)?; + if !snapshot + .transaction_for_command(&transaction.command_id) + .is_empty() + { + return Ok(()); + } + if let Some(command) = snapshot + .commands + .iter() + .find(|stored| stored.value.command_id == transaction.command_id) + { + SingleFileMessagePackBackingStore::new(&history_store_path(state_store)) + .insert_entry_if_absent(command.envelope.clone()) + .context("archiving a consumed command to history")?; + SingleFileMessagePackBackingStore::new(state_store) + .delete_batch_if_unchanged(std::slice::from_ref(&command.envelope)) + .context("retiring a consumed command from the control store")?; + } + Ok(()) +} + +/// What `history.cc` yielded: the transactions it decoded and the entries it +/// could not, each with its key and reason. +struct HistoryRead { + transactions: Vec, + undecodable: Vec<(String, String)>, +} + +impl HistoryRead { + /// What is missing from `transactions`, or `None` when nothing is. + fn report(&self) -> Option { + if self.undecodable.is_empty() { + return None; + } + let keys = self + .undecodable + .iter() + .map(|(key, error)| format!("{key} ({error})")) + .collect::>() + .join("; "); + Some(format!( + "Idunn history: {} archived transaction(s) could not be decoded and are not counted \ + by status or continuity backoff: {keys}", + self.undecodable.len() + )) + } +} + +/// Finished transactions. `control.cc` keeps the byte-exact canonical check +/// because its records still gate decisions. History describes what already +/// happened, so an undecodable entry is reported and skipped rather than +/// allowed to refuse the read -- that asymmetry is the whole reason the two +/// files are separate. A file that cannot be read at all is an error: the +/// caller decides whether an empty answer is safe. +fn read_history(state_store: &Path) -> Result { + let path = history_store_path(state_store); + if !path.exists() { + return Ok(HistoryRead { + transactions: Vec::new(), + undecodable: Vec::new(), + }); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .with_context(|| format!("Idunn cannot read {}", path.display()))?; + let (transactions, undecodable) = decode_history_transactions(envelopes); + Ok(HistoryRead { + transactions, + undecodable, + }) +} + +/// History for display only (status). An unreadable file shows as empty and +/// says so; decisions use `Engine::history_for_decision`, which does not. +fn read_history_transactions(state_store: &Path) -> Vec { + match read_history(state_store) { + Ok(history) => { + if let Some(report) = history.report() { + eprintln!("{report}"); + } + history.transactions + } + Err(error) => { + eprintln!("{error:#}; every archived transaction is invisible to status"); + Vec::new() + } + } +} + +/// How long to wait after `failures` consecutive failures: one poll interval +/// after the first, doubling each time, never past the ceiling. +fn backoff_wait(base_millis: u64, failures: u32, ceiling_millis: u64) -> u64 { + base_millis + .saturating_mul(1u64 << failures.min(20)) + .min(ceiling_millis) +} + +/// Whether an attempt due at `not_before` must still wait at `now`. A due time +/// further ahead than any wait its owner can set means the clock stepped back, +/// so the attempt is due: a step stalls nothing longer than the wait itself. +fn is_waiting(now: u64, not_before: u64, longest_wait: u64) -> bool { + now < not_before && not_before - now <= longest_wait +} + +/// Whether a transaction moved between two reads. An error note (`last_error`) +/// and its timestamp are commentary on a stuck step, not a step. +fn state_advanced(before: &DeploymentTransaction, after: &DeploymentTransaction) -> bool { + let mut after = after.clone(); + after.last_error.clone_from(&before.last_error); + after.updated_at_unix_millis = before.updated_at_unix_millis; + &after != before +} + +/// Says a fault once on stderr while it lasts. A scheduler tick that hits the same +/// unreadable file every half second must not print it every half second. +/// +/// It is a trace and owns nothing else: it is in memory, keyed by free-form +/// strings, and an Idunn restart forgets it. Whether the operator is told +/// belongs to the incident record (`incident`), never to this map. +#[derive(Default)] +struct ReportOnce { + last: Mutex>, +} + +impl ReportOnce { + /// The text to print now: `report` when it differs from the last one + /// offered, nothing while the same fault stands or once it has cleared. + fn offer(&self, report: Option) -> Option { + let mut last = self.last.lock().expect("report mutex"); + if *last == report { + return None; + } + last.clone_from(&report); + report + } +} + +/// Decode every archived transaction. An entry that fails is returned with its +/// key and reason instead of vanishing, so the reader can say how many records +/// its answer is missing. +/// +/// `LegacyDeploymentTransaction` decodes the v2 and v3 layouts. `history.cc` is +/// never migrated or pruned, so it dies only when history is pruned or +/// migrated, not at boot. +fn decode_history_transactions( + envelopes: Vec, +) -> (Vec, Vec<(String, String)>) { + let mut transactions = Vec::new(); + let mut undecodable = Vec::new(); + for envelope in envelopes + .into_iter() + .filter(|envelope| envelope.r#type == DeploymentTransaction::TYPE) + { + let decoded = match envelope.schema_id.as_deref() { + Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) => { + lift_legacy_transaction(&envelope) + } + _ => rmp_serde::from_slice::(&envelope.payload) + .map_err(anyhow::Error::from), + }; + match decoded { + Ok(transaction) => transactions.push(transaction), + Err(error) => undecodable.push((envelope.key, format!("{error:#}"))), + } + } + (transactions, undecodable) +} + +/// Consumed commands, read as leniently as their transactions. +fn read_history_commands(state_store: &Path) -> Vec { + let path = history_store_path(state_store); + if !path.exists() { + return Vec::new(); + } + let Ok(envelopes) = SingleFileMessagePackBackingStore::new(&path).pull_all_read_only_snapshot() + else { + return Vec::new(); + }; + envelopes + .into_iter() + .filter(|envelope| envelope.r#type == DeploymentCommand::TYPE) + .filter_map(|envelope| rmp_serde::from_slice::(&envelope.payload).ok()) + .collect() +} + +fn replace_transaction( + store_path: &Path, + current: &Stored, + next: &DeploymentTransaction, +) -> Result<()> { + next.validate()?; + ensure!( + current.value.transaction_id == next.transaction_id, + "transaction replacement changes identity" + ); + let envelope = transaction_envelope(next, next.updated_at_unix_millis)?; + ensure!( + SingleFileMessagePackBackingStore::new(store_path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: current.value.transaction_id.clone(), + current: Some(current.envelope.clone()), + }], + std::slice::from_ref(&envelope), + )?, + "deployment transaction changed before its compare-exchange" + ); + if next.is_terminal() { + archive_terminal_transaction(store_path, &envelope)?; + } + Ok(()) +} + +#[derive(Clone)] +struct LoadedBinding { + binding: OperatorBinding, + bytes: Vec, +} + +fn load_bindings(directory: &Path) -> Result> { + let mut paths = fs::read_dir(directory) + .with_context(|| format!("reading Idunn binding directory {}", directory.display()))? + .map(|entry| entry.map(|entry| entry.path())) + .collect::>>()?; + paths.retain(|path| path.extension() == Some(std::ffi::OsStr::new("toml"))); + paths.sort(); + let mut bindings = BTreeMap::new(); + for path in paths { + let bytes = fs::read(&path) + .with_context(|| format!("reading operator binding {}", path.display()))?; + let text = std::str::from_utf8(&bytes) + .with_context(|| format!("operator binding {} is not UTF-8", path.display()))?; + let binding = OperatorBinding::parse(text) + .with_context(|| format!("validating operator binding {}", path.display()))?; + let target = binding.target.clone(); + ensure!( + bindings + .insert(target.clone(), LoadedBinding { binding, bytes }) + .is_none(), + "operator binding target {target} is duplicated" + ); + } + ensure!(!bindings.is_empty(), "Idunn binding directory is empty"); + validate_route_bindings(&bindings)?; + Ok(bindings) +} + +fn validate_route_bindings(bindings: &BTreeMap) -> Result<()> { + let routes = bindings + .iter() + .filter_map(|(target, loaded)| { + loaded + .binding + .route + .as_ref() + .map(|route| (target.as_str(), route)) + }) + .collect::>(); + validate_route_binding_set(&routes) +} + +fn validate_route_binding_set(routes: &[(&str, &RouteBinding)]) -> Result<()> { + for (index, (target, route)) in routes.iter().enumerate() { + let (stable_host, stable_port) = route.stable_socket()?; + let private_host = route + .private_host + .parse::() + .context("validated route private host stopped being an IP address")?; + ensure!( + stable_host != private_host + || !(route.private_port_start..=route.private_port_end).contains(&stable_port), + "route {target} stable socket overlaps its candidate port range" + ); + + for (other_target, other) in &routes[index + 1..] { + ensure!( + route.route_id != other.route_id, + "route id {} is shared by targets {target} and {other_target}", + route.route_id + ); + ensure!( + route.config_path != other.config_path, + "route fragment {} is shared by targets {target} and {other_target}", + route.config_path.display() + ); + let other_stable = other.stable_socket()?; + ensure!( + route.driver != other.driver || (stable_host, stable_port) != other_stable, + "stable route socket {}:{} is shared by targets {target} and {other_target}", + stable_host, + stable_port + ); + if route.driver != other.driver { + continue; + } + let other_private_host = other + .private_host + .parse::() + .context("validated route private host stopped being an IP address")?; + let ranges_overlap = route.private_host == other.private_host + && route.private_port_start <= other.private_port_end + && other.private_port_start <= route.private_port_end; + ensure!( + !ranges_overlap, + "candidate port ranges overlap for targets {target} and {other_target}" + ); + ensure!( + stable_host != other_private_host + || !(other.private_port_start..=other.private_port_end).contains(&stable_port), + "stable route socket for {target} overlaps {other_target}'s candidate range" + ); + ensure!( + other_stable.0 != private_host + || !(route.private_port_start..=route.private_port_end) + .contains(&other_stable.1), + "stable route socket for {other_target} overlaps {target}'s candidate range" + ); + } + } + Ok(()) +} + +fn resolve_selector( + bindings: &BTreeMap, + selector: &str, +) -> Result> { + let mut targets = if let Some(profile) = selector.strip_prefix("profile:") { + bindings + .values() + .filter(|binding| binding.binding.profiles.contains(profile)) + .map(|binding| binding.binding.target.clone()) + .collect::>() + } else { + ensure!( + bindings.contains_key(selector), + "unknown deployment target {selector}" + ); + vec![selector.to_owned()] + }; + ensure!( + !targets.is_empty(), + "unknown or empty deployment profile {selector}" + ); + targets.sort_by(|left, right| { + (left != "odin") + .cmp(&(right != "odin")) + .then_with(|| left.cmp(right)) + }); + Ok(targets) +} + +fn submit( + store_path: &Path, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + if let Some(parent) = store_path.parent() { + fs::create_dir_all(parent)?; + } + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: format!("up-{}", Uuid::new_v4()), + kind: CommandKind::Deploy, + selector: selector.into(), + requested_by: requested_by.into(), + requested_at_unix_millis: now, + }; + command.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(store_path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, now)?], + )?, + "deployment command id collided" + ); + println!("{}", command.command_id); + if !wait { + return Ok(()); + } + let deadline = now.saturating_add(timeout_seconds.saturating_mul(1000)); + loop { + let snapshot = ControlSnapshot::read(store_path)?; + ensure!( + snapshot + .commands + .iter() + .any(|stored| stored.value.command_id == command.command_id), + "submitted deployment command disappeared" + ); + let transactions = snapshot.transaction_for_command(&command.command_id); + if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { + if let Some(error) = + transactions + .iter() + .find_map(|transaction| match &transaction.completion { + Some(TransactionCompletion::FailedBeforeFencing { error }) + | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => { + Some(error.as_str()) + } + _ => None, + }) + { + bail!("deployment failed: {error}") + } + println!("succeeded {} target(s)", transactions.len()); + return Ok(()); + } + if now_millis()? >= deadline { + bail!("deployment command timed out") + } + thread::sleep(Duration::from_millis(250)); + } +} + +fn status( + out: &mut impl Write, + store_path: &Path, + incident_store: &Path, + command_id: Option<&str>, +) -> Result<()> { + let snapshot = ControlSnapshot::read(store_path)?; + // Commands stay resident; their finished transactions do not. Without + // history a completed command would report as though it had never run, + // which is the operator surface R11 refused to trade away for a smaller + // live set. + let archived = read_history_transactions(store_path); + let archived_commands = read_history_commands(store_path); + let mut commands = snapshot + .commands + .iter() + .map(|stored| &stored.value) + .chain(archived_commands.iter()) + .collect::>(); + commands.sort_by_key(|command| command.requested_at_unix_millis); + if let Some(command_id) = command_id { + commands.retain(|command| command.command_id == command_id); + ensure!(!commands.is_empty(), "deployment command is unknown"); + } + for command in commands { + let mut transactions = snapshot.transaction_for_command(&command.command_id); + transactions.extend( + archived + .iter() + .filter(|value| value.command_id == command.command_id), + ); + transactions.sort_by_key(|value| value.ordinal); + let (state, detail) = derived_command_status(&transactions); + writeln!( + out, + "{} {} {} {}", + command.command_id, command.selector, state, detail + )?; + // Naming one command asks about that command, so print what a stuck + // transaction is actually waiting on. A gate reason lives in + // `last_error` and was never rendered anywhere, which left "Sealing" + // looking identical whether the brake had not authorized the + // transaction or the phase was simply slow. The release and deployment + // ids are here because they are exactly what a brake release must name. + if command_id.is_some() { + for transaction in transactions { + writeln!(out, " transaction {}", transaction.transaction_id)?; + writeln!( + out, + " target {} phase {:?}", + transaction.target, transaction.phase + )?; + if let Some(expected) = &transaction.expected { + writeln!(out, " runtime {}", expected.runtime_id)?; + writeln!(out, " release {}", expected.sealed_release_id)?; + } + writeln!( + out, + " odin publisher cursor {}", + transaction.odin_publisher_sequence_cursor + )?; + if let Some(evidence) = &transaction.latest_odin_observation { + writeln!( + out, + " latest odin observation sequence {}", + evidence.publisher_sequence + )?; + } + if let Some(reason) = &transaction.last_error { + writeln!(out, " waiting on {reason}")?; + } + // A Complete transaction that is not terminal still owns its + // target through unfinished cleanup. Without this an operator + // sees "Complete" and a stale reason and cannot tell what is + // holding the target. + if let Some(completion) = &transaction.completion { + writeln!(out, " completion {completion:?}")?; + } + if let Some(abort) = &transaction.pre_fencing_abort { + writeln!(out, " pre-fencing abort {abort:?}")?; + } + if let Some(abort) = &transaction.post_fencing_abort { + writeln!(out, " post-fencing abort {abort:?}")?; + } + if let Some(cleanup) = &transaction.post_commit_cleanup { + writeln!(out, " post-commit cleanup {cleanup:?}")?; + } + writeln!( + out, + " terminal {} owns-target {}", + transaction.is_terminal(), + transaction.blocks_new_target_mutation() + )?; + } + } + } + // What supervision holds for each target: the union of the admitted + // generations and the metered targets, so a target that has only been + // deployed to (no generation yet) still shows what it has used. + for line in render_targets(&snapshot, incident_store, now_millis()?) { + writeln!(out, "{line}")?; + } + Ok(()) +} + +/// Every supervised target with its meters and its incidents. An incident +/// store that cannot be read costs the incident lines and nothing else. +fn render_targets(snapshot: &ControlSnapshot, incident_store: &Path, now: u64) -> Vec { + let incidents = IncidentStore::new(incident_store).read(); + let mut lines = Vec::new(); + for target in supervised_targets(snapshot) { + lines.extend(render_supervision( + target, + snapshot.admitted_for(target).map(|stored| &stored.value), + snapshot.supervision_for(target).map(|stored| &stored.value), + now, + )); + if let Ok(records) = &incidents { + lines.extend(render_incidents(records, target, now)); + } + } + if let Err(error) = &incidents { + lines.push(format!("incidents unreadable: {}", error.root_cause())); + } + lines +} + +/// Every target supervision holds something for: the admitted generations +/// and the metered targets. +fn supervised_targets(snapshot: &ControlSnapshot) -> BTreeSet<&str> { + snapshot + .admitted + .iter() + .map(|stored| stored.value.target.as_str()) + .chain(snapshot.targets.iter().map(|stored| stored.value.target.as_str())) + .collect() +} + +/// One target's meters and route pacing, one line each, times in unix +/// milliseconds. The counts are what is still inside the window at `now`. +fn render_supervision( + target: &str, + generation: Option<&AdmittedGeneration>, + supervision: Option<&TargetSupervision>, + now: u64, +) -> Vec { + let at = |time: Option| time.map_or("none".to_owned(), |millis| millis.to_string()); + let meters = supervision.cloned().unwrap_or_else(|| TargetSupervision::new(target)); + let mut lines = vec![format!( + "target {target} {}", + generation.map_or("no-admitted-generation", |value| value.generation_id.as_str()) + )]; + if let Some(reason) = generation.and_then(|value| value.last_error.as_deref()) { + lines.push(format!(" held: {reason}")); + } + lines.push(format!( + " continuity attempts-in-hour {} pages-at {} streak {} next-attempt-at {}", + meters.restarts_used(now), + CONTINUITY_PAGING_RESTARTS, + meters.continuity_attempt_streak, + at(generation.and_then(|value| meters.backoff_due(value.admitted_at_unix_millis))) + )); + if let Some(rematerialized) = meters.continuity_rematerialized_at_unix_millis { + lines.push(format!(" continuity rematerialized-at {rematerialized}")); + } + if let (Some(until), Some(reason)) = ( + meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + ) { + lines.push(format!(" continuity deferred until {until}: {reason}")); + } + if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { + lines.push(format!( + " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", + match route.degraded_since_unix_millis { + Some(since) => format!("degraded-since {since}"), + None => "healthy".to_owned(), + }, + route.consecutive_failures, + at(route.last_challenge_at_unix_millis), + at(route.next_challenge_at_unix_millis) + )); + } + if generation.is_none_or(|value| value.route_supervision.is_some()) + || !meters.route_actuations.is_empty() + || !meters.route_survivals.is_empty() + { + lines.push(format!( + " route actuations {}/{} in window, reopens-at {}", + meters.route_used(now), + ROUTE_ACTUATION_CEILING, + at(meters.route_reopens_at(now)) + )); + // A record, not a meter: only the newest are kept, so a saturated + // count says "at least". + let survivals = meters.survivals_in_window(now); + lines.push(format!( + " route survival actuations {}{} in window (uncounted), last-at {}", + if survivals == ROUTE_SURVIVAL_RECORD { "at least " } else { "" }, + survivals, + at(meters.route_survivals.last().copied()) + )); + } + lines +} + +fn derived_command_status(transactions: &[&DeploymentTransaction]) -> (&'static str, String) { + if transactions.is_empty() { + return ("queued", String::new()); + } + // A live transaction is the command's present tense. Reporting an older + // attempt's failure while a new one is sealing hid a running deployment + // behind the word "failed". + let current = transactions + .iter() + .filter(|transaction| !transaction.is_terminal()) + .map(|transaction| format!("{}:{:?}", transaction.target, transaction.phase)) + .collect::>(); + if !current.is_empty() { + return ("running", current.join(",")); + } + if let Some(error) = transactions + .iter() + .find_map(|transaction| match &transaction.completion { + Some(TransactionCompletion::FailedBeforeFencing { error }) + | Some(TransactionCompletion::FailedAfterFencing { error, .. }) => Some(error.clone()), + _ => None, + }) + { + return ("failed", error); + } + if transactions.iter().all(|transaction| { + transaction.is_terminal() + && matches!( + transaction.completion, + Some(TransactionCompletion::Admitted { .. }) + ) + }) { + return ( + "succeeded", + transactions + .iter() + .map(|transaction| transaction.target.as_str()) + .collect::>() + .join(","), + ); + } + ("complete", String::new()) +} + +struct ProcessLock { + file: File, + #[cfg(not(unix))] + path: PathBuf, +} + +impl ProcessLock { + fn acquire(store_path: &Path) -> Result { + let path = sibling_path(store_path, ".daemon.lock"); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + #[cfg(unix)] + { + use std::os::fd::AsRawFd; + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(&path) + .with_context(|| format!("opening Idunn daemon lock {}", path.display()))?; + let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + ensure!(result == 0, "another Idunn daemon owns this control store"); + Ok(Self { file }) + } + #[cfg(not(unix))] + { + let file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .with_context(|| format!("another Idunn daemon owns {}", path.display()))?; + Ok(Self { file, path }) + } + } +} + +impl Drop for ProcessLock { + fn drop(&mut self) { + #[cfg(unix)] + { + use std::os::fd::AsRawFd; + let _ = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; + } + #[cfg(not(unix))] + { + let _ = fs::remove_file(&self.path); + } + } +} + +/// One target's in-memory challenge wait: when it is due, and the stored +/// records of the target as they were when the record failed to land. +struct ChallengeWait { + due_unix_millis: u64, + stamp: Vec, +} + +struct Engine { + options: RuntimeOptions, + idunn_signer: ServiceIdentitySigner, + idunn_anchor: ServiceIdentityTrustAnchor, + bootstrap_odin_authority: AdmittedOdinAuthority, + source: Arc, + docker_runner: DockerRunnerDriver, + systemd_workload: Arc, + host_actuators: Option, + history_report: ReportOnce, + /// One `ReportOnce` per faulting record, so one wedged transaction says + /// its fault once and never drowns another's. + fault_reports: Mutex>, + /// Retry pacing for a transaction whose step keeps failing, by + /// transaction id: consecutive failures and the earliest next attempt. + /// Process-local on purpose: a restart retries at once, and no schema + /// carries a clock that only pacing reads. + resume_backoff: Mutex>, + /// When each routed target's next challenge is due, held only when the + /// stored `RouteSupervisionState` could not record the last one. The stored + /// value owns the schedule and survives restarts; this spaces attempts + /// while the record did not land. An entry ends the moment any write for + /// its target lands (its stamp no longer matches the store) or the target + /// is retired or withdrawn: it never outlives the episode that made it. + challenge_wait: Mutex>, +} + +/// The route driver's door to the actuation ceiling: every group of host +/// mutations a driver makes for `target` is charged through here. The gate +/// owns which charge a request is: a driver asks for the change it is making, +/// and the command that owns the transaction decides whether that change is +/// deployment (`Forward`, refusable) or the continuity of the admitted +/// generation (`Survival`, never refused). +struct EngineRouteGate<'a> { + engine: &'a Engine, + target: &'a str, + command: CommandKind, +} + +impl RouteActuationGate for EngineRouteGate<'_> { + fn admit(&self, requested: RouteActuation) -> Result<()> { + let kind = match self.command { + CommandKind::Deploy => requested, + CommandKind::Continuity => RouteActuation::Survival, + }; + self.engine.charge_route_actuation(self.target, kind) + } +} + +#[derive(Clone, Copy)] +struct ResumeBackoff { + failures: u32, + not_before_unix_millis: u64, +} + +/// Longest wait between attempts at a failing transaction step. +const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; + +impl Engine { + fn open(options: RuntimeOptions) -> Result { + Self::open_with_systemd_workload( + options, + Arc::new(SystemdTransientWorkloadDriver::default()), + ) + } + + /// `open` with the systemd workload port supplied. It is the seam that + /// lets the phase engine run past Fencing without a systemd to talk to. + fn open_with_systemd_workload( + options: RuntimeOptions, + systemd_workload: Arc, + ) -> Result { + let idunn_signer = + open_service_identity_at::(&options.idunn_identity_store) + .context("opening Idunn activation identity")?; + let idunn_anchor = idunn_signer.trust_anchor()?; + let odin_anchor = read_trust_anchor::(&options.odin_trust_anchor) + .context("reading bootstrap Odin topology anchor")?; + let bootstrap_odin_authority = AdmittedOdinAuthority::from_anchor(&odin_anchor)?; + let source: Arc = Arc::new(GitSourceDriver::new( + &options.source_root, + options.staging_root.join("frozen-sources"), + options.source_identity, + )); + let host_actuators = options + .host_actuator_bind + .map(|bind| HostActuatorHub::bind(bind).map(|hub| Arc::new(Mutex::new(hub)))) + .transpose()?; + if let Some(hub) = &host_actuators { + eprintln!( + "Idunn host actuator hub listening on {}", + hub.lock().expect("hub mutex").local_addr()? + ); + let service_signer = + open_service_identity_at::(&options.idunn_identity_store) + .context("opening Idunn identity for the host actuator hub thread")?; + let bindings_dir = options.bindings_dir.clone(); + spawn_hub_service(Arc::clone(hub), service_signer, move || { + host_anchors_from(&bindings_dir) + }); + } + Ok(Self { + options, + idunn_signer, + idunn_anchor, + bootstrap_odin_authority, + source, + docker_runner: DockerRunnerDriver::default(), + systemd_workload, + host_actuators, + history_report: ReportOnce::default(), + fault_reports: Mutex::default(), + resume_backoff: Mutex::default(), + challenge_wait: Mutex::default(), + }) + } + + fn host_anchors(&self) -> BTreeMap { + host_anchors_from(&self.options.bindings_dir) + } + + /// Every route driver comes from here, so the actuator paths are the + /// options' and nothing else's. + fn route_driver(&self, binding: RouteBinding) -> NginxRouteDriver { + NginxRouteDriver::with_actuators(binding, &self.options.route_actuators) + } + + fn route_gate<'a>(&'a self, target: &'a str, command: CommandKind) -> EngineRouteGate<'a> { + EngineRouteGate { + engine: self, + target, + command, + } + } + + /// Charge one route actuation to the target's `TargetSupervision` before + /// the driver acts. The record is created by the first charge, so a first + /// deployment is metered like any other. A `Forward` is counted against + /// the sliding ceiling; a refused one is the typed `RouteActuationRefused`, + /// and one that cannot be recorded is refused too: no charge, no + /// actuation. A `Survival` is never counted or refused, only recorded when + /// it can be, and runs regardless when it cannot: an unwritable store must + /// not stop a rollback, a withdrawal or a repair. + fn charge_route_actuation(&self, target: &str, kind: RouteActuation) -> Result<()> { + match self.record_route_actuation(target, kind) { + Ok(charged) => charged.map_err(anyhow::Error::new), + Err(error) if kind == RouteActuation::Survival => { + eprintln!( + "Idunn could not record a survival route actuation of {target}; it proceeds unrecorded: {error:#}" + ); + Ok(()) + } + Err(error) => Err(error), + } + } + + /// The outer error is the ledger failing to read or write; the inner one + /// is the ceiling refusing. + fn record_route_actuation( + &self, + target: &str, + kind: RouteActuation, + ) -> Result> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let seen = snapshot.supervision_for(target); + let mut next = snapshot.supervision_or_new(target); + let now = now_millis()?; + let charged = next.charge_route(now, kind); + // A refusal writes nothing unless the clock had stepped back and the + // log needed settling. + if charged.is_ok() || seen.is_some_and(|stored| stored.value != next) { + self.write_target_supervision(seen, &next, now)?; + } + Ok(charged) + } + + /// Replace a target's meters, or create them when `seen` is `None`, by + /// compare-exchange against exactly what was read. + fn write_target_supervision( + &self, + seen: Option<&Stored>, + next: &TargetSupervision, + now: u64, + ) -> Result<()> { + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: TargetSupervision::TYPE.into(), + key: next.target.clone(), + current: seen.map(|stored| stored.envelope.clone()), + }], + &[target_supervision_envelope(next, now)?], + )?, + "target supervision changed before its meters were written" + ); + Ok(()) + } + + /// Replace an admitted generation by compare-exchange against the envelope + /// that was read. `Ok(false)` is a lost race: another writer moved the + /// record, so the caller's snapshot is stale. `Err` is a write the store + /// refused, which changed nothing. + fn replace_generation( + &self, + seen: &Stored, + next: &AdmittedGeneration, + now: u64, + ) -> Result { + Ok(SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: seen.value.target.clone(), + current: Some(seen.envelope.clone()), + }], + &[admitted_envelope(next, now)?], + )?) + } + + /// Say a fault once while it lasts, under `key`. + fn report_once(&self, key: &str, message: String) { + let offered = self + .fault_reports + .lock() + .expect("fault report mutex") + .entry(key.to_string()) + .or_default() + .offer(Some(message)); + if let Some(message) = offered { + eprintln!("{message}"); + } + } + + fn incidents(&self) -> IncidentStore { + IncidentStore::new(&self.options.incident_store) + } + + /// Settle the outcome of one incident operation. A failure never leaves + /// this function: it is traced once per episode, under the operation and + /// the subject it was decided for (`*` for a pass over the whole store), + /// and continuity decides exactly as it would have with a healthy store. + /// Each decision site has its own key, so one site's success never ends + /// another site's episode. The message names the operation and the root + /// cause and never a path or a value. Incident writes are never progress. + fn record_incident(&self, operation: &str, subject: &str, outcome: Result) { + let key = format!("incident-store:{operation}:{subject}"); + match outcome { + Ok(_) => self.clear_fault(&key), + Err(error) => self.report_once( + &key, + format!( + "Idunn could not {operation} an operator incident: {}", + error.root_cause() + ), + ), + } + } + + /// Close every open incident whose subject is a target that is no longer + /// admitted, then retire what has been closed past retention. + fn reconcile_incidents(&self, snapshot: &ControlSnapshot) { + let store = self.incidents(); + let close_unadmitted = || -> Result<()> { + let now = now_millis()?; + for record in store.read()? { + if record.is_open() + && record.condition.ends_with_admission() + && snapshot.admitted_for(&record.subject).is_none() + { + store.close( + record.condition, + &record.subject, + CloseReason::NoLongerAdmitted, + now, + )?; + } + } + Ok(()) + }; + self.record_incident("close", "*", close_unadmitted()); + self.record_incident("retire", "*", now_millis().and_then(|now| store.retire_closed(now))); + } + + fn host_access(&self) -> Result> { + let hub = self + .host_actuators + .as_ref() + .context("this Idunn serves no host actuators (no --host-actuator-bind)")?; + Ok(HostActuatorAccess { + hub: Arc::clone(hub), + anchors: self.host_anchors(), + signer: &self.idunn_signer, + }) + } +} + +/// The trust anchor of every host a binding names. A binding whose anchor +/// cannot be read leaves its host unattachable and is logged; the other +/// hosts are unaffected. +fn host_anchors_from(bindings_dir: &Path) -> BTreeMap { + { + let mut anchors = BTreeMap::new(); + let bindings = match load_bindings(bindings_dir) { + Ok(bindings) => bindings, + Err(error) => { + eprintln!("Idunn cannot read bindings for host anchors: {error:#}"); + return anchors; + } + }; + for loaded in bindings.values() { + let WorkloadBinding::HostActuator(workload) = &loaded.binding.workload else { + continue; + }; + match read_trust_anchor::(&workload.host_trust_anchor_store) + { + Ok(anchor) => { + anchors.insert(workload.host.clone(), anchor); + } + Err(error) => eprintln!( + "Idunn cannot read the actuator anchor for host {}: {error:#}", + workload.host + ), + } + } + anchors + } +} + +impl Engine { + /// The runner the plan's binding declares. Every runner in one binding + /// is of one kind; the binding validated that. + fn runner_for(&self, plan: &CompiledDeploymentPlan) -> Result> { + let (_, binding) = plan.parsed_inputs()?; + Ok(match &binding.workload { + WorkloadBinding::SystemdTransient(_) => Box::new(self.docker_runner.clone()), + WorkloadBinding::HostActuator(_) => Box::new(HostActuatorRunnerDriver { + access: self.host_access()?, + }), + }) + } + + /// The workload driver the plan's binding declares. + fn workload_for(&self, plan: &CompiledDeploymentPlan) -> Result> { + let (_, binding) = plan.parsed_inputs()?; + Ok(match &binding.workload { + WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), + WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { + access: self.host_access()?, + }), + }) + } + + /// History for a decision that would be wrong on an empty answer: the + /// continuity restart ceiling and the check that a command is consumed. + /// `None` means `history.cc` cannot be read, so the decision is not made; + /// a history that merely has no entries is `Some`. The fault is printed + /// once while it lasts, not on every tick. + fn history_for_decision(&self) -> Option> { + let (report, transactions) = match read_history(&self.options.state_store) { + Ok(history) => (history.report(), Some(history.transactions)), + Err(error) => ( + Some(format!( + "{error:#}; continuity and command retirement stop until it is readable" + )), + None, + ), + }; + if let Some(text) = self.history_report.offer(report) { + eprintln!("{text}"); + } + transactions + } + + fn topology(&self) -> CultCacheTopologyDriver { + CultCacheTopologyDriver { + projection_store: self.options.topology_store.clone(), + correlation_store: self.options.odin_correlation_store.clone(), + } + } + + fn current_odin_authority(&self, snapshot: &ControlSnapshot) -> Result { + // The bootstrap key stands only until Odin itself is admitted. Once it + // is, its own generation names the authority; an admitted Odin that + // carries none has no authority to name, and falling back to the + // bootstrap key would let a stale key vouch for the Verse. + let authority = match snapshot.admitted_odin() { + None => self.bootstrap_odin_authority.clone(), + Some(stored) => stored.value.odin_authority.clone().with_context(|| { + "admitted Odin generation carries no Odin authority (route-proof readiness)" + })?, + }; + authority.validate()?; + Ok(authority) + } + + fn trusted_topology_context(&self, now: u64) -> OdinTopologyAuthenticationContext { + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: now, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + } + } + + /// Re-prove the evidence that still gates a decision. + /// + /// Terminal transactions are skipped: they describe what already happened + /// and authorize nothing further, so re-proving them only creates ways for + /// history to refuse a boot. On yggdrasil that is 259 of 260 records. + /// + /// The operator anchor is read on demand rather than up front. Reading it + /// unconditionally meant an absent brake artifact refused startup against + /// an *empty* store -- a deployment brake gating Idunn itself, which + /// `F:\Projects\CLAUDE.md` forbids outright: a target brake "may never gate + /// Idunn itself or unrelated service lifecycle". + fn validate_durable_authority(&self, snapshot: &ControlSnapshot) -> Result<()> { + let mut operator_anchor = None; + for stored in &snapshot.transactions { + let transaction = &stored.value; + if transaction.is_terminal() { + continue; + } + if let Some(authorization) = &transaction.deployment_authorization { + authorization.validate_shape()?; + let record: IdunnDeploymentBrakeRecord = + rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + if operator_anchor.is_none() { + operator_anchor = + Some(read_trust_anchor::( + &self.options.deployment_brake_operator_anchor, + )?); + } + verify_idunn_deployment_brake_authorization( + &record, + operator_anchor.as_ref().expect("read directly above"), + )?; + let expected = required(&transaction.expected, "authorized Expected projection")?; + ensure!( + record.authorized_release_id.as_deref() + == Some(expected.sealed_release_id.as_str()) + && record.authorized_deployment_id.as_deref() + == Some(transaction.transaction_id.as_str()) + && record.runtime_id == expected.runtime_id, + "durable deployment authorization names another release or transaction" + ); + } + let lease = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + if let Some(evidence) = &transaction.latest_odin_observation { + let authenticated = self.authenticate_topology_bytes( + snapshot, + transaction, + &evidence.canonical_bytes, + lease, + evidence.admitted_at_unix_millis, + )?; + validate_authenticated_evidence(evidence, &authenticated)?; + } + if let Some(evidence) = &transaction.warming { + match evidence { + WarmingEvidence::OdinTopology { evidence } => { + let authenticated = self.authenticate_topology_bytes( + snapshot, + transaction, + &evidence.canonical_bytes, + None, + evidence.admitted_at_unix_millis, + )?; + validate_authenticated_evidence(evidence, &authenticated)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(snapshot, transaction)?; + ensure!( + is_semantic_warming( + required(&transaction.expected, "Warming Expected projection",)?, + required(&transaction.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?, + "durable Warming gate is not supported by current runtime evidence" + ); + } + WarmingEvidence::FirstOdinDirect { evidence } => { + self.authenticate_first_odin_warming_presence( + transaction, + &evidence.message_id, + evidence.challenged_at_unix_millis, + evidence.admitted_at_unix_millis, + &evidence.canonical_bytes, + )?; + } + WarmingEvidence::RouteProofDirect { evidence } => { + self.reauthenticate_route_proof( + transaction, + evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + evidence.admitted_at_unix_millis, + )?; + } + } + } + if let Some(ReadinessEvidence::RouteProof { evidence }) = &transaction.ready { + self.reauthenticate_route_proof( + transaction, + evidence, + &["active"], + lease, + evidence.admitted_at_unix_millis, + )?; + } + if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) { + let authenticated = self.authenticate_topology_bytes( + snapshot, + transaction, + &evidence.canonical_bytes, + lease, + evidence.admitted_at_unix_millis, + )?; + validate_authenticated_evidence(evidence, &authenticated)?; + ensure!( + is_semantic_ready(&authenticated), + "durable Ready label is not exact semantic Ready" + ); + } + } + let odin_authority = self.current_odin_authority(snapshot)?; + for stored in &snapshot.admitted { + let generation = &stored.value; + // Only a generation whose own Expected says Odin vouches for it has + // Odin receipts to re-prove. One whose evidence disagrees with its + // class is held and reported by supervision, not re-proved here. + if !matches!(generation.readiness(), Ok(class) if class != ReadinessClass::RouteProof) + { + continue; + } + let odin = generation.odin_receipts()?; + let authority = self.runtime_authority_parts( + &generation.plan, + &generation.expected, + &generation.activation, + )?; + let latest = authenticate_odin_runtime_topology_correlation( + &odin.latest.canonical_bytes, + &authority, + generation.leasing.lease_sha256(), + &odin_authority.signer_public_key, + self.trusted_topology_context(odin.latest.admitted_at_unix_millis), + )?; + validate_authenticated_evidence(odin.latest, &latest)?; + let ready = authenticate_odin_runtime_topology_correlation( + &odin.ready.canonical_bytes, + &authority, + generation.leasing.lease_sha256(), + &odin_authority.signer_public_key, + self.trusted_topology_context(odin.ready.admitted_at_unix_millis), + )?; + validate_authenticated_evidence(odin.ready, &ready)?; + ensure!( + is_semantic_ready(&ready), + "admitted generation Ready label is not exact semantic Ready" + ); + } + Ok(()) + } +} + +/// What the boot reconciliation of failed continuity projections found. +#[derive(Debug, Default, PartialEq, Eq)] +struct ProjectionReconciliation { + /// Failed continuity transactions whose exact activation was demoted. + demoted: Vec, + /// Targets whose projection names an activation that neither the admitted + /// generation, a live transaction, nor a failed issuer accounts for. They + /// are reported and left alone. + unexplained: Vec, +} + +impl Engine { + /// Once at boot: demote the activation a failed continuity left standing, + /// where the projection still names exactly the activation that failed + /// transaction issued. A continuity abort before the single resolution + /// rule left such residue (and the lift owes the same demotion to any + /// abort still resident). The issuer is identified by exact activation, + /// from history and from resident terminal records not yet retired to it. + /// + /// Nothing is adopted: an activation with no failed issuer is never + /// demoted or kept on anyone's behalf, only reported. + fn reconcile_failed_continuity_projections(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut outcome = ProjectionReconciliation::default(); + // Resident issuers need no history; only the history issuers wait on it. + let history = self.history_for_decision(); + if history.is_none() { + eprintln!( + "Idunn cannot read history at boot: failed continuities archived there are not \ + reconciled, resident ones still are" + ); + } + let topology = self.topology(); + let issuers = history + .iter() + .flatten() + .chain(snapshot.transactions.iter().map(|stored| &stored.value)) + .filter(|transaction| { + transaction.command_kind == CommandKind::Continuity + && matches!( + transaction.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + | Some(TransactionCompletion::FailedAfterFencing { .. }) + ) + }); + for issuer in issuers { + let (Some(expected), Some(activation), Some(plan)) = + (&issuer.expected, &issuer.activation, &issuer.plan) + else { + continue; + }; + let demoted = (|| -> Result { + if topology.projected_activation(expected)?.as_ref() != Some(activation) { + return Ok(false); + } + let provider_anchor = self.provider_anchor_for_plan(plan)?; + topology.demote_to_expected_only(expected, &provider_anchor, activation, None)?; + Ok(true) + })(); + match demoted { + Ok(true) => outcome.demoted.push(issuer.transaction_id.clone()), + Ok(false) => {} + Err(error) => eprintln!( + "Idunn could not reconcile the projection of failed continuity {}: {error:#}", + issuer.transaction_id + ), + } + } + for generation in &snapshot.admitted { + let projected = match topology.projected_activation(&generation.value.expected) { + Ok(projected) => projected, + Err(error) => { + eprintln!( + "Idunn cannot read the projected activation of {}: {error:#}", + generation.value.target + ); + continue; + } + }; + let Some(projected) = projected else { + continue; + }; + let accounted = projected == generation.value.activation + || snapshot.transactions.iter().any(|stored| { + stored.value.target == generation.value.target + && !stored.value.is_terminal() + && stored.value.activation.as_ref() == Some(&projected) + }); + if !accounted { + outcome.unexplained.push(generation.value.target.clone()); + } + } + Ok(outcome) + } +} + +/// Everything the daemon does once, before its loop: lock, migrate, validate, +/// reconcile the projection. The lock is returned so the caller holds it for as +/// long as the engine runs. +fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { + for path in [ + &options.state_store, + &options.topology_store, + &options.staging_root, + ] { + let directory = if path.extension().is_some() { + path.parent() + .context("configured Idunn path has no parent")? + } else { + path.as_path() + }; + fs::create_dir_all(directory) + .with_context(|| format!("creating Idunn directory {}", directory.display()))?; + } + let lock = ProcessLock::acquire(&options.state_store)?; + let migrated = migrate_control_store_to_current_schema(&options.state_store) + .context("migrating Idunn control records to the current schema")?; + if migrated > 0 { + println!( + "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA}" + ); + } + ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; + let engine = Engine::open(options)?; + engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; + match engine.reconcile_failed_continuity_projections() { + Ok(outcome) => { + for transaction_id in &outcome.demoted { + println!("demoted the activation failed continuity {transaction_id} left projected"); + } + for target in &outcome.unexplained { + eprintln!( + "Idunn found an activation projected for {target} that no admitted generation, \ + live transaction or failed continuity accounts for; left as it is" + ); + } + } + Err(error) => eprintln!("Idunn boot projection reconciliation failed: {error:#}"), + } + Ok((lock, engine)) +} + +fn serve(options: RuntimeOptions) -> Result<()> { + let (_lock, engine) = boot(options)?; + serve_while(&engine, || true) +} + +/// The daemon's loop: tick, sleep, and go again while `again` says so. `serve` +/// says so forever; a test says so a few times. +fn serve_while(engine: &Engine, mut again: impl FnMut() -> bool) -> Result<()> { + while again() { + match engine.run_scheduler_tick() { + Ok(true) => continue, + Ok(false) => {} + // A fault in one transaction is that transaction's problem. Idunn + // runs under Restart=always, so propagating it here turns a single + // unreadable record into a crashloop that takes every unrelated + // target's continuity down with it -- the daemon-survival organ + // killed by the thing it exists to survive. Log against the tick + // and keep going; the fault recurs every poll until it is fixed, + // which is louder than a restart loop and cheaper than an outage. + Err(error) => eprintln!("Idunn scheduler tick failed: {error:#}"), + } + thread::sleep(Duration::from_millis(engine.options.poll_millis)); + } + Ok(()) +} + +impl Engine { + /// One resident terminal transaction goes to history per tick. + /// + /// Transactions that finished before finished transactions travelled to + /// history are still resident with their commands, gating nothing and + /// consuming their commands correctly, but growing the store every decision + /// reads. They leave the same way a transaction finishing today does. + fn retire_one_terminal_transaction(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + // A record that cannot be archived (history unreadable, a torn write) + // is that record's fault: the next terminal record is still tried, and + // the tick goes on to supervise and freeze. + for stored in snapshot + .transactions + .iter() + .filter(|stored| stored.value.is_terminal()) + { + match archive_terminal_transaction(&self.options.state_store, &stored.envelope) { + Ok(()) => { + eprintln!( + "Idunn retired resident terminal transaction {} to history", + stored.value.transaction_id + ); + return Ok(true); + } + Err(error) => self.note_fault( + "could not retire terminal transaction", + &stored.value.transaction_id, + &error, + ), + } + } + Ok(false) + } + + /// One pass of the scheduler. An unreadable control store is the tick's + /// fault and is returned; anything one transaction or one admitted + /// generation does wrong is logged against it and never stops the others. + fn run_scheduler_tick(&self) -> Result { + if self.retire_one_terminal_transaction()? { + return Ok(true); + } + let transaction_progress = self.resume_one_transaction()?; + let continuity_progress = self.supervise_one_admitted_generation()?; + // Freezing is not gated on the others' progress: a queued command + // whose target is free must not wait behind an unrelated target's + // wedge. `freeze_command` already refuses a busy target, so this is + // still at most one freeze per tick and never a second claimant. + let froze = self.freeze_one_queued_command()?; + Ok(transaction_progress || continuity_progress || froze) + } + + /// Say a per-record fault once while it lasts and leave it in the + /// record's `last_error`, which status renders. Recording is best effort: + /// a store too broken to take the note is already in stderr. + fn note_fault(&self, what: &str, transaction_id: &str, error: &anyhow::Error) { + let detail = truncate(&format!("{error:#}"), 2048); + let offered = { + let mut reports = self.fault_reports.lock().expect("fault report mutex"); + reports + .entry(transaction_id.to_string()) + .or_default() + .offer(Some(detail.clone())) + }; + let recorded = self.record_last_error(transaction_id, &detail); + if offered.is_some() { + eprintln!("Idunn {what} {transaction_id}: {detail}"); + if let Err(error) = recorded { + eprintln!("Idunn could not leave that in the record of {transaction_id}: {error:#}"); + } + } + } + + fn clear_fault(&self, transaction_id: &str) { + self.fault_reports + .lock() + .expect("fault report mutex") + .remove(transaction_id); + } + + /// Write `detail` as the record's `last_error`, touching nothing else and + /// not archiving: a terminal record whose archive is the fault stays put. + fn record_last_error(&self, transaction_id: &str, detail: &str) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let Some(stored) = snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == transaction_id) + else { + return Ok(()); + }; + if stored.value.last_error.as_deref() == Some(detail) { + return Ok(()); + } + let envelope = cleanup_evidence::last_error_envelope(&stored.value, detail)?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction_id.to_string(), + current: Some(stored.envelope.clone()), + }], + &[envelope], + )?, + "transaction changed before its fault note" + ); + Ok(()) + } + + /// Startup and every later loop use the same order: unfinished ownership + /// work first, admitted-body continuity second, new commands last. A + /// waiting transaction yields without relaxing ordinal order inside its + /// own command, so one target brake cannot suspend unrelated continuity. + fn resume_one_transaction(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; + let mut candidates = snapshot + .transactions + .iter() + .filter(|stored| !stored.value.is_terminal()) + .collect::>(); + candidates.sort_by_key(|stored| { + ( + stored.value.created_at_unix_millis, + stored.value.ordinal, + stored.value.transaction_id.as_str(), + ) + }); + for current in candidates { + if snapshot.has_earlier_authority_sibling(¤t.value) { + continue; + } + match self.resume_candidate(current) { + Ok(moved) => progressed |= moved, + Err(error) => { + self.note_fault( + "could not resume transaction", + ¤t.value.transaction_id, + &error, + ); + } + } + } + Ok(progressed) + } + + /// Advance one transaction, turning its failure into the durable record + /// its phase calls for. Returns whether its record changed. + /// + /// "Moved" means the transaction's state advanced: a phase change, a + /// completion, or a durable abort step. Rewriting `last_error` to say the + /// same step failed again in new words is not movement, or a wedged + /// transaction would report progress every tick. + fn resume_candidate(&self, current: &Stored) -> Result { + let id = current.value.transaction_id.as_str(); + if let Some(moved) = self.resolve_phase_deadline(current)? { + return Ok(moved); + } + if self + .resume_backoff + .lock() + .expect("resume backoff mutex") + .get(id) + .is_some_and(|backoff| { + now_millis().is_ok_and(|now| { + is_waiting(now, backoff.not_before_unix_millis, RESUME_BACKOFF_CEILING_MILLIS) + }) + }) + { + return Ok(false); + } + if let Some(disagreement) = current.value.held_disagreement() { + self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); + return Ok(false); + } + let advanced = self.advance_transaction(current); + if advanced.is_ok() { + // The step ran and succeeded: the fault, if any, has recovered, so + // a later failure is news. A skipped or failed attempt clears nothing. + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + self.clear_fault(id); + } + if let Err(error) = advanced { + let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; + let latest = latest_snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared while recording an execution error")?; + if latest.value.is_terminal() { + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + return Ok(true); + } + self.back_off(id)?; + if latest.value.phase < DeploymentPhase::Fencing { + // Before the fence an abort that cannot finish a step is + // resumable, never a post-fence abort: that path refuses a + // pre-fence phase, so choosing it only wedged the tick. + // + // A publish that lost to a held lock is not a failed step: the + // holder lets go, so the transaction stays in its phase with + // the error recorded and the backoff above retries it. The + // phase deadline, not an abort, bounds the wait. + if latest.value.pre_fencing_abort.is_none() + && error.downcast_ref::().is_none() + { + self.begin_pre_fencing_abort(latest, error)?; + } else { + self.record_resumable_error(latest, &error)?; + } + } else if latest.value.post_fencing_abort.is_none() + && self.candidate_is_permanently_stopped(&latest.value)? + { + // Past the fence an error is resumable while the candidate + // can still recover. This one cannot: its transient unit + // has failed and carries Restart=no, so retrying would hold + // the target forever behind a transaction that can never + // finish. + self.begin_post_fencing_abort(latest, error)?; + } else { + // Resumable only until the phase deadline: the resolver ends + // the phase before the next step once it has passed. + self.record_resumable_error(latest, &error)?; + } + } + let after = ControlSnapshot::read(&self.options.state_store)?; + let live = after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared while checking scheduler progress")?; + Ok(state_advanced(¤t.value, &live.value)) + } + + /// Wait twice as long after each consecutive failure of a transaction's + /// step, from one poll interval up to a ceiling. + fn back_off(&self, transaction_id: &str) -> Result<()> { + let now = now_millis()?; + let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); + let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); + let wait = backoff_wait(self.options.poll_millis, failures, RESUME_BACKOFF_CEILING_MILLIS); + all.insert( + transaction_id.to_string(), + ResumeBackoff { + failures: failures.saturating_add(1), + not_before_unix_millis: now.saturating_add(wait), + }, + ); + Ok(()) + } + + fn freeze_one_queued_command(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let live_commands = snapshot + .transactions + .iter() + .map(|stored| stored.value.command_id.clone()) + .collect::>(); + let mut candidates = snapshot + .commands + .iter() + .filter(|stored| { + stored.value.kind == CommandKind::Deploy + && !live_commands.contains(&stored.value.command_id) + }) + .collect::>(); + if candidates.is_empty() { + return Ok(false); + } + // A command is consumed by any transaction, live or retired. Commands + // whose transactions were retired to history before commands travelled + // with them are still resident; they are not queued, they are + // history that never moved. Retire them here and never freeze them. + // With history unreadable, whether a command was consumed is unknown: + // freezing it again could run a finished deployment twice. + let Some(history) = self.history_for_decision() else { + return Ok(false); + }; + let historical_commands = history + .into_iter() + .map(|transaction| transaction.command_id) + .collect::>(); + let mut retired = false; + candidates.retain(|stored| { + if !historical_commands.contains(&stored.value.command_id) { + return true; + } + match SingleFileMessagePackBackingStore::new(&history_store_path( + &self.options.state_store, + )) + .insert_entry_if_absent(stored.envelope.clone()) + .and_then(|_| { + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .delete_batch_if_unchanged(std::slice::from_ref(&stored.envelope)) + }) { + Ok(_) => { + eprintln!( + "Idunn retired consumed command {}: its transactions are history", + stored.value.command_id + ); + retired = true; + } + Err(error) => eprintln!( + "Idunn left consumed command {} resident: {error:#}", + stored.value.command_id + ), + } + false + }); + if retired { + return Ok(true); + } + candidates.sort_by_key(|stored| stored.value.requested_at_unix_millis); + // Oldest first, but a command whose target is busy does not block the + // ones behind it for other targets. + for command in candidates { + if self.freeze_command(&snapshot, command)? { + return Ok(true); + } + } + Ok(false) + } + + /// Freeze one queued command into its transactions. `Ok(false)` means its + /// target is busy and nothing was written. + fn freeze_command( + &self, + snapshot: &ControlSnapshot, + command: &Stored, + ) -> Result { + let bindings = match load_bindings(&self.options.bindings_dir) { + Ok(bindings) => bindings, + Err(error) => { + eprintln!( + "Idunn left command {} queued because operator bindings are invalid: {error:#}", + command.value.command_id + ); + return Ok(false); + } + }; + let targets = match resolve_selector(&bindings, &command.value.selector) { + Ok(targets) => targets, + Err(error) => { + let now = now_millis()?; + let rejected = DeploymentTransaction::rejected(&command.value, error, now)?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: rejected.transaction_id.clone(), + current: None, + }], + &[transaction_envelope(&rejected, now)?], + )?, + "bad selector changed before refusal was recorded" + ); + return Ok(true); + } + }; + let busy = snapshot + .transactions + .iter() + .filter(|stored| stored.value.blocks_new_target_mutation()) + .map(|stored| stored.value.target.as_str()) + .collect::>(); + if targets.iter().any(|target| busy.contains(target.as_str())) { + return Ok(false); + } + let now = now_millis()?; + let transactions = targets + .into_iter() + .enumerate() + .map(|(ordinal, target)| { + DeploymentTransaction::new( + &command.value, + target.clone(), + u32::try_from(ordinal)?, + snapshot.admitted_for(&target).map(|stored| &stored.value), + now, + ) + }) + .collect::>>()?; + let expected = transactions + .iter() + .map(|transaction| CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }) + .collect::>(); + let next = transactions + .iter() + .map(|transaction| transaction_envelope(transaction, now)) + .collect::>>()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange(&expected, &next)?, + "queued command lost its ordered transaction creation CAS" + ); + Ok(true) + } + + fn supervise_one_admitted_generation(&self) -> Result { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + // A wait belongs to the failed record it spaces. Once any write for its + // target has landed, or the target is no longer admitted, the stored + // schedule owns the target again. + self.challenge_wait + .lock() + .expect("challenge wait mutex") + .retain(|target, wait| snapshot.target_stamp(target) == wait.stamp); + self.reconcile_incidents(&snapshot); + let mut progressed = false; + let mut admitted = snapshot.admitted.iter().collect::>(); + admitted.sort_by_key(|stored| stored.value.target.as_str()); + for current in admitted { + if let Err(disagreement) = current.value.readiness() { + self.note_fault( + "holds admitted generation", + &format!("generation:{}", current.value.target), + &anyhow::Error::new(disagreement), + ); + } + let blocker = snapshot.transactions.iter().find(|stored| { + stored.value.target == current.value.target + && stored.value.blocks_new_target_mutation() + }); + if blocker.is_some_and(|stored| { + let owns_from = if stored.value.rollout_stops_incumbent_first() { + // The incumbent is stopped at Starting by design; a + // continuity restart in that window would fight the + // candidate for the same host resources. + DeploymentPhase::Starting + } else { + DeploymentPhase::Fencing + }; + stored.value.phase >= owns_from && stored.value.phase < DeploymentPhase::Complete + }) { + continue; + } + match self.supervise_admitted_route(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + } + match self.refresh_admitted_topology(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn preserved admitted {} after rejecting topology observation: {error:#}", + current.value.target + ), + } + match self.retire_stale_incarnations(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn preserved a stale {} incarnation projection: {error:#}", + current.value.target + ), + } + // Restore the admitted Expected before anything is asked of the + // workload, and only the Expected. + // + // A target that cannot see its own Expected cannot start, and an + // aborted deployment leaves exactly that gap, so this cannot wait + // for an observation the dead process will never provide. The + // activation is deliberately not restored: it names one running + // incarnation, every restart issues a fresh one, and republishing + // the admitted one leaves the projection naming an incarnation that + // no longer exists -- which the workload rightly refuses to adopt. + // That record is published from observation once the process is up. + if blocker.is_none() + || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) + { + let repair = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.admitted_expected_projection_is_exact( + ¤t.value.expected, + &provider_anchor, + )? { + ensure!( + topology.publish_expected(¤t.value.expected, &provider_anchor)? + == current.value.expected.canonical_sha256()?, + "admitted Expected projection repair differs" + ); + } + Ok(()) + })(); + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); + } + } + let mut operational_error = None; + let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { + workload.observe( + ¤t.value.expected, + ¤t.value.activation, + ¤t.value.workload, + ) + }) { + Ok(observation) => { + let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { + let lease_health = (|| -> Result { + let lease_path = current + .value + .plan + .parsed_inputs()? + .1 + .process_write_lease + .context("admitted lease has no operator binding")? + .record_path; + let driver = + CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + if driver.observe_exact(lease)? { + return Ok(true); + } + if driver.observe_empty()? { + return Ok(false); + } + bail!("write-lease store contains unexpected authority") + })(); + match lease_health { + Ok(true) => false, + Ok(false) => { + operational_error = + Some(anyhow!("admitted physical write lease is missing")); + true + } + Err(error) => { + eprintln!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target + ); + continue; + } + } + } else { + false + }; + if lease_is_missing { + None + } else { + Some(observation) + } + } + Err(error) if error.downcast_ref::().is_some() => { + // The host cannot be asked right now (its actuator is + // between sessions, or Idunn just restarted). That is + // silence, not a death; counting it as one burned every + // continuity attempt in the reattach window on + // 2026-09-11 and left a healthy Muninn demoted. + eprintln!( + "Idunn cannot observe admitted {} this tick: {error:#}", + current.value.target + ); + continue; + } + Err(error) => { + operational_error = Some(error); + None + } + }; + if observation.is_some() { + let seen = snapshot.supervision_for(¤t.value.target); + if let Some(stored) = seen.filter(|stored| { + stored.value.continuity_backoff_from_unix_millis.is_some() + }) { + let mut next = stored.value.clone(); + next.end_backoff(); + self.write_target_supervision(Some(stored), &next, now_millis()?)?; + } + // The admitted workload was seen running: the only evidence + // that ends an exhaustion incident, once the window no longer holds it. + // The window sliding alone is not health and never closes one. + self.record_incident( + "close", + ¤t.value.target, + now_millis().and_then(|now| { + // Running is not enough while the window still holds + // the restarts that exhausted it: a release that + // survives one poll after each restart is still the + // same incident. + if snapshot + .supervision_or_new(¤t.value.target) + .restarts_page(now) + { + return Ok(false); + } + self.incidents().close( + IncidentCondition::ContinuityExhausted, + ¤t.value.target, + CloseReason::Recovered, + now, + ) + }), + ); + continue; + } + let workload_error = operational_error + .context("admitted operational state has neither observation nor error")?; + // A held generation is a record the recipe never declared + // readiness for. Idunn reports it and does not decide for it: a + // continuity minted over it copies the undeclared Expected, is + // held at once, and owns the target forever, so the declaring + // redeploy that would clear the hold could never freeze. The dead + // held target stays down and free. + if current.value.readiness().is_err() { + self.note_fault( + "holds a dead admitted generation down; declare readiness in its recipe and redeploy", + &format!("generation-down:{}", current.value.target), + &anyhow!("admitted generation {} is not running", current.value.generation_id), + ); + continue; + } + // Continuity restarts a release; it cannot repair one, and it + // never stops (budget-never-stop). Every charged attempt is spaced + // by a doubling, capped wait, so a queued deployment always gets + // a free gap (map B10). The log outlives every generation and only + // pages. Nothing here reads history: an unreadable history file + // must not stop crash recovery. + let now = now_millis()?; + let supervision = snapshot.supervision_or_new(¤t.value.target); + // A stepped-back clock leaves entries in the future. Settle them + // before deciding from them; the write ends this target's pass. + let mut settled = supervision.clone(); + settled.settle(now); + if settled != supervision { + self.write_target_supervision( + snapshot.supervision_for(¤t.value.target), + &settled, + now, + )?; + progressed = true; + continue; + } + let continuity_key = format!("continuity:{}", current.value.target); + + // Continuity defers to any live transaction for its target and + // never aborts one (handover-phase-split). + if blocker.is_some() { + continue; + } + if supervision.restarts_page(now) { + // The record comes first and the trace after it: the trace is + // this record's shadow, never its substitute. + self.record_incident( + "open", + ¤t.value.target, + self.incidents().open( + IncidentCondition::ContinuityExhausted, + ¤t.value.target, + now, + ), + ); + self.report_once( + &continuity_key, + format!( + "Idunn has restarted admitted {} {} times inside the hour and keeps restarting it on a capped backoff (streak {}): {workload_error:#}", + current.value.target, + supervision.restarts_used(now), + supervision.continuity_attempt_streak, + ), + ); + } else { + self.clear_fault(&continuity_key); + } + if supervision.continuity_deferred(now) { + continue; + } + // A unit the host lost to a reboot is re-materialized once per + // generation, uncharged and with no wait; the driver owns the + // evidence and supervision owns the budget. An unanswerable + // driver is not evidence of a reboot. + let admitted_at = current.value.admitted_at_unix_millis; + let rematerialize = !supervision.rematerialization_spent(admitted_at) + && match self.workload_for(¤t.value.plan).and_then(|workload| { + workload.lost_to_host_reboot(¤t.value.workload, admitted_at) + }) { + Ok(lost) => lost, + Err(error) => { + self.note_fault( + "could not tell whether the host rebooted under", + &format!("reboot:{}", current.value.target), + &error, + ); + false + } + }; + let mut streak = 0; + if !rematerialize { + match supervision.continuity_backoff(now, admitted_at) { + ContinuityBackoff::Waiting => continue, + ContinuityBackoff::Begin => { + let mut began = supervision.clone(); + began.begin_backoff(now); + self.write_target_supervision( + snapshot.supervision_for(¤t.value.target), + &began, + now, + )?; + progressed = true; + continue; + } + ContinuityBackoff::Due { streak: due } => streak = due, + } + } + + // The projected activation names an incarnation that is gone. Every + // restart is issued a fresh one, so leaving the old record standing + // makes the projection describe a process that no longer exists -- + // and a workload that reads it refuses to adopt an activation that + // is not its own, which is a restart loop rather than a recovery. + // Demote to Expected-only, the same shape an aborted deployment + // leaves, and let the restart publish its own activation once it is + // observed. + let demotion = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.projected_activation_is_present(¤t.value.expected)? { + return Ok(()); + } + topology.demote_to_expected_only( + ¤t.value.expected, + &provider_anchor, + ¤t.value.activation, + current.value.leasing.lease(), + )?; + Ok(()) + })(); + // Expected-only is the precondition of a restart: a continuity + // minted over a projection that still names an activation would + // fail between preparing and publishing its own, and its abort + // could not resolve. So a failed demotion mints nothing. The + // failure is written to the generation, and the next attempt is + // not before the deferral ends. + if let Err(error) = demotion { + eprintln!( + "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", + current.value.target + ); + let mut deferred = supervision.clone(); + deferred.defer_continuity(now, &truncate(&format!("{error:#}"), 2048)); + self.write_target_supervision( + snapshot.supervision_for(¤t.value.target), + &deferred, + now, + )?; + progressed = true; + continue; + } + // An engaged lifecycle brake means no continuity transaction is + // minted at all. A transaction that exists and waits on the brake + // owns the target, and a target owned by a parked restart accepts + // no deployment -- which is the one thing a lifecycle brake must + // never gate. The Sealing-phase check remains as the guard for a + // transaction minted just before the brake was engaged. + if !self.lifecycle_allows_generation(current, now)? { + continue; + } + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: format!("continuity-{}", Uuid::new_v4()), + kind: CommandKind::Continuity, + selector: current.value.target.clone(), + requested_by: "idunn-continuity".into(), + requested_at_unix_millis: now, + }; + command.validate()?; + let transaction = + DeploymentTransaction::from_continuity(&command, ¤t.value, now)?; + // The attempt is charged, or recorded as the one re-materialization, + // in the same CAS that mints it, so a crash cannot schedule one the + // meters never saw. + let mut counted = supervision; + if rematerialize { + counted.record_rematerialization(now); + } else { + counted.record_restart(now, streak); + } + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: TargetSupervision::TYPE.into(), + key: counted.target.clone(), + current: snapshot + .supervision_for(&counted.target) + .map(|stored| stored.envelope.clone()), + }, + ], + &[ + command_envelope(&command, now)?, + transaction_envelope(&transaction, now)?, + target_supervision_envelope(&counted, now)?, + ], + )?, + "continuity scheduling lost its command/transaction/meter CAS" + ); + progressed = true; + } + Ok(progressed) + } + + fn supervise_admitted_route( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + ) -> Result { + let Some(expected_route) = current.value.expected.route.as_ref() else { + ensure!( + matches!(¤t.value.routing, RoutingEvidence::SkippedUnrouted), + "unrouted admitted generation carries route authority" + ); + return Ok(false); + }; + let RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + } = ¤t.value.routing + else { + bail!("routed admitted generation has no promoted route receipt") + }; + let now = now_millis()?; + // A route whose proofs keep failing is challenged on a widening + // schedule, not every tick. + if current + .value + .route_supervision + .as_ref() + .is_some_and(|state| state.is_waiting(now, self.options.topology_maximum_age_millis)) + { + return Ok(false); + } + if self + .challenge_wait + .lock() + .expect("challenge wait mutex") + .get(¤t.value.target) + .is_some_and(|wait| { + is_waiting( + now, + wait.due_unix_millis, + ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS.max(self.options.topology_maximum_age_millis), + ) + }) + { + return Ok(false); + } + ensure!( + observation.route_id == expected_route.route_id + && observation.runtime_instance_id == current.value.activation.runtime_instance_id, + "admitted route receipt names another incarnation" + ); + let binding = current.value.plan.parsed_inputs()?.1; + let lease_driver = if let Some(lease) = current.value.leasing.lease() { + let lease_path = &binding + .process_write_lease + .as_ref() + .context("stateful admitted generation has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + ensure!( + driver.observe_exact(lease)?, + "admitted process write lease is no longer exact" + ); + Some(driver) + } else { + None + }; + let route_binding = binding + .route + .context("routed admitted generation has no operator route binding")?; + let driver = self.route_driver(route_binding); + let gate = self.route_gate(¤t.value.target, CommandKind::Continuity); + + if driver.observe_membership(¤t.value.expected, &observation.membership_sha256)? + && route_observation_is_current( + observation.observed_at_unix_millis, + now, + self.options.topology_maximum_age_millis, + self.options.topology_maximum_future_skew_millis, + ) + { + return Ok(false); + } + let route_key = format!("route:{}", current.value.target); + // A failed repair or a failed proof changes observation state only: it + // marks the route degraded and widens the wait. That record is written + // like a proved one: when it lands the pass ends (`Ok(true)`), because + // the caller's snapshot no longer matches the generation; when the store + // refuses it the pass reports no progress and the in-memory wait below + // spaces the next attempt. + let challenge = (|| -> Result { + // Actuates only when the fragment on disk differs from the + // admitted membership; an exact fragment makes this a no-op. It is + // survival: restoring the admitted route is never refused. + driver.restore_admitted_membership( + ¤t.value.expected, + &observation.membership_sha256, + &gate, + )?; + let authority = self.runtime_authority_parts( + ¤t.value.plan, + ¤t.value.expected, + ¤t.value.activation, + )?; + let refreshed = self.prove_stable_route_against( + ¤t.value.expected, + ¤t.value.activation, + &authority, + current.value.leasing.lease_sha256(), + &driver, + observation.membership_sha256.clone(), + )?; + ensure!( + driver.observe_membership(¤t.value.expected, &refreshed.membership_sha256)?, + "admitted route membership changed during its continuity challenge" + ); + if let (Some(lease_driver), Some(lease)) = (&lease_driver, current.value.leasing.lease()) + { + ensure!( + lease_driver.observe_exact(lease)?, + "admitted process write lease changed during its continuity challenge" + ); + } + Ok(refreshed) + })(); + let mut next = current.value.clone(); + // The wait the record carries, also held in memory until a record lands. + let (recorded, wait_until) = match challenge { + Ok(refreshed) => { + next.routing = RoutingEvidence::Promoted { + observation: refreshed, + promoted_at_unix_millis: *promoted_at_unix_millis, + }; + if let Some(state) = next.route_supervision.as_mut() { + state.record_proved_challenge(now); + } + self.clear_fault(&route_key); + ( + "proved", + now.saturating_add(self.options.topology_maximum_age_millis), + ) + } + Err(error) => { + if let Some(state) = next.route_supervision.as_mut() { + state.record_failed_challenge(now, self.options.topology_maximum_age_millis); + } + self.report_once( + &route_key, + format!( + "Idunn could not prove admitted {} on its stable route: {error:#}", + current.value.target + ), + ); + let due = next + .route_supervision + .as_ref() + .and_then(|state| state.next_challenge_at_unix_millis) + .unwrap_or(now); + ("failed", due) + } + }; + // Progress is a store that changed: our record landed, or another + // writer beat it (the snapshot is stale either way, so the pass ends). + // A write the store refused changed nothing: no progress, the scheduler + // sleeps if nothing else moved, and the in-memory wait spaces the next + // challenge. + let record_key = format!("route-record:{}", current.value.target); + match self.replace_generation(current, &next, now) { + Ok(_) => { + self.challenge_wait + .lock() + .expect("challenge wait mutex") + .remove(¤t.value.target); + self.clear_fault(&record_key); + Ok(true) + } + Err(record) => { + self.challenge_wait + .lock() + .expect("challenge wait mutex") + .insert( + current.value.target.clone(), + ChallengeWait { + due_unix_millis: wait_until, + stamp: snapshot.target_stamp(¤t.value.target), + }, + ); + self.report_once( + &record_key, + format!( + "Idunn could not record the {recorded} route challenge of {}: {}", + current.value.target, + // The root cause only: the error chain names a fresh temp file each + // attempt, which would defeat once-per-fault. + record.root_cause() + ), + ); + Ok(false) + } + } + } + + fn refresh_admitted_topology( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + ) -> Result { + if !matches!(current.value.readiness(), Ok(class) if class != ReadinessClass::RouteProof) { + return Ok(false); + } + let odin = current.value.odin_receipts()?; + let Some(received) = self.topology().receive( + ¤t.value.target, + ¤t.value.expected.canonical_sha256()?, + )? + else { + return Ok(false); + }; + ensure!( + received.target == current.value.target, + "topology transport substituted admitted target" + ); + let now = now_millis()?; + let authority = self.runtime_authority_parts( + ¤t.value.plan, + ¤t.value.expected, + ¤t.value.activation, + )?; + let odin_authority = self.current_odin_authority(snapshot)?; + let authenticated = match authenticate_odin_runtime_topology_correlation( + &received.canonical_bytes, + &authority, + current.value.leasing.lease_sha256(), + &odin_authority.signer_public_key, + self.trusted_topology_context(now), + ) { + Ok(authenticated) => authenticated, + // An aged-out correlation is nothing new about the admitted + // generation, not a rejected observation. + Err(error) if is_stale_observation(&error) => return Ok(false), + Err(error) => return Err(error), + }; + let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; + if !sequence_requires_admission( + Some(odin.latest), + snapshot.max_odin_sequence(¤t.value.target, &evidence.signer_identity_id), + &evidence, + )? { + return Ok(false); + } + let mut next = current.value.clone(); + next.latest_odin_observation = Some(evidence.clone()); + next.odin_publisher_sequence_cursor = evidence.publisher_sequence; + if is_semantic_ready(&authenticated) { + next.ready = ReadinessEvidence::OdinCorrelated { evidence }; + } + next.validate()?; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: current.value.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&next, now)?], + )?, + "admitted generation changed before topology sequence CAS" + ); + Ok(true) + } + + fn advance_transaction(&self, current: &Stored) -> Result<()> { + if current.value.post_fencing_abort.is_some() && current.value.completion.is_none() { + return self.advance_post_fencing_abort(current); + } + if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() { + return self.advance_pre_fencing_abort(current); + } + match current.value.phase { + DeploymentPhase::Sealing => self.advance_sealing(current), + DeploymentPhase::Starting => self.advance_starting(current), + DeploymentPhase::Warming => self.advance_warming(current), + DeploymentPhase::Fencing => self.advance_fencing(current), + DeploymentPhase::Leasing => self.advance_leasing(current), + DeploymentPhase::AwaitingReady => self.advance_awaiting_ready(current), + DeploymentPhase::Routing => self.advance_routing(current), + DeploymentPhase::Committing => self.advance_committing(current), + DeploymentPhase::Complete => self.advance_post_commit_cleanup(current), + } + } + + fn advance_sealing(&self, current: &Stored) -> Result<()> { + if current.value.plan.is_none() { + ensure!( + current.value.command_kind == CommandKind::Deploy, + "continuity transaction lost its admitted plan" + ); + let bindings = load_bindings(&self.options.bindings_dir)?; + let loaded = bindings + .get(¤t.value.target) + .context("operator binding disappeared before sealing")?; + let now = now_millis()?; + let resolved = + self.source + .resolve(&loaded.binding, ¤t.value.transaction_id, now)?; + let provider_snapshot = ControlSnapshot::read(&self.options.state_store)?; + let providers = self.current_ready_provider_tokens(&provider_snapshot)?; + let candidate_port = self.select_candidate_port(&loaded.binding)?; + let plan = compile_deployment_plan( + &resolved.recipe_bytes, + &loaded.bytes, + resolved.facts, + format!("incarnation-{}", current.value.transaction_id), + candidate_port, + now, + &providers, + )?; + // Idunn infers no way to prove readiness. A recipe that declares + // none is refused here, before anything is frozen, built or installed. + plan.readiness_class()?; + return self.persist_same_phase(current, |next| { + next.plan = Some(plan); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + + if current.value.command_kind == CommandKind::Deploy + && current.value.frozen_source.is_none() + { + let now = now_millis()?; + let plan = required(¤t.value.plan, "transaction plan")?; + let receipt = self.source.freeze(¤t.value.transaction_id, plan)?; + return self.persist_same_phase(current, |next| { + next.frozen_source = Some(receipt); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + + if current.value.sealed_release.is_none() + || current.value.installed_release.is_none() + || current.value.expected.is_none() + { + ensure!( + current.value.command_kind == CommandKind::Deploy, + "continuity transaction lost its admitted release evidence" + ); + let now = now_millis()?; + let plan = required(¤t.value.plan, "transaction plan")?; + let frozen_receipt = required(¤t.value.frozen_source, "frozen source")?; + let frozen = self.source.observe_frozen(plan, frozen_receipt)?; + let materialized = self.runner_for(plan)?.materialize( + &frozen, + plan, + &self.options.staging_root, + now, + )?; + let installed = self.workload_for(plan)?.install(plan, &materialized)?; + let expected = materialized.release.expected_projection(plan)?; + return self.persist_same_phase(current, |next| { + next.sealed_release = Some(materialized.release); + next.installed_release = Some(installed); + next.expected = Some(expected); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + + // The candidate's Expected is published in Starting, after the brake + // has admitted this exact transaction. Publishing it is the first + // Verse-visible change a deployment makes, and the brake gates changes. + let now = now_millis()?; + let mut next = current.value.clone(); + match current.value.command_kind { + CommandKind::Deploy => { + let Some(authorization) = self.deployment_authorization(current, now)? else { + return self.record_gate_wait( + current, + "deployment brake has not released this exact transaction", + ); + }; + next.deployment_authorization = Some(authorization); + } + CommandKind::Continuity => { + if !self.lifecycle_allows(current, now)? { + return self + .record_gate_wait(current, "lifecycle brake denies continuity restart"); + } + next.lifecycle_authorized_at_unix_millis = Some(now); + } + } + validate_live_providers_for_deploy(current.value.command_kind, || { + self.validate_selected_providers_current(required( + ¤t.value.plan, + "transaction plan", + )?) + })?; + next.enter_phase(DeploymentPhase::Starting, now); + next.last_error = None; + replace_transaction(&self.options.state_store, current, &next) + } + + fn advance_starting(&self, current: &Stored) -> Result<()> { + let expected = required(¤t.value.expected, "Expected projection")?; + let plan = required(¤t.value.plan, "transaction plan")?; + let release = required(¤t.value.sealed_release, "sealed release")?; + let installed = required(¤t.value.installed_release, "installed release")?; + if current.value.expected_publication_sha256.is_none() { + // Published under the candidate's own incarnation key, beside + // whatever the target's admitted incarnation currently projects. + // Nothing the incumbent reads about itself changes here. + let now = now_millis()?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; + let digest = self + .topology() + .publish_expected(expected, &provider_anchor)?; + return self.persist_same_phase(current, |next| { + next.expected_publication_sha256 = Some(digest); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + if current.value.activation.is_none() { + validate_live_providers_for_deploy(current.value.command_kind, || { + self.validate_selected_providers_current(plan) + })?; + let now = now_millis()?; + let runtime_instance_id = runtime_instance_id(¤t.value.transaction_id)?; + let launch = IdunnRuntimeActivationLaunch::issue( + expected, + runtime_instance_id, + now, + &self.idunn_signer, + )?; + let activation = self + .workload_for(plan)? + .prepare_activation(plan, expected, launch)?; + return self.persist_same_phase(current, |next| { + next.activation = Some(activation); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + if current.value.workload.is_none() { + let now = now_millis()?; + let activation = required(¤t.value.activation, "activation")?; + if current.value.rollout_stops_incumbent_first() { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(incumbent) = self.exact_incumbent(&snapshot, ¤t.value)? { + self.workload_for(&incumbent.value.plan)? + .stop(&incumbent.value.workload) + .context("stopping the incumbent before the candidate starts")?; + } + } + let observation = self + .workload_for(plan)? + .start_prepared(plan, release, installed, expected, activation)?; + return self.persist_same_phase(current, |next| { + next.workload = Some(observation); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + if current.value.activation_publication_sha256.is_none() { + let now = now_millis()?; + let activation = required(¤t.value.activation, "activation")?; + let workload = required(¤t.value.workload, "workload")?; + self.workload_for(required(¤t.value.plan, "transaction plan")?)? + .observe(expected, activation, workload)?; + let digest = self + .topology() + .publish_observed_activation(expected, activation, workload)?; + return self.persist_same_phase(current, |next| { + next.activation_publication_sha256 = Some(digest); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + } + let mut next = current.value.clone(); + next.enter_phase(DeploymentPhase::Warming, now_millis()?); + next.last_error = None; + replace_transaction(&self.options.state_store, current, &next) + } + + fn advance_warming(&self, current: &Stored) -> Result<()> { + let expected = required(¤t.value.expected, "Expected projection")?; + let activation = required(¤t.value.activation, "activation")?; + let workload = required(¤t.value.workload, "workload")?; + self.workload_for(required(¤t.value.plan, "transaction plan")?)? + .observe(expected, activation, workload)?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; + let class = ReadinessClass::of(expected)?; + if current.value.warming.is_none() && class == ReadinessClass::RouteProof { + // A route-proof target's warming is Idunn's own challenge to the + // candidate endpoint. No Odin is read: the target declared no + // reason to be aware of one. + return match self.challenge_candidate( + ¤t.value, + route_proof_warming_states(expected), + None, + )? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { evidence, present } => { + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + self.persist_same_phase(current, |next| { + next.warming = Some(warming); + Ok(()) + }) + } + }; + } + if current.value.warming.is_none() { + // Odin's warming is observed directly when nothing else can observe + // it. That is the first bootstrap, and it is also every continuity + // restart of Odin: the incumbent that would do the observing is the + // very process being restarted, so it cannot report on its own + // return. A deployment of Odin is not included -- there the healthy + // incumbent observes the candidate, which is the point. + // Odin's warming is observed directly exactly when no Odin can + // observe it: the first bootstrap, a continuity restart of Odin + // itself, or any moment the admitted Odin has no usable correlation + // to report through. The rule is the same one that justified the + // bootstrap case -- "observed through Odin" is not available -- and + // it stays scoped to this one target. Every other target keeps + // waiting for Odin, which is what makes Odin the root of the chain. + // + // Asked once. Admitting the latest topology persists the sequence + // cursor, so asking twice with the same record fails the second + // time as "transaction changed before topology admission". + let is_odin = class == ReadinessClass::OdinSelf; + let odin_observation = if is_odin + && snapshot.admitted_odin().is_some() + && current.value.command_kind != CommandKind::Continuity + { + self.admit_latest_topology(current, None)? + } else { + None + }; + let odin_observes_itself = is_odin && odin_observation.is_none(); + if odin_observes_itself { + ensure!( + expected.write_lease_required, + "direct Odin warming must be a stateful incarnation" + ); + let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + return self.persist_same_phase(current, |next| { + next.warming = Some(warming); + Ok(()) + }); + } + let observation = match odin_observation { + Some(observation) => Some(observation), + None => self.admit_latest_topology(current, None)?, + }; + let Some((admitted, authenticated)) = observation else { + // Only the first-Odin bootstrap above observes presence + // directly. Every other target's warming presence arrives as + // Odin's authenticated runtime topology correlation, so until + // Odin is admitted and publishing, a candidate warms forever + // with nothing to say why. + return self.record_gate_wait( + current, + "no authenticated Odin topology correlation yet: warming presence for a \ + non-Odin target is observed through Odin, which must be admitted first", + ); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + let semantic_warming = is_semantic_warming( + expected, + activation, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?; + let latest = required( + &admitted.value.latest_odin_observation, + "sequence-admitted warming evidence", + )?; + if semantic_warming { + let evidence = latest.clone(); + let _token = SequenceAdmittedWarming::from_topology( + admitted.value.transaction_id.clone(), + authenticated, + )?; + return self.persist_same_phase(&admitted, |next| { + next.warming = Some(WarmingEvidence::OdinTopology { evidence }); + Ok(()) + }); + } else { + return Ok(()); + } + } + let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; + ensure!( + warming.transaction_id() == current.value.transaction_id + && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + "durable Warming evidence belongs to another candidate" + ); + + if expected.route.is_some() && current.value.route_preflight.is_none() { + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; + let incumbent_route = + incumbent.and_then(|generation| generation.value.routing.observation().cloned()); + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let route_binding = binding + .route + .context("routed Expected lost route binding")?; + let driver = self.route_driver(route_binding); + let receipt = driver.preflight( + expected, + &activation.runtime_instance_id, + incumbent_route.as_ref(), + &self.route_gate(¤t.value.target, current.value.command_kind), + )?; + return self.persist_same_phase(current, |next| { + next.route_preflight = Some(receipt); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + + if current.value.isolation.is_none() { + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; + // Isolation is a property of two processes running at once. A + // stopped incumbent has already released its DynamicUser UID, and + // systemd is free to hand that same UID to the candidate -- so + // comparing the candidate against the incumbent's *recorded* + // identity turns ordinary UID reuse into a permanent refusal to + // fence. There is nothing left to be isolated from. + let incumbent_workload = match incumbent { + Some(value) + if self + .workload_for(&value.value.plan)? + .is_permanently_stopped(&value.value.workload)? => + { + None + } + other => other.map(|value| &value.value.workload), + }; + let isolation = WorkloadObservation::prove_isolation(workload, incumbent_workload)?; + return self.persist_same_phase(current, |next| { + next.isolation = Some(isolation); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + + self.transition(current, DeploymentPhase::Fencing) + } + + fn advance_fencing(&self, current: &Stored) -> Result<()> { + if current.value.fencing.is_none() { + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent = self.exact_incumbent(&snapshot, ¤t.value)?; + let incumbent_lease = incumbent.and_then(|generation| generation.value.leasing.lease()); + let expected = required(¤t.value.expected, "Expected projection")?; + let evidence = if expected.write_lease_required || incumbent_lease.is_some() { + let incumbent_lease_path = incumbent_lease + .map(|_| -> Result { + Ok(incumbent + .context("incumbent lease has no admitted generation")? + .value + .plan + .parsed_inputs()? + .1 + .process_write_lease + .context("incumbent lease has no admitted binding")? + .record_path) + }) + .transpose()?; + if let (Some(lease), Some(path)) = (incumbent_lease, &incumbent_lease_path) { + let incumbent = incumbent.context("incumbent lease lost its generation")?; + self.workload_for(&incumbent.value.plan)? + .stop(&incumbent.value.workload) + .context("stopping the exact incumbent before revoking its lifetime-held write lease")?; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); + driver.revoke_exact(Some(lease))?; + ensure!( + driver.observe_empty()?, + "incumbent write lease remained after exact fencing" + ); + self.topology().withdraw_process_write_lease( + &incumbent.value.expected, + &incumbent.value.activation, + Some(lease), + )?; + } + let candidate_lease_path = if expected.write_lease_required { + Some( + current + .value + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .process_write_lease + .context("stateful candidate has no write-lease binding")? + .record_path, + ) + } else { + None + }; + if let Some(path) = &candidate_lease_path { + if incumbent_lease_path.as_ref() != Some(path) { + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, path); + driver.revoke_exact(None)?; + ensure!( + driver.observe_empty()?, + "candidate write-lease path was not empty before grant" + ); + } + } + self.topology().withdraw_process_write_lease( + expected, + required(¤t.value.activation, "candidate activation")?, + None, + )?; + FencingEvidence::Revoked { + incumbent_lease_sha256: incumbent_lease + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose()?, + candidate_lease_path_verified_empty: candidate_lease_path.is_some(), + } + } else { + FencingEvidence::SkippedStateless + }; + return self.persist_same_phase(current, |next| { + next.fencing = Some(evidence); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + self.transition(current, DeploymentPhase::Leasing) + } + + fn advance_leasing(&self, current: &Stored) -> Result<()> { + let expected = required(¤t.value.expected, "Expected projection")?; + if !expected.write_lease_required { + if current.value.leasing.is_none() { + let now = now_millis()?; + return self.persist_same_phase(current, |next| { + next.leasing = Some(LeasingEvidence::SkippedStateless); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + return self.transition(current, DeploymentPhase::AwaitingReady); + } + + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let lease_path = binding + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + + if matches!( + current.value.leasing.as_ref(), + Some(LeasingEvidence::Granted { .. }) + ) { + let activation = required(¤t.value.activation, "activation")?; + let warming = self.rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease) + .context("granted Leasing evidence has no write lease")?; + let recorded_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("granted Leasing evidence has no lease digest")?; + ensure!( + driver.observe_exact(lease)?, + "physical write lease disappeared after Granted became durable" + ); + self.workload_for(required(¤t.value.plan, "transaction plan")?)? + .observe( + expected, + activation, + required(¤t.value.workload, "candidate workload")?, + )?; + ensure!( + driver.grant(expected, activation, &warming, lease)? == recorded_sha256, + "replayed physical write lease differs from Granted evidence" + ); + ensure!( + self.topology() + .publish_process_write_lease(expected, activation, lease)? + == recorded_sha256, + "replayed write-lease projection differs from Granted evidence" + ); + return self.transition(current, DeploymentPhase::AwaitingReady); + } + + if let Some((lease, prepared_sha256)) = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::prepared_lease) + { + let now = now_millis()?; + let activation = required(¤t.value.activation, "activation")?; + let historical_warming = self.rehydrate_warming_token(¤t.value, now, false)?; + let physical_is_exact = driver.observe_exact(lease)?; + let warming = if physical_is_exact { + historical_warming + } else { + ensure!( + driver.observe_empty()?, + "candidate write-lease store contains authority other than its durable prepared lease" + ); + match self.rehydrate_warming_token(¤t.value, now, true) { + Ok(warming) => warming, + Err(_) => { + let Some((admitted, fresh_evidence, fresh_warming)) = + self.fresh_warming_for_lease(current, now)? + else { + return Ok(()); + }; + let replacement = self.prepare_candidate_write_lease( + &admitted.value, + &fresh_warming, + now, + )?; + let replacement_sha256 = replacement.canonical_sha256()?; + return self.persist_same_phase(&admitted, |next| { + next.warming = Some(fresh_evidence); + next.leasing = Some(LeasingEvidence::Prepared { + lease: replacement, + lease_sha256: replacement_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + } + }; + self.workload_for(required(¤t.value.plan, "transaction plan")?)? + .observe( + expected, + activation, + required(¤t.value.workload, "candidate workload")?, + )?; + if !physical_is_exact { + ensure!( + driver.observe_empty()?, + "candidate write-lease store changed before physical grant" + ); + } + let granted_sha256 = driver.grant(expected, activation, &warming, lease)?; + ensure!( + granted_sha256 == prepared_sha256, + "granted write lease differs from the durable prepared lease" + ); + let projected_sha256 = self + .topology() + .publish_process_write_lease(expected, activation, lease)?; + ensure!( + projected_sha256 == granted_sha256, + "projected write lease differs from the granted process authority" + ); + let lease = lease.clone(); + return self.persist_same_phase(current, |next| { + next.leasing = Some(LeasingEvidence::Granted { + lease, + lease_sha256: projected_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + + ensure!( + current.value.leasing.is_none(), + "stateful Leasing phase carries invalid lease evidence" + ); + ensure!( + driver.observe_empty()?, + "candidate write-lease store contains authority before lease preparation" + ); + let now = now_millis()?; + let Some((admitted, fresh_evidence, fresh_warming)) = + self.fresh_warming_for_lease(current, now)? + else { + return Ok(()); + }; + let lease = self.prepare_candidate_write_lease(&admitted.value, &fresh_warming, now)?; + let lease_sha256 = lease.canonical_sha256()?; + self.persist_same_phase(&admitted, |next| { + next.warming = Some(fresh_evidence); + next.leasing = Some(LeasingEvidence::Prepared { + lease, + lease_sha256, + }); + next.updated_at_unix_millis = now; + Ok(()) + }) + } + + fn advance_awaiting_ready(&self, current: &Stored) -> Result<()> { + let now = now_millis()?; + let expected = required(¤t.value.expected, "Expected projection")?; + self.observe_candidate_before_waiting(¤t.value)?; + if expected.write_lease_required { + let activation = required(¤t.value.activation, "activation")?; + let warming = self.rehydrate_warming_token(¤t.value, now, false)?; + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease) + .context("stateful transaction has no process write lease")?; + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let path = binding + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + ensure!( + CultCacheWriteLeaseDriver::new(¤t.value.target, path) + .observe(expected, activation, &warming, lease)?, + "candidate process write lease is no longer exact" + ); + } + if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { + // Ready is the candidate answering Idunn's own challenge Active, + // holding the exact lease Idunn granted. Once recorded it is + // history; the stable route's challenge is what keeps proving it. + if matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ) { + return self.transition(current, DeploymentPhase::Routing); + } + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + return match self.challenge_candidate(¤t.value, &["warming", "active"], lease)? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { present, .. } if present.record().state == "warming" => { + self.record_gate_wait(current, "candidate is still warming") + } + CandidateAnswer::Answered { evidence, .. } => { + let adoption = LeaseAdoptionEvidence::with_ready( + lease, + &evidence.canonical_sha256, + AdoptionSource::Direct, + evidence.admitted_at_unix_millis, + ); + self.persist_same_phase(current, |next| { + next.ready = Some(ReadinessEvidence::RouteProof { evidence }); + next.lease_adoption = adoption; + Ok(()) + }) + } + }; + } + { + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + let semantic_ready = is_semantic_ready(&authenticated); + let latest = required( + &admitted.value.latest_odin_observation, + "sequence-admitted Ready evidence", + )?; + if current.value.ready.as_ref().and_then(ReadinessEvidence::odin) == Some(latest) { + ensure!(semantic_ready, "stored Ready receipt changed meaning"); + } else if semantic_ready { + let evidence = latest.clone(); + // A Ready stateful correlation carries exactly the current + // lease (the authenticator refuses one that does not). + let adoption = authenticated + .record() + .signed_presence_sha256 + .as_deref() + .and_then(|presence| { + LeaseAdoptionEvidence::with_ready( + current_lease, + presence, + AdoptionSource::OdinTopology, + evidence.admitted_at_unix_millis, + ) + }); + let _token = SequenceAdmittedReady { + transaction_id: admitted.value.transaction_id.clone(), + evidence: evidence.clone(), + expected: expected.clone(), + authenticated, + }; + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence }); + next.lease_adoption = adoption; + Ok(()) + }); + } else { + return Ok(()); + } + } + self.transition(current, DeploymentPhase::Routing) + } + + fn advance_routing(&self, current: &Stored) -> Result<()> { + let expected = required(¤t.value.expected, "Expected projection")?; + let activation = required(¤t.value.activation, "activation")?; + self.observe_candidate_before_waiting(¤t.value)?; + if current.value.routing.is_none() { + let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { + // Ready is the candidate's own proof, recorded in AwaitingReady. + // Validation refuses Routing without a Ready receipt, and a + // receipt the other party vouched for is held before any step + // runs. No Odin correlation is read to admit a route. + current.clone() + } else { + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at route admission" + ); + let latest = required( + &admitted.value.latest_odin_observation, + "current route-admission topology evidence", + )?; + if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) + != Some(latest) + { + let latest = latest.clone(); + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); + Ok(()) + }); + } + let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; + ensure!( + ready.transaction_id() == admitted.value.transaction_id, + "Ready token belongs to another transaction" + ); + admitted + }; + validate_live_providers_for_deploy(admitted.value.command_kind, || { + self.validate_selected_providers_current(required( + &admitted.value.plan, + "transaction plan", + )?) + })?; + self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; + let evidence = if expected.route.is_some() { + let binding = admitted.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let route_binding = binding.route.context("routed plan has no route binding")?; + let preflight = required(&admitted.value.route_preflight, "route preflight")?; + let driver = self.route_driver(route_binding); + let gate = self.route_gate(&admitted.value.target, admitted.value.command_kind); + ensure!( + preflight.candidate_runtime_instance_id == activation.runtime_instance_id, + "route preflight belongs to another runtime instance" + ); + let rollback_allowed = may_rollback_route_after_failed_proof(required( + &admitted.value.fencing, + "route fencing evidence", + )?); + let membership_sha256 = driver.install( + expected, + &activation.runtime_instance_id, + preflight, + rollback_allowed, + &gate, + )?; + let observation = match self.prove_stable_route( + &admitted.value, + &driver, + membership_sha256, + ) { + Ok(observation) => observation, + Err(proof_error) if !rollback_allowed => { + return Err(proof_error).context( + "incumbent authority was fenced; candidate route remains installed for fail-closed retry", + ); + } + Err(proof_error) => { + return match driver.rollback( + expected, + &activation.runtime_instance_id, + preflight, + &gate, + ) { + Ok(()) => Err(proof_error) + .context("candidate did not answer its stable route challenge"), + Err(rollback_error) => Err(proof_error).context(format!( + "candidate did not answer its stable route challenge; exact route rollback also failed: {rollback_error:#}" + )), + }; + } + }; + ensure!( + driver.observe_membership(expected, &observation.membership_sha256)?, + "route membership changed during its signed stable-listener observation" + ); + self.ensure_transaction_write_lease_current(&admitted.value, now_millis()?)?; + let promoted_at_unix_millis = observation.observed_at_unix_millis; + RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + } + } else { + RoutingEvidence::SkippedUnrouted + }; + let now = now_millis()?; + return self.persist_same_phase(&admitted, |next| { + next.routing = Some(evidence); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + self.transition(current, DeploymentPhase::Committing) + } + + fn advance_committing(&self, current: &Stored) -> Result<()> { + self.observe_candidate_before_waiting(¤t.value)?; + let route_proof = ReadinessClass::of(required( + ¤t.value.expected, + "Expected projection", + )?)? == ReadinessClass::RouteProof; + let ready_current = if route_proof { + // The candidate's Ready proof is durable, and the stable route's + // challenge below is its currency. No Odin is read to commit. + current.clone() + } else { + let current_lease_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + // A newer sequence admitted here changed the record, and commit + // goes on from that record rather than returning: returning let + // Odin's next re-stamp abort every attempt (F16). + let Some((ready_current, authenticated)) = + self.admit_latest_topology(current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at admission commit" + ); + // Not require_current: the Ready receipt is durable evidence, so + // authenticating it against `now` asks a stored record to be live and + // refuses it once it ages past the 30s observation window. Route + // admission (:3906) refreshes the receipt to the latest observation and + // can therefore demand currency; by Committing the receipt is history. + // Currency here is answered by the latest observation, authenticated + // and checked for semantic readiness immediately above. + self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; + ready_current + }; + validate_live_providers_for_deploy(ready_current.value.command_kind, || { + self.validate_selected_providers_current(required( + &ready_current.value.plan, + "transaction plan", + )?) + })?; + + let expected = required(&ready_current.value.expected, "Expected projection")?; + let activation = required(&ready_current.value.activation, "activation")?; + let workload = required(&ready_current.value.workload, "workload")?; + self.workload_for(required(&ready_current.value.plan, "transaction plan")?)? + .observe(expected, activation, workload)?; + self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; + if let Some(route) = ready_current + .value + .routing + .as_ref() + .and_then(RoutingEvidence::observation) + { + let binding = ready_current + .value + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1; + let driver = + self.route_driver(binding.route.context("routed plan has no route binding")?); + ensure!( + driver.observe_membership(expected, &route.membership_sha256)?, + "route membership changed before admission commit" + ); + let current_route = self.prove_stable_route( + &ready_current.value, + &driver, + route.membership_sha256.clone(), + )?; + ensure!( + current_route.route_id == route.route_id + && current_route.runtime_instance_id == route.runtime_instance_id, + "stable route changed incarnation before admission commit" + ); + ensure!( + driver.observe_membership(expected, &route.membership_sha256)?, + "route membership changed during the final signed admission challenge" + ); + self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; + } + + let commit_current = if route_proof { + ready_current + } else { + let current_lease_sha256 = ready_current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + // The final proof's own presence makes Odin re-stamp, so this + // admission usually writes. Commit is from the record it wrote. + let Some((commit_current, authenticated)) = + self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready after the final admission challenge" + ); + self.rehydrate_ready_token(&commit_current.value, now_millis()?, false)?; + commit_current + }; + let now = now_millis()?; + validate_live_providers_for_deploy(commit_current.value.command_kind, || { + self.validate_selected_providers_current(required( + &commit_current.value.plan, + "transaction plan", + )?) + })?; + self.workload_for(required(&commit_current.value.plan, "transaction plan")?)? + .observe( + required(&commit_current.value.expected, "Expected projection")?, + required(&commit_current.value.activation, "activation")?, + required(&commit_current.value.workload, "workload")?, + )?; + self.ensure_transaction_write_lease_current(&commit_current.value, now)?; + + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; + let odin_authority = (!route_proof) + .then(|| self.current_odin_authority(&snapshot)) + .transpose()?; + let generation = AdmittedGeneration::from_transaction( + &commit_current.value, + odin_authority, + now, + )?; + let post_commit_cleanup = PostCommitCleanup { + incumbent: match incumbent { + Some(incumbent) + if incumbent_was_stopped_during_fencing(required( + &commit_current.value.fencing, + "commit fencing evidence", + )?) => + { + IncumbentCleanupEvidence::Complete { + generation_id: incumbent.value.generation_id.clone(), + } + } + Some(incumbent) => IncumbentCleanupEvidence::Pending { + generation_id: incumbent.value.generation_id.clone(), + workload: incumbent.value.workload.clone(), + }, + None => IncumbentCleanupEvidence::SkippedNoIncumbent, + }, + source: match commit_current.value.command_kind { + CommandKind::Deploy => SourceCleanupEvidence::Pending, + CommandKind::Continuity => SourceCleanupEvidence::SkippedContinuity, + }, + }; + let mut complete = commit_current.value.clone(); + complete.enter_phase(DeploymentPhase::Complete, now); + complete.last_error = None; + complete.completion = Some(TransactionCompletion::Admitted { + generation_id: generation.generation_id.clone(), + }); + complete.post_commit_cleanup = Some(post_commit_cleanup); + complete.validate()?; + + let admitted_expected = CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: generation.target.clone(), + current: incumbent.map(|stored| stored.envelope.clone()), + }; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: commit_current.value.transaction_id.clone(), + current: Some(commit_current.envelope.clone()), + }, + admitted_expected, + ], + &[ + transaction_envelope(&complete, now)?, + admitted_envelope(&generation, now)?, + ], + )?, + "incumbent or transaction changed before atomic admission commit" + ); + Ok(()) + } + + fn exact_incumbent<'a>( + &self, + snapshot: &'a ControlSnapshot, + transaction: &DeploymentTransaction, + ) -> Result>> { + let current = snapshot.admitted_for(&transaction.target); + match (&transaction.incumbent_generation_id, current) { + (None, None) => Ok(None), + (Some(expected), Some(current)) if current.value.generation_id == *expected => { + Ok(Some(current)) + } + _ => bail!("target incumbent changed after transaction creation"), + } + } + + fn incumbent_lease_sha256_for_warming( + &self, + snapshot: &ControlSnapshot, + transaction: &DeploymentTransaction, + ) -> Result> { + if let Some(fencing) = &transaction.fencing { + return match fencing { + FencingEvidence::SkippedStateless => Ok(None), + FencingEvidence::Revoked { + incumbent_lease_sha256, + .. + } => Ok(incumbent_lease_sha256.clone()), + }; + } + self.exact_incumbent(snapshot, transaction)? + .and_then(|incumbent| incumbent.value.leasing.lease()) + .map(IdunnProcessWriteLeaseRecord::canonical_sha256) + .transpose() + } + + fn fresh_warming_for_lease( + &self, + current: &Stored, + now: u64, + ) -> Result< + Option<( + Stored, + WarmingEvidence, + SequenceAdmittedWarming, + )>, + > { + let prior = self.rehydrate_warming_token(¤t.value, now, false)?; + match required(¤t.value.warming, "pre-fence Warming evidence")?.clone() { + WarmingEvidence::FirstOdinDirect { .. } => { + // Whether this transaction may observe Odin directly was + // settled when the Warming evidence was recorded, and that + // decision is durable in the transaction. Re-deciding it here + // with the narrower "no admitted Odin" test contradicted the + // caller and failed the refresh. What the refresh owes is + // freshness, and the replay check below is what provides it. + // That the target is Odin was already required by the + // rehydration above, which authenticates the recorded answer. + let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "first Odin replayed its pre-fence Warming presence" + ); + Ok(Some((current.clone(), warming, token))) + } + WarmingEvidence::RouteProofDirect { .. } => { + // A stateful route-proof candidate is warming until it holds + // the lease this refresh is about to bind, so the fresh + // presence must be warming again, and must be a new one. + let CandidateAnswer::Answered { evidence, present } = + self.challenge_candidate(¤t.value, &["warming"], None)? + else { + return Ok(None); + }; + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "route-proof candidate replayed its pre-fence Warming presence" + ); + Ok(Some((current.clone(), warming, token))) + } + WarmingEvidence::OdinTopology { + evidence: prior_evidence, + } => { + let Some((admitted, authenticated)) = self.admit_latest_topology(current, None)? + else { + return Ok(None); + }; + let fresh_evidence = required( + &admitted.value.latest_odin_observation, + "post-fence Odin Warming observation", + )? + .clone(); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, &admitted.value)?; + if !is_semantic_warming( + required(&admitted.value.expected, "Warming Expected projection")?, + required(&admitted.value.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )? { + return Ok(None); + } + let token = SequenceAdmittedWarming::from_topology( + admitted.value.transaction_id.clone(), + authenticated, + )?; + if !provider_warming_advanced( + prior_evidence.publisher_sequence, + prior.signed_presence_sha256(), + fresh_evidence.publisher_sequence, + token.signed_presence_sha256(), + ) { + return Ok(None); + } + Ok(Some(( + admitted, + WarmingEvidence::OdinTopology { + evidence: fresh_evidence, + }, + token, + ))) + } + } + } + + fn prepare_candidate_write_lease( + &self, + transaction: &DeploymentTransaction, + warming: &SequenceAdmittedWarming, + now: u64, + ) -> Result { + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + ensure!( + expected.write_lease_required + && warming.transaction_id() == transaction.transaction_id + && warming.runtime_instance_id() == activation.runtime_instance_id, + "fresh Warming token does not own this stateful candidate" + ); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let epoch = self + .exact_incumbent(&snapshot, transaction)? + .and_then(|generation| generation.value.leasing.lease()) + .map_or(1, |lease| lease.lease_epoch.saturating_add(1)); + let lease = IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: activation.canonical_sha256()?, + state_schema_generation: expected + .state_schema_generation + .clone() + .context("stateful Expected has no schema generation")?, + state_contract_sha256: expected + .state_contract_sha256 + .clone() + .context("stateful Expected has no state contract")?, + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + warming_presence_sha256: warming.signed_presence_sha256().to_owned(), + lease_epoch: epoch, + issued_at_unix_millis: now, + }; + lease.validate()?; + Ok(lease) + } + + fn ensure_transaction_write_lease_current( + &self, + transaction: &DeploymentTransaction, + now: u64, + ) -> Result<()> { + let expected = required(&transaction.expected, "Expected projection")?; + let lease = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease); + ensure!( + expected.write_lease_required == lease.is_some(), + "transaction write-lease disposition differs from Expected" + ); + let Some(lease) = lease else { + return Ok(()); + }; + let activation = required(&transaction.activation, "activation")?; + let warming = self.rehydrate_warming_token(transaction, now, false)?; + let lease_path = required(&transaction.plan, "transaction plan")? + .parsed_inputs()? + .1 + .process_write_lease + .context("stateful target has no write-lease binding")? + .record_path; + ensure!( + CultCacheWriteLeaseDriver::new(&transaction.target, lease_path) + .observe(expected, activation, &warming, lease,)?, + "process write lease changed across the authority boundary" + ); + Ok(()) + } + + fn select_candidate_port(&self, binding: &OperatorBinding) -> Result> { + let Some(route) = &binding.route else { + return Ok(None); + }; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut used = BTreeSet::new(); + for plan in snapshot + .admitted + .iter() + .map(|stored| &stored.value.plan) + .chain( + snapshot + .transactions + .iter() + .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter_map(|stored| stored.value.plan.as_ref()), + ) + { + if let Some(port) = plan.candidate_port { + used.insert(port); + } + } + (route.private_port_start..=route.private_port_end) + .find(|port| !used.contains(port)) + .map(Some) + .context("no private candidate port remains in the operator range") + } + + fn deployment_authorization( + &self, + current: &Stored, + now: u64, + ) -> Result> { + let expected = required(¤t.value.expected, "Expected projection")?; + let plan = required(¤t.value.plan, "transaction plan")?; + let binding = plan.parsed_inputs()?.1; + let Some((record, canonical_bytes)) = + read_deployment_brake(&binding.brakes.deployment_store)? + else { + return Ok(None); + }; + let anchor = read_trust_anchor::( + &self.options.deployment_brake_operator_anchor, + )?; + if evaluate_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&record), + &anchor, + &expected.runtime_id, + &expected.sealed_release_id, + ¤t.value.transaction_id, + now, + ) + .is_err() + { + return Ok(None); + } + let authorization_id = record + .authorization_id + .clone() + .context("released brake has no authorization id")?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + ensure!( + !snapshot.transactions.iter().any(|stored| { + stored + .value + .deployment_authorization + .as_ref() + .is_some_and(|used| used.authorization_id == authorization_id) + }), + "deployment authorization was already consumed" + ); + Ok(Some(DeploymentAuthorization { + authorization_id, + brake_sha256: sha256_id(&canonical_bytes), + canonical_brake_bytes: canonical_bytes, + authorized_at_unix_millis: now, + })) + } + + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { + let plan = required(¤t.value.plan, "continuity plan")?; + let binding = plan.parsed_inputs()?.1; + let expected = required(¤t.value.expected, "continuity Expected")?; + match read_lifecycle_brake(&binding.brakes.lifecycle_store) { + Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Present(&record), + &expected.runtime_id, + ¤t.value.target, + now, + ) + .is_ok()), + Ok(None) => Ok(evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Missing, + &expected.runtime_id, + ¤t.value.target, + now, + ) + .is_ok()), + Err(_) => Ok(false), + } + } + + /// A phase that waits on Odin's evidence about the candidate must first + /// confirm the candidate still exists. Otherwise a candidate that died + /// while its correlation was pending is never observed again: the phase + /// returns early on "no evidence yet" every tick, the error that would have + /// triggered the post-fencing abort never happens, and the transaction + /// owns the target forever behind a process that is gone. + fn observe_candidate_before_waiting(&self, transaction: &DeploymentTransaction) -> Result<()> { + self.workload_for(required(&transaction.plan, "transaction plan")?)? + .observe( + required(&transaction.expected, "Expected projection")?, + required(&transaction.activation, "activation")?, + required(&transaction.workload, "candidate workload")?, + ) + .map(|_| ()) + } + + /// Withdraw one projected incarnation of this target that nothing owns: + /// not the admitted generation, not any live transaction. A replaced + /// incumbent, an aborted candidate whose reconciliation was skipped, or a + /// record left by the projection migration all end here. One per tick; + /// the projection is a shared file and each withdrawal is one CAS. + fn retire_stale_incarnations( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + ) -> Result { + let topology = self.topology(); + let target = current.value.target.as_str(); + let owned = snapshot + .transactions + .iter() + .filter(|stored| stored.value.target == target && !stored.value.is_terminal()) + .filter_map(|stored| stored.value.expected.as_ref()) + .map(IdunnExpectedIncarnationRecord::canonical_sha256) + .collect::>>()?; + let admitted_sha256 = current.value.expected.canonical_sha256()?; + let Some(stale) = topology + .projected_incarnations(target)? + .into_iter() + .find(|expected| { + expected + .canonical_sha256() + .is_ok_and(|sha256| sha256 != admitted_sha256 && !owned.contains(&sha256)) + }) + else { + return Ok(false); + }; + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + topology.withdraw_stale_incarnation(&stale, &provider_anchor)?; + eprintln!( + "Idunn withdrew stale {} incarnation {} from the projection", + target, stale.incarnation_id + ); + Ok(true) + } + + /// The lifecycle brake as it applies to restarting one admitted + /// generation: the same evaluation `lifecycle_allows` makes for a + /// continuity transaction, asked before one is minted. + fn lifecycle_allows_generation( + &self, + current: &Stored, + now: u64, + ) -> Result { + let binding = current.value.plan.parsed_inputs()?.1; + let expected = ¤t.value.expected; + match read_lifecycle_brake(&binding.brakes.lifecycle_store) { + Ok(Some(record)) => Ok(evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Present(&record), + &expected.runtime_id, + ¤t.value.target, + now, + ) + .is_ok()), + Ok(None) => Ok(evaluate_idunn_continuity_restart( + IdunnLifecycleBrakeObservation::Missing, + &expected.runtime_id, + ¤t.value.target, + now, + ) + .is_ok()), + Err(_) => Ok(false), + } + } + + fn record_gate_wait( + &self, + current: &Stored, + detail: &str, + ) -> Result<()> { + if current.value.last_error.as_deref() == Some(detail) { + return Ok(()); + } + let mut next = current.value.clone(); + next.last_error = Some(detail.into()); + next.updated_at_unix_millis = now_millis()?; + replace_transaction(&self.options.state_store, current, &next) + } + + fn runtime_authority( + &self, + transaction: &DeploymentTransaction, + ) -> Result { + self.runtime_authority_parts( + required(&transaction.plan, "transaction plan")?, + required(&transaction.expected, "Expected projection")?, + required(&transaction.activation, "activation")?, + ) + } + + /// The stable route's proof: a challenged presence that is Active. + fn authenticate_routed_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result<(String, u64)> { + let present = self.authenticate_challenged_presence( + authority, + &["active"], + current_write_lease_sha256, + message_id, + challenged_at_unix_millis, + received_at_unix_millis, + canonical_presence, + )?; + Ok(( + present.signed_presence_sha256().to_owned(), + received_at_unix_millis, + )) + } + + /// A presence Idunn challenged for itself, from the stable endpoint or the + /// candidate's: signed by the provider and the launch's activation key, + /// bound to current authority, minted after the challenge, answering that + /// exact challenge, in one of `states`. A warming presence holds no write + /// lease; a presence in any other state holds exactly the current one. + /// Every disagreement with authority is named, so a capacity below the + /// Expected minimum reads as that and not as a generic refusal. + fn authenticate_challenged_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + states: &[&str], + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { + ensure!( + received_at_unix_millis >= challenged_at_unix_millis, + "route proof predates its challenge" + ); + let authenticated = authenticate_runtime_presence_claim( + canonical_presence, + authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: received_at_unix_millis, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + }, + )?; + let correlation = correlate_runtime_presence_claim(authenticated, authority)?; + if !correlation.disagreements().is_empty() { + return Err(PresenceDisagrees { + disagreements: correlation.disagreements().to_vec(), + } + .into()); + } + let present = correlation.into_undisputed_present()?; + let presence = present.record(); + ensure!( + presence.observed_at_unix_millis >= challenged_at_unix_millis, + "route proof returned a presence minted before its challenge" + ); + ensure!( + states.contains(&presence.state.as_str()), + "route proof runtime is {}, not {}", + presence.state, + states.join(" or ") + ); + ensure!( + presence.detail == format!("route-observation:{message_id}"), + "route proof response is not bound to the exact challenge" + ); + // A warming presence is exempt from the lease comparison: it is + // Idunn's own lease that a warming candidate cannot hold yet, and the + // authenticator above already refuses one that claims a lease. + let lease_is_current = presence.state == "warming" + || presence.write_lease_sha256.as_deref() == current_write_lease_sha256; + ensure!( + lease_is_current, + "route proof runtime does not hold the exact current process write lease" + ); + Ok(present) + } + + fn authenticate_first_odin_warming_presence( + &self, + transaction: &DeploymentTransaction, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { + // Whether direct observation is warranted at all is decided in + // advance_warming, which asks the question that matters: can any Odin + // report on this one. Re-deciding it here with the narrower "has an + // Odin ever been admitted" test only contradicted that caller. What is + // still checked is what this function itself depends on -- the target + // is Odin, and the incarnation is stateful. + let expected = required(&transaction.expected, "Odin Expected")?; + ensure!( + ReadinessClass::of(expected)? == ReadinessClass::OdinSelf + && expected.write_lease_required, + "direct Warming presence is reserved for a stateful Odin incarnation" + ); + ensure!( + received_at_unix_millis >= challenged_at_unix_millis, + "first Odin Warming observation predates its challenge" + ); + let authority = self.runtime_authority(transaction)?; + let authenticated = authenticate_runtime_presence_claim( + canonical_presence, + &authority, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: received_at_unix_millis, + maximum_age_millis: self.options.topology_maximum_age_millis, + maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, + }, + )?; + let correlation = correlate_runtime_presence_claim(authenticated, &authority)?; + ensure!( + correlation.disagreements().is_empty(), + "first Odin Warming presence disagrees with its Expected activation" + ); + let present = correlation.into_undisputed_present()?; + let presence = present.record(); + ensure!( + presence.observed_at_unix_millis >= challenged_at_unix_millis + && presence.state == "warming" + && presence.write_lease_sha256.is_none() + && presence.detail == format!("idunn-warming:{message_id}"), + "first Odin candidate did not return exact fresh pre-lease Warming evidence" + ); + Ok(present) + } + + fn observe_first_odin_warming( + &self, + transaction: &DeploymentTransaction, + ) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + let expected = required(&transaction.expected, "first Odin Expected")?; + let binding = required(&transaction.plan, "first Odin plan")? + .parsed_inputs()? + .1; + let driver = self.route_driver( + binding + .route + .context("first Odin bootstrap has no candidate route binding")?, + ); + let message_id = format!("warming-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = driver.request_candidate_runtime_presence(expected, &message_id)?; + ensure!( + response.message_id == message_id, + "first Odin candidate transport substituted its challenge identity" + ); + let admitted_at_unix_millis = now_millis()?; + let present = self.authenticate_first_odin_warming_presence( + transaction, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok((evidence, present)) + } + + fn prove_stable_route( + &self, + transaction: &DeploymentTransaction, + driver: &NginxRouteDriver, + membership_sha256: String, + ) -> Result { + let expected = required(&transaction.expected, "route Expected projection")?; + let activation = required(&transaction.activation, "route activation")?; + let authority = self.runtime_authority(transaction)?; + let current_write_lease_sha256 = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + self.prove_stable_route_against( + expected, + activation, + &authority, + current_write_lease_sha256, + driver, + membership_sha256, + ) + } + + fn prove_stable_route_against( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + current_write_lease_sha256: Option<&str>, + driver: &NginxRouteDriver, + membership_sha256: String, + ) -> Result { + let message_id = format!("route-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = driver.request_runtime_presence(expected, &message_id)?; + ensure!( + response.message_id == message_id, + "route transport substituted its challenge identity" + ); + let received_at_unix_millis = now_millis()?; + let (signed_presence_sha256, observed_at_unix_millis) = self.authenticate_routed_presence( + authority, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + received_at_unix_millis, + &response.canonical_presence, + )?; + let observation = RouteObservation { + route_id: driver.binding.route_id.clone(), + runtime_instance_id: activation.runtime_instance_id.clone(), + membership_sha256, + signed_presence_sha256, + observed_at_unix_millis, + }; + observation.validate()?; + Ok(observation) + } + + fn runtime_authority_parts( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + let provider_anchor = self.provider_anchor_for_plan(plan)?; + verify_runtime_authority( + expected, + activation, + &self.idunn_anchor, + &provider_anchor.public_key, + ) + } + + fn provider_anchor_for_plan( + &self, + plan: &CompiledDeploymentPlan, + ) -> Result { + let binding = plan.parsed_inputs()?.1; + let provider_anchor = read_trust_anchor::( + &binding.runtime_identity.trust_anchor_store, + )?; + ensure!( + provider_anchor.schema_version + == ::TRUST_ANCHOR_SCHEMA, + "provider runtime presence trust anchor schema is unsupported" + ); + Ok(provider_anchor) + } + + fn authenticate_topology_bytes( + &self, + snapshot: &ControlSnapshot, + transaction: &DeploymentTransaction, + canonical_bytes: &[u8], + current_write_lease_sha256: Option<&str>, + trusted_received_at: u64, + ) -> Result { + let authority = self.runtime_authority(transaction)?; + let odin_authority = self.current_odin_authority(snapshot)?; + authenticate_odin_runtime_topology_correlation( + canonical_bytes, + &authority, + current_write_lease_sha256, + &odin_authority.signer_public_key, + self.trusted_topology_context(trusted_received_at), + ) + } + + /// Persist every newly authenticated publisher sequence before deciding + /// whether it means Warming, Ready, degraded, or disagreement. + fn admit_latest_topology( + &self, + current: &Stored, + current_write_lease_sha256: Option<&str>, + ) -> Result< + Option<( + Stored, + AuthenticatedOdinRuntimeTopologyCorrelation, + )>, + > { + // Odin keeps one correlation per incarnation, so the transaction's own + // Expected selects exactly the evidence about it. A correlation about + // the incarnation this one is replacing lives under another key and is + // never handed back here. + let expected_sha256 = + required(¤t.value.expected, "Expected projection")?.canonical_sha256()?; + let Some(received) = self + .topology() + .receive(¤t.value.target, &expected_sha256)? + else { + return Ok(None); + }; + ensure!( + received.target == current.value.target, + "topology transport substituted target" + ); + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let live = snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == current.value.transaction_id) + .context("transaction disappeared before topology admission")?; + ensure!( + live.envelope == current.envelope, + "transaction changed before topology admission" + ); + let authenticated = match self.authenticate_topology_bytes( + &snapshot, + &live.value, + &received.canonical_bytes, + current_write_lease_sha256, + now, + ) { + Ok(authenticated) => authenticated, + // Odin re-stamps a correlation only when its facts change, and a + // live presence changes them every heartbeat. A correlation that + // has aged out therefore says one thing: nothing about this + // incarnation has arrived lately. That is absence to wait on, not + // evidence to refuse; it failed every Heimdall deployment and the + // first observed Odin deployment as a fault. + Err(error) if is_stale_observation(&error) => return Ok(None), + Err(error) => return Err(error), + }; + let evidence = TopologyEvidence::from_authenticated(&authenticated, now)?; + if !sequence_requires_admission( + live.value.latest_odin_observation.as_ref(), + snapshot.max_odin_sequence(&live.value.target, &evidence.signer_identity_id), + &evidence, + )? { + return Ok(Some((live.clone(), authenticated))); + } + let mut next = live.value.clone(); + next.latest_odin_observation = Some(evidence.clone()); + next.odin_publisher_sequence_cursor = evidence.publisher_sequence; + next.updated_at_unix_millis = now; + next.last_error = None; + replace_transaction(&self.options.state_store, live, &next)?; + let admitted = ControlSnapshot::read(&self.options.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == next.transaction_id) + .context("sequence-admitted transaction disappeared")?; + ensure!( + admitted.value.latest_odin_observation.as_ref() == Some(&evidence) + && admitted.value.odin_publisher_sequence_cursor == evidence.publisher_sequence, + "topology sequence was not durably admitted" + ); + Ok(Some((admitted, authenticated))) + } + + fn rehydrate_warming_token( + &self, + transaction: &DeploymentTransaction, + now: u64, + require_current: bool, + ) -> Result { + let evidence = required(&transaction.warming, "durable warming evidence")?.clone(); + match evidence { + WarmingEvidence::OdinTopology { evidence } => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let authenticated = self.authenticate_topology_bytes( + &snapshot, + transaction, + &evidence.canonical_bytes, + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + validate_authenticated_evidence(&evidence, &authenticated)?; + let incumbent_lease_sha256 = + self.incumbent_lease_sha256_for_warming(&snapshot, transaction)?; + ensure!( + is_semantic_warming( + required(&transaction.expected, "Warming Expected projection")?, + required(&transaction.activation, "Warming activation")?, + incumbent_lease_sha256.as_deref(), + &authenticated, + )?, + "durable warming evidence no longer satisfies the Warming gate" + ); + SequenceAdmittedWarming::from_topology( + transaction.transaction_id.clone(), + authenticated, + ) + } + WarmingEvidence::FirstOdinDirect { evidence } => { + let present = self.authenticate_first_odin_warming_presence( + transaction, + &evidence.message_id, + evidence.challenged_at_unix_millis, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + &evidence.canonical_bytes, + )?; + SequenceAdmittedWarming::from_direct_presence( + transaction.transaction_id.clone(), + WarmingEvidence::FirstOdinDirect { evidence }, + present, + ) + } + WarmingEvidence::RouteProofDirect { evidence } => { + let present = self.reauthenticate_route_proof( + transaction, + &evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + SequenceAdmittedWarming::from_direct_presence( + transaction.transaction_id.clone(), + WarmingEvidence::RouteProofDirect { evidence }, + present, + ) + } + } + } + + /// A route-proof presence Idunn recorded, authenticated again as of `at`. + fn reauthenticate_route_proof( + &self, + transaction: &DeploymentTransaction, + evidence: &RuntimePresenceEvidence, + states: &[&str], + current_write_lease_sha256: Option<&str>, + at_unix_millis: u64, + ) -> Result { + let authority = self.runtime_authority(transaction)?; + self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &evidence.message_id, + evidence.challenged_at_unix_millis, + at_unix_millis, + &evidence.canonical_bytes, + ) + } + + /// One direct challenge to a route-proof target's candidate endpoint. + /// Failing to reach or hear the candidate is only silence: a process that + /// is still starting does not answer, and that is waiting, not a fault. + /// What does answer is authenticated, and a bad answer is an error, as is + /// a challenge Idunn could not make: those stay a `ChallengeFailure` in the + /// error's chain. + fn challenge_candidate( + &self, + transaction: &DeploymentTransaction, + states: &[&str], + current_write_lease_sha256: Option<&str>, + ) -> Result { + let expected = required(&transaction.expected, "candidate Expected")?; + let binding = required(&transaction.plan, "candidate plan")? + .parsed_inputs()? + .1; + let driver = self.route_driver( + binding + .route + .context("route-proof candidate has no route binding")?, + ); + let message_id = format!("candidate-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = match driver.request_candidate_runtime_presence(expected, &message_id) { + Ok(response) => response, + Err(ChallengeFailure::Silent(error)) => { + return Ok(CandidateAnswer::Silent(format!( + "candidate endpoint did not answer its challenge: {error:#}" + ))); + } + Err(bad @ ChallengeFailure::Refused(_)) => return Err(bad.into()), + }; + let admitted_at_unix_millis = now_millis()?; + let authority = self.runtime_authority(transaction)?; + let present = self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok(CandidateAnswer::Answered { evidence, present }) + } + + fn rehydrate_ready_token( + &self, + transaction: &DeploymentTransaction, + now: u64, + require_current: bool, + ) -> Result { + let evidence = required(&transaction.ready, "durable Ready evidence")? + .odin() + .context("durable Ready evidence is not an Odin receipt")? + .clone(); + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let current_lease = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let authenticated = self.authenticate_topology_bytes( + &snapshot, + transaction, + &evidence.canonical_bytes, + current_lease, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + validate_authenticated_evidence(&evidence, &authenticated)?; + ensure!( + is_semantic_ready(&authenticated), + "durable Ready evidence no longer authenticates as Ready" + ); + Ok(SequenceAdmittedReady { + transaction_id: transaction.transaction_id.clone(), + evidence, + expected: required(&transaction.expected, "Expected projection")?.clone(), + authenticated, + }) + } + + fn current_ready_provider_tokens( + &self, + snapshot: &ControlSnapshot, + ) -> Result> { + let mut providers = Vec::new(); + for stored in &snapshot.admitted { + // No pre-filter on ready == latest: a provider that published again + // after going ready is still ready. rehydrate_admitted_ready owns + // that judgment now, and reports why when it refuses. + match self.rehydrate_admitted_ready(snapshot, &stored.value) { + Ok(provider) => providers.push(provider), + Err(error) => eprintln!( + "Idunn excluded non-current provider {}: {error:#}", + stored.value.target + ), + } + } + Ok(providers) + } + + fn rehydrate_admitted_ready( + &self, + snapshot: &ControlSnapshot, + generation: &AdmittedGeneration, + ) -> Result { + let odin = generation.odin_receipts()?; + let authority = self.runtime_authority_parts( + &generation.plan, + &generation.expected, + &generation.activation, + )?; + let odin_authority = self.current_odin_authority(snapshot)?; + let current_lease = generation.leasing.lease_sha256(); + let authenticated = authenticate_odin_runtime_topology_correlation( + &odin.ready.canonical_bytes, + &authority, + current_lease, + &odin_authority.signer_public_key, + self.trusted_topology_context(odin.ready.admitted_at_unix_millis), + )?; + validate_authenticated_evidence(odin.ready, &authenticated)?; + ensure!( + is_semantic_ready(&authenticated), + "admitted provider's Ready receipt is not Ready at its admission time" + ); + // A provider that published after going ready is the ordinary case, not + // a stale one, so the cursor is authenticated on its own terms rather + // than compared to the receipt. Authentication is what binds it to this + // incarnation -- it requires the record's current_activation_sha256 and + // runtime_instance_id to match this generation's activation -- so a + // separate identity check here would only restate it more weakly. + // Note this is the cursor as frozen at admission, not a live read of + // Odin. Both receipts are historical proofs; current workload health + // is owned by continuity and route observation, not receipt age. + let authenticated_latest = authenticate_odin_runtime_topology_correlation( + &odin.latest.canonical_bytes, + &authority, + current_lease, + &odin_authority.signer_public_key, + self.trusted_topology_context(odin.latest.admitted_at_unix_millis), + )?; + validate_authenticated_evidence(odin.latest, &authenticated_latest)?; + ensure!( + is_semantic_ready(&authenticated_latest), + "admitted provider's latest receipt is not Ready at its admission time" + ); + Ok(SequenceAdmittedReady { + transaction_id: generation.transaction_id.clone(), + evidence: odin.ready.clone(), + expected: generation.expected.clone(), + authenticated, + }) + } + + fn validate_selected_providers_current( + &self, + plan: &CompiledDeploymentPlan, + ) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + for selection in &plan.dependencies { + let Some(provider) = &selection.provider else { + ensure!( + selection.requirement.kind == DependencyKind::Optional, + "required dependency has no provider" + ); + continue; + }; + let DependencyProviderAuthority::ManagedReady { + target, + incarnation_id, + plan_id, + sealed_release_id, + expected_projection_sha256, + odin_topology_correlation_sha256, + odin_topology_publisher_sequence, + } = &provider.authority + else { + continue; + }; + let admitted = snapshot + .admitted_for(target) + .context("selected managed dependency is no longer admitted")?; + let token = self.rehydrate_admitted_ready(&snapshot, &admitted.value)?; + ensure!( + admitted.value.expected.incarnation_id == *incarnation_id + && admitted.value.plan.plan_id == *plan_id + && admitted.value.sealed_release.sealed_release_id == *sealed_release_id + && admitted.value.expected.canonical_sha256()? == *expected_projection_sha256 + && token.publisher_sequence() >= *odin_topology_publisher_sequence, + "selected dependency authority changed before actuation" + ); + ensure!( + token.publisher_sequence() != *odin_topology_publisher_sequence + || token.evidence_sha256() == odin_topology_correlation_sha256, + "selected dependency Odin sequence changed evidence" + ); + ensure!( + token + .authenticated() + .record() + .observed_capabilities + .iter() + .any(|capability| { + capability_compatible( + &selection.requirement.capability, + &selection.requirement.schema, + &selection.requirement.compatibility, + &capability.capability, + &capability.schema, + &capability.compatibility, + ) && capability.capacity >= selection.requirement.minimum_capacity + }), + "selected dependency no longer provides its required capability" + ); + } + Ok(()) + } + + fn persist_same_phase( + &self, + current: &Stored, + mutation: F, + ) -> Result<()> + where + F: FnOnce(&mut DeploymentTransaction) -> Result<()>, + { + let mut next = current.value.clone(); + mutation(&mut next)?; + ensure!( + next.phase == current.value.phase, + "same-phase evidence update changed phase" + ); + next.updated_at_unix_millis = now_millis()?; + next.last_error = None; + replace_transaction(&self.options.state_store, current, &next) + } + + fn transition( + &self, + current: &Stored, + next_phase: DeploymentPhase, + ) -> Result<()> { + ensure!( + next_phase as u8 == current.value.phase as u8 + 1, + "deployment phase transition is not adjacent" + ); + let mut next = current.value.clone(); + next.enter_phase(next_phase, now_millis()?); + next.last_error = None; + replace_transaction(&self.options.state_store, current, &next) + } + + /// Every post-fencing phase ends. Before a live transaction past the + /// fence runs its step, a phase whose deadline has passed, or whose + /// command an operator asked to expire, is resolved as `phase_end` says. + /// Returns `Some(moved)` when it decided the tick, `None` to run the step. + /// + /// The deadline is written only by `enter_phase`, so nothing here or in a + /// resume path extends it. A record lifted from before deadlines existed + /// has none; its phase is timed once, from the first time this sees it. + fn resolve_phase_deadline( + &self, + current: &Stored, + ) -> Result> { + let transaction = ¤t.value; + if !transaction.resolves_by_deadline() { + return Ok(None); + } + let now = now_millis()?; + let Some(deadline) = transaction.phase_deadline else { + let mut next = transaction.clone(); + next.enter_phase(transaction.phase, now); + if next.phase_deadline.is_none() { + return Ok(None); + } + replace_transaction(&self.options.state_store, current, &next)?; + return Ok(Some(true)); + }; + let requested = ControlSnapshot::read(&self.options.state_store)? + .expiry_requests + .iter() + .any(|request| request.value.command_id == transaction.command_id); + let cause = if now >= deadline.deadline_at_unix_millis { + format!( + "the {:?} phase passed its deadline at unix ms {}", + transaction.phase, deadline.deadline_at_unix_millis + ) + } else if requested { + format!("the {:?} phase was expired by the operator", transaction.phase) + } else { + return Ok(None); + }; + match transaction.phase_end() { + PhaseEnd::Abort { lease_not_adopted } => { + let error = if lease_not_adopted { + anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") + } else { + anyhow!("{cause}") + }; + self.begin_post_fencing_abort(current, error)?; + Ok(Some(true)) + } + PhaseEnd::OperatorRequired => { + self.report_once( + &format!("deadline:{}", transaction.transaction_id), + format!( + "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", + transaction.transaction_id + ), + ); + Ok(None) + } + } + } + + /// Delete expiry requests whose command has no phase left to end: it was + /// resolved, it finished, or its command is gone. + fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { + let spent = snapshot + .expiry_requests + .iter() + .filter(|request| { + !snapshot.transactions.iter().any(|stored| { + stored.value.command_id == request.value.command_id + && stored.value.resolves_by_deadline() + }) + }) + .map(|request| request.envelope.clone()) + .collect::>(); + if spent.is_empty() { + return Ok(false); + } + SingleFileMessagePackBackingStore::new(&self.options.state_store) + .delete_batch_if_unchanged(&spent) + } + + fn record_resumable_error( + &self, + current: &Stored, + error: &anyhow::Error, + ) -> Result<()> { + let detail = truncate(&format!("{error:#}"), 2048); + if current.value.last_error.as_deref() == Some(detail.as_str()) { + return Ok(()); + } + let mut next = current.value.clone(); + next.last_error = Some(detail); + next.updated_at_unix_millis = now_millis()?; + replace_transaction(&self.options.state_store, current, &next) + } + + fn begin_pre_fencing_abort( + &self, + current: &Stored, + error: anyhow::Error, + ) -> Result<()> { + ensure!( + current.value.phase < DeploymentPhase::Fencing, + "post-fence transaction cannot terminal-fail" + ); + ensure!( + current.value.pre_fencing_abort.is_none(), + "pre-fencing abort intent is already durable" + ); + let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); + self.persist_same_phase(current, |next| { + next.pre_fencing_abort = Some(abort); + Ok(()) + }) + } + + /// Whether this transaction's candidate can no longer run at all. + /// + /// A transaction that has never started one has no candidate to be dead, so + /// it is not permanently stopped -- it is simply not there yet. + fn candidate_is_permanently_stopped( + &self, + transaction: &DeploymentTransaction, + ) -> Result { + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(required(&transaction.plan, "transaction plan")?)? + .is_permanently_stopped(workload) + } + + fn begin_post_fencing_abort( + &self, + current: &Stored, + error: anyhow::Error, + ) -> Result<()> { + ensure!( + current.value.phase >= DeploymentPhase::Fencing, + "pre-fence transaction must abort through the pre-fencing path" + ); + ensure!( + current.value.post_fencing_abort.is_none(), + "post-fencing abort intent is already durable" + ); + let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); + self.persist_same_phase(current, |next| { + next.post_fencing_abort = Some(abort); + Ok(()) + }) + } + + /// Withdraw the candidate in the one order that never leaves two writers: + /// take away its route, then its write lease, then the process itself, and + /// only then reconcile the projection. + fn advance_post_fencing_abort(&self, current: &Stored) -> Result<()> { + let abort = required( + ¤t.value.post_fencing_abort, + "post-fencing abort intent", + )?; + if abort.route_restoration == CleanupEvidence::Pending { + let preflight = required(¤t.value.route_preflight, "route preflight receipt")?; + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let driver = + self.route_driver(binding.route.context("routed plan has no route binding")?); + driver + .withdraw_candidate_membership( + preflight, + &self.route_gate(¤t.value.target, current.value.command_kind), + ) + .context("restoring the route the candidate found")?; + return self.persist_same_phase(current, |next| { + next.post_fencing_abort.as_mut().unwrap().route_restoration = + CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.lease_withdrawal == CleanupEvidence::Pending { + let expected = required(¤t.value.expected, "Expected projection")?; + let activation = required(¤t.value.activation, "activation")?; + // A prepared lease is withdrawn too: its physical grant may have + // landed before `Granted` was durable. Revoking an absent lease, + // and withdrawing an absent projection, are no-ops. + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + .context("post-fencing abort lost the lease it must withdraw")?; + let binding = current.value.plan.as_ref().unwrap().parsed_inputs()?.1; + let lease_path = binding + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + driver + .revoke_exact(Some(lease)) + .context("revoking the abandoned candidate write lease")?; + ensure!( + driver.observe_empty()?, + "candidate write lease remained after post-fencing withdrawal" + ); + self.topology() + .withdraw_process_write_lease(expected, activation, Some(lease)) + .context("withdrawing the abandoned candidate write-lease projection")?; + return self.persist_same_phase(current, |next| { + next.post_fencing_abort.as_mut().unwrap().lease_withdrawal = + CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.candidate_cleanup == CleanupEvidence::Pending { + let plan = required(¤t.value.plan, "abandoned candidate plan")?; + if let Some(workload) = ¤t.value.workload { + self.workload_for(plan)? + .stop(workload) + .context("stopping the abandoned candidate")?; + } + self.workload_for(plan)? + .discard_prepared( + required(¤t.value.plan, "abandoned candidate plan")?, + required(¤t.value.expected, "abandoned Expected projection")?, + required(¤t.value.activation, "abandoned activation")?, + ) + .context("discarding the abandoned activation material")?; + return self.persist_same_phase(current, |next| { + next.post_fencing_abort.as_mut().unwrap().candidate_cleanup = + CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.topology_reconciliation == CleanupEvidence::Pending { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + // The failed candidate's projection is resolved first. For a + // deployment, fencing also stopped the incumbent and revoked its + // lease, so the incumbent is demoted to Expected-only under its + // key with its own exact activation, which is what lets + // continuity bring it back. A continuity candidate is the + // incumbent's key: it has nothing further to demote. + let topology = self.topology(); + let failed_provider_anchor = self.provider_anchor_for_plan(required( + ¤t.value.plan, + "failed transaction plan", + )?)?; + reconcile_failed_candidate_projection( + &topology, + ¤t.value, + &failed_provider_anchor, + )?; + if let Some(incumbent) = self + .exact_incumbent(&snapshot, ¤t.value)? + .filter(|_| current.value.command_kind == CommandKind::Deploy) + { + let admitted_provider_anchor = + self.provider_anchor_for_plan(&incumbent.value.plan)?; + let expected_sha256 = topology.demote_to_expected_only( + &incumbent.value.expected, + &admitted_provider_anchor, + &incumbent.value.activation, + incumbent.value.leasing.lease(), + )?; + ensure!( + expected_sha256 == incumbent.value.expected.canonical_sha256()?, + "restored incumbent Expected receipt differs" + ); + } + return self.persist_same_phase(current, |next| { + next.post_fencing_abort + .as_mut() + .unwrap() + .topology_reconciliation = CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.source_cleanup == CleanupEvidence::Pending { + self.source + .cleanup( + ¤t.value.transaction_id, + current.value.frozen_source.as_ref(), + ) + .context("cleaning the abandoned source")?; + return self.persist_same_phase(current, |next| { + next.post_fencing_abort.as_mut().unwrap().source_cleanup = + CleanupEvidence::Complete; + Ok(()) + }); + } + ensure!( + abort.is_complete(), + "post-fencing abort cleanup is incomplete" + ); + let mut next = current.value.clone(); + next.enter_phase(DeploymentPhase::Complete, now_millis()?); + next.last_error = Some(abort.error.clone()); + next.completion = Some(TransactionCompletion::FailedAfterFencing { + error: abort.error.clone(), + recovery: current.value.terminal_recovery(), + }); + replace_transaction(&self.options.state_store, current, &next) + } + + fn advance_pre_fencing_abort(&self, current: &Stored) -> Result<()> { + ensure!( + current.value.phase < DeploymentPhase::Fencing, + "pre-fencing abort crossed the fencing boundary" + ); + let abort = required(¤t.value.pre_fencing_abort, "pre-fencing abort intent")?; + if abort.candidate_cleanup == CleanupEvidence::Pending { + let plan = required(¤t.value.plan, "transaction plan")?; + if let Some(workload) = ¤t.value.workload { + self.workload_for(plan)? + .stop(workload) + .context("stopping exact pre-fence candidate")?; + } + self.workload_for(plan)? + .discard_prepared( + required(¤t.value.plan, "pre-fencing candidate plan")?, + required(¤t.value.expected, "pre-fencing Expected projection")?, + required(¤t.value.activation, "pre-fencing activation")?, + ) + .context("discarding exact pre-fence activation material")?; + return self.persist_same_phase(current, |next| { + next.pre_fencing_abort.as_mut().unwrap().candidate_cleanup = + CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.topology_reconciliation == CleanupEvidence::Pending { + // Before fencing a deployment never touched its incumbent: it is + // still running under its own key with its activation and lease, + // so only the candidate's records are withdrawn. A continuity + // candidate shares the admitted key and demotes what it issued. + let plan = required(¤t.value.plan, "failed transaction plan")?; + let provider_anchor = self.provider_anchor_for_plan(plan)?; + reconcile_failed_candidate_projection( + &self.topology(), + ¤t.value, + &provider_anchor, + )?; + return self.persist_same_phase(current, |next| { + next.pre_fencing_abort + .as_mut() + .unwrap() + .topology_reconciliation = CleanupEvidence::Complete; + Ok(()) + }); + } + if abort.source_cleanup == CleanupEvidence::Pending { + self.source + .cleanup( + ¤t.value.transaction_id, + current.value.frozen_source.as_ref(), + ) + .context("cleaning failed pre-fence source")?; + return self.persist_same_phase(current, |next| { + next.pre_fencing_abort.as_mut().unwrap().source_cleanup = CleanupEvidence::Complete; + Ok(()) + }); + } + ensure!( + abort.is_complete(), + "pre-fencing abort cleanup is incomplete" + ); + let mut next = current.value.clone(); + next.enter_phase(DeploymentPhase::Complete, now_millis()?); + next.last_error = Some(abort.error.clone()); + next.completion = Some(TransactionCompletion::FailedBeforeFencing { + error: abort.error.clone(), + }); + replace_transaction(&self.options.state_store, current, &next) + } + + fn advance_post_commit_cleanup(&self, current: &Stored) -> Result<()> { + let cleanup = required( + ¤t.value.post_commit_cleanup, + "post-commit cleanup evidence", + )?; + if let IncumbentCleanupEvidence::Pending { + generation_id, + workload, + } = &cleanup.incumbent + { + let binding = required(¤t.value.plan, "committed transaction plan")? + .parsed_inputs()? + .1; + if let Some(promoted_at_unix_millis) = required( + ¤t.value.routing, + "committed transaction routing evidence", + )? + .promoted_at_unix_millis() + { + let retire_not_before = + route_drain_deadline(promoted_at_unix_millis, binding.rollout.drain_seconds)?; + if now_millis()? < retire_not_before { + return Ok(()); + } + } + // The incumbent is the same target under the same binding kind; + // an incumbent of another kind is refused by the driver, not + // guessed at. + self.workload_for(required(¤t.value.plan, "committed transaction plan")?)? + .stop(workload) + .with_context(|| format!("retiring admitted incumbent {generation_id}"))?; + let generation_id = generation_id.clone(); + return self.persist_same_phase(current, |next| { + next.post_commit_cleanup.as_mut().unwrap().incumbent = + IncumbentCleanupEvidence::Complete { generation_id }; + Ok(()) + }); + } + if cleanup.source == SourceCleanupEvidence::Pending { + self.source + .cleanup( + ¤t.value.transaction_id, + current.value.frozen_source.as_ref(), + ) + .context("cleaning committed deployment source")?; + return self.persist_same_phase(current, |next| { + next.post_commit_cleanup.as_mut().unwrap().source = SourceCleanupEvidence::Complete; + Ok(()) + }); + } + ensure!(cleanup.is_complete(), "post-commit cleanup is incomplete"); + Ok(()) + } +} + +fn route_drain_deadline(promoted_at_unix_millis: u64, drain_seconds: u32) -> Result { + promoted_at_unix_millis + .checked_add( + u64::from(drain_seconds) + .checked_mul(1_000) + .context("route drain duration overflows milliseconds")?, + ) + .context("route drain deadline overflows Unix milliseconds") +} + +/// CultLib refuses a correlation older than the trusted window with this +/// text. Idunn reads that one refusal as "nothing new", everywhere it waits +/// on Odin's evidence; every other refusal stays a refusal. +fn is_stale_observation(error: &anyhow::Error) -> bool { + format!("{error:#}").contains("outside the trusted observation window") +} + +fn route_observation_is_current( + observed_at_unix_millis: u64, + now_unix_millis: u64, + maximum_age_millis: u64, + maximum_future_skew_millis: u64, +) -> bool { + observed_at_unix_millis <= now_unix_millis.saturating_add(maximum_future_skew_millis) + && now_unix_millis.saturating_sub(observed_at_unix_millis) <= maximum_age_millis +} + +fn provider_warming_advanced( + prior_odin_sequence: u64, + prior_signed_presence_sha256: &str, + candidate_odin_sequence: u64, + candidate_signed_presence_sha256: &str, +) -> bool { + candidate_odin_sequence > prior_odin_sequence + && candidate_signed_presence_sha256 != prior_signed_presence_sha256 +} + +/// Resolve the projection a failed candidate left. A deployment candidate owns +/// its own incarnation key and withdraws it whole. A continuity candidate +/// shares the admitted key: the admitted Expected is never withdrawn, and the +/// only thing demoted is the activation this transaction issued, exactly. +fn reconcile_failed_candidate_projection( + topology: &CultCacheTopologyDriver, + transaction: &DeploymentTransaction, + provider_anchor: &ServiceIdentityTrustAnchor, +) -> Result<()> { + let expected = required(&transaction.expected, "failed Expected projection")?; + match transaction.command_kind { + CommandKind::Deploy => topology + .withdraw_incarnation( + expected, + provider_anchor, + transaction.activation.as_ref(), + None, + ) + .context("withdrawing the failed candidate projection"), + CommandKind::Continuity => topology + .demote_to_expected_only( + expected, + provider_anchor, + required(&transaction.activation, "failed continuity activation")?, + None, + ) + .map(drop) + .context("demoting the failed continuity activation"), + } +} + +fn may_rollback_route_after_failed_proof(fencing: &FencingEvidence) -> bool { + matches!(fencing, FencingEvidence::SkippedStateless) +} + +fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { + matches!( + fencing, + FencingEvidence::Revoked { + incumbent_lease_sha256: Some(_), + .. + } + ) +} + +fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { + PreFencingAbort { + error: truncate(error, 2048), + candidate_cleanup: transaction.candidate_cleanup_owed(), + topology_reconciliation: transaction.abort_topology_reconciliation(), + source_cleanup: if transaction.command_kind == CommandKind::Deploy { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + } +} + +fn post_fencing_abort_intent( + transaction: &DeploymentTransaction, + error: &str, +) -> PostFencingAbort { + PostFencingAbort { + error: truncate(error, 2048), + route_restoration: if transaction.route_preflight.is_some() { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + lease_withdrawal: if transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + .is_some() + { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + candidate_cleanup: transaction.candidate_cleanup_owed(), + topology_reconciliation: transaction.abort_topology_reconciliation(), + source_cleanup: if transaction.command_kind == CommandKind::Deploy { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + } +} + +fn sequence_requires_admission( + latest: Option<&TopologyEvidence>, + maximum_admitted_sequence: u64, + candidate: &TopologyEvidence, +) -> Result { + candidate.validate_shape()?; + if latest.is_some_and(|existing| { + existing.canonical_bytes == candidate.canonical_bytes + && existing.canonical_sha256 == candidate.canonical_sha256 + && existing.signer_identity_id == candidate.signer_identity_id + && existing.publisher_sequence == candidate.publisher_sequence + }) { + return Ok(false); + } + ensure!( + candidate.publisher_sequence > maximum_admitted_sequence, + "Odin topology publisher sequence was replayed or reordered" + ); + Ok(true) +} + +fn validate_live_providers_for_deploy(command_kind: CommandKind, validation: F) -> Result<()> +where + F: FnOnce() -> Result<()>, +{ + match command_kind { + CommandKind::Deploy => validation(), + CommandKind::Continuity => Ok(()), + } +} + +fn validate_authenticated_evidence( + evidence: &TopologyEvidence, + authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, +) -> Result<()> { + let record = authenticated.record(); + ensure!( + authenticated.canonical_bytes() == evidence.canonical_bytes.as_slice() + && sha256_id(authenticated.canonical_bytes()) == evidence.canonical_sha256 + && record.signer_identity_id == evidence.signer_identity_id + && record.publisher_sequence == evidence.publisher_sequence, + "durable topology evidence differs from its authenticated receipt" + ); + Ok(()) +} + +fn is_semantic_warming( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + incumbent_lease_sha256: Option<&str>, + authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation, +) -> Result { + let record = authenticated.record(); + if !record.present || record.observed_write_lease_sha256.is_some() { + return Ok(false); + } + let expected_projection_detail = format!( + "expected:{};activation:{}", + expected.canonical_sha256()?, + activation.canonical_sha256()? + ); + if !warming_disagreements_match_incumbent( + &expected_projection_detail, + incumbent_lease_sha256, + &record.disagreements, + ) { + return Ok(false); + } + Ok(if expected.write_lease_required { + !record.ready && record.observed_presence_state.as_deref() == Some("warming") + } else { + matches!( + record.observed_presence_state.as_deref(), + Some("warming" | "active") + ) + }) +} + +fn warming_disagreements_match_incumbent( + expected_projection_detail: &str, + incumbent_lease_sha256: Option<&str>, + disagreements: &[OdinTopologyDisagreement], +) -> bool { + if disagreements.is_empty() { + true + } else if let (Some(incumbent_lease_sha256), [disagreement]) = + (incumbent_lease_sha256, disagreements) + { + disagreement.code == "projected-write-lease" + && disagreement.expected.as_deref() == Some(expected_projection_detail) + && disagreement.observed.as_deref() == Some(incumbent_lease_sha256) + } else { + false + } +} + +fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation) -> bool { + let record = authenticated.record(); + record.present + && record.ready + && record.observed_presence_state.as_deref() == Some("active") + && record.disagreements.is_empty() +} + +fn read_deployment_brake(path: &Path) -> Result)>> { + let Some(envelope) = read_single_envelope(path)? else { + return Ok(None); + }; + ensure!( + envelope.r#type == IdunnDeploymentBrakeRecord::TYPE + && envelope.schema_id.as_deref() == Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA), + "deployment brake store contains a foreign record" + ); + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&envelope.payload)?; + record.validate()?; + ensure!( + rmp_serde::to_vec(&record)? == envelope.payload && envelope.key == record.brake_id, + "deployment brake is noncanonical or keyed by another authority" + ); + Ok(Some((record, envelope.payload))) +} + +fn read_lifecycle_brake(path: &Path) -> Result> { + let Some(envelope) = read_single_envelope(path)? else { + return Ok(None); + }; + ensure!( + envelope.r#type == IdunnLifecycleBrakeRecord::TYPE + && envelope.schema_id.as_deref() == Some(IDUNN_LIFECYCLE_BRAKE_SCHEMA), + "lifecycle brake store contains a foreign record" + ); + IdunnLifecycleBrakeRecord::decode_canonical(&envelope.payload).map(Some) +} + +fn read_single_envelope(path: &Path) -> Result> { + if !path.exists() { + return Ok(None); + } + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + match entries.as_slice() { + [] => Ok(None), + [entry] => Ok(Some(entry.clone())), + _ => bail!("single-record authority store is ambiguous"), + } +} + +pub(crate) fn read_trust_anchor( + path: &Path, +) -> Result { + let envelope = read_single_envelope(path)? + .with_context(|| format!("service identity trust anchor {} is absent", path.display()))?; + ensure!( + envelope.r#type == P::TRUST_ANCHOR_TYPE + && envelope.key == P::TRUST_ANCHOR_KEY + && envelope.schema_id.as_deref() == Some(P::TRUST_ANCHOR_SCHEMA), + "service identity trust anchor belongs to another profile" + ); + let anchor: ServiceIdentityTrustAnchor = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&anchor)? == envelope.payload + && derive_service_identity_id::

(&anchor.public_key)? == anchor.identity_id, + "service identity trust anchor is noncanonical or self-inconsistent" + ); + Ok(anchor) +} + +fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { + value.as_ref().with_context(|| format!("{label} is absent")) +} + +fn now_millis() -> Result { + Ok(SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock predates Unix epoch")? + .as_millis() + .try_into()?) +} + +fn rfc3339_millis(millis: u64) -> Result { + chrono::DateTime::from_timestamp_millis(i64::try_from(millis)?) + .context("timestamp is out of range") + .map(|value| value.to_rfc3339()) +} + +fn sibling_path(path: &Path, suffix: &str) -> PathBuf { + let mut value = path.as_os_str().to_os_string(); + value.push(suffix); + PathBuf::from(value) +} + +fn string_value(args: &mut impl Iterator, name: &str) -> Result { + args.next() + .ok_or_else(|| anyhow!("{name} requires a value")) +} + +fn path_value(args: &mut impl Iterator, name: &str) -> Result { + Ok(PathBuf::from(string_value(args, name)?)) +} + +fn u32_value(args: &mut impl Iterator, name: &str) -> Result { + string_value(args, name)? + .parse() + .with_context(|| format!("{name} requires a u32")) +} + +fn u64_value(args: &mut impl Iterator, name: &str) -> Result { + string_value(args, name)? + .parse() + .with_context(|| format!("{name} requires a u64")) +} + +fn require_selector(value: &str) -> Result<()> { + let value = value.strip_prefix("profile:").unwrap_or(value); + require_id(value, "deployment selector") +} + +pub(crate) fn require_id(value: &str, label: &str) -> Result<()> { + ensure!( + !value.is_empty() + && value.len() <= 256 + && value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/') + }), + "{label} is invalid" + ); + Ok(()) +} + +fn require_value(value: &str, label: &str) -> Result<()> { + ensure!( + !value.trim().is_empty() + && value == value.trim() + && value.len() <= 1024 + && !value.contains('\0'), + "{label} is invalid" + ); + Ok(()) +} + +fn require_detail(value: &str, label: &str) -> Result<()> { + ensure!( + !value.is_empty() && value.len() <= 2048 && !value.contains('\0'), + "{label} is invalid" + ); + Ok(()) +} + +fn truncate(value: &str, length: usize) -> String { + value.chars().take(length).collect() +} + +fn sha256_id(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut encoded = String::with_capacity(71); + encoded.push_str("sha256-"); + for byte in digest { + use std::fmt::Write as _; + write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); + } + encoded +} + +fn runtime_instance_id(transaction_id: &str) -> Result { + require_id(transaction_id, "runtime transaction id")?; + Ok(sha256_id( + format!("gamecult.idunn.runtime-instance.v1:{transaction_id}").as_bytes(), + )) +} + +fn usage() -> &'static str { + "Idunn deployment, admission, and continuity control plane\n\n\ + idunn serve [runtime options] [--host-actuator-bind ADDR]\n\ + idunn up [--state-store PATH] [--no-wait]\n\ + idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ + idunn cancel [--state-store PATH]\n\ + idunn expire [--state-store PATH]\n\ + idunn validate --recipe PATH [--binding PATH]\n\n\ + Recipes describe capability and process requirements. Idunn seals exact\n\ + source and artifacts, admits one incarnation, and delegates execution to\n\ + systemd and routing mechanics to the configured proxy." +} + +#[cfg(test)] +mod tests { + use cultnet_rs::{ + GameCultProviderHealthIdentity, OdinRuntimeTopologyCorrelationPurpose, + OdinRuntimeTopologyCorrelationRecord, enroll_service_identity_at, + export_service_identity_trust_anchor, + }; + use tempfile::TempDir; + + use super::*; + use crate::drivers::{SystemdWorkloadObservation, incarnation_key}; + use crate::incident::INCIDENT_RETENTION_MILLIS; + + /// Fixed clock for the signing fixture; correlations must land inside + /// DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS of it to authenticate. + const NOW: u64 = 1_700_000_000_000; + + fn digest(byte: char) -> String { + byte.to_string().repeat(64) + } + + fn route_binding( + route_id: &str, + stable_port: u16, + private_port_start: u16, + private_port_end: u16, + ) -> RouteBinding { + RouteBinding { + driver: crate::deployment::RouteDriver::NginxStreamTcp, + route_id: route_id.into(), + stable_endpoint: format!("tcp://127.0.0.1:{stable_port}"), + private_host: "127.0.0.1".into(), + private_port_start, + private_port_end, + config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), + reload_unit: "nginx.service".into(), + } + } + + fn command(kind: CommandKind) -> DeploymentCommand { + DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: match kind { + CommandKind::Deploy => "up-test".into(), + CommandKind::Continuity => "continuity-test".into(), + }, + kind, + selector: "ghostlight".into(), + requested_by: "operator".into(), + requested_at_unix_millis: 100, + } + } + + /// Drives the replacement gate against the real `control.cc`. The point is + /// not that the unit tests pass but that the record actually stuck on the + /// host would now commit. Run with: + /// IDUNN_LIVE_CONTROL=/path/to/control.cc cargo test live_committing -- --ignored --nocapture + #[test] + #[ignore] + fn live_committing_record_is_unstuck_by_the_replacement_gate() -> Result<()> { + let Ok(path) = std::env::var("IDUNN_LIVE_CONTROL") else { + eprintln!("IDUNN_LIVE_CONTROL unset"); + return Ok(()); + }; + let snapshot = ControlSnapshot::read(Path::new(&path))?; + for stored in &snapshot.transactions { + let value = &stored.value; + if value.completion.is_some() { + continue; + } + let (Some(ready), Some(latest)) = ( + value.ready.as_ref().and_then(ReadinessEvidence::odin), + &value.latest_odin_observation, + ) else { + continue; + }; + println!( + "LIVE {} target={} phase={:?} ready_seq={} latest_seq={}", + value.transaction_id, + value.target, + value.phase, + ready.publisher_sequence, + latest.publisher_sequence + ); + println!( + " old gate (receipt == cursor): {}", + if ready == latest { "PASS" } else { "REFUSE" } + ); + let (latest_record, _) = + OdinRuntimeTopologyCorrelationRecord::decode_canonical_signed_payload( + &latest.canonical_bytes, + )?; + println!( + " cursor says: projection={} instance={:?} ready={} present={} disagreements={}", + latest_record.expected_projection_sha256, + latest_record.runtime_instance_id, + latest_record.ready, + latest_record.present, + latest_record.disagreements.len() + ); + let (Some(expected), Some(activation)) = (&value.expected, &value.activation) else { + println!(" transaction has not declared an incarnation yet"); + continue; + }; + println!( + " declared: projection={} instance={}", + expected.canonical_sha256()?, + activation.runtime_instance_id + ); + let binds = latest_record.expected_projection_sha256 == expected.canonical_sha256()? + && latest_record.runtime_instance_id.as_deref() + == Some(activation.runtime_instance_id.as_str()); + println!(" binds declared incarnation (what authentication enforces): {binds}"); + } + Ok(()) + } + + /// The smallest real `Engine` the scheduler needs. Idunn had no + /// Engine-level test at all, which is why "does one bad record kill the + /// daemon" was arguable rather than measurable. + struct EngineFixture { + _temp: TempDir, + engine: Engine, + state_store: PathBuf, + odin_signer: ServiceIdentitySigner, + root: PathBuf, + /// The host programs every route driver of this Engine actuates + /// through: logged stubs, so nothing touches the workstation's nginx. + #[cfg(unix)] + route_stubs: RouteStubs, + } + + /// Stub nginx, systemd-run, systemctl and ufw that append each call to one + /// log. `reload-fails` makes `systemctl reload` exit non-zero. + #[cfg(unix)] + struct RouteStubs { + calls: PathBuf, + reload_fails: PathBuf, + ufw_delete_fails: PathBuf, + } + + #[cfg(unix)] + impl RouteStubs { + fn new(root: &Path) -> Result<(RouteActuators, Self)> { + use std::os::unix::fs::PermissionsExt; + + let dir = root.join("route-stubs"); + std::fs::create_dir_all(&dir)?; + let calls = dir.join("calls"); + let reload_fails = dir.join("reload-fails"); + let ufw_delete_fails = dir.join("ufw-delete-fails"); + let program = |name: &str, refuse: &str| -> Result { + let path = dir.join(name); + std::fs::write( + &path, + format!( + "#!/bin/sh\necho \"{name} $*\" >> '{}'\n{refuse}exit 0\n", + calls.display() + ), + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))?; + Ok(path) + }; + let actuators = RouteActuators { + nginx: program("nginx", "")?, + systemd_run: program("systemd-run", "")?, + systemctl: program( + "systemctl", + &format!( + "if [ \"$1\" = reload ] && [ -e '{}' ]; then exit 1; fi\n", + reload_fails.display() + ), + )?, + ufw: program( + "ufw", + &format!( + "if [ \"$1\" = delete ] && [ -e '{}' ]; then exit 1; fi\n", + ufw_delete_fails.display() + ), + )?, + preflight_root: root.join("route-preflight"), + }; + Ok(( + actuators, + Self { + calls, + reload_fails, + ufw_delete_fails, + }, + )) + } + + fn refuse_reloads(&self) -> Result<()> { + std::fs::write(&self.reload_fails, b"x")?; + Ok(()) + } + + /// `ufw delete` exits non-zero with an error that is not "non-existent". + fn refuse_ufw_deletes(&self) -> Result<()> { + std::fs::write(&self.ufw_delete_fails, b"x")?; + Ok(()) + } + + /// How many logged calls begin with `prefix`, e.g. `systemctl reload`. + fn count(&self, prefix: &str) -> usize { + std::fs::read_to_string(&self.calls) + .unwrap_or_default() + .lines() + .filter(|line| line.starts_with(prefix)) + .count() + } + } + + impl EngineFixture { + fn new() -> Result { + Self::build(None) + } + + /// An Engine whose systemd workload port is `workload`, so the phase + /// machine can run without systemd. This is the whole seam: nginx, + /// systemctl and the network are never reached on an unrouted, + /// stateless target, and the Odin evidence is signed by the key this + /// fixture enrolled. + fn with_workload(workload: Arc) -> Result { + Self::build(Some(workload)) + } + + fn build(workload: Option>) -> Result { + let temp = TempDir::new()?; + let root = temp.path(); + std::fs::create_dir_all(root.join("identities"))?; + let idunn = root.join("identities/idunn.cc"); + enroll_service_identity_at::(&idunn)?; + let odin_private = root.join("identities/odin.cc"); + let odin_signer = enroll_service_identity_at::(&odin_private)?; + let odin_anchor = root.join("identities/odin-anchor.cc"); + export_service_identity_trust_anchor(&odin_signer, &odin_anchor)?; + + let state_store = root.join("control.cc"); + #[cfg(unix)] + let (route_actuators, route_stubs) = RouteStubs::new(root)?; + let options = RuntimeOptions { + state_store: state_store.clone(), + bindings_dir: root.join("bindings"), + source_root: root.join("sources"), + staging_root: root.join("staging"), + topology_store: root.join("topology.cc"), + incident_store: root.join("incidents.cc"), + odin_correlation_store: root.join("odin-correlation.cc"), + odin_trust_anchor: odin_anchor, + idunn_identity_store: idunn, + deployment_brake_operator_anchor: root.join("brake-anchor.cc"), + #[cfg(unix)] + route_actuators, + ..RuntimeOptions::default() + }; + let engine = match workload { + Some(workload) => Engine::open_with_systemd_workload(options, workload)?, + None => Engine::open(options)?, + }; + Ok(Self { + root: root.to_path_buf(), + _temp: temp, + engine, + state_store, + odin_signer, + #[cfg(unix)] + route_stubs, + }) + } + } + + fn terminal_transaction(target: &str) -> Result { + Ok(terminal_transaction_with_command(target)?.0) + } + + fn terminal_transaction_with_command( + target: &str, + ) -> Result<(DeploymentTransaction, DeploymentCommand)> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: format!("up-{target}"), + kind: CommandKind::Deploy, + selector: target.into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, target.into(), 0, None, 100)?; + transaction.phase = DeploymentPhase::Complete; + transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { + error: "sealed source vanished".into(), + }); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "sealed source vanished".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + assert!(transaction.is_terminal(), "fixture must be terminal"); + Ok((transaction, command)) + } + + #[test] + fn a_finished_transaction_leaves_the_live_set_and_stays_observable() -> Result<()> { + let world = EngineFixture::new()?; + let history = history_store_path(&world.state_store); + + let (live, command) = terminal_transaction_with_command("ghostlight")?; + // A transaction is only readable beside its immutable command. + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope( + &command, + command.requested_at_unix_millis + )?], + )? + ); + let mut opening = live.clone(); + opening.phase = DeploymentPhase::Sealing; + opening.completion = None; + opening.pre_fencing_abort = None; + let opening_envelope = transaction_envelope(&opening, opening.updated_at_unix_millis)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: opening.transaction_id.clone(), + current: None, + }], + std::slice::from_ref(&opening_envelope), + )? + ); + let stored = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("seeded transaction")?; + + replace_transaction(&world.state_store, &stored, &live)?; + + // Gone from the set that still gates decisions... + assert!( + ControlSnapshot::read(&world.state_store)? + .transactions + .is_empty(), + "a finished transaction stayed resident" + ); + // ...and still answerable, which is what R11 refused to trade away. + let archived = read_history_transactions(&world.state_store); + assert_eq!(archived.len(), 1); + assert_eq!(archived[0].transaction_id, live.transaction_id); + assert!(history.exists()); + // The command went with it. Left resident it would read as queued and + // be frozen again next tick; consumed, it is history beside its + // transaction and `status` still answers for it. + assert!( + ControlSnapshot::read(&world.state_store)? + .commands + .is_empty(), + "a consumed command stayed resident" + ); + let archived_commands = read_history_commands(&world.state_store); + assert_eq!(archived_commands.len(), 1); + assert_eq!(archived_commands[0].command_id, command.command_id); + assert!(!world.engine.freeze_one_queued_command()?); + Ok(()) + } + + #[test] + fn a_resident_command_whose_transactions_are_history_is_retired_not_frozen() -> Result<()> { + // Before commands travelled to history with their transactions, a + // command outlived its retired transaction in the control store and + // read as queued again. Freeze must retire it, never re-freeze it. + let world = EngineFixture::new()?; + let (finished, command) = terminal_transaction_with_command("ghostlight")?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + command_envelope(&command, command.requested_at_unix_millis)?, + )?; + SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) + .insert_entry_if_absent(transaction_envelope( + &finished, + finished.updated_at_unix_millis, + )?)?; + assert_eq!(ControlSnapshot::read(&world.state_store)?.commands.len(), 1); + + assert!(world.engine.freeze_one_queued_command()?); + + assert!( + ControlSnapshot::read(&world.state_store)? + .commands + .is_empty(), + "consumed command stayed resident" + ); + assert_eq!(read_history_commands(&world.state_store).len(), 1); + assert!( + ControlSnapshot::read(&world.state_store)? + .transactions + .is_empty(), + "a consumed command was frozen again" + ); + assert!(!world.engine.freeze_one_queued_command()?); + Ok(()) + } + + #[test] + fn a_resident_terminal_transaction_is_retired_by_the_tick_with_its_command() -> Result<()> { + let world = EngineFixture::new()?; + let (finished, command) = terminal_transaction_with_command("ghostlight")?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + command_envelope(&command, command.requested_at_unix_millis)?, + )?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + transaction_envelope(&finished, finished.updated_at_unix_millis)?, + )?; + assert!(world.engine.retire_one_terminal_transaction()?); + let live = ControlSnapshot::read(&world.state_store)?; + assert!(live.transactions.is_empty() && live.commands.is_empty()); + assert_eq!(read_history_transactions(&world.state_store).len(), 1); + assert_eq!(read_history_commands(&world.state_store).len(), 1); + assert!(!world.engine.retire_one_terminal_transaction()?); + Ok(()) + } + + #[test] + fn re_archiving_after_a_crash_window_is_a_no_op() -> Result<()> { + // History is written before the live copy is removed, so a crash + // between the two leaves the record in both. The repeat must not + // duplicate it or fail. + let world = EngineFixture::new()?; + let live = terminal_transaction("odin")?; + let envelope = transaction_envelope(&live, live.updated_at_unix_millis)?; + + archive_terminal_transaction(&world.state_store, &envelope)?; + archive_terminal_transaction(&world.state_store, &envelope)?; + + let archived = read_history_transactions(&world.state_store); + assert_eq!(archived.len(), 1, "the repeat duplicated the record"); + Ok(()) + } + + #[test] + fn an_unreadable_history_entry_is_skipped_rather_than_fatal() -> Result<()> { + // The asymmetry that justifies two files: control.cc refuses a record + // it cannot re-encode byte for byte, because those records still gate + // decisions. History describes what already happened and must never be + // able to refuse a read. + let world = EngineFixture::new()?; + let live = terminal_transaction("voidbot")?; + let envelope = transaction_envelope(&live, live.updated_at_unix_millis)?; + archive_terminal_transaction(&world.state_store, &envelope)?; + + let history = history_store_path(&world.state_store); + assert!( + SingleFileMessagePackBackingStore::new(&history).insert_entry_if_absent( + CultCacheEnvelope { + key: "tx-corrupt".into(), + r#type: DeploymentTransaction::TYPE.into(), + payload: vec![0xc1], + stored_at: rfc3339_millis(1)?, + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), + } + )? + ); + + let archived = read_history_transactions(&world.state_store); + assert_eq!(archived.len(), 1, "the readable record must survive"); + assert_eq!(archived[0].transaction_id, live.transaction_id); + Ok(()) + } + + #[test] + fn an_absent_brake_anchor_does_not_gate_an_empty_store() -> Result<()> { + // `F:\Projects\CLAUDE.md`: a target brake "may never gate Idunn itself + // or unrelated service lifecycle". Reading the operator anchor up front + // meant a missing brake artifact refused startup against a store with + // nothing in it -- the deployment authority held hostage by an artifact + // describing one target's consent. + let world = EngineFixture::new()?; + assert!( + !world + .engine + .options + .deployment_brake_operator_anchor + .exists(), + "fixture must not have written a brake anchor" + ); + let snapshot = ControlSnapshot::read(&world.state_store)?; + world.engine.validate_durable_authority(&snapshot)?; + Ok(()) + } + + #[test] + fn one_unreadable_record_faults_its_tick_without_touching_the_store() -> Result<()> { + let world = EngineFixture::new()?; + + // An empty store schedules nothing and must not fault. + assert!(!world.engine.run_scheduler_tick()?); + + // A record the control store cannot read. Idunn runs under + // Restart=always, so before run_scheduler_tick existed this + // propagated out of the loop and crashlooped the daemon -- taking + // every unrelated target's continuity with it. + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: "tx-unreadable".into(), + current: None, + }], + &[CultCacheEnvelope { + key: "tx-unreadable".into(), + r#type: DeploymentTransaction::TYPE.into(), + // 0xc1 is msgpack's never-used byte: decodable by nothing. + payload: vec![0xc1], + stored_at: rfc3339_millis(1)?, + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), + }], + )? + ); + let poisoned = std::fs::read(&world.state_store)?; + + for _ in 0..3 { + assert!( + world.engine.run_scheduler_tick().is_err(), + "the tick must report the fault rather than swallow it" + ); + } + + // Faulting is not the same as flailing: a tick that cannot read must + // not rewrite, truncate or repair the store behind the operator. + assert_eq!( + std::fs::read(&world.state_store)?, + poisoned, + "a faulting tick rewrote the control store" + ); + Ok(()) + } + + /// A real signing chain for topology gates. Until this existed no test in + /// this file could construct a signed Odin correlation -- every topology + /// fixture was `vec![byte]` -- so every gate between Idunn and Odin was + /// asserted against bytes that could not be authenticated. That is how a + /// comparison against Idunn's own wall clock survived in the admission + /// path. Modelled on `odin-daemon`'s TestWorld, which already had to build + /// this chain to test the other side of the same contract. + struct TopologyFixture { + _temp: TempDir, + idunn_anchor: ServiceIdentityTrustAnchor, + odin_signer: ServiceIdentitySigner, + provider_public_key: Vec, + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + } + + impl TopologyFixture { + fn new(target: &str) -> Result { + let temp = TempDir::new()?; + let root = temp.path().join("identities"); + std::fs::create_dir_all(&root)?; + let idunn_signer = + enroll_service_identity_at::(&root.join("idunn.cc"))?; + let idunn_anchor = idunn_signer.trust_anchor()?; + let odin_signer = + enroll_service_identity_at::(&root.join("odin.cc"))?; + let provider_signer = enroll_service_identity_at::( + &root.join("provider.cc"), + )?; + + let expected = IdunnExpectedIncarnationRecord { + schema_version: cultnet_rs::IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: target.into(), + plan_id: sha256_id(b"plan"), + incarnation_id: format!("{target}/generation-1"), + sealed_release_id: sha256_id(b"release"), + source_repository: format!("github.com/GameCult/{target}"), + source_revision: "3".repeat(40), + recipe_sha256: sha256_id(b"recipe"), + runtime_id: format!("{target}-runtime"), + expected_signer_identity_id: provider_signer.entry().identity_id.clone(), + health_contract: format!("{target}.runtime-health.v1"), + artifact_sha256: sha256_id(b"artifact"), + state_schema_generation: None, + state_contract_sha256: None, + write_lease_required: false, + route: None, + capabilities: Vec::new(), + dependencies: Vec::new(), + }; + expected.validate()?; + + // Idunn-signed, exactly as the real activation path issues it: the + // authority chain is what makes a correlation authenticable at all. + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + sha256_id(b"artifact-witness"), + NOW - 20, + &idunn_signer, + )?; + let activation = launch.activation().clone(); + + Ok(Self { + _temp: temp, + idunn_anchor, + odin_signer, + provider_public_key: provider_signer.entry().public_key.clone(), + expected, + activation, + }) + } + + fn authority(&self) -> Result { + verify_runtime_authority( + &self.expected, + &self.activation, + &self.idunn_anchor, + &self.provider_public_key, + ) + } + + fn correlation( + &self, + sequence: u64, + ready: bool, + ) -> Result { + Ok(OdinRuntimeTopologyCorrelationRecord { + schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: self.expected.target.clone(), + expected_projection_sha256: self.expected.canonical_sha256()?, + expected: true, + current_activation_sha256: Some(self.activation.canonical_sha256()?), + signed_presence_sha256: Some(sha256_id(b"presence")), + observed_presence_state: Some("active".into()), + observed_presence_publisher_sequence: Some(sequence), + observed_write_lease_sha256: None, + observed_capabilities: Vec::new(), + runtime_id: self.expected.runtime_id.clone(), + runtime_instance_id: Some(self.activation.runtime_instance_id.clone()), + present: true, + ready, + dependencies: Vec::new(), + disagreements: Vec::new(), + signer_identity_id: self.odin_signer.entry().identity_id.clone(), + publisher_sequence: sequence, + observed_at_unix_millis: NOW, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }) + } + + fn sign(&self, record: &mut OdinRuntimeTopologyCorrelationRecord) -> Result> { + record.signature = self + .odin_signer + .sign::( + &record.unsigned_signature_payload()?, + ) + .signature; + record.canonical_bytes() + } + + fn authenticate( + &self, + canonical: &[u8], + ) -> Result { + authenticate_odin_runtime_topology_correlation( + canonical, + &self.authority()?, + None, + &self.odin_signer.entry().public_key, + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis: NOW, + maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, + maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, + }, + ) + } + } + + #[test] + fn the_fixture_actually_authenticates() -> Result<()> { + // First: prove the chain can REFUSE. Without this the tests below + // establish nothing -- a fixture that authenticates anything would + // make every gate look satisfied. + let world = TopologyFixture::new("odin")?; + let mut record = world.correlation(297_765, true)?; + let canonical = world.sign(&mut record)?; + world.authenticate(&canonical)?; + + // Re-encoded with the old signature over changed content. + let mut forged = record.clone(); + forged.publisher_sequence = 297_766; + assert!(world.authenticate(&forged.canonical_bytes()?).is_err()); + + let mut unsigned = world.correlation(297_765, true)?; + unsigned.signature = vec![0; 64]; + assert!(world.authenticate(&unsigned.canonical_bytes()?).is_err()); + + // Stale beyond the trusted observation window. + let mut old = world.correlation(297_765, true)?; + old.observed_at_unix_millis = NOW - DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS - 1; + let old_canonical = world.sign(&mut old)?; + assert!(world.authenticate(&old_canonical).is_err()); + Ok(()) + } + + #[test] + fn a_signed_observation_far_past_the_receipt_admits_this_incarnation() -> Result<()> { + // The yggdrasil wedge, with a real signature rather than vec![byte]: + // the cursor has run hundreds of thousands of sequences past the Ready + // receipt and still describes the incarnation the transaction declared. + let world = TopologyFixture::new("odin")?; + let mut record = world.correlation(501_713, true)?; + let canonical = world.sign(&mut record)?; + let authenticated = world.authenticate(&canonical)?; + + // Authenticating at all is the incarnation check: the authority is + // built from the declared expected and activation, and the correlation + // had to bind both to get here. Nothing further to compare. + assert!(is_semantic_ready(&authenticated)); + assert_eq!(authenticated.record().publisher_sequence, 501_713); + Ok(()) + } + + #[test] + fn authentication_binds_a_correlation_to_the_declared_incarnation() -> Result<()> { + // The failure that matters: Odin's signature is valid and the record is + // fresh, but the process it describes is not the one we activated. + // Authentication already refuses this -- the authority is built from + // this transaction's own expected and activation, and the correlation + // must bind both. Pinned here because a redundant identity gate sat on + // top of this for want of a test that could reach it. + let world = TopologyFixture::new("odin")?; + let mut restarted = world.correlation(501_713, true)?; + restarted.runtime_instance_id = Some(sha256_id(b"another-instance")); + let canonical = world.sign(&mut restarted)?; + let error = world + .authenticate(&canonical) + .expect_err("correlation must bind the current activation"); + assert!( + format!("{error:#}").contains("does not bind the current activation"), + "{error:#}" + ); + + // Same for a correlation about another Expected projection: the + // authority carries this transaction's declaration, so a correlation + // that is not about it cannot authenticate against it. + let mut reprojected = world.correlation(501_714, true)?; + reprojected.expected_projection_sha256 = sha256_id(b"another-projection"); + let canonical = world.sign(&mut reprojected)?; + assert!(world.authenticate(&canonical).is_err()); + Ok(()) + } + + #[test] + fn a_signed_observation_that_is_not_ready_does_not_admit() -> Result<()> { + let world = TopologyFixture::new("odin")?; + let mut degraded = world.correlation(501_713, false)?; + let canonical = world.sign(&mut degraded)?; + assert!(!is_semantic_ready(&world.authenticate(&canonical)?)); + + // Ready alongside a disagreement is not merely "not ready" -- the + // correlation contract refuses to encode it at all, so a publisher + // cannot assert readiness over its own dissent. Worth pinning: it is + // why is_semantic_ready's disagreement clause is belt to the schema's + // braces rather than the only thing standing there. + let mut disputed = world.correlation(501_714, true)?; + disputed.disagreements.push(OdinTopologyDisagreement { + code: "route-membership-differs".into(), + expected: Some("a".into()), + observed: Some("b".into()), + }); + assert!(world.sign(&mut disputed).is_err()); + Ok(()) + } + + // --------------------------------------------------------------------- + // Admission audit (soul/admission-audit). Each test below is a negative + // check on one claim made by c09993c..8448f83; none asserts that the + // happy path works, the tests above already do that. + // --------------------------------------------------------------------- + + impl TopologyFixture { + /// Authenticate with an explicit trusted time, against this fixture's + /// own authority and Odin key. Mirrors `rehydrate_ready_token`'s two + /// callers: `now` when require_current, `admitted_at` otherwise. + fn authenticate_at( + &self, + canonical: &[u8], + trusted_received_at_unix_millis: u64, + ) -> Result { + authenticate_odin_runtime_topology_correlation( + canonical, + &self.authority()?, + None, + &self.odin_signer.entry().public_key, + OdinTopologyAuthenticationContext { + trusted_received_at_unix_millis, + maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, + maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, + }, + ) + } + } + + /// Claim 1: deleting `observation_describes_declared_incarnation` opened + /// nothing because authentication binds the same identity. The strongest + /// attack is a correlation that is internally perfect for ANOTHER + /// incarnation of the same target -- its own projection, its own activation + /// digest, its own instance id -- signed by OUR admitted Odin, so the only + /// thing left to refuse it is the incarnation binding itself. + #[test] + fn a_perfect_correlation_about_another_incarnation_of_this_target_is_refused() -> Result<()> { + let ours = TopologyFixture::new("odin")?; + let theirs = TopologyFixture::new("odin")?; + assert_ne!( + ours.expected.canonical_sha256()?, + theirs.expected.canonical_sha256()?, + "fixtures must declare distinct incarnations for this test to mean anything" + ); + // Sanity: it is a real, Ready correlation for the other incarnation. + let mut genuine = theirs.correlation(501_713, true)?; + let genuine = theirs.sign(&mut genuine)?; + assert!(is_semantic_ready(&theirs.authenticate(&genuine)?)); + + let mut record = theirs.correlation(501_713, true)?; + let canonical = ours.sign(&mut record)?; + let error = ours + .authenticate(&canonical) + .expect_err("another incarnation's correlation must not authenticate against ours"); + assert!( + format!("{error:#}").contains("substitutes or omits Expected authority"), + "{error:#}" + ); + Ok(()) + } + + /// Claim 1, second face: same Expected, different activation. This is the + /// case the deleted gate could not see at all -- it compared instance ids + /// only -- and the one that matters when a process is re-activated under + /// the same projection. Authentication compares the activation digest. + #[test] + fn a_correlation_bound_to_another_activation_of_the_same_expected_is_refused() -> Result<()> { + let world = TopologyFixture::new("odin")?; + let other = TopologyFixture::new("odin")?; + + let mut reactivated = world.correlation(501_713, true)?; + reactivated.current_activation_sha256 = Some(other.activation.canonical_sha256()?); + let canonical = world.sign(&mut reactivated)?; + let error = world + .authenticate(&canonical) + .expect_err("activation digest must bind"); + assert!( + format!("{error:#}").contains("does not bind the current activation"), + "{error:#}" + ); + + // A Present record with no activation cannot even be encoded: the + // schema refuses it upstream of authentication ("Present topology + // state lacks an authenticated runtime session"). Either refusal is + // the right answer; what must not happen is an absent field reading as + // a match for ours. + let mut anonymous = world.correlation(501_714, true)?; + anonymous.current_activation_sha256 = None; + let error = world + .sign(&mut anonymous) + .and_then(|canonical| world.authenticate(&canonical)) + .expect_err("absent activation must not read as ours"); + assert!( + format!("{error:#}").contains("lacks an authenticated runtime session"), + "{error:#}" + ); + + let mut nameless = world.correlation(501_715, true)?; + nameless.runtime_instance_id = None; + let error = world + .sign(&mut nameless) + .and_then(|canonical| world.authenticate(&canonical)) + .expect_err("absent instance id must not read as ours"); + // Likewise refused at encoding: a Present record must carry its + // instance id ("runtime instance identity and observation evidence are + // partial"). + assert!(format!("{error:#}").contains("are partial"), "{error:#}"); + Ok(()) + } + + /// Claim 1, third face: the deleted gate also compared `target`. The + /// projection digest already covers the target, but a record can carry a + /// foreign `target` field beside our projection digest; prove + /// `validate_against_expected` refuses that on its own. + #[test] + fn a_correlation_naming_another_target_beside_our_projection_is_refused() -> Result<()> { + let world = TopologyFixture::new("odin")?; + let mut stranger = world.correlation(501_713, true)?; + stranger.target = "ghostlight".into(); + let canonical = world.sign(&mut stranger)?; + let error = world + .authenticate(&canonical) + .expect_err("target must bind"); + assert!( + format!("{error:#}").contains("substitutes or omits Expected authority"), + "{error:#}" + ); + Ok(()) + } + + /// Claim 4: the fixture's chain can refuse at every link, not only at the + /// correlation signature. If any of these passed, the tests above would be + /// asserting against a rubber stamp. + #[test] + fn the_fixture_chain_refuses_at_every_link() -> Result<()> { + let ours = TopologyFixture::new("odin")?; + let theirs = TopologyFixture::new("odin")?; + + // Authority link: activation must bind this Expected ... + assert!( + verify_runtime_authority( + &ours.expected, + &theirs.activation, + &ours.idunn_anchor, + &ours.provider_public_key + ) + .is_err(), + "another incarnation's activation verified against our Expected" + ); + // ... be signed by the Idunn we trust ... + assert!( + verify_runtime_authority( + &ours.expected, + &ours.activation, + &theirs.idunn_anchor, + &ours.provider_public_key + ) + .is_err(), + "activation verified against a foreign Idunn anchor" + ); + // ... and name the provider key the Expected selected. + assert!( + verify_runtime_authority( + &ours.expected, + &ours.activation, + &ours.idunn_anchor, + &theirs.provider_public_key + ) + .is_err(), + "a foreign provider key was accepted as the Expected signer" + ); + + // Odin link: a correlation honestly signed by another Odin identity + // (its own id, its own signature) is refused against our admitted key. + let mut record = ours.correlation(501_713, true)?; + record.signer_identity_id = theirs.odin_signer.entry().identity_id.clone(); + let canonical = theirs.sign(&mut record)?; + let error = ours + .authenticate(&canonical) + .expect_err("foreign Odin signer accepted"); + assert!( + format!("{error:#}").contains("names a different identity"), + "{error:#}" + ); + + // Odin link: our identity id, their signature bytes. + let mut record = ours.correlation(501_713, true)?; + record.signature = theirs + .odin_signer + .sign::(&record.unsigned_signature_payload()?) + .signature; + let error = ours + .authenticate(&record.canonical_bytes()?) + .expect_err("forged signature accepted"); + assert!( + format!("{error:#}").contains("signature verification failed"), + "{error:#}" + ); + Ok(()) + } + + /// Claim 2: `rehydrate_ready_token(.., require_current = false)` replays the + /// receipt's authentication at its own `admitted_at`, so the window it + /// widens is exactly the one the receipt already passed at admission. Pin + /// both halves: at `admitted_at` the receipt authenticates however old it + /// is now; at `now` past the window it does not. An `admitted_at` earlier + /// than the observation is refused by the future-skew bound, so the field + /// cannot be pulled backwards to launder a later observation either. + #[test] + fn a_durable_receipt_replays_at_its_admission_time_and_not_at_now() -> Result<()> { + let world = TopologyFixture::new("odin")?; + let mut record = world.correlation(3_841, true)?; + let canonical = world.sign(&mut record)?; + let admitted_at = NOW + 5; + + world.authenticate_at(&canonical, admitted_at)?; + let much_later = admitted_at + DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS + 1; + assert!( + world.authenticate_at(&canonical, much_later).is_err(), + "require_current would refuse" + ); + world.authenticate_at(&canonical, admitted_at)?; + + let before_observation = NOW - DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS - 1; + assert!( + world + .authenticate_at(&canonical, before_observation) + .is_err(), + "an admitted_at earlier than the observation must not authenticate" + ); + Ok(()) + } + + /// Claim 5 companion: what the live store holds, not only whether the + /// stuck record would commit. Run with: + /// IDUNN_LIVE_CONTROL=/path/to/control.cc cargo test live_store_inventory -- --ignored --nocapture + #[test] + #[ignore] + fn live_store_inventory() -> Result<()> { + let Ok(path) = std::env::var("IDUNN_LIVE_CONTROL") else { + eprintln!("IDUNN_LIVE_CONTROL unset"); + return Ok(()); + }; + let snapshot = ControlSnapshot::read(Path::new(&path))?; + println!( + "INVENTORY commands={} transactions={} admitted={}", + snapshot.commands.len(), + snapshot.transactions.len(), + snapshot.admitted.len() + ); + for stored in &snapshot.transactions { + let value = &stored.value; + println!( + "TX {} target={} kind={:?} phase={:?} complete={} ready_seq={:?} latest_seq={:?} incumbent={:?}", + value.transaction_id, + value.target, + value.command_kind, + value.phase, + value.completion.is_some(), + value + .ready + .as_ref() + .and_then(ReadinessEvidence::odin) + .map(|evidence| evidence.publisher_sequence), + value + .latest_odin_observation + .as_ref() + .map(|evidence| evidence.publisher_sequence), + value.incumbent_generation_id, + ); + } + for stored in &snapshot.admitted { + let value = &stored.value; + println!( + "ADMITTED target={} generation={} tx={} admitted_at={} ready_seq={:?} ready_admitted_at={:?} latest_seq={:?} latest_admitted_at={:?} instance={} {}", + value.target, + value.generation_id, + value.transaction_id, + value.admitted_at_unix_millis, + value.ready.odin().map(|evidence| evidence.publisher_sequence), + value.ready.odin().map(|evidence| evidence.admitted_at_unix_millis), + value + .latest_odin_observation + .as_ref() + .map(|evidence| evidence.publisher_sequence), + value + .latest_odin_observation + .as_ref() + .map(|evidence| evidence.admitted_at_unix_millis), + value.activation.runtime_instance_id, + value.workload.describe(), + ); + } + Ok(()) + } + + fn topology(sequence: u64, byte: u8) -> TopologyEvidence { + let canonical_bytes = vec![byte]; + TopologyEvidence { + canonical_sha256: sha256_id(&canonical_bytes), + canonical_bytes, + signer_identity_id: "odin-signer".into(), + publisher_sequence: sequence, + admitted_at_unix_millis: 100, + } + } + + fn workload(uid: u32, pid_namespace_id: u64, mount_namespace_id: u64) -> WorkloadObservation { + WorkloadObservation::Systemd(SystemdWorkloadObservation { + unit: format!("idunn-{uid}.service"), + unit_description: format!("Idunn test {uid}"), + invocation_id: format!("invocation-{uid}"), + exec_main_start_timestamp_monotonic: 1, + service_type: "exec".into(), + restart_policy: "always".into(), + kill_mode: "control-group".into(), + dynamic_user: true, + systemd_user: format!("u{uid}"), + systemd_group: format!("u{uid}"), + supplementary_groups: String::new(), + capability_bounding_set: String::new(), + ambient_capabilities: String::new(), + private_mounts: true, + private_pids: true, + protect_proc: "invisible".into(), + proc_subset: "pid".into(), + no_new_privileges: true, + umask: "0077".into(), + inaccessible_paths: String::new(), + load_credential: String::new(), + main_pid: uid, + process_start_time: 1, + process_uids: [uid; 4], + process_gids: [uid; 4], + process_groups: vec![uid], + process_cap_inheritable: 0, + process_cap_permitted: 0, + process_cap_effective: 0, + process_cap_bounding: 0, + process_cap_ambient: 0, + process_no_new_privileges: true, + process_namespace_pids: vec![1], + mount_namespace_id, + pid_namespace_id, + executable: PathBuf::from("/opt/test/bin/service"), + executable_device: 1, + executable_inode: 1, + executable_sha256: sha256_id(&[1]), + runtime_instance_id: sha256_id(&uid.to_be_bytes()), + working_directory: PathBuf::from("/opt/test"), + runtime_bundle: PathBuf::from("/run/test"), + command_line_sha256: sha256_id(&[2]), + environment_names: Vec::new(), + environment_contract_sha256: sha256_id(&[3]), + control_group: format!("/system.slice/idunn-{uid}.service"), + credentials_directory: None, + parent_only_file_descriptors: Vec::new(), + activation_signer_identity_id: "activation".into(), + activation_signer_public_key: vec![1; 32], + service_credentials: Vec::new(), + }) + } + + fn write_lease() -> IdunnProcessWriteLeaseRecord { + IdunnProcessWriteLeaseRecord { + schema_version: IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: "ghostlight".into(), + expected_projection_sha256: sha256_id(&[1]), + plan_id: sha256_id(&[2]), + incarnation_id: "incarnation-test".into(), + sealed_release_id: sha256_id(&[3]), + activation_witness_sha256: sha256_id(&[4]), + state_schema_generation: "ghostlight-state-v1".into(), + state_contract_sha256: sha256_id(&[5]), + runtime_id: "ghostlight".into(), + runtime_instance_id: sha256_id(&[6]), + warming_presence_sha256: sha256_id(&[7]), + lease_epoch: 1, + issued_at_unix_millis: 100, + } + } + + #[test] + fn validate_is_offline_and_declarative() { + // It must accept a recipe alone, accept a recipe plus a binding, and + // refuse to be handed anything executable. It opens no store and + // actuates nothing, which is what makes it safe to run on a candidate + // binding before that binding is installed. + let parsed = parse( + ["validate", "--recipe", "/tmp/recipe.toml"] + .into_iter() + .map(str::to_owned), + ) + .unwrap(); + assert!(matches!(parsed, Command::Validate { binding: None, .. })); + + assert!( + parse( + ["validate", "--recipe", "/r.toml", "--binding", "/b.toml"] + .into_iter() + .map(str::to_owned) + ) + .is_ok() + ); + assert!(parse(["validate"].into_iter().map(str::to_owned)).is_err()); + for rejected in [ + vec![ + "validate", + "--recipe", + "/r.toml", + "--deploy-command", + "sh -c bad", + ], + vec!["validate", "--recipe", "/r.toml", "--state-store", "/s.cc"], + ] { + assert!(parse(rejected.into_iter().map(str::to_owned)).is_err()); + } + } + + #[test] + fn cli_exposes_only_declarative_commands() { + for arguments in [ + vec!["up", "ghostlight", "--deploy-command", "sh -c bad"], + vec!["serve", "--swarm-profile", "yggdrasil-local"], + vec!["serve", "--restart-command", "bad"], + vec!["validate-runtime-admission"], + ] { + assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); + } + } + + #[test] + fn garden_path_accepts_service_and_profile_selectors() { + for selector in ["ghostlight", "profile:aetheria", "profile:full-gamecult"] { + let parsed = + parse(["up", selector, "--no-wait"].into_iter().map(str::to_owned)).unwrap(); + let Command::Up { + selector: actual, .. + } = parsed + else { + panic!("expected up command") + }; + assert_eq!(actual, selector); + } + } + + #[test] + fn route_bindings_are_one_global_socket_and_candidate_authority_map() { + let first = route_binding("first", 4103, 14103, 14111); + let second = route_binding("second", 8831, 18831, 18839); + validate_route_binding_set(&[("first", &first), ("second", &second)]).unwrap(); + + let mut duplicate_id = second.clone(); + duplicate_id.route_id = first.route_id.clone(); + assert!( + validate_route_binding_set(&[("first", &first), ("second", &duplicate_id)]).is_err() + ); + + let mut overlapping_candidates = second.clone(); + overlapping_candidates.private_port_start = 14111; + overlapping_candidates.private_port_end = 14120; + assert!( + validate_route_binding_set(&[("first", &first), ("second", &overlapping_candidates),]) + .is_err() + ); + + let mut stable_inside_other_range = second; + stable_inside_other_range.stable_endpoint = "tcp://127.0.0.1:14105".into(); + assert!( + validate_route_binding_set(&[ + ("first", &first), + ("second", &stable_inside_other_range), + ]) + .is_err() + ); + } + + #[test] + fn deployment_command_is_immutable_positional_fact() -> Result<()> { + let value = command(CommandKind::Deploy); + value.validate()?; + let encoded = rmp_serde::to_vec(&value)?; + assert_eq!(encoded[0], 0x96); + assert!(!encoded.windows(7).any(|window| window == b"running")); + assert!(!encoded.windows(5).any(|window| window == b"owner")); + Ok(()) + } + + #[test] + fn phases_are_one_exact_forward_chain() { + let phases = [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Warming, + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + DeploymentPhase::Complete, + ]; + assert!( + phases + .windows(2) + .all(|pair| pair[1] as u8 == pair[0] as u8 + 1) + ); + } + + #[test] + fn only_live_stateless_deployments_can_be_cancelled_after_fencing() { + let command = command(CommandKind::Deploy); + let mut stateless = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100).unwrap(); + stateless.phase = DeploymentPhase::Routing; + stateless.fencing = Some(FencingEvidence::SkippedStateless); + assert!(cancel_is_safe_for_live_stateless(&stateless)); + + let mut stateful = stateless.clone(); + stateful.fencing = Some(FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + }); + assert!(!cancel_is_safe_for_live_stateless(&stateful)); + + let mut unfenced = stateless.clone(); + unfenced.phase = DeploymentPhase::Sealing; + assert!(!cancel_is_safe_for_live_stateless(&unfenced)); + + let mut committed = stateless; + committed.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-test".into(), + }); + assert!(!cancel_is_safe_for_live_stateless(&committed)); + } + + #[test] + fn every_new_odin_sequence_must_advance_and_exact_retry_is_idempotent() -> Result<()> { + let first = topology(7, 1); + assert!(sequence_requires_admission(None, 6, &first)?); + assert!(!sequence_requires_admission( + Some(&first), + 7, + &topology(7, 1) + )?); + assert!(sequence_requires_admission( + Some(&first), + 7, + &topology(8, 2) + )?); + assert!(sequence_requires_admission(Some(&first), 7, &topology(7, 2)).is_err()); + assert!(sequence_requires_admission(Some(&first), 7, &topology(6, 3)).is_err()); + Ok(()) + } + + #[test] + fn odin_sequence_cursors_are_scoped_by_target_and_signer() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut ghostlight = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + ghostlight.latest_odin_observation = Some(topology(7, 1)); + ghostlight.odin_publisher_sequence_cursor = 7; + let mut odin = DeploymentTransaction::new(&command, "odin".into(), 1, None, 100)?; + odin.latest_odin_observation = Some(topology(41, 2)); + odin.odin_publisher_sequence_cursor = 41; + let envelope = |key: &str| CultCacheEnvelope { + key: key.into(), + r#type: DeploymentTransaction::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()), + }; + let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), + commands: Vec::new(), + transactions: vec![ + Stored { + envelope: envelope("ghostlight"), + value: ghostlight, + }, + Stored { + envelope: envelope("odin"), + value: odin, + }, + ], + admitted: Vec::new(), + targets: Vec::new(), + }; + + assert_eq!(snapshot.max_odin_sequence("ghostlight", "odin-signer"), 7); + assert_eq!(snapshot.max_odin_sequence("odin", "odin-signer"), 41); + assert_eq!(snapshot.max_odin_sequence("ghostlight", "other-signer"), 0); + + // A transaction that failed was never built upon, so its readings must + // not raise the bar for the next attempt. Counting them is how a target + // that has failed once becomes a target that can never deploy. + let mut abandoned = snapshot; + abandoned.transactions[0].value.completion = + Some(TransactionCompletion::FailedBeforeFencing { + error: "sealed source was rejected".into(), + }); + assert_eq!(abandoned.max_odin_sequence("ghostlight", "odin-signer"), 0); + abandoned.transactions[1].value.completion = + Some(TransactionCompletion::FailedAfterFencing { + error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + assert_eq!(abandoned.max_odin_sequence("odin", "odin-signer"), 0); + Ok(()) + } + + #[test] + fn stale_odin_provider_receipt_cannot_gate_continuity_restart() -> Result<()> { + let touched = std::cell::Cell::new(false); + validate_live_providers_for_deploy(CommandKind::Continuity, || { + touched.set(true); + bail!("Odin provider receipt is stale") + })?; + assert!(!touched.get()); + assert!( + validate_live_providers_for_deploy(CommandKind::Deploy, || { + bail!("Odin provider receipt is stale") + }) + .is_err() + ); + Ok(()) + } + + #[test] + fn prepared_write_lease_is_durable_but_not_admitted_authority() -> Result<()> { + let lease = write_lease(); + lease.validate()?; + let prepared = LeasingEvidence::Prepared { + lease: lease.clone(), + lease_sha256: lease.canonical_sha256()?, + }; + let encoded = rmp_serde::to_vec(&prepared)?; + let decoded: LeasingEvidence = rmp_serde::from_slice(&encoded)?; + assert_eq!(decoded, prepared); + assert!(decoded.lease().is_none()); + assert!(decoded.lease_sha256().is_none()); + assert_eq!(decoded.prepared_lease().unwrap().0, &lease); + Ok(()) + } + + #[test] + fn runtime_instance_identity_is_stable_for_activation_prepare_replay() -> Result<()> { + let first = runtime_instance_id("tx-one")?; + assert_eq!(runtime_instance_id("tx-one")?, first); + assert_ne!(runtime_instance_id("tx-two")?, first); + Ok(()) + } + + #[test] + fn pre_fencing_abort_is_durable_before_cleanup_and_cannot_resume_as_success() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut transaction = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "sealed source was rejected".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Pending, + }); + transaction.validate()?; + assert!(!transaction.is_terminal()); + + transaction + .pre_fencing_abort + .as_mut() + .unwrap() + .source_cleanup = CleanupEvidence::Complete; + transaction.phase = DeploymentPhase::Complete; + transaction.last_error = Some("sealed source was rejected".into()); + transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { + error: "sealed source was rejected".into(), + }); + transaction.validate()?; + assert!(transaction.is_terminal()); + Ok(()) + } + + #[test] + fn post_fencing_abort_is_terminal_only_when_every_cleanup_is_complete() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut transaction = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + transaction.phase = DeploymentPhase::Complete; + transaction.post_fencing_abort = Some(PostFencingAbort { + error: "candidate died after the fence".into(), + route_restoration: CleanupEvidence::Skipped, + lease_withdrawal: CleanupEvidence::Pending, + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Pending, + }); + transaction.last_error = Some("candidate died after the fence".into()); + transaction.completion = Some(TransactionCompletion::FailedAfterFencing { + error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + // The evidence exists so that a target is released only once the + // candidate's route, lease, process and projection are actually gone. + assert!(transaction.validate().is_err()); + assert!(!transaction.is_terminal()); + + let abort = transaction.post_fencing_abort.as_mut().unwrap(); + abort.lease_withdrawal = CleanupEvidence::Complete; + abort.source_cleanup = CleanupEvidence::Complete; + transaction.validate()?; + assert!(transaction.is_terminal()); + + // The recorded error and the completion must name the same failure. + transaction.completion = Some(TransactionCompletion::FailedAfterFencing { + error: "a different story".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + assert!(transaction.validate().is_err()); + Ok(()) + } + + #[test] + fn a_pre_fence_transaction_cannot_carry_a_post_fencing_abort() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut transaction = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + assert!(transaction.phase < DeploymentPhase::Fencing); + transaction.post_fencing_abort = Some(PostFencingAbort { + error: "candidate died after the fence".into(), + route_restoration: CleanupEvidence::Skipped, + lease_withdrawal: CleanupEvidence::Skipped, + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Skipped, + }); + assert!(transaction.validate().is_err()); + Ok(()) + } + #[test] + fn candidate_cleanup_is_owed_when_an_activation_or_a_workload_exists() -> Result<()> { + // (activation issued, workload observed, expected cleanup) + let rows = [ + (false, false, CleanupEvidence::Skipped), + (true, false, CleanupEvidence::Pending), + (false, true, CleanupEvidence::Pending), + (true, true, CleanupEvidence::Pending), + ]; + for (has_activation, has_workload, expected) in rows { + let world = ContinuityProjection::after_candidate_observed()?; + let mut transaction = world.continuity_transaction()?; + if !has_activation { + transaction.activation = None; + } + transaction.workload = has_workload.then(|| workload(1000, 1, 2)); + assert_eq!( + transaction.candidate_cleanup_owed(), + expected, + "activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + pre_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "pre-fencing intent, activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "post-fencing intent, activation {has_activation}, workload {has_workload}" + ); + } + Ok(()) + } + + #[test] + fn a_stored_abort_whose_candidate_cleanup_disagrees_with_the_owed_rule_is_refused() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + // (activation issued, recorded cleanup, accepted). + // Rows with a workload are omitted: a transaction holding one is refused on + // other grounds, so they could not speak for this rule. The workload operand + // of the owed rule is pinned by the method-level test above. + let rows = [ + (false, CleanupEvidence::Skipped, true), + (false, CleanupEvidence::Pending, false), + (false, CleanupEvidence::Complete, false), + (true, CleanupEvidence::Skipped, false), + (true, CleanupEvidence::Pending, true), + (true, CleanupEvidence::Complete, true), + ]; + for (has_activation, recorded, accepted) in rows { + let label = format!("activation {has_activation}, {recorded:?}"); + let shape = |mut transaction: DeploymentTransaction| -> DeploymentTransaction { + transaction.activation = has_activation.then(|| world.candidate_activation.clone()); + transaction + }; + + let mut pre = shape(world.continuity_transaction()?); + pre.pre_fencing_abort = Some(PreFencingAbort { + error: "candidate refused".into(), + candidate_cleanup: recorded, + topology_reconciliation: pre.abort_topology_reconciliation(), + source_cleanup: CleanupEvidence::Skipped, + }); + assert_eq!(pre.validate().is_ok(), accepted, "pre-fencing, {label}"); + + let mut post = shape(DeploymentTransaction::new( + &command(CommandKind::Deploy), + "ghostlight".into(), + 0, + None, + 100, + )?); + post.phase = DeploymentPhase::Complete; + post.post_fencing_abort = Some(PostFencingAbort { + error: "candidate died after the fence".into(), + route_restoration: CleanupEvidence::Skipped, + lease_withdrawal: CleanupEvidence::Skipped, + candidate_cleanup: recorded, + topology_reconciliation: post.abort_topology_reconciliation(), + source_cleanup: CleanupEvidence::Complete, + }); + post.last_error = Some("candidate died after the fence".into()); + post.completion = Some(TransactionCompletion::FailedAfterFencing { + error: "candidate died after the fence".into(), + recovery: TerminalRecovery::RestoreIncumbent, + }); + // A Pending cleanup is not terminal, so this completed shape is + // only judged for the recorded values a terminal abort can carry. + if recorded != CleanupEvidence::Pending { + assert_eq!(post.validate().is_ok(), accepted, "post-fencing, {label}"); + } + } + Ok(()) + } + + #[test] + fn bad_selector_becomes_one_terminal_refusal_record() -> Result<()> { + let command = command(CommandKind::Deploy); + let transaction = DeploymentTransaction::rejected( + &command, + anyhow!("deployment selector is unknown"), + 100, + )?; + assert!(transaction.is_terminal()); + assert_eq!(transaction.target, command.selector); + assert!(matches!( + transaction.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + Ok(()) + } + + #[test] + fn post_commit_cleanup_keeps_exact_incumbent_work_owned_until_retired() -> Result<()> { + let incumbent = workload(1001, 40, 50); + let cleanup = PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Pending { + generation_id: "generation-old".into(), + workload: incumbent.clone(), + }, + source: SourceCleanupEvidence::Pending, + }; + let encoded = rmp_serde::to_vec(&cleanup)?; + let mut replayed: PostCommitCleanup = rmp_serde::from_slice(&encoded)?; + assert_eq!(replayed, cleanup); + assert!(!replayed.is_complete()); + replayed.incumbent = IncumbentCleanupEvidence::Complete { + generation_id: "generation-old".into(), + }; + assert!(!replayed.is_complete()); + replayed.source = SourceCleanupEvidence::Complete; + assert!(replayed.is_complete()); + Ok(()) + } + + #[test] + fn routed_incumbent_retirement_waits_for_the_declared_drain_deadline() -> Result<()> { + assert_eq!(route_drain_deadline(1_000, 30)?, 31_000); + assert!(route_drain_deadline(u64::MAX, 1).is_err()); + Ok(()) + } + + #[test] + fn admitted_route_receipt_refreshes_when_stale_or_implausibly_future_dated() { + assert!(route_observation_is_current(900, 1_000, 100, 10)); + assert!(route_observation_is_current(1_010, 1_000, 100, 10)); + assert!(!route_observation_is_current(899, 1_000, 100, 10)); + assert!(!route_observation_is_current(1_011, 1_000, 100, 10)); + } + + #[test] + fn lease_warming_requires_both_new_odin_and_new_provider_evidence() { + let old = digest('1'); + let fresh = digest('2'); + assert!(!provider_warming_advanced(7, &old, 7, &fresh)); + assert!(!provider_warming_advanced(7, &old, 8, &old)); + assert!(provider_warming_advanced(7, &old, 8, &fresh)); + } + + #[test] + fn failed_route_proof_never_restores_any_fenced_incumbent() { + assert!(!may_rollback_route_after_failed_proof( + &FencingEvidence::Revoked { + incumbent_lease_sha256: Some(digest('1')), + candidate_lease_path_verified_empty: false, + } + )); + assert!(!may_rollback_route_after_failed_proof( + &FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + } + )); + assert!(may_rollback_route_after_failed_proof( + &FencingEvidence::SkippedStateless + )); + } + + #[test] + fn fenced_writer_is_already_retired_before_post_commit_cleanup() { + assert!(incumbent_was_stopped_during_fencing( + &FencingEvidence::Revoked { + incumbent_lease_sha256: Some(digest('1')), + candidate_lease_path_verified_empty: false, + } + )); + assert!(!incumbent_was_stopped_during_fencing( + &FencingEvidence::Revoked { + incumbent_lease_sha256: None, + candidate_lease_path_verified_empty: true, + } + )); + assert!(!incumbent_was_stopped_during_fencing( + &FencingEvidence::SkippedStateless + )); + } + + #[test] + fn warming_accepts_only_the_exact_projected_incumbent_lease_disagreement() { + let expected_detail = format!("expected:{};activation:{}", digest('1'), digest('2')); + let incumbent = digest('3'); + let exact = OdinTopologyDisagreement { + code: "projected-write-lease".into(), + expected: Some(expected_detail.clone()), + observed: Some(incumbent.clone()), + }; + + assert!(warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[exact.clone()], + )); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + None, + &[exact.clone()], + )); + + let mut substituted = exact.clone(); + substituted.observed = Some(digest('4')); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[substituted], + )); + assert!(!warming_disagreements_match_incumbent( + &expected_detail, + Some(&incumbent), + &[exact.clone(), exact], + )); + } + + #[test] + fn candidate_and_incumbent_must_differ_in_all_three_native_boundaries() { + let incumbent = workload(1001, 40, 50); + let prove = |candidate: &WorkloadObservation| { + WorkloadObservation::prove_isolation(candidate, Some(&incumbent)) + }; + assert!(prove(&workload(1002, 41, 51)).is_ok()); + assert!(prove(&workload(1001, 41, 51)).is_err()); + assert!(prove(&workload(1002, 40, 51)).is_err()); + assert!(prove(&workload(1002, 41, 50)).is_err()); + } + + #[test] + fn exactly_one_inflight_transaction_may_own_a_target() -> Result<()> { + let command = command(CommandKind::Deploy); + let one = DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + let two = DeploymentTransaction::new(&command, "ghostlight".into(), 1, None, 100)?; + let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), + commands: vec![Stored { + envelope: command_envelope(&command, 100)?, + value: command, + }], + transactions: vec![ + Stored { + envelope: transaction_envelope(&one, 100)?, + value: one, + }, + Stored { + envelope: transaction_envelope(&two, 100)?, + value: two, + }, + ], + admitted: Vec::new(), + targets: Vec::new(), + }; + assert!(snapshot.validate_relations().is_err()); + Ok(()) + } + + #[test] + fn command_ordinals_do_not_turn_one_target_wait_into_a_global_brake() -> Result<()> { + let first_command = command(CommandKind::Deploy); + let first = DeploymentTransaction::new(&first_command, "odin".into(), 0, None, 100)?; + let second = DeploymentTransaction::new(&first_command, "ghostlight".into(), 1, None, 100)?; + let mut unrelated_command = command(CommandKind::Deploy); + unrelated_command.command_id = "up-unrelated".into(); + let unrelated = + DeploymentTransaction::new(&unrelated_command, "huginn".into(), 0, None, 101)?; + let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), + commands: Vec::new(), + transactions: [&first, &second, &unrelated] + .into_iter() + .map(|value| Stored { + envelope: transaction_envelope(value, 100).unwrap(), + value: value.clone(), + }) + .collect(), + admitted: Vec::new(), + targets: Vec::new(), + }; + assert!(!snapshot.has_earlier_authority_sibling(&first)); + assert!(snapshot.has_earlier_authority_sibling(&second)); + assert!(!snapshot.has_earlier_authority_sibling(&unrelated)); + Ok(()) + } + + #[test] + fn complete_cleanup_remains_retryable_without_owning_the_current_incarnation() -> Result<()> { + let command = command(CommandKind::Deploy); + let mut transaction = + DeploymentTransaction::new(&command, "ghostlight".into(), 0, None, 100)?; + transaction.phase = DeploymentPhase::Complete; + transaction.completion = Some(TransactionCompletion::Admitted { + generation_id: format!("generation-{}", transaction.transaction_id), + }); + transaction.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Pending, + }); + + assert!(!transaction.is_terminal()); + assert!(!transaction.owns_target_authority()); + assert!(transaction.blocks_new_target_mutation()); + Ok(()) + } + + #[test] + fn legacy_mutable_command_schema_is_rejected() -> Result<()> { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let legacy = CultCacheEnvelope { + key: "up-legacy".into(), + r#type: DeploymentCommand::TYPE.into(), + payload: rmp_serde::to_vec(&(DEPLOYMENT_COMMAND_SCHEMA, "up-legacy", "running"))?, + stored_at: rfc3339_millis(100)?, + schema_id: Some("idunn.deployment_command.v1".into()), + }; + assert!( + SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: "up-legacy".into(), + current: None, + }], + &[legacy], + )? + ); + assert!(ControlSnapshot::read(&path).is_err()); + Ok(()) + } + + #[test] + fn loadcredential_era_control_schemas_are_rejected_before_rehydration() -> Result<()> { + for (record_type, key, schema) in [ + ( + DeploymentTransaction::TYPE, + "tx-legacy-workload-observation", + "idunn.deployment_transaction.v1", + ), + ( + AdmittedGeneration::TYPE, + "legacy-admitted-target", + "idunn.admitted_generation.v1", + ), + ] { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let legacy = CultCacheEnvelope { + key: key.into(), + r#type: record_type.into(), + payload: Vec::new(), + stored_at: rfc3339_millis(100)?, + schema_id: Some(schema.into()), + }; + assert!( + SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: record_type.into(), + key: key.into(), + current: None, + }], + &[legacy], + )? + ); + assert!(ControlSnapshot::read(&path).is_err()); + } + Ok(()) + } + + #[test] + fn source_identity_is_explicit_and_atomic() { + let partial = parse( + ["serve", "--source-uid", "1001"] + .into_iter() + .map(str::to_owned), + ) + .unwrap_err() + .to_string(); + assert!(partial.contains("supplied together")); + let Command::Serve(options) = parse( + ["serve", "--source-uid", "1001", "--source-gid", "1002"] + .into_iter() + .map(str::to_owned), + ) + .unwrap() else { + panic!("expected serve") + }; + assert_eq!( + options.source_identity, + Some(ProcessIdentity { + uid: 1001, + gid: 1002 + }) + ); + } + + // ---- v2/v3 control records lifted to the v3 generation / v4 transaction ---- + // + // The bytes under tests/fixtures/idunn-control-legacy were written by the + // encoder at Idunn 9001b58 (see the README there). Nothing below encodes a + // legacy record: the legacy layout is decoded from those bytes, and the + // lifted record is compared with what the decoded legacy record says. + + const FIXTURE_GENERATION: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-with-receipts.hex"); + const FIXTURE_GENERATION_REPAIRING: &str = include_str!( + "../tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex" + ); + const FIXTURE_GENERATION_V3_ODIN: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-odin.hex"); + const FIXTURE_GENERATION_V3_ROUTE_PROOF: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex"); + const FIXTURE_TRANSACTIONS: [(&str, &str); 6] = [ + ( + "fencing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-fencing.hex"), + ), + ( + "leasing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-leasing.hex"), + ), + ( + "awaiting-ready", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-awaiting-ready.hex"), + ), + ( + "routing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-routing.hex"), + ), + ( + "committing", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing.hex"), + ), + ( + "failed-after-fencing", + include_str!( + "../tests/fixtures/idunn-control-legacy/transaction-failed-after-fencing.hex" + ), + ), + ]; + const FIXTURE_TRANSACTION_V2: &str = + include_str!("../tests/fixtures/idunn-control-legacy/transaction-committing-v2.hex"); + + fn fixture_envelope(text: &str, record_type: &str) -> Result { + let mut lines = text.lines(); + let schema = lines.next().context("fixture has no schema line")?; + let hex = lines.next().context("fixture has no payload line")?.trim(); + let payload = (0..hex.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&hex[index..index + 2], 16)) + .collect::, _>>()?; + let key = if record_type == AdmittedGeneration::TYPE && schema == ADMITTED_GENERATION_SCHEMA_V3 { + rmp_serde::from_slice::(&payload)?.target + } else if record_type == AdmittedGeneration::TYPE { + rmp_serde::from_slice::(&payload)?.target + } else { + rmp_serde::from_slice::(&payload)?.transaction_id + }; + Ok(CultCacheEnvelope { + key, + r#type: record_type.into(), + payload, + stored_at: rfc3339_millis(1_700_000_123_456)?, + schema_id: Some(schema.into()), + }) + } + + fn fixture_generation(text: &str) -> Result<(LegacyAdmittedGeneration, AdmittedGeneration)> { + let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; + let legacy = rmp_serde::from_slice(&envelope.payload)?; + Ok((legacy, read_generation_record(&envelope)?)) + } + + fn fixture_transaction( + text: &str, + ) -> Result<(LegacyDeploymentTransaction, DeploymentTransaction)> { + let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; + let legacy = rmp_serde::from_slice(&envelope.payload)?; + Ok((legacy, read_transaction_record(&envelope)?)) + } + + #[test] + fn a_v2_generation_lifts_to_the_decisions_it_encoded() -> Result<()> { + for text in [FIXTURE_GENERATION, FIXTURE_GENERATION_REPAIRING] { + let (legacy, lifted) = fixture_generation(text)?; + // Every receipt the old decisions read is the same receipt, now + // tagged Odin-correlated. + let odin = lifted.odin_receipts()?; + assert_eq!(odin.ready, &legacy.ready); + assert_eq!(odin.latest, &legacy.latest_odin_observation); + assert_eq!(lifted.odin_authority.as_ref(), Some(&legacy.odin_authority)); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.activation, legacy.activation); + assert_eq!(lifted.workload, legacy.workload); + assert_eq!(lifted.generation_id, legacy.generation_id); + // The new state starts empty and the plan is still a v2 plan. + assert_eq!(lifted.route_supervision, Some(RouteSupervisionState::default())); + assert_eq!(lifted.last_error, None); + assert_eq!(lifted.plan.schema, crate::deployment_plan::COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2); + assert_eq!( + lifted.plan.phase_deadlines(), + crate::deployment_plan::PhaseDeadlines::IDUNN_DEFAULTS + ); + } + Ok(()) + } + + #[test] + fn v2_and_v3_transactions_lift_to_the_decisions_they_encoded() -> Result<()> { + let mut phases = Vec::new(); + for (name, text) in FIXTURE_TRANSACTIONS { + let (legacy, lifted) = fixture_transaction(text)?; + phases.push((name, lifted.phase)); + assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + assert_eq!(lifted.phase, legacy.phase); + assert_eq!(lifted.completion, legacy.completion); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.fencing, legacy.fencing); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.warming, legacy.warming); + assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!( + lifted.ready.as_ref().and_then(ReadinessEvidence::odin), + legacy.ready.as_ref() + ); + assert_eq!(lifted.phase_deadline, None); + assert_eq!(lifted.lease_adoption, None); + assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); + } + assert_eq!( + phases.iter().map(|(_, phase)| *phase).collect::>(), + [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + DeploymentPhase::Complete, + ] + ); + + // The genuine v2 layout lifts to the same record as the v3 one. + let (_, from_v2) = fixture_transaction(FIXTURE_TRANSACTION_V2)?; + let (_, from_v3) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + assert_eq!(from_v2, from_v3); + + // A terminal failure predating recovery names the recovery it always did. + let (_, failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; + assert!(matches!( + failed.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + assert!(failed.is_terminal()); + Ok(()) + } + + #[test] + fn the_boot_migration_rewrites_every_legacy_record_once() -> Result<()> { + // The fixtures were cut from one build, so their records share keys + // and each migrates in a store of its own. + let mut fixtures = vec![ + fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?, + fixture_envelope(FIXTURE_GENERATION_REPAIRING, AdmittedGeneration::TYPE)?, + fixture_envelope(FIXTURE_TRANSACTION_V2, DeploymentTransaction::TYPE)?, + ]; + for (_, text) in FIXTURE_TRANSACTIONS { + fixtures.push(fixture_envelope(text, DeploymentTransaction::TYPE)?); + } + for envelope in fixtures { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + assert!(store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: envelope.r#type.clone(), + key: envelope.key.clone(), + current: None, + }], + std::slice::from_ref(&envelope), + )?); + let debug_of = |envelope: &CultCacheEnvelope| -> Result { + Ok(if envelope.r#type == AdmittedGeneration::TYPE { + format!("{:?}", read_generation_record(envelope)?) + } else { + format!("{:?}", read_transaction_record(envelope)?) + }) + }; + let before = debug_of(&envelope)?; + + assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); + + let after = store.pull_all_read_only_snapshot()?; + assert_eq!(after.len(), 1); + let expected_schema = if envelope.r#type == AdmittedGeneration::TYPE { + ADMITTED_GENERATION_SCHEMA + } else { + DEPLOYMENT_TRANSACTION_SCHEMA + }; + assert_eq!(after[0].schema_id.as_deref(), Some(expected_schema)); + assert_eq!(after[0].stored_at, envelope.stored_at); + // The rewritten record is canonical and means what the legacy one meant. + assert_eq!(debug_of(&after[0])?, before); + } + Ok(()) + } + + #[test] + fn history_still_reads_legacy_transactions() -> Result<()> { + let temporary = tempfile::tempdir()?; + let state_store = temporary.path().join("control.cc"); + let envelope = fixture_envelope(FIXTURE_TRANSACTIONS[5].1, DeploymentTransaction::TYPE)?; + SingleFileMessagePackBackingStore::new(&history_store_path(&state_store)) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: envelope.key.clone(), + current: None, + }], + std::slice::from_ref(&envelope), + )?; + let archived = read_history_transactions(&state_store); + assert_eq!(archived.len(), 1); + assert_eq!(archived[0].transaction_id, envelope.key); + assert_eq!(archived[0].schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + Ok(()) + } + + #[test] + fn readiness_class_follows_the_expected_odin_dependency() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut expected = generation.expected.clone(); + assert!(expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + })); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + for dependency in &mut expected.dependencies { + // Same capability, other kind: only the shared-infrastructure + // declaration makes a target Odin-correlated. + dependency.kind = "required".into(); + } + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + expected.dependencies.clear(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + // Nothing to challenge and no Odin dependency: the recipe has declared + // no way to prove readiness, and Idunn does not pick one. + expected.route = None; + assert_eq!( + ReadinessClass::of(&expected), + Err(UndeclaredReadiness { + target: expected.target.clone() + }) + ); + // Declaring the Odin dependency is the declaration, routed or not: a + // routed target that only publishes to Odin is Odin-correlated. + let odin_dependency = generation.expected.dependencies.clone(); + expected.dependencies = odin_dependency.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + expected.route = generation.expected.route.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + // Providing the rendezvous is being Odin, whatever else is declared. + expected.dependencies.clear(); + expected.route = None; + provide_odin(&mut expected); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); + Ok(()) + } + + #[test] + fn warming_evidence_is_shape_checked_whatever_its_source() { + let good = |evidence: &ReadinessEvidence| match evidence { + ReadinessEvidence::RouteProof { evidence } => evidence.clone(), + ReadinessEvidence::OdinCorrelated { .. } => unreachable!("a route-proof fixture"), + }; + let presence = good(&route_proof_evidence()); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: presence.clone() }, + WarmingEvidence::RouteProofDirect { evidence: presence.clone() }, + WarmingEvidence::OdinTopology { evidence: topology(3, 1) }, + ] { + warming.validate_shape().expect("well-formed evidence"); + } + let mut damaged = presence; + damaged.canonical_bytes.push(1); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: damaged.clone() }, + WarmingEvidence::RouteProofDirect { evidence: damaged }, + WarmingEvidence::OdinTopology { + evidence: TopologyEvidence { + canonical_sha256: "sha256:not-the-digest".into(), + ..topology(3, 1) + }, + }, + ] { + assert!(warming.validate_shape().is_err(), "{warming:?}"); + } + } + + #[test] + fn a_presence_disagreement_names_each_disagreement() { + let shortfall = PresenceDisagrees { + disagreements: vec![ + OdinTopologyDisagreement { + code: "capacity-below-minimum".into(), + expected: Some("2".into()), + observed: Some("1".into()), + }, + OdinTopologyDisagreement { + code: "state".into(), + expected: None, + observed: None, + }, + ], + }; + assert_eq!( + shortfall.to_string(), + "route proof answered with a runtime that disagrees with current authority: \ + capacity-below-minimum (expected 2, observed 1); state (expected none, observed none)" + ); + } + + fn route_proof_evidence() -> ReadinessEvidence { + let canonical_bytes = vec![7, 7, 7]; + ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + canonical_sha256: sha256_id(&canonical_bytes), + canonical_bytes, + message_id: "challenge-1".into(), + challenged_at_unix_millis: 10, + admitted_at_unix_millis: 11, + }, + } + } + + #[test] + fn route_proof_readiness_is_refused_for_a_target_that_declares_odin() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut proof = generation.clone(); + proof.ready = route_proof_evidence(); + proof.latest_odin_observation = None; + proof.odin_authority = None; + // The fixture's Expected declares Odin, so route proof is not its class. + assert!(proof.validate().is_err()); + + // Mixed shapes are refused whatever the class. + let mut mixed = generation.clone(); + mixed.latest_odin_observation = None; + assert!(mixed.validate().is_err()); + let mut mixed = generation; + mixed.odin_authority = None; + assert!(mixed.validate().is_err()); + + let mut transaction = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?.1; + transaction.validate()?; + transaction.ready = Some(route_proof_evidence()); + assert!(transaction.validate().is_err()); + Ok(()) + } + + #[test] + fn route_supervision_state_exists_exactly_for_a_promoted_route() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let mut missing = generation.clone(); + missing.route_supervision = None; + assert!(missing.validate().is_err()); + + let mut unrouted = generation; + unrouted.route_supervision = None; + unrouted.routing = RoutingEvidence::SkippedUnrouted; + // Unrouted while Expected still names a route: refused for that reason + // too, so the test above pins the supervision rule on its own. + assert!(unrouted.validate().is_err()); + Ok(()) + } + + #[test] + fn a_new_generation_starts_unobserved_and_undegraded() -> Result<()> { + let committing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1; + let (_, incumbent) = fixture_generation(FIXTURE_GENERATION)?; + let authority = incumbent + .odin_authority + .clone() + .context("fixture generation has no Odin authority")?; + let next = AdmittedGeneration::from_transaction( + &committing, + Some(authority), + 1_700_000_200_000, + )?; + assert_eq!(next.route_supervision, Some(RouteSupervisionState::default())); + assert_eq!(next.last_error, None); + Ok(()) + } + + #[test] + fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + let plan = &generation.plan; + let defaults = plan.phase_deadlines(); + for (phase, seconds) in [ + (DeploymentPhase::Fencing, defaults.fencing_seconds), + (DeploymentPhase::Leasing, defaults.leasing_seconds), + (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), + (DeploymentPhase::Routing, defaults.routing_seconds), + (DeploymentPhase::Committing, defaults.committing_seconds), + ] { + let deadline = PhaseDeadline::entering(phase, plan, 1_000).context("no deadline")?; + assert_eq!(deadline.phase, phase); + assert_eq!(deadline.entered_at_unix_millis, 1_000); + assert_eq!(deadline.deadline_at_unix_millis, 1_000 + u64::from(seconds) * 1000); + } + for phase in [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Warming, + DeploymentPhase::Complete, + ] { + assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); + } + Ok(()) + } + + #[test] + fn transition_stamps_the_deadline_of_the_phase_it_enters() -> Result<()> { + let world = EngineFixture::new()?; + // Routing, already holding the route receipt Committing requires. + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[3].1)?; + assert_eq!(transaction.phase, DeploymentPhase::Routing); + transaction.routing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1.routing; + let envelope = transaction_envelope(&transaction, transaction.updated_at_unix_millis)?; + let command = command_envelope(&command(CommandKind::Continuity), 100)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.key.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[command, envelope], + )? + ); + let stored = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("stored transaction vanished")?; + world.engine.transition(&stored, DeploymentPhase::Committing)?; + let advanced = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("advanced transaction vanished")?; + let deadline = advanced.value.phase_deadline.context("no deadline stamped")?; + assert_eq!(deadline.phase, DeploymentPhase::Committing); + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + u64::from(advanced.value.plan.as_ref().unwrap().phase_deadlines().committing_seconds) + * 1000 + ); + assert_eq!(deadline.entered_at_unix_millis, advanced.value.updated_at_unix_millis); + Ok(()) + } + + #[test] + fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let plan = transaction.plan.clone().unwrap(); + let now = transaction.updated_at_unix_millis; + transaction.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now); + transaction.validate()?; + let mut wrong_phase = transaction.clone(); + wrong_phase.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::Routing, &plan, now); + assert!(wrong_phase.validate().is_err()); + let mut backwards = transaction.clone(); + backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; + assert!(backwards.validate().is_err()); + // Entry is not ordered against the wall-clock stamp of later writes. + let mut entered_after_the_stamp = transaction; + entered_after_the_stamp.phase_deadline = + PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); + entered_after_the_stamp.validate()?; + Ok(()) + } + + // --------------------------------------------------------------------- + // The phase machine, driven by the real Engine past Fencing. + // --------------------------------------------------------------------- + + /// A workload port with no systemd behind it: observing a workload + /// returns what was observed, stopping and discarding succeed, and + /// anything that would launch a process refuses. + struct StillWorkload; + + impl WorkloadPort for StillWorkload { + fn install( + &self, + _: &CompiledDeploymentPlan, + _: &crate::drivers::MaterializedRelease, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn prepare_activation( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: IdunnRuntimeActivationLaunch, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn start_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &SealedRelease, + _: &crate::drivers::InstalledReleaseObservation, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result { + bail!("StillWorkload launches nothing") + } + fn discard_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + Ok(()) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + Ok(prior.clone()) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(false) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } + } + + /// A Continuity transaction for an unrouted, stateless target with no + /// incumbent: the shape most Engine tests need. + fn transaction_at( + world: &EngineFixture, + phase: DeploymentPhase, + ) -> Result { + seeded_transaction(world, phase, CommandKind::Continuity, None) + } + + /// The Odin publisher sequence a transaction seeded over `incumbent` + /// warms at; its Ready is the next one. Above anything an earlier + /// generation of the world admitted. + fn seeded_sequence(incumbent: Option<&AdmittedGeneration>) -> u64 { + if incumbent.is_some() { 14 } else { 4 } + } + + /// A released deployment brake receipt, signed by an operator identity + /// enrolled in the world: what a Deploy's `deployment_authorization` holds. + fn deployment_authorization( + world: &EngineFixture, + expected: &IdunnExpectedIncarnationRecord, + transaction_id: &str, + now: u64, + ) -> Result { + use cultnet_rs::{ + IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, + IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, + IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, + }; + let signer = enroll_service_identity_at::( + &world.root.join("identities/brake-operator.cc"), + )?; + export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + let mut record = IdunnDeploymentBrakeRecord { + schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), + brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), + runtime_id: expected.runtime_id.clone(), + status: "released".into(), + scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), + reason: "test rollout".into(), + observed_at_unix_millis: now, + expires_at_unix_millis: Some(now + 600_000), + authorization_id: Some("authorization-test".into()), + authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), + authorized_release_id: Some(expected.sealed_release_id.clone()), + authorized_deployment_id: Some(transaction_id.to_owned()), + authorized_by: Some(signer.trust_anchor()?.identity_id), + authorization_issued_at_unix_millis: Some(now), + authorization_expires_at_unix_millis: Some(now + 600_000), + signature_algorithm: Some("ed25519".into()), + signature: None, + private_state_exposed: false, + updated_by: "operator/test".into(), + }; + record.signature = Some( + signer + .sign::(&rmp_serde::to_vec(&record)?) + .signature, + ); + record.validate()?; + let canonical_brake_bytes = rmp_serde::to_vec(&record)?; + Ok(DeploymentAuthorization { + authorization_id: "authorization-test".into(), + brake_sha256: sha256_id(&canonical_brake_bytes), + canonical_brake_bytes, + authorized_at_unix_millis: now, + }) + } + + /// What makes a target Odin: it provides the rendezvous. + fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + }); + } + + /// The declaration that makes an unrouted fixture target Odin-correlated. + /// Idunn infers no way to prove readiness, and a target with no route has + /// nothing for Idunn to challenge, so the world says outright that Odin + /// reports on it. The plan selects no provider: these worlds never plan + /// against Odin, they only read its correlation, which names the provider + /// Expected does. + fn declare_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.dependencies.push(cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: Some("odin".into()), + provider_authority: Some("managed-incarnation".into()), + provider_expected_projection_sha256: Some(sha256_id(b"odin expected projection")), + provider_endpoint: None, + }); + } + + /// A transaction for an unrouted, stateless target, sitting at `phase` + /// (Warming or later) with every earlier phase's evidence in place. The + /// plan, sealed release, Expected, activation and Warming evidence are the + /// real constructions; only the workload and isolation observations are + /// borrowed from a recorded transaction. + /// + /// `incumbent` is an admitted generation already in the world's store. A + /// Continuity over it restarts that generation's own release, so it shares + /// the incumbent's Expected key. A Deploy over it builds a second + /// incarnation. A Deploy carries what phases past Fencing never read but + /// validation requires: a three-field frozen-source receipt and a + /// well-formed release receipt from the brake. + fn seeded_transaction( + world: &EngineFixture, + phase: DeploymentPhase, + kind: CommandKind, + incumbent: Option<&AdmittedGeneration>, + ) -> Result { + use crate::deployment_plan::tests::{ + BINDING, RECIPE, artifact_receipt, external_input_receipt, source, + }; + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = if provider_path.exists() { + open_service_identity_at::(&provider_path)? + } else { + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + provider + }; + + let (binding_head, binding_tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ) + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id); + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ); + let (plan, release) = match (kind, incumbent) { + (CommandKind::Continuity, Some(incumbent)) => ( + incumbent.plan.clone(), + incumbent.sealed_release.clone(), + ), + _ => { + let incarnation = if incumbent.is_some() { + "service-incarnation-2" + } else { + "service-incarnation-1" + }; + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + incarnation, + None, + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + (plan, release) + } + }; + let mut expected = release.expected_projection(&plan)?; + assert!(!expected.write_lease_required && expected.route.is_none()); + declare_odin(&mut expected); + + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: format!( + "{}-service{}", + match kind { + CommandKind::Deploy => "up", + CommandKind::Continuity => "continuity", + }, + if incumbent.is_some() { "-again" } else { "" } + ), + kind, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = match (kind, incumbent) { + (CommandKind::Continuity, Some(incumbent)) => { + DeploymentTransaction::from_continuity(&command, incumbent, now)? + } + _ => DeploymentTransaction::new(&command, "service".into(), 0, incumbent, now)?, + }; + let activation = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&transaction.transaction_id)?, + now, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("recorded transaction has no workload")?; + match kind { + CommandKind::Continuity => { + transaction.lifecycle_authorized_at_unix_millis = Some(now); + } + CommandKind::Deploy => { + transaction.frozen_source = Some(FrozenSourceReceipt { + transaction_id: transaction.transaction_id.clone(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + transaction.deployment_authorization = Some(deployment_authorization( + world, + &expected, + &transaction.transaction_id, + now, + )?); + } + } + transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); + transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; + transaction.isolation = recorded.isolation; + transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + transaction.sealed_release = Some(release); + transaction.expected = Some(expected); + transaction.activation = Some(activation); + transaction.workload = Some(workload); + transaction.plan = Some(plan); + // Odin's first word about the candidate, before it was Ready. + let sequence = seeded_sequence(incumbent); + let warming = signed_correlation(world, &transaction, sequence, false)?; + let authenticated = world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&world.state_store)?, + &transaction, + &warming, + None, + now, + )?; + transaction.warming = Some(WarmingEvidence::OdinTopology { + evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, + }); + transaction.odin_publisher_sequence_cursor = sequence; + transaction.enter_phase(phase, now); + transaction.validate()?; + + let store = SingleFileMessagePackBackingStore::new(&world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )?); + Ok(transaction) + } + + /// Odin's correlation for the transaction's incarnation, signed by the key + /// the Engine trusts and stamped now. + fn signed_correlation( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + ) -> Result> { + signed_correlation_as(world, transaction, sequence, ready, "active", None, b"presence") + } + + /// Odin's correlation about the transaction's incarnation, observing a + /// presence in `state` that holds `lease`, identified by `presence`. + fn signed_correlation_as( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + state: &str, + lease: Option<&str>, + presence: &[u8], + ) -> Result> { + let expected = transaction.expected.as_ref().context("no Expected")?; + let activation = transaction.activation.as_ref().context("no activation")?; + let mut record = OdinRuntimeTopologyCorrelationRecord { + schema_version: cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: expected.canonical_sha256()?, + expected: true, + current_activation_sha256: Some(activation.canonical_sha256()?), + signed_presence_sha256: Some(sha256_id(presence)), + observed_presence_state: Some(state.into()), + observed_presence_publisher_sequence: Some(sequence), + observed_write_lease_sha256: lease.map(str::to_owned), + observed_capabilities: expected + .capabilities + .iter() + .map(|capability| cultnet_rs::GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: 1, + }) + .collect(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: Some(activation.runtime_instance_id.clone()), + present: true, + ready, + dependencies: expected + .dependencies + .iter() + .map(|dependency| cultnet_rs::OdinRuntimeDependencyEvidence { + kind: dependency.kind.clone(), + capability: dependency.capability.clone(), + schema: dependency.schema.clone(), + compatibility: dependency.compatibility.clone(), + provider_id: dependency.provider_id.clone(), + provider_authority: dependency.provider_authority.clone(), + provider_expected_projection_sha256: dependency + .provider_expected_projection_sha256 + .clone(), + provider_endpoint: dependency.provider_endpoint.clone(), + observed_capacity: Some(dependency.minimum_capacity), + provider_evidence_sha256: Some(sha256_id(b"odin provider evidence")), + ready: true, + }) + .collect(), + disagreements: Vec::new(), + signer_identity_id: world.odin_signer.entry().identity_id.clone(), + publisher_sequence: sequence, + observed_at_unix_millis: now_millis()?, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + }; + record.signature = world + .odin_signer + .sign::(&record.unsigned_signature_payload()?) + .signature; + record.canonical_bytes() + } + + /// Odin publishes its Ready correlation for the transaction incarnation. + fn odin_reports_ready( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let expected = transaction.expected.as_ref().context("no Expected")?; + SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store) + .insert_entry_if_absent(CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &expected.target, + &expected.canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(world, transaction, sequence, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + })?; + Ok(()) + } + + fn resident(world: &EngineFixture) -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .next() + .context("the transaction left the live set") + } + + /// The transaction wherever it now lives: live, or archived once terminal. + fn latest(world: &EngineFixture) -> Result { + match resident(world) { + Ok(stored) => Ok(stored.value), + Err(_) => read_history_transactions(&world.state_store) + .into_iter() + .next() + .context("the transaction is in neither the live set nor history"), + } + } + + /// One engine step at a time until `stop` holds, recording each phase + /// entered and checking after every step that the deadline on the record + /// is exactly the current post-fencing phase's, and absent outside one. + fn drive( + world: &EngineFixture, + stop: impl Fn(&DeploymentTransaction) -> bool, + ) -> Result> { + let mut phases = Vec::new(); + for _ in 0..40 { + let Ok(current) = resident(world) else { + return Ok(phases); + }; + if stop(¤t.value) { + return Ok(phases); + } + world.engine.advance_transaction(¤t)?; + let after = latest(world)?; + match &after.phase_deadline { + Some(deadline) => assert_eq!(deadline.phase, after.phase), + None => assert!( + !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) + .contains(&after.phase), + "a post-fencing record lost its deadline" + ), + } + if phases.last() != Some(&after.phase) { + phases.push(after.phase); + } + } + bail!("the transaction did not reach the stop condition in 40 steps") + } + + #[test] + fn an_admission_runs_from_fencing_to_complete_and_commits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + + let phases = drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + assert_eq!( + phases, + [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + DeploymentPhase::Complete, + ] + ); + let finished = latest(&world)?; + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::Admitted { .. }) + )); + let snapshot = ControlSnapshot::read(&world.state_store)?; + let admitted = snapshot + .admitted_for("service") + .context("commit wrote no admitted generation")?; + assert_eq!(admitted.value.transaction_id, finished.transaction_id); + assert_eq!(Some(&admitted.value.expected), seeded.expected.as_ref()); + Ok(()) + } + + #[test] + fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Warming)?; + let warming = resident(&world)?; + assert_eq!(warming.value.phase_deadline, None); + + world + .engine + .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + let finished = latest(&world)?; + assert_eq!(finished.phase, DeploymentPhase::Complete); + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + Ok(()) + } + + #[test] + fn a_post_fence_abort_runs_to_complete() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Routing + })?; + let routing = resident(&world)?; + assert!(routing.value.phase_deadline.is_some()); + + world + .engine + .begin_post_fencing_abort(&routing, anyhow!("candidate refused"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + let finished = latest(&world)?; + assert_eq!(finished.phase, DeploymentPhase::Complete); + assert_eq!(finished.phase_deadline, None); + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { .. }) + )); + assert!( + ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .is_none(), + "an aborted candidate was admitted" + ); + Ok(()) + } + + #[test] + fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + let plan = transaction.plan.clone().unwrap(); + let base = transaction.created_at_unix_millis; + transaction.enter_phase(DeploymentPhase::Routing, base + 1_000); + let deadline = transaction.phase_deadline.context("no deadline on entry")?; + assert_eq!(deadline.phase, DeploymentPhase::Routing); + assert_eq!( + deadline.deadline_at_unix_millis, + base + 1_000 + u64::from(plan.phase_deadlines().routing_seconds) * 1000 + ); + // A same-phase write after the wall clock stepped backwards is still a + // valid record: the deadline is fixed at entry, not ordered against + // later stamps, and neither is the creation time. + transaction.updated_at_unix_millis = deadline.entered_at_unix_millis - 1; + transaction.validate()?; + transaction.updated_at_unix_millis = base - 1; + transaction.validate()?; + transaction.enter_phase(DeploymentPhase::Committing, base + 2_000); + assert_eq!( + transaction.phase_deadline.unwrap().phase, + DeploymentPhase::Committing + ); + transaction.enter_phase(DeploymentPhase::Complete, base + 3_000); + assert_eq!(transaction.phase_deadline, None); + Ok(()) + } + + #[test] + fn an_undecodable_history_record_is_counted_and_never_dropped_silently() -> Result<()> { + let (finished, _) = terminal_transaction_with_command("ghostlight")?; + let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; + let mut torn = good.clone(); + torn.key = "tx-torn".into(); + // 0xc1 is msgpack's never-used byte: decodable by nothing. + torn.payload = vec![0xc1]; + let mut foreign = good.clone(); + foreign.key = "tx-foreign".into(); + foreign.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA_V3.into()); + foreign.payload = vec![0xc1]; + + let (decoded, undecodable) = + decode_history_transactions(vec![good, torn, foreign]); + assert_eq!(decoded, [finished]); + assert_eq!( + undecodable.iter().map(|(key, _)| key.as_str()).collect::>(), + ["tx-torn", "tx-foreign"] + ); + assert!(undecodable.iter().all(|(_, error)| !error.is_empty())); + Ok(()) + } + + /// The layout `IsolationEvidence` had before c848459 turned it into an + /// untagged Linux/Host enum. That commit claimed old records decode + /// unchanged; this is the check with the old shape itself. + #[test] + fn isolation_evidence_from_before_the_host_variant_still_decodes() -> Result<()> { + #[derive(Serialize)] + struct BeforeTheHostVariant { + candidate_uid: u32, + candidate_pid_namespace_id: u64, + candidate_mount_namespace_id: u64, + incumbent_uid: Option, + incumbent_pid_namespace_id: Option, + incumbent_mount_namespace_id: Option, + } + for incumbent in [None, Some(7)] { + let old = BeforeTheHostVariant { + candidate_uid: 61_000, + candidate_pid_namespace_id: 4_026_531_836, + candidate_mount_namespace_id: 4_026_531_841, + incumbent_uid: incumbent, + incumbent_pid_namespace_id: incumbent.map(|_| 8), + incumbent_mount_namespace_id: incumbent.map(|_| 9), + }; + for bytes in [rmp_serde::to_vec(&old)?, rmp_serde::to_vec_named(&old)?] { + let decoded: IsolationEvidence = rmp_serde::from_slice(&bytes)?; + let IsolationEvidence::Linux(linux) = decoded else { + bail!("an old Linux record decoded as a host record"); + }; + assert_eq!(linux.candidate_uid, 61_000); + assert_eq!(linux.incumbent_uid, incumbent); + } + } + Ok(()) + } + + #[test] + fn an_admitted_odin_without_odin_authority_is_refused_not_replaced_by_the_bootstrap_key() -> Result<()> { + let world = EngineFixture::new()?; + let empty = ControlSnapshot::read(&world.state_store)?; + assert_eq!( + world.engine.current_odin_authority(&empty)?, + world.engine.bootstrap_odin_authority + ); + + let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; + odin.target = "odin".into(); + // Odin is the target that provides the rendezvous, whatever it is named. + odin.expected.target = "odin".into(); + odin.expected.dependencies.clear(); + provide_odin(&mut odin.expected); + odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( + &world.odin_signer.trust_anchor()?, + )?); + let mut with_authority = ControlSnapshot::default(); + with_authority.admitted.push(Stored { + value: odin.clone(), + envelope: CultCacheEnvelope { + key: "odin".into(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + }); + assert_eq!( + world.engine.current_odin_authority(&with_authority)?, + odin.odin_authority.clone().unwrap() + ); + + odin.odin_authority = None; + let mut without = ControlSnapshot::default(); + without.admitted.push(Stored { + value: odin.clone(), + envelope: CultCacheEnvelope { + key: "odin".into(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + }); + let error = world + .engine + .current_odin_authority(&without) + .expect_err("a route-proof Odin must not fall back to the bootstrap key"); + assert!(format!("{error:#}").contains("no Odin authority"), "{error:#}"); + Ok(()) + } + + fn error_text(result: Result<()>) -> String { + format!("{:#}", result.expect_err("the record must be refused")) + } + + /// A route-proof generation from a real admission: the world's Expected + /// declares no Odin dependency, so route proof is its class. + fn route_proof_generation() -> Result { + route_proof::committed_generation() + } + + #[test] + fn a_route_proof_generation_is_admitted_only_in_its_exact_shape() -> Result<()> { + let generation = route_proof_generation()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.odin_authority.is_none()); + assert!(generation.latest_odin_observation.is_none()); + assert!(generation.odin_receipts().is_err()); + + // Route proof carries no Odin receipt and no Odin authority. + let mut with_receipt = generation.clone(); + with_receipt.latest_odin_observation = Some(topology(1, 1)); + assert!(with_receipt.validate().is_err()); + let mut with_authority = generation.clone(); + with_authority.odin_authority = Some(AdmittedOdinAuthority { + signer_identity_id: "odin-signer".into(), + signer_public_key: vec![1; 32], + }); + assert!(with_authority.validate().is_err()); + + // Its readiness evidence is checked by shape. + let ReadinessEvidence::RouteProof { evidence } = &generation.ready else { + bail!("not route proof"); + }; + let mut tampered = generation.clone(); + tampered.ready = ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + canonical_sha256: sha256_id(b"another"), + ..evidence.clone() + }, + }; + assert!(error_text(tampered.validate()).contains("runtime presence evidence")); + let mut untimed = generation.clone(); + untimed.ready = ReadinessEvidence::RouteProof { + evidence: RuntimePresenceEvidence { + admitted_at_unix_millis: evidence.challenged_at_unix_millis - 1, + ..evidence.clone() + }, + }; + assert!(error_text(untimed.validate()).contains("timeline")); + Ok(()) + } + + #[test] + fn odin_correlated_readiness_evidence_is_checked_by_shape() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.validate()?; + let ReadinessEvidence::OdinCorrelated { evidence } = &generation.ready else { + bail!("fixture is not Odin-correlated"); + }; + for broken in [ + TopologyEvidence { + canonical_sha256: sha256_id(b"another"), + ..evidence.clone() + }, + TopologyEvidence { + publisher_sequence: 0, + ..evidence.clone() + }, + TopologyEvidence { + admitted_at_unix_millis: 0, + ..evidence.clone() + }, + ] { + let mut tampered = generation.clone(); + tampered.ready = ReadinessEvidence::OdinCorrelated { evidence: broken }; + assert!(tampered.validate().is_err()); + } + Ok(()) + } + + #[test] + fn lease_adoption_evidence_is_checked_by_shape_on_the_transaction() -> Result<()> { + let (_, transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let adoption = LeaseAdoptionEvidence { + write_lease_sha256: sha256_id(b"lease"), + signed_presence_sha256: sha256_id(b"presence"), + source: AdoptionSource::Direct, + observed_at_unix_millis: 100, + }; + let refused = |change: fn(&mut LeaseAdoptionEvidence)| { + let mut adopting = transaction.clone(); + let mut evidence = adoption.clone(); + change(&mut evidence); + adopting.lease_adoption = Some(evidence); + error_text(adopting.validate()) + }; + assert!(refused(|evidence| evidence.write_lease_sha256.clear()) + .contains("adopted lease digest")); + assert!(refused(|evidence| evidence.signed_presence_sha256 = "no spaces allowed".into()) + .contains("presence digest")); + assert!(refused(|evidence| evidence.observed_at_unix_millis = 0) + .contains("no observation time")); + // Well-formed but naming no granted lease: the record is refused for + // that reason and no other. + assert!(refused(|_| {}).contains("does not name the granted lease")); + Ok(()) + } + + #[test] + fn route_supervision_is_checked_on_the_generation_and_the_meters_on_their_own_record() -> Result<()> { + let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.validate()?; + let supervised = |change: fn(&mut RouteSupervisionState)| { + let mut tampered = generation.clone(); + change(tampered.route_supervision.as_mut().unwrap()); + error_text(tampered.validate()) + }; + assert!(supervised(|state| state.degraded_since_unix_millis = Some(0)) + .contains("no start time")); + + let meters = |change: fn(&mut TargetSupervision)| { + let mut meters = TargetSupervision::new("service"); + change(&mut meters); + error_text(meters.validate()) + }; + TargetSupervision::new("service").validate()?; + assert!(meters(|m| m.route_actuations = vec![0]).contains("ascending")); + assert!(meters(|m| m.route_actuations = vec![9, 5]).contains("ascending")); + assert!( + meters(|m| m.route_actuations = (1..=13).collect()).contains("exceeds its bound") + ); + assert!(meters(|m| m.continuity_restarts = (1..=7).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.route_survivals = (1..=13).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.route_survivals = vec![9, 5]).contains("ascending")); + assert!(meters(|m| m.continuity_deferred_until = Some(5)).contains("time or a reason")); + assert!(meters(|m| m.continuity_deferral_reason = Some("x".into())).contains("time or a reason")); + assert!(meters(|m| m.schema_version = "idunn.target_supervision.v0".into()) + .contains("unsupported")); + Ok(()) + } + + #[test] + fn max_odin_sequence_counts_admitted_generations_by_target_and_signer() -> Result<()> { + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + let authority = generation.odin_authority.clone().context("no authority")?; + generation.odin_publisher_sequence_cursor = 88; + let target = generation.target.clone(); + let snapshot = ControlSnapshot { + expiry_requests: Vec::new(), + commands: Vec::new(), + transactions: Vec::new(), + admitted: vec![Stored { + envelope: CultCacheEnvelope { + key: target.clone(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + value: generation, + }], + targets: Vec::new(), + }; + assert_eq!(snapshot.max_odin_sequence(&target, &authority.signer_identity_id), 88); + assert_eq!(snapshot.max_odin_sequence(&target, "another-signer"), 0); + assert_eq!( + snapshot.max_odin_sequence("another-target", &authority.signer_identity_id), + 0 + ); + Ok(()) + } + + #[test] + fn a_store_holding_a_lifted_legacy_generation_reads_and_keeps_its_key_honest() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let envelope = fixture_envelope(FIXTURE_GENERATION, AdmittedGeneration::TYPE)?; + let backing = SingleFileMessagePackBackingStore::new(&store); + backing.insert_entry_if_absent(envelope.clone())?; + + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.admitted.len(), 1); + let lifted = &snapshot.admitted[0].value; + assert_eq!(lifted.target, envelope.key); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); + assert!(snapshot.admitted_for(&envelope.key).is_some()); + + // The lifted record is held to the same key rule as a current one. + let other = temp.path().join("misfiled.cc"); + let mut misfiled = envelope; + misfiled.key = "not-its-target".into(); + SingleFileMessagePackBackingStore::new(&other).insert_entry_if_absent(misfiled)?; + assert!(error_text(ControlSnapshot::read(&other).map(|_| ())) + .contains("admitted generation key is not its target")); + Ok(()) + } + + /// Replace Odin's one correlation for the incarnation with a newer one. + fn odin_republishes( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let path = &world.engine.options.odin_correlation_store; + let store = SingleFileMessagePackBackingStore::new(path); + let existing = store.pull_all_read_only_snapshot()?; + assert!(store.delete_batch_if_unchanged(&existing)?); + odin_reports_ready(world, transaction, sequence) + } + + fn ready_sequence(transaction: &DeploymentTransaction) -> Option { + transaction + .ready + .as_ref() + .and_then(ReadinessEvidence::odin) + .map(|evidence| evidence.publisher_sequence) + } + + #[test] + fn a_newer_odin_reading_refreshes_the_receipt_before_awaiting_ready_or_routing_advance() + -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + + // AwaitingReady: the receipt is written, then confirmed, then the + // phase advances only on a receipt equal to the latest reading. + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::AwaitingReady + && ready_sequence(transaction) == Some(5) + })?; + odin_republishes(&world, &seeded, 6)?; + world.engine.advance_transaction(&resident(&world)?)?; // admits 6 + let admitted = resident(&world)?.value; + assert_eq!(admitted.phase, DeploymentPhase::AwaitingReady); + assert_eq!(admitted.latest_odin_observation.as_ref().unwrap().publisher_sequence, 6); + assert_eq!(ready_sequence(&admitted), Some(5)); + world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 6 + let refreshed = resident(&world)?.value; + assert_eq!(refreshed.phase, DeploymentPhase::AwaitingReady); + assert_eq!(ready_sequence(&refreshed), Some(6)); + world.engine.advance_transaction(&resident(&world)?)?; + assert_eq!(resident(&world)?.value.phase, DeploymentPhase::Routing); + + // Routing: route admission demands a receipt equal to the latest + // reading, so a newer one is adopted before anything is routed. + odin_republishes(&world, &seeded, 7)?; + world.engine.advance_transaction(&resident(&world)?)?; // admits 7 + assert_eq!(ready_sequence(&resident(&world)?.value), Some(6)); + world.engine.advance_transaction(&resident(&world)?)?; // refreshes Ready to 7 + let routing = resident(&world)?.value; + assert_eq!(routing.phase, DeploymentPhase::Routing); + assert_eq!(ready_sequence(&routing), Some(7)); + assert!(routing.routing.is_none()); + world.engine.advance_transaction(&resident(&world)?)?; + assert!(resident(&world)?.value.routing.is_some()); + Ok(()) + } + + #[test] + fn lease_adoption_names_only_the_granted_lease() -> Result<()> { + let lease = write_lease(); + let lease_sha256 = lease.canonical_sha256()?; + let adoption = LeaseAdoptionEvidence { + write_lease_sha256: lease_sha256.clone(), + signed_presence_sha256: sha256_id(b"adopting-presence"), + source: AdoptionSource::Direct, + observed_at_unix_millis: 100, + }; + adoption.validate_shape()?; + let granted = LeasingEvidence::Granted { + lease: lease.clone(), + lease_sha256: lease_sha256.clone(), + }; + assert!(adoption.names(&granted)); + // Prepared is not granted, stateless has nothing to adopt, and another + // lease's digest is not this lease. + assert!(!adoption.names(&LeasingEvidence::Prepared { + lease, + lease_sha256, + })); + assert!(!adoption.names(&LeasingEvidence::SkippedStateless)); + let mut other = adoption.clone(); + other.write_lease_sha256 = sha256_id(b"another-lease"); + assert!(!other.names(&granted)); + + // On a transaction, adoption without a granted lease is refused. + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + transaction.lease_adoption = Some(adoption); + assert!(transaction.validate().is_err()); + Ok(()) + } + + /// Q3 as a table: which ended phases abort, which name the unadopted + /// lease, which are left to the operator, and which recovery the terminal + /// record names. + #[test] + fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let lease = write_lease(); + let sha256 = lease.canonical_sha256()?; + let granted = LeasingEvidence::Granted { + lease: lease.clone(), + lease_sha256: sha256.clone(), + }; + let prepared = LeasingEvidence::Prepared { + lease, + lease_sha256: sha256.clone(), + }; + let adoption = LeaseAdoptionEvidence::with_ready( + Some(&sha256), + &sha256_id(b"adopting-presence"), + AdoptionSource::Direct, + 1, + ); + assert!(adoption.is_some()); + assert_eq!( + LeaseAdoptionEvidence::with_ready(None, &sha256_id(b"p"), AdoptionSource::Direct, 1), + None + ); + let abort = |lease_not_adopted| PhaseEnd::Abort { lease_not_adopted }; + use CommandKind::{Continuity, Deploy}; + let cases = [ + (Deploy, None, None, abort(false)), + (Deploy, Some(LeasingEvidence::SkippedStateless), None, abort(false)), + (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), + (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), + (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), + (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), + (Continuity, Some(prepared), None, abort(false)), + (Continuity, Some(granted.clone()), None, abort(true)), + (Continuity, Some(granted), adoption.clone(), abort(false)), + ]; + for (kind, leasing, lease_adoption, expected) in cases { + transaction.command_kind = kind; + transaction.leasing = leasing.clone(); + transaction.lease_adoption = lease_adoption.clone(); + assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); + let recovery = if kind == Continuity && lease_adoption.is_some() { + TerminalRecovery::RestartAdmitted + } else { + TerminalRecovery::RestoreIncumbent + }; + assert_eq!(transaction.terminal_recovery(), recovery, "{kind:?} {lease_adoption:?}"); + } + Ok(()) + } + + #[test] + fn an_operator_required_recovery_must_say_why() -> Result<()> { + let (_, mut failed) = fixture_transaction(FIXTURE_TRANSACTIONS[5].1)?; + let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = + failed.completion.as_mut() + else { + panic!("fixture is not a post-fencing failure") + }; + *recovery = TerminalRecovery::OperatorRequired { + reason: "adopted-lease".into(), + }; + failed.validate()?; + assert!(failed.is_terminal()); + let Some(TransactionCompletion::FailedAfterFencing { recovery, .. }) = + failed.completion.as_mut() + else { + unreachable!() + }; + *recovery = TerminalRecovery::OperatorRequired { + reason: String::new(), + }; + assert!(failed.validate().is_err()); + Ok(()) + } + /// The persisted projection a continuity restart leaves behind, read at + /// the file the daemon writes. + struct ContinuityProjection { + _temp: TempDir, + topology: CultCacheTopologyDriver, + provider_anchor: ServiceIdentityTrustAnchor, + expected: IdunnExpectedIncarnationRecord, + admitted_activation: IdunnRuntimeActivationRecord, + candidate_activation: IdunnRuntimeActivationRecord, + } + + impl ContinuityProjection { + /// The production sequence: the admitted incarnation dies and is + /// demoted to Expected-only, then the continuity candidate publishes + /// its own activation under the same key once it is observed. + fn after_candidate_observed() -> Result { + let temp = TempDir::new()?; + let root = temp.path(); + let idunn = enroll_service_identity_at::(&root.join("idunn.cc"))?; + let provider = enroll_service_identity_at::( + &root.join("provider.cc"), + )?; + let provider_anchor = provider.trust_anchor()?; + let mut expected = TopologyFixture::new("ghostlight")?.expected; + expected.expected_signer_identity_id = provider.entry().identity_id.clone(); + expected.artifact_sha256 = sha256_id(&[1]); + expected.validate()?; + let topology = CultCacheTopologyDriver { + projection_store: root.join("topology.cc"), + correlation_store: root.join("correlation.cc"), + }; + let admitted = workload(1, 40, 50); + let candidate = workload(2, 41, 51); + let issue = |observation: &WorkloadObservation| -> Result<_> { + Ok(IdunnRuntimeActivationLaunch::issue( + &expected, + observation.runtime_instance_id().to_owned(), + NOW, + &idunn, + )? + .activation() + .clone()) + }; + let admitted_activation = issue(&admitted)?; + let candidate_activation = issue(&candidate)?; + assert_ne!(admitted_activation, candidate_activation); + + topology.publish_expected(&expected, &provider_anchor)?; + topology.publish_observed_activation(&expected, &admitted_activation, &admitted)?; + topology.demote_to_expected_only( + &expected, + &provider_anchor, + &admitted_activation, + None, + )?; + topology.publish_expected(&expected, &provider_anchor)?; + topology.publish_observed_activation(&expected, &candidate_activation, &candidate)?; + Ok(Self { + _temp: temp, + topology, + provider_anchor, + expected, + admitted_activation, + candidate_activation, + }) + } + + fn continuity_transaction(&self) -> Result { + let mut transaction = DeploymentTransaction::new( + &command(CommandKind::Continuity), + "ghostlight".into(), + 0, + None, + 100, + )?; + transaction.expected = Some(self.expected.clone()); + transaction.activation = Some(self.candidate_activation.clone()); + Ok(transaction) + } + + /// Types of the records projected under the shared incarnation key. + fn projected_types(&self) -> Result> { + let key = incarnation_key(&self.expected)?; + Ok( + SingleFileMessagePackBackingStore::new(&self.topology.projection_store) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.key == key) + .map(|entry| entry.r#type) + .collect(), + ) + } + } + + #[test] + fn a_failed_continuity_demotes_its_own_activation_and_keeps_the_expected() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + assert!( + world + .projected_types()? + .contains(&IdunnRuntimeActivationRecord::TYPE.to_owned()) + ); + let transaction = world.continuity_transaction()?; + assert_eq!( + transaction.abort_topology_reconciliation(), + CleanupEvidence::Pending + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "candidate died").topology_reconciliation, + CleanupEvidence::Pending + ); + + reconcile_failed_candidate_projection( + &world.topology, + &transaction, + &world.provider_anchor, + )?; + + assert_eq!( + world.projected_types()?, + vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()], + "the shared key must keep its Expected and nothing else" + ); + Ok(()) + } + + #[test] + fn a_continuity_that_issued_no_activation_owes_no_projection_change() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + let mut transaction = world.continuity_transaction()?; + transaction.activation = None; + assert_eq!( + transaction.abort_topology_reconciliation(), + CleanupEvidence::Skipped + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "candidate never started") + .topology_reconciliation, + CleanupEvidence::Skipped + ); + Ok(()) + } + + #[test] + fn a_deploy_fencing_after_a_failed_continuity_resolves_with_the_admitted_activation() + -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + reconcile_failed_candidate_projection( + &world.topology, + &world.continuity_transaction()?, + &world.provider_anchor, + )?; + let settled = std::fs::read(&world.topology.projection_store)?; + + // The deploy's post-fence rollback demotes the admitted generation's + // exact activation. The failed continuity left nothing standing, so it + // resolves instead of refusing as substituted. + world.topology.demote_to_expected_only( + &world.expected, + &world.provider_anchor, + &world.admitted_activation, + None, + )?; + assert_eq!(std::fs::read(&world.topology.projection_store)?, settled); + Ok(()) + } + + #[test] + fn no_path_adopts_an_activation_the_transaction_did_not_issue() -> Result<()> { + let world = ContinuityProjection::after_candidate_observed()?; + let before = std::fs::read(&world.topology.projection_store)?; + + // The candidate's activation is still projected. The admitted + // generation's exact activation is not what stands there, so its + // demotion refuses rather than taking the newer one. + assert!( + world + .topology + .demote_to_expected_only( + &world.expected, + &world.provider_anchor, + &world.admitted_activation, + None, + ) + .is_err() + ); + + // Nor may a continuity transaction demote an activation it did not + // issue. + let mut foreign = world.continuity_transaction()?; + foreign.activation = Some(world.admitted_activation.clone()); + assert!( + reconcile_failed_candidate_projection( + &world.topology, + &foreign, + &world.provider_anchor + ) + .is_err() + ); + assert_eq!(std::fs::read(&world.topology.projection_store)?, before); + Ok(()) + } + + // --------------------------------------------------------------------- + // B1 fix batch: the transition, the precondition, the boot reconciliation, + // history that fails closed, and the abort paths through the Engine. + // --------------------------------------------------------------------- + + /// A workload port whose process can be killed: observing it succeeds + /// until `kill`, then fails the way a dead unit does. Everything else is + /// `StillWorkload`'s. + struct SwitchWorkload { + alive: std::sync::atomic::AtomicBool, + rebooted: std::sync::atomic::AtomicBool, + } + + impl SwitchWorkload { + fn new() -> Arc { + Arc::new(Self { + alive: std::sync::atomic::AtomicBool::new(true), + rebooted: std::sync::atomic::AtomicBool::new(false), + }) + } + + fn revive(&self) { + self.alive + .store(true, std::sync::atomic::Ordering::SeqCst); + self.rebooted + .store(false, std::sync::atomic::Ordering::SeqCst); + } + + /// The host rebooted under the unit: it is gone and the driver can tell. + fn reboot(&self) { + self.rebooted + .store(true, std::sync::atomic::Ordering::SeqCst); + self.kill(); + } + + fn kill(&self) { + self.alive + .store(false, std::sync::atomic::Ordering::SeqCst); + } + } + + impl WorkloadPort for SwitchWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + Ok(()) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + ensure!( + self.alive.load(std::sync::atomic::Ordering::SeqCst), + "the workload is gone" + ); + Ok(prior.clone()) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(false) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) + } + } + + /// Admit a generation for "service" through the real phase machine and + /// retire its transaction, leaving the world with an incumbent. + fn admit_incumbent(world: &EngineFixture) -> Result { + let seeded = transaction_at(world, DeploymentPhase::Fencing)?; + odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; + drive(world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + assert!(world.engine.retire_one_terminal_transaction()?); + Ok(ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone()) + } + + #[test] + fn an_admitted_odin_correlated_generation_takes_a_newer_odin_reading_once() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let first = seeded_sequence(None) + 1; + odin_reports_ready(&world, &seeded, first)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + assert!(world.engine.retire_one_terminal_transaction()?); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + assert_eq!(admitted.readiness(), Ok(ReadinessClass::OdinCorrelated)); + assert_eq!(admitted.odin_publisher_sequence_cursor, first); + + // Nothing newer than what is held: nothing is written. + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + let envelope = current.envelope.clone(); + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + assert_eq!(incumbent_envelope(&world)?, envelope); + + // A newer reading is adopted, once. + odin_republishes(&world, &seeded, first + 4)?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(world.engine.refresh_admitted_topology(&snapshot, current)?); + let refreshed = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no generation")? + .value + .clone(); + assert_eq!(refreshed.odin_publisher_sequence_cursor, first + 4); + assert_eq!( + refreshed.latest_odin_observation.as_ref().map(|latest| latest.publisher_sequence), + Some(first + 4) + ); + assert_eq!(refreshed.ready.odin().map(|ready| ready.publisher_sequence), Some(first + 4)); + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + Ok(()) + } + + /// Change the incumbent in the store, compare-and-swap. + fn edit_incumbent( + world: &EngineFixture, + edit: impl FnOnce(&mut AdmittedGeneration), + ) -> Result { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let stored = snapshot + .admitted_for("service") + .context("no admitted generation")?; + let mut next = stored.value.clone(); + edit(&mut next); + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: "service".into(), + current: Some(stored.envelope.clone()), + }], + &[admitted_envelope(&next, now_millis()?)?], + )? + ); + Ok(next) + } + + /// Replace or create a target's meters in the store. + fn set_meters(world: &EngineFixture, meters: &TargetSupervision) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + world.engine.write_target_supervision( + snapshot.supervision_for(&meters.target), + meters, + now_millis()?, + ) + } + + fn meters_of(world: &EngineFixture, target: &str) -> Result { + Ok(ControlSnapshot::read(&world.state_store)?.supervision_or_new(target)) + } + + /// Meters for "service" whose restarts happened these many milliseconds ago. + fn restarted_ago(ages: &[u64]) -> Result { + let now = now_millis()?; + let mut meters = TargetSupervision::new("service"); + meters.continuity_restarts = ages.iter().map(|age| now - age).collect(); + meters.continuity_restarts.sort_unstable(); + // Every logged attempt belongs to one streak. + meters.continuity_attempt_streak = ages.len() as u32; + Ok(meters) + } + + /// Drop every live transaction and command: what a finished restart leaves. + fn clear_live_transactions(world: &EngineFixture) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let envelopes = snapshot + .transactions + .iter() + .map(|stored| stored.envelope.clone()) + .chain(snapshot.commands.iter().map(|stored| stored.envelope.clone())) + .collect::>(); + if envelopes.is_empty() { + return Ok(()); + } + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&envelopes)? + ); + Ok(()) + } + + fn incumbent_envelope(world: &EngineFixture) -> Result { + Ok(ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone()) + } + + /// Publish what a transaction's candidate projects: its Expected and, + /// when asked, its observed activation. + fn project_candidate( + world: &EngineFixture, + transaction: &DeploymentTransaction, + with_activation: bool, + ) -> Result<()> { + let expected = transaction.expected.as_ref().context("no Expected")?; + let anchor = world + .engine + .provider_anchor_for_plan(transaction.plan.as_ref().context("no plan")?)?; + let topology = world.engine.topology(); + topology.publish_expected(expected, &anchor)?; + if with_activation { + // The observation must name the release the Expected declares; + // the borrowed workload is patched to. + let mut workload = transaction.workload.clone().context("no workload")?; + if let WorkloadObservation::Systemd(observed) = &mut workload { + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + topology.publish_observed_activation( + expected, + transaction.activation.as_ref().context("no activation")?, + &workload, + )?; + } + Ok(()) + } + + /// Publish what an admitted generation projects while it runs. + fn project_admitted(world: &EngineFixture, generation: &AdmittedGeneration) -> Result<()> { + let anchor = world.engine.provider_anchor_for_plan(&generation.plan)?; + let topology = world.engine.topology(); + topology.publish_expected(&generation.expected, &anchor)?; + topology.publish_observed_activation( + &generation.expected, + &generation.activation, + &generation.workload, + )?; + Ok(()) + } + + /// The types of the records projected under one incarnation key, sorted. + fn projected_under( + world: &EngineFixture, + expected: &IdunnExpectedIncarnationRecord, + ) -> Result> { + let path = &world.engine.options.topology_store; + if !path.exists() { + return Ok(Vec::new()); + } + let key = incarnation_key(expected)?; + let mut types = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot()? + .into_iter() + .filter(|entry| entry.key == key) + .map(|entry| entry.r#type) + .collect::>(); + types.sort(); + Ok(types) + } + + /// One transaction by id, wherever it now lives. A failed transaction is + /// archived the moment it finishes, so it is in history, not resident. + fn record_of(world: &EngineFixture, transaction_id: &str) -> Result { + if let Some(stored) = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == transaction_id) + { + return Ok(stored.value); + } + read_history_transactions(&world.state_store) + .into_iter() + .find(|transaction| transaction.transaction_id == transaction_id) + .context("the transaction is in neither the live set nor history") + } + + fn expected_only() -> Vec { + vec![IdunnExpectedIncarnationRecord::TYPE.to_owned()] + } + + // ---- the Engine's abort paths, Deploy and incumbent ---- + + #[test] + fn a_deploy_that_fences_commits_and_retires_the_incumbent_it_replaced() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + admit_incumbent(&world)?; + let meters = restarted_ago(&[600_000, 60_000])?; + set_meters(&world, &meters)?; + let incumbent = edit_incumbent(&world, |_| {})?; + + let candidate = seeded_transaction( + &world, + DeploymentPhase::Fencing, + CommandKind::Deploy, + Some(&incumbent), + )?; + odin_reports_ready(&world, &candidate, seeded_sequence(Some(&incumbent)) + 1)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Complete + })?; + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the deploy admitted nothing")? + .value + .clone(); + assert_eq!(admitted.transaction_id, candidate.transaction_id); + assert_ne!(admitted.generation_id, incumbent.generation_id); + // The restart log belongs to the target: a commit must not reset it. + assert_eq!(meters_of(&world, "service")?, meters); + assert_eq!(admitted.route_supervision, incumbent.route_supervision); + + // The commit did not stop the incumbent: it is retired after. + let committed = record_of(&world, &candidate.transaction_id)?; + let cleanup = committed.post_commit_cleanup.clone().context("no cleanup")?; + assert_eq!( + cleanup.incumbent, + IncumbentCleanupEvidence::Pending { + generation_id: incumbent.generation_id.clone(), + workload: incumbent.workload.clone(), + } + ); + assert_eq!(cleanup.source, SourceCleanupEvidence::Pending); + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + + drive(&world, |transaction| transaction.is_terminal())?; + let retired = record_of(&world, &candidate.transaction_id)? + .post_commit_cleanup + .context("no cleanup")?; + assert_eq!( + retired.incumbent, + IncumbentCleanupEvidence::Complete { + generation_id: incumbent.generation_id + } + ); + assert_eq!(retired.source, SourceCleanupEvidence::Complete); + assert!(!frozen.exists(), "the frozen source outlived its deploy"); + Ok(()) + } + + #[test] + fn a_deploy_post_fence_abort_cleans_up_and_restores_its_incumbent_exactly() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let candidate = seeded_transaction( + &world, + DeploymentPhase::Fencing, + CommandKind::Deploy, + Some(&incumbent), + )?; + project_candidate(&world, &candidate, true)?; + let candidate_expected = candidate.expected.clone().context("no Expected")?; + assert_eq!( + projected_under(&world, &candidate_expected)?.len(), + 2, + "the candidate did not project" + ); + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + let before = incumbent_envelope(&world)?; + + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + assert!(resident(&world)?.value.post_fencing_abort.is_some()); + drive(&world, |transaction| transaction.completion.is_some())?; + + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + let abort = finished.post_fencing_abort.context("no abort evidence")?; + assert_eq!(abort.topology_reconciliation, CleanupEvidence::Complete); + assert_eq!(abort.source_cleanup, CleanupEvidence::Complete); + assert!(!frozen.exists(), "the abandoned source was not cleaned"); + assert!( + projected_under(&world, &candidate_expected)?.is_empty(), + "the abandoned candidate is still projected" + ); + // Fencing stopped the incumbent: it stands as Expected-only, with the + // exact activation it ran under withdrawn, ready for continuity. + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + assert_eq!(incumbent_envelope(&world)?, before); + Ok(()) + } + + #[test] + fn a_deploy_pre_fence_abort_cleans_up_and_leaves_its_incumbent_running() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let candidate = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Deploy, + Some(&incumbent), + )?; + project_candidate(&world, &candidate, true)?; + let candidate_expected = candidate.expected.clone().context("no Expected")?; + let frozen = world + .engine + .options + .staging_root + .join("frozen-sources") + .join(&candidate.transaction_id); + std::fs::create_dir_all(&frozen)?; + std::fs::write(frozen.join("tree"), b"frozen")?; + let before = incumbent_envelope(&world)?; + + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let intent = resident(&world)?.value.pre_fencing_abort.context("no intent")?; + assert_eq!(intent.topology_reconciliation, CleanupEvidence::Pending); + assert_eq!(intent.source_cleanup, CleanupEvidence::Pending); + drive(&world, |transaction| transaction.completion.is_some())?; + + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + assert!(!frozen.exists(), "the abandoned source was not cleaned"); + assert!(projected_under(&world, &candidate_expected)?.is_empty()); + // Before fencing a deployment never touched its incumbent. + assert_eq!( + projected_under(&world, &incumbent.expected)?, + { + let mut types = vec![ + IdunnExpectedIncarnationRecord::TYPE.to_owned(), + IdunnRuntimeActivationRecord::TYPE.to_owned(), + ]; + types.sort(); + types + } + ); + assert_eq!(incumbent_envelope(&world)?, before); + Ok(()) + } + + #[test] + fn a_continuity_abort_over_its_incumbent_demotes_only_the_activation_it_issued() -> Result<()> { + for post_fence in [false, true] { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let candidate = seeded_transaction( + &world, + if post_fence { + DeploymentPhase::Fencing + } else { + DeploymentPhase::Warming + }, + CommandKind::Continuity, + Some(&incumbent), + )?; + // The candidate shares the incumbent's key: same Expected, its own + // activation. + assert_eq!(candidate.expected.as_ref(), Some(&incumbent.expected)); + assert_ne!(candidate.activation.as_ref(), Some(&incumbent.activation)); + project_candidate(&world, &candidate, true)?; + let before = incumbent_envelope(&world)?; + + let stored = resident(&world)?; + if post_fence { + world + .engine + .begin_post_fencing_abort(&stored, anyhow!("candidate died"))?; + assert_eq!( + resident(&world)? + .value + .post_fencing_abort + .context("no intent")? + .topology_reconciliation, + CleanupEvidence::Pending + ); + } else { + world + .engine + .begin_pre_fencing_abort(&stored, anyhow!("candidate died"))?; + assert_eq!( + resident(&world)? + .value + .pre_fencing_abort + .context("no intent")? + .topology_reconciliation, + CleanupEvidence::Pending + ); + } + drive(&world, |transaction| transaction.completion.is_some())?; + + assert_eq!( + projected_under(&world, &incumbent.expected)?, + expected_only(), + "post_fence={post_fence}: the shared key must keep its Expected and nothing else" + ); + let finished = record_of(&world, &candidate.transaction_id)?; + assert!(finished.is_terminal()); + assert!(matches!( + finished.completion, + Some( + TransactionCompletion::FailedBeforeFencing { .. } + | TransactionCompletion::FailedAfterFencing { .. } + ) + )); + assert_eq!(incumbent_envelope(&world)?, before); + } + Ok(()) + } + + // ---- fix 1: the legacy lift owns the transition ---- + + const FIXTURE_PRE_B1_ABORTS: [(&str, &str); 3] = [ + ( + "pre-fence abort in flight", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort.hex"), + ), + ( + "pre-fence abort already terminal", + include_str!( + "../tests/fixtures/idunn-control-legacy/transaction-pre-fence-abort-terminal.hex" + ), + ), + ( + "post-fence abort in flight", + include_str!("../tests/fixtures/idunn-control-legacy/transaction-post-fence-abort.hex"), + ), + ]; + const FIXTURE_PROVIDER_ANCHOR: &str = + include_str!("../tests/fixtures/idunn-control-legacy/provider-anchor.hex"); + /// Where the fixtures' plan binding names the provider trust anchor. + const FIXTURE_ANCHOR_PATH: &str = "/tmp/idunn-b1-legacy/provider-anchor.cc"; + + fn hex_bytes(text: &str) -> Vec { + let hex = text.trim(); + (0..hex.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("hex fixture")) + .collect() + } + + fn write_fixture_anchor() -> Result<()> { + let path = Path::new(FIXTURE_ANCHOR_PATH); + std::fs::create_dir_all(path.parent().context("anchor path has no parent")?)?; + std::fs::write(path, hex_bytes(FIXTURE_PROVIDER_ANCHOR))?; + Ok(()) + } + + fn abort_owed(transaction: &DeploymentTransaction) -> Option { + transaction + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + .or_else(|| { + transaction + .post_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }) + } + + /// Put a continuity's record and its command in the control store as they + /// stood: `envelope` is written as given, in whatever schema it is in. + fn make_resident( + world: &EngineFixture, + transaction: &DeploymentTransaction, + envelope: CultCacheEnvelope, + ) -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: transaction.command_id.clone(), + kind: CommandKind::Continuity, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: envelope.key.clone(), + current: None, + }, + ], + &[command_envelope(&command, 100)?, envelope], + )? + ); + Ok(()) + } + + #[test] + fn a_pre_b1_continuity_abort_lifts_without_reopening_and_boot_resolves_its_residue() + -> Result<()> { + write_fixture_anchor()?; + for (name, text) in FIXTURE_PRE_B1_ABORTS { + let envelope = fixture_envelope(text, DeploymentTransaction::TYPE)?; + + // The record is what the old rule wrote: an activation was issued + // and the abort says it owes the projection nothing. + let legacy: LegacyDeploymentTransaction = rmp_serde::from_slice(&envelope.payload)?; + assert_eq!(legacy.command_kind, CommandKind::Continuity, "{name}"); + assert!(legacy.activation.is_some(), "{name}"); + + // History describes what happened, so it lifts the record as written. + let archived = lift_legacy_transaction(&envelope)?; + assert_eq!(abort_owed(&archived), Some(CleanupEvidence::Skipped), "{name}"); + + // The control store lifts it without reopening anything. + let lifted = read_transaction_record(&envelope)?; + let terminal = archived.completion.is_some(); + if terminal { + assert!( + abort_owed(&lifted).is_some_and(CleanupEvidence::is_legacy_marker), + "{name}" + ); + assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); + assert_eq!(lifted.phase, archived.phase, "{name}"); + assert_eq!(lifted.completion, archived.completion, "{name}"); + assert!(!lifted.owns_target_authority(), "{name}"); + } else { + assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); + assert!(!lifted.is_terminal(), "{name}"); + assert_eq!(lifted.completion, None, "{name}"); + } + + // Boot: the store reads. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + make_resident(&world, &lifted, envelope.clone())?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 1, "{name}"); + project_candidate(&world, &lifted, true)?; + let expected = lifted.expected.clone().context("no Expected")?; + assert_eq!(projected_under(&world, &expected)?.len(), 2, "{name}"); + + if terminal { + // A finished record owes nothing and is not touched: boot + // reconciliation demotes the residue by its exact activation. + let before = resident(&world)?.envelope; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted, vec![lifted.transaction_id.clone()], "{name}"); + assert_eq!(resident(&world)?.envelope, before, "{name}"); + } else { + // A record in flight resolves through its own abort. + for _ in 0..12 { + let Ok(current) = resident(&world) else { break }; + if current.value.completion.is_some() { + break; + } + world.engine.advance_transaction(¤t)?; + } + assert!( + record_of(&world, &envelope.key)?.completion.is_some(), + "{name}: the abort never resolved" + ); + } + assert_eq!( + projected_under(&world, &expected)?, + expected_only(), + "{name}: the projection was not demoted to Expected-only" + ); + } + Ok(()) + } + + #[test] + fn resident_terminal_pre_b1_aborts_never_claim_target_authority() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + let first = read_transaction_record(&envelope)?; + assert_eq!(first.target, live.target); + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + + // A second resident abort of the same shape for the same target, + // already written as the marker the migration persists. + let mut second = first.clone(); + second.transaction_id = "tx-second-legacy".into(); + second.command_id = "continuity-second-legacy".into(); + make_resident( + &world, + &second, + cleanup_evidence::last_error_envelope(&second, "second")?, + )?; + + // The store reads: three records, one of them live, none of the + // finished ones claiming the target. + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 3); + let live_before = snapshot + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope + .clone(); + + // Reconciliation cleans the residue once, by exact activation. + let expected = first.expected.clone().context("no Expected")?; + project_candidate(&world, &first, true)?; + assert_eq!(projected_under(&world, &expected)?.len(), 2); + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted.len(), 1); + assert_eq!(projected_under(&world, &expected)?, expected_only()); + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!( + after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope, + live_before + ); + Ok(()) + } + + #[test] + fn boot_never_republishes_a_withdrawn_incarnation_and_history_keeps_one_copy() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let lifted = read_transaction_record(&envelope)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + make_resident(&world, &lifted, envelope.clone())?; + // A crash between the two archive steps: history already holds it. + assert!( + SingleFileMessagePackBackingStore::new(&history_store_path(&world.state_store)) + .insert_entry_if_absent(envelope)? + ); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + + // The incarnation was withdrawn: nothing is projected for it. + let expected = lifted.expected.clone().context("no Expected")?; + assert!(projected_under(&world, &expected)?.is_empty()); + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + assert!( + world + .engine + .reconcile_failed_continuity_projections()? + .demoted + .is_empty() + ); + assert!( + projected_under(&world, &expected)?.is_empty(), + "a withdrawn incarnation's Expected was published again" + ); + + // One copy in history, finished, and nothing resident. + let history = read_history(&world.state_store)?; + assert_eq!(history.transactions.len(), 1); + assert!(history.transactions[0].is_terminal()); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + Ok(()) + } + + #[test] + fn boot_runs_the_projection_reconciliation_serve_starts_from() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let failed = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + project_candidate(&world, &failed, true)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + // A pre-B1 abort's residue: the failed candidate's activation stands. + project_candidate(&world, &failed, true)?; + assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); + + let (_lock, _engine) = boot(world.engine.options.clone())?; + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + /// A workload whose candidate can neither be stopped nor recover: an abort + /// that cannot finish its first step. + struct WedgedWorkload; + + impl WorkloadPort for WedgedWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + StillWorkload.observe(expected, activation, prior) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + bail!("the unit will not stop") + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } + } + + #[test] + fn one_wedged_pre_fence_abort_does_not_stop_the_scheduler() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; + let wedged = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + + // Recording the error is not progress, and neither is repeating it. + assert!(!world.engine.resume_one_transaction()?); + assert!(!world.engine.resume_one_transaction()?); + // Every tick answers: the wedged abort records its error and waits, + // and the rest of the scheduler goes on to freeze the queued command. + write_service_binding(&world)?; + for _ in 0..6 { + world.engine.run_scheduler_tick()?; + } + let stuck = record_of(&world, &wedged.transaction_id)?; + assert!(!stuck.is_terminal()); + assert!(stuck.post_fencing_abort.is_none()); + assert!(stuck.last_error.is_some()); + assert!( + read_history_transactions(&world.state_store) + .iter() + .any(|transaction| transaction.command_id == command.command_id), + "the queued command was never frozen" + ); + Ok(()) + } + + #[test] + fn a_wedged_post_fence_abort_only_waits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(WedgedWorkload))?; + let dead = transaction_at(&world, DeploymentPhase::Fencing)?; + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; + + // The abort cannot stop the unit. The abort is already durable, so + // that is a resumable error, and the tick answers every time. + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + let stuck = record_of(&world, &dead.transaction_id)?; + assert!(!stuck.is_terminal()); + assert!(stuck.last_error.is_some()); + Ok(()) + } + + /// A wedged abort whose `stop()` error carries a changing counter, as + /// systemctl output with a PID or timestamp would. + struct ChangingErrorWorkload(std::sync::atomic::AtomicU32, std::sync::atomic::AtomicBool); + + impl WorkloadPort for ChangingErrorWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + WedgedWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + WedgedWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + WedgedWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + WedgedWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + WedgedWorkload.observe(expected, activation, prior) + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + let n = self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.1.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } + bail!("the unit will not stop (main pid {n})") + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } + } + + fn queue_deploy_for_nowhere(world: &EngineFixture) -> Result { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + Ok(command) + } + + #[test] + fn a_wedge_whose_error_text_changes_every_tick_does_not_starve_other_targets() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), + )))?; + transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let command = queue_deploy_for_nowhere(&world)?; + write_service_binding(&world)?; + for _ in 0..20 { + world.engine.run_scheduler_tick()?; + } + assert!( + read_history_transactions(&world.state_store) + .iter() + .any(|transaction| transaction.command_id == command.command_id), + "the queued command was starved by another target's wedge" + ); + Ok(()) + } + + #[test] + fn rewriting_last_error_is_not_progress() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let before = transaction_at(&world, DeploymentPhase::Warming)?; + let mut noted = before.clone(); + noted.last_error = Some("the unit will not stop (main pid 7)".into()); + noted.updated_at_unix_millis += 1; + assert!(!state_advanced(&before, ¬ed)); + let mut moved = noted.clone(); + moved.phase = DeploymentPhase::AwaitingReady; + assert!(state_advanced(&before, &moved)); + Ok(()) + } + + #[test] + fn a_failing_step_is_retried_after_a_backoff_not_every_tick() -> Result<()> { + let workload = Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), + )); + let world = EngineFixture::with_workload(workload.clone())?; + transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let attempts = || workload.0.load(std::sync::atomic::Ordering::SeqCst); + let before = now_millis()?; + world.engine.run_scheduler_tick()?; + let after = now_millis()?; + assert_eq!(attempts(), 1, "the step did not run"); + // The first failure waits one poll interval: the wait back_off wrote. + let poll = world.engine.options.poll_millis; + let written = world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&resident(&world)?.value.transaction_id) + .map(|backoff| backoff.not_before_unix_millis) + .expect("the failure set no backoff"); + assert!( + (before + poll..=after + poll).contains(&written), + "back_off wrote {written}, not one poll ({poll} ms) after the failure" + ); + // The wait is set from the wall clock, and a loaded host can spend it + // inside a few ticks. So the test owns the wait: held open, no tick + // retries; once due, the next tick does. + let id = resident(&world)?.value.transaction_id; + let set_due = |due: u64| { + world + .engine + .resume_backoff + .lock() + .unwrap() + .get_mut(&id) + .expect("the failure set no backoff") + .not_before_unix_millis = due; + }; + set_due(now_millis()? + RESUME_BACKOFF_CEILING_MILLIS / 2); + for _ in 0..10 { + world.engine.run_scheduler_tick()?; + } + assert_eq!(attempts(), 1, "the wedged step was retried inside its backoff"); + // Each further failure doubles the wait back_off writes. + for (attempt, polls) in [(2, 2), (3, 4)] { + set_due(1); + let before = now_millis()?; + world.engine.run_scheduler_tick()?; + let after = now_millis()?; + assert_eq!(attempts(), attempt, "the step was not retried once its backoff was due"); + let written = world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&id) + .map(|backoff| backoff.not_before_unix_millis) + .expect("the failure set no backoff"); + assert!( + (before + polls * poll..=after + polls * poll).contains(&written), + "failure {attempt} waits {written}, not {polls} polls ({poll} ms each) after it" + ); + } + Ok(()) + } + + /// A candidate that fails to advance and is dead for good. + struct DeadCandidateWorkload; + + impl WorkloadPort for DeadCandidateWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + _: &WorkloadObservation, + ) -> Result { + bail!("the candidate is gone") + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + Ok(()) + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + Ok(true) + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + Ok(false) + } + } + + #[test] + fn a_permanently_dead_candidate_past_the_fence_begins_its_post_fence_abort() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(DeadCandidateWorkload))?; + let dead = transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(record_of(&world, &dead.transaction_id)?.post_fencing_abort.is_none()); + for _ in 0..8 { + world.engine.resume_one_transaction()?; + } + assert!( + record_of(&world, &dead.transaction_id)? + .post_fencing_abort + .is_some(), + "a dead candidate past the fence was retried instead of aborted" + ); + Ok(()) + } + + #[test] + fn a_resume_fault_is_reported_once_and_left_in_the_record() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = transaction_at(&world, DeploymentPhase::Warming)?; + let error = anyhow!("store hiccup 1"); + world + .engine + .note_fault("could not resume transaction", &live.transaction_id, &error); + assert_eq!( + record_of(&world, &live.transaction_id)?.last_error.as_deref(), + Some("store hiccup 1") + ); + let reports = world.engine.fault_reports.lock().unwrap(); + assert_eq!(reports.len(), 1); + assert!(reports[&live.transaction_id].offer(Some("store hiccup 1".into())).is_none()); + Ok(()) + } + + #[test] + fn a_new_record_cannot_carry_the_legacy_marker() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let lifted = read_transaction_record(&envelope)?; + assert!(lifted.carries_legacy_marker()); + let error = error_text(transaction_envelope(&lifted, 1).map(|_| ())); + assert!(error.contains("legacy lift"), "{error}"); + // The migration's own path still writes it, and it reads back. + let (_, migrated) = cleanup_evidence::migrate_transaction_record(&envelope)?; + assert!(read_transaction_record(&migrated)?.carries_legacy_marker()); + Ok(()) + } + + #[test] + fn unreadable_history_does_not_skip_resident_residue_at_boot() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let first = read_transaction_record(&envelope)?; + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + let expected = first.expected.clone().context("no Expected")?; + project_candidate(&world, &first, true)?; + assert_eq!(projected_under(&world, &expected)?.len(), 2); + corrupt_history(&world)?; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!(outcome.demoted.len(), 1); + assert_eq!(projected_under(&world, &expected)?, expected_only()); + Ok(()) + } + + #[test] + fn backoff_doubles_caps_and_waits_exactly() { + let ceiling = RESUME_BACKOFF_CEILING_MILLIS; + assert_eq!(backoff_wait(500, 0, ceiling), 500); + assert_eq!(backoff_wait(500, 1, ceiling), 1000); + assert_eq!(backoff_wait(500, 2, ceiling), 2000); + assert_eq!(backoff_wait(500, 3, ceiling), 4000); + assert_eq!(backoff_wait(500, 30, ceiling), ceiling); + assert_eq!(backoff_wait(u64::MAX, 5, ceiling), ceiling); + // The ceiling is the caller's: continuity's is lower than the resume path's. + assert_eq!(backoff_wait(5_000, 5, 300_000), 160_000); + assert_eq!(backoff_wait(5_000, 6, 300_000), 300_000); + assert_eq!(backoff_wait(5_000, 20, 300_000), 300_000); + assert!(is_waiting(9, 10, 5)); + assert!(!is_waiting(10, 10, 5)); + assert!(!is_waiting(11, 10, 5)); + // Due further ahead than the longest wait the owner can set: the clock + // stepped back, so the attempt is due, not stalled for the step. + assert!(is_waiting(5, 10, 5)); + assert!(!is_waiting(4, 10, 5)); + } + + #[test] + fn a_resume_backoff_set_by_a_stepped_back_clock_is_due_and_a_real_one_waits() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let id = transaction_at(&world, DeploymentPhase::Warming)?.transaction_id; + let not_before = |world: &EngineFixture| { + world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&id) + .map(|backoff| backoff.not_before_unix_millis) + }; + let set = |world: &EngineFixture, not_before_unix_millis: u64| { + world.engine.resume_backoff.lock().unwrap().insert( + id.clone(), + ResumeBackoff { + failures: 3, + not_before_unix_millis, + }, + ); + }; + + // Half a minute ahead is a wait the backoff can set: the step is left alone. + let waiting = now_millis()? + 30_000; + set(&world, waiting); + let _ = world.engine.resume_candidate(&resident(&world)?); + assert_eq!(not_before(&world), Some(waiting)); + + // A day ahead is a clock that stepped back: the step is attempted, and + // whatever it does, it replaces the backoff. + let ahead = now_millis()? + 86_400_000; + set(&world, ahead); + let _ = world.engine.resume_candidate(&resident(&world)?); + assert!(not_before(&world).is_none_or(|due| due < ahead)); + Ok(()) + } + + #[test] + fn a_persistent_fault_is_reported_once_and_again_only_after_recovery() -> Result<()> { + let workload = Arc::new(ChangingErrorWorkload( + std::sync::atomic::AtomicU32::new(0), + std::sync::atomic::AtomicBool::new(false), + )); + let world = EngineFixture::with_workload(workload.clone())?; + let wedged = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let id = wedged.transaction_id; + let expire = || { + world.engine.resume_backoff.lock().unwrap().clear(); + }; + let reported = || world.engine.fault_reports.lock().unwrap().contains_key(&id); + // A fault already reported for this transaction (an Err from the + // resume itself); a handled, still-failing step must not re-arm it. + world + .engine + .note_fault("could not resume transaction", &id, &anyhow!("store hiccup")); + assert!(reported()); + world.engine.resume_one_transaction()?; + assert!(reported(), "a failing handled step re-armed the report"); + // Skipped inside the backoff: the fault is still the same fault. + world.engine.resume_one_transaction()?; + assert!(reported(), "a backoff skip re-armed the report"); + // Still failing after the backoff expires: still the same fault. + expire(); + world.engine.resume_one_transaction()?; + assert!(reported()); + // A real recovery clears it. + workload.1.store(true, std::sync::atomic::Ordering::SeqCst); + expire(); + world.engine.resume_one_transaction()?; + assert!(!reported(), "a step that ran and succeeded left the fault armed"); + Ok(()) + } + + #[test] + fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { + write_fixture_anchor()?; + let envelope = fixture_envelope(FIXTURE_PRE_B1_ABORTS[1].1, DeploymentTransaction::TYPE)?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let first = read_transaction_record(&envelope)?; + make_resident(&world, &first, envelope)?; + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + assert!(record_of(&world, &first.transaction_id)?.carries_legacy_marker()); + corrupt_history(&world)?; + assert!(!world.engine.retire_one_terminal_transaction()?); + let noted = record_of(&world, &first.transaction_id)?; + assert!( + noted.last_error.is_some() && noted.last_error != first.last_error, + "the retire fault never reached the record" + ); + assert!(noted.carries_legacy_marker()); + Ok(()) + } + + #[test] + fn a_tick_that_only_freezes_or_only_advances_reports_progress() -> Result<()> { + // Only a freeze. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + queue_deploy_for_nowhere(&world)?; + write_service_binding(&world)?; + assert!(world.engine.run_scheduler_tick()?, "a freeze-only tick reported no progress"); + + // Only a phase advance. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(world.engine.resume_one_transaction()?, "a real advance reported none"); + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + transaction_at(&world, DeploymentPhase::Fencing)?; + assert!(world.engine.run_scheduler_tick()?); + Ok(()) + } + + #[test] + fn a_terminal_transaction_that_cannot_be_archived_does_not_stop_the_scheduler() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let live = transaction_at(&world, DeploymentPhase::Warming)?; + odin_reports_ready(&world, &live, seeded_sequence(None) + 1)?; + let (finished, command) = terminal_transaction_with_command("ghostlight")?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: finished.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, 100)?, + transaction_envelope(&finished, finished.updated_at_unix_millis)?, + ], + )? + ); + corrupt_history(&world)?; + let before = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope; + + // The finished record cannot be archived, and the live one still moves. + world.engine.run_scheduler_tick()?; + let after = ControlSnapshot::read(&world.state_store)?; + assert!( + after + .transactions + .iter() + .any(|stored| stored.value.transaction_id == finished.transaction_id) + ); + assert_ne!( + after + .transactions + .iter() + .find(|stored| stored.value.transaction_id == live.transaction_id) + .context("live transaction")? + .envelope, + before, + "the live transaction was not resumed" + ); + Ok(()) + } + + // ---- fix 2: supervision owns the Expected-only precondition ---- + + #[test] + fn continuity_mints_no_restart_over_a_projection_it_cannot_demote() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + + // The projection names an activation that is not the incumbent's. + let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; + let topology = world.engine.topology(); + topology.publish_expected(&incumbent.expected, &anchor)?; + let stranger = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + topology.publish_observed_activation( + &incumbent.expected, + stranger.activation.as_ref().context("no activation")?, + stranger.workload.as_ref().context("no workload")?, + )?; + // Remove the seeded stranger transaction: only the projection matters. + let seeded = resident(&world)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged( + &[ + seeded.envelope.clone(), + ControlSnapshot::read(&world.state_store)? + .commands + .into_iter() + .find(|stored| stored.value.command_id == seeded.value.command_id) + .context("seeded command")? + .envelope, + ] + )? + ); + workload.kill(); + + let before = now_millis()?; + assert!(world.engine.supervise_one_admitted_generation()?); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!( + snapshot.transactions.is_empty(), + "continuity minted a restart over a projection it could not demote" + ); + let recorded = snapshot + .supervision_for("service") + .context("the failed demotion was not recorded")? + .value + .clone(); + let deferred = recorded.continuity_deferred_until.context("no deferral time")?; + assert!(recorded.continuity_deferral_reason.is_some()); + assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); + let after = now_millis()?; + assert!( + (before + CONTINUITY_DEFERRAL_MILLIS..=after + CONTINUITY_DEFERRAL_MILLIS) + .contains(&deferred) + ); + + // Deferred: the next tick neither retries nor writes. + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(meters_of(&world, "service")?, recorded); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + + // The projection is repaired and the deferral has run out: the restart + // proceeds, from an Expected-only projection. + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_admitted(&world, &incumbent)?; + let mut ran_out = recorded.clone(); + ran_out.continuity_deferred_until = Some(1); + set_meters(&world, &ran_out)?; + assert!(world.engine.supervise_one_admitted_generation()?); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!(snapshot.transactions.len(), 1); + let minted = snapshot.supervision_for("service").context("no meters")?; + assert_eq!(minted.value.continuity_restarts.len(), 1); + assert_eq!(minted.value.continuity_deferred_until, None); + assert_eq!(minted.value.continuity_deferral_reason, None); + assert_eq!(snapshot.transactions[0].value.command_kind, CommandKind::Continuity); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + // ---- fix 3: one-time boot reconciliation, by exact issuer ---- + + #[test] + fn boot_demotes_the_activation_a_failed_continuity_left_and_only_that() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + let failed = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + project_candidate(&world, &failed, true)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate died"))?; + drive(&world, |transaction| transaction.completion.is_some())?; + // A failed transaction is archived the moment it finishes. + let aborted = record_of(&world, &failed.transaction_id)?; + assert!(aborted.is_terminal()); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + + // Drift a pre-B1 abort left: the failed candidate's activation stands. + project_candidate(&world, &failed, true)?; + assert_eq!(projected_under(&world, &incumbent.expected)?.len(), 2); + + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!( + outcome, + ProjectionReconciliation { + demoted: vec![failed.transaction_id.clone()], + unexplained: Vec::new(), + } + ); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + + // Once: nothing is left to demote. + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + Ok(()) + } + + #[test] + fn boot_never_adopts_an_activation_no_failed_transaction_issued() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + + // The incumbent's own activation is accounted for. + project_admitted(&world, &incumbent)?; + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + + // Some other activation, issued by nothing in history, is reported and + // left exactly as it is. + let stranger = seeded_transaction( + &world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; + let topology = world.engine.topology(); + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_candidate(&world, &stranger, true)?; + // A live transaction accounts for its own activation... + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?, + ProjectionReconciliation::default() + ); + // ...and for nobody else's. + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + let mut other = stranger.clone(); + let other_activation = IdunnRuntimeActivationLaunch::issue( + &incumbent.expected, + runtime_instance_id("tx-someone-else")?, + now_millis()?, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + if let Some(WorkloadObservation::Systemd(observed)) = other.workload.as_mut() { + observed.runtime_instance_id = other_activation.runtime_instance_id.clone(); + } + other.activation = Some(other_activation); + project_candidate(&world, &other, true)?; + assert_eq!( + world.engine.reconcile_failed_continuity_projections()?.unexplained, + vec!["service".to_owned()] + ); + topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; + project_candidate(&world, &stranger, true)?; + // ...and once it is gone, nothing does. + let seeded = resident(&world)?; + let command = ControlSnapshot::read(&world.state_store)? + .commands + .into_iter() + .find(|stored| stored.value.command_id == seeded.value.command_id) + .context("seeded command")?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&[seeded.envelope.clone(), command.envelope])? + ); + let projection = std::fs::read(&world.engine.options.topology_store)?; + let outcome = world.engine.reconcile_failed_continuity_projections()?; + assert_eq!( + outcome, + ProjectionReconciliation { + demoted: Vec::new(), + unexplained: vec!["service".into()], + } + ); + assert_eq!(std::fs::read(&world.engine.options.topology_store)?, projection); + Ok(()) + } + + // ---- fix 4: history that cannot be read stops actuation ---- + + /// An operator binding for the target "service", so the command queue can + /// resolve selectors. + fn write_service_binding(world: &EngineFixture) -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let (head, tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{head}[brakes]{}", + tail.split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ); + std::fs::create_dir_all(&world.engine.options.bindings_dir)?; + std::fs::write(world.engine.options.bindings_dir.join("service.toml"), binding)?; + Ok(()) + } + + fn corrupt_history(world: &EngineFixture) -> Result<()> { + std::fs::write(history_store_path(&world.state_store), b"\xc1\xc1 torn")?; + Ok(()) + } + + #[test] + fn history_that_cannot_be_read_is_an_error_and_missing_history_is_empty() -> Result<()> { + let world = EngineFixture::new()?; + let history = read_history(&world.state_store)?; + assert!(history.transactions.is_empty() && history.report().is_none()); + + corrupt_history(&world)?; + let error = read_history(&world.state_store).err().context("torn history read")?; + assert!(format!("{error:#}").contains("history"), "{error:#}"); + assert!(world.engine.history_for_decision().is_none()); + // The display path still answers, with nothing. + assert!(read_history_transactions(&world.state_store).is_empty()); + Ok(()) + } + + #[test] + fn undecodable_history_names_what_it_is_missing() -> Result<()> { + let (finished, _) = terminal_transaction_with_command("ghostlight")?; + let good = transaction_envelope(&finished, finished.updated_at_unix_millis)?; + let mut torn = good.clone(); + torn.key = "tx-torn".into(); + torn.payload = vec![0xc1]; + let (transactions, undecodable) = decode_history_transactions(vec![good, torn]); + let history = HistoryRead { + transactions, + undecodable, + }; + let report = history.report().context("nothing reported")?; + assert!(report.contains("1 archived transaction"), "{report}"); + assert!(report.contains("tx-torn"), "{report}"); + + let complete = HistoryRead { + transactions: history.transactions, + undecodable: Vec::new(), + }; + assert_eq!(complete.report(), None); + Ok(()) + } + + #[test] + fn a_fault_is_said_once_while_it_lasts() { + let once = ReportOnce::default(); + assert_eq!(once.offer(None), None); + assert_eq!(once.offer(Some("torn".into())), Some("torn".into())); + assert_eq!(once.offer(Some("torn".into())), None); + assert_eq!(once.offer(Some("torn".into())), None); + assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); + assert_eq!(once.offer(None), None); + assert_eq!(once.offer(Some("worse".into())), Some("worse".into())); + } + + #[test] + fn unreadable_history_does_not_stop_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + corrupt_history(&world)?; + + // Crash recovery reads the target's own log, never history: the dead + // workload is restarted, and the restart is counted there. + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + #[test] + fn unreadable_history_stops_command_retirement_and_freezing() -> Result<()> { + let world = EngineFixture::new()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-nowhere".into(), + kind: CommandKind::Deploy, + selector: "nowhere".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, 100)?], + )? + ); + write_service_binding(&world)?; + corrupt_history(&world)?; + + // Whether the command was consumed is unknown, so it is not acted on. + assert!(!world.engine.freeze_one_queued_command()?); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + + // Readable, it is acted on: the unknown selector is refused. + std::fs::remove_file(history_store_path(&world.state_store))?; + assert!(world.engine.freeze_one_queued_command()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + Ok(()) + } + + #[test] + fn the_legacy_lift_owes_only_a_continuity_that_issued_an_activation() -> Result<()> { + let envelope = fixture_envelope( + FIXTURE_PRE_B1_ABORTS[0].1, + DeploymentTransaction::TYPE, + )?; + let written = lift_legacy_transaction(&envelope)?; + let owed = |transaction: &DeploymentTransaction| { + transaction + .pre_fencing_abort + .as_ref() + .map(|abort| abort.topology_reconciliation) + }; + assert_eq!(owed(&written), Some(CleanupEvidence::Skipped)); + + let mut lifted = written.clone(); + cleanup_evidence::owe_legacy_continuity_projection(&mut lifted); + assert_eq!(owed(&lifted), Some(CleanupEvidence::Pending)); + + // A deployment published its own key and recorded what it owed. + let mut deploy = written.clone(); + deploy.command_kind = CommandKind::Deploy; + cleanup_evidence::owe_legacy_continuity_projection(&mut deploy); + assert_eq!(owed(&deploy), Some(CleanupEvidence::Skipped)); + + // A continuity that issued nothing owes nothing. + let mut unissued = written; + unissued.activation = None; + cleanup_evidence::owe_legacy_continuity_projection(&mut unissued); + assert_eq!(owed(&unissued), Some(CleanupEvidence::Skipped)); + Ok(()) + } + + // ---- operator incidents ---------------------------------------------- + + fn incident_path(world: &EngineFixture) -> PathBuf { + world.root.join("incidents.cc") + } + + fn incidents_of(world: &EngineFixture) -> Result> { + IncidentStore::new(&incident_path(world)).read() + } + + /// The control store never holds an incident: it would stop every older + /// binary that reads it. + fn assert_control_store_holds_no_incident(world: &EngineFixture) -> Result<()> { + ControlSnapshot::read(&world.state_store)?; + for envelope in SingleFileMessagePackBackingStore::new(&world.state_store) + .pull_all_read_only_snapshot()? + { + assert_ne!(envelope.r#type, "idunn.operator_incident"); + } + Ok(()) + } + + /// Six restarts inside the window, the last long enough ago that its wait is + /// over: only the ceiling holds. + fn spend_the_window(world: &EngineFixture) -> Result<()> { + set_meters( + world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + ) + } + + /// One entry has left the window, and the newest restart is recent enough + /// that continuity is still waiting: exhaustion is over, nothing restarts. + fn slide_the_window(world: &EngineFixture) -> Result<()> { + set_meters( + world, + &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 10_000])?, + ) + } + + #[test] + fn an_exhausted_target_opens_one_incident_across_ticks_and_restarts() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + assert!(!incident_path(&world).exists()); + + // The first pass over the dead target starts its wait: progress, no mint. + assert!(world.engine.supervise_one_admitted_generation()?); + let opened = incidents_of(&world)?; + assert_eq!(opened.len(), 1); + assert_eq!(opened[0].subject, "service"); + assert_eq!(opened[0].condition, IncidentCondition::ContinuityExhausted); + assert!(opened[0].is_open()); + let written = std::fs::read(incident_path(&world))?; + + for _ in 0..3 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + // An Idunn restart forgets its in-memory trace; the durable record + // keeps the decision from being written again. + let restarted = + Engine::open_with_systemd_workload(world.engine.options.clone(), workload.clone())?; + assert!(restarted.fault_reports.lock().unwrap().is_empty()); + assert!(!restarted.supervise_one_admitted_generation()?); + assert_eq!(std::fs::read(incident_path(&world))?, written); + assert_eq!(incidents_of(&world)?, opened); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn an_exhaustion_incident_closes_on_observed_health_not_on_the_window() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let first = incidents_of(&world)?; + assert_eq!(first.len(), 1); + + // The window slides while the workload is still dead: exhaustion is + // over, the target is not healthy, and the incident stays open. + slide_the_window(&world)?; + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(incidents_of(&world)?, first); + + // Seen running, it closes as recovered. + workload.revive(); + assert!(!world.engine.supervise_one_admitted_generation()?); + let closed = incidents_of(&world)?; + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].incident_key, first[0].incident_key); + assert!( + closed[0] + .closed_at_unix_millis + .is_some_and(|at| at >= closed[0].opened_at_unix_millis) + ); + assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + + // A later exhaustion is a new record with a new opening time. + std::thread::sleep(Duration::from_millis(5)); + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let again = incidents_of(&world)?; + assert_eq!(again.len(), 2); + let reopened = again + .iter() + .find(|record| record.is_open()) + .context("no new incident")?; + assert!(reopened.opened_at_unix_millis > first[0].opened_at_unix_millis); + assert_ne!(reopened.incident_key, first[0].incident_key); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn an_incident_whose_target_is_no_longer_admitted_closes() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert!(incidents_of(&world)?[0].is_open()); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone(); + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&[admitted])? + ); + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::NoLongerAdmitted)); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn a_tick_retires_incidents_closed_past_retention() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let store = IncidentStore::new(&incident_path(&world)); + let condition = IncidentCondition::ContinuityExhausted; + let long_ago = now_millis()? - INCIDENT_RETENTION_MILLIS - 10_000; + assert!(store.open(condition, "gone", long_ago)?); + assert!(store.close(condition, "gone", CloseReason::Recovered, long_ago + 1)?); + assert!(store.open(condition, "recent", now_millis()?)?); + assert!(store.close(condition, "recent", CloseReason::Recovered, now_millis()?)?); + + world.engine.supervise_one_admitted_generation()?; + let live = store.read()?; + assert_eq!(live.len(), 1); + assert_eq!(live[0].subject, "recent"); + assert!(world.root.join("incident-history.cc").exists()); + Ok(()) + } + + /// The same exhaustion and recovery sequence, reduced to what continuity + /// decided: each tick's return, and the transactions and restart log it left. + fn continuity_decisions( + world: &EngineFixture, + engine: &Engine, + workload: &SwitchWorkload, + ) -> Result> { + let mut decided = Vec::new(); + let tick = |decided: &mut Vec<(bool, usize, usize)>| -> Result<()> { + let moved = engine.supervise_one_admitted_generation()?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + decided.push(( + moved, + snapshot.transactions.len(), + snapshot.supervision_or_new("service").continuity_restarts.len(), + )); + Ok(()) + }; + workload.kill(); + spend_the_window(world)?; + tick(&mut decided)?; + tick(&mut decided)?; + slide_the_window(world)?; + tick(&mut decided)?; + workload.revive(); + tick(&mut decided)?; + workload.kill(); + spend_the_window(world)?; + tick(&mut decided)?; + Ok(decided) + } + + #[test] + fn an_unwritable_incident_store_changes_nothing_continuity_decides() -> Result<()> { + const CANARY: &str = "CANARY-6b1d-secret-segment"; + let healthy_workload = SwitchWorkload::new(); + let healthy = EngineFixture::with_workload(healthy_workload.clone())?; + admit_incumbent(&healthy)?; + let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; + assert!(incidents_of(&healthy)?.iter().any(|record| record.is_open())); + + // Not permission-dependent: a store path beneath a regular file cannot + // be created for any uid, root included. + let broken_workload = SwitchWorkload::new(); + let mut broken = EngineFixture::with_workload(broken_workload.clone())?; + admit_incumbent(&broken)?; + let blocker = broken.root.join(CANARY); + std::fs::write(&blocker, b"a file, not a directory")?; + let mut options = broken.engine.options.clone(); + options.incident_store = blocker.join("incidents.cc"); + broken.engine = Engine::open_with_systemd_workload(options, broken_workload.clone())?; + + let actual = continuity_decisions(&broken, &broken.engine, &broken_workload)?; + assert_eq!(actual, expected, "a broken incident store changed a continuity decision"); + assert!(!blocker.join("incidents.cc").exists()); + + // The fault is traced once, under its operation, and echoes no path. + let reports = broken.engine.fault_reports.lock().unwrap(); + let faults = reports + .iter() + .filter(|(key, _)| key.starts_with("incident-store:")) + .collect::>(); + assert_eq!( + faults.len(), + 1, + "{:?}", + faults.iter().map(|(key, _)| key).collect::>() + ); + let message = faults[0].1.last.lock().unwrap().clone().context("fault cleared")?; + assert!(message.contains("open an operator incident"), "{message}"); + assert!(!message.contains(CANARY), "{message}"); + drop(reports); + assert_control_store_holds_no_incident(&broken) + } + + #[test] + fn status_renders_incidents_and_survives_an_unreadable_store() -> Result<()> { + const CANARY: &str = "CANARY-90c2-secret-segment"; + let temp = TempDir::new()?; + let now = now_millis()?; + let mut snapshot = ControlSnapshot::default(); + let meters = TargetSupervision::new("service"); + snapshot.targets.push(Stored { + envelope: target_supervision_envelope(&meters, now)?, + value: meters, + }); + + let readable = temp.path().join("incidents.cc"); + IncidentStore::new(&readable).open(IncidentCondition::ContinuityExhausted, "service", now)?; + let lines = render_targets(&snapshot, &readable, now); + assert!(lines[0].starts_with("target service "), "{lines:?}"); + assert_eq!( + lines.last().map(String::as_str), + Some(format!(" incident continuity-exhausted opened-at {now}").as_str()) + ); + + let unreadable = temp.path().join(CANARY); + std::fs::write(&unreadable, b"not a cultcache file")?; + let lines = render_targets(&snapshot, &unreadable, now); + assert!(lines[0].starts_with("target service "), "{lines:?}"); + let last = lines.last().context("no lines")?; + assert!(last.starts_with("incidents unreadable: "), "{last}"); + assert!(!lines.iter().any(|line| line.contains(CANARY)), "{lines:?}"); + Ok(()) + } + + #[test] + fn the_incident_store_is_named_by_serve_and_status() { + let Command::Serve(options) = + parse(["serve", "--incident-store", "/tmp/i.cc"].into_iter().map(str::to_owned)) + .unwrap() + else { + panic!("expected serve") + }; + assert_eq!(options.incident_store, PathBuf::from("/tmp/i.cc")); + assert!(usage().contains("idunn status [--state-store PATH] [--incident-store PATH]")); + let Command::Status { incident_store, .. } = + parse(["status", "--incident-store", "/tmp/j.cc"].into_iter().map(str::to_owned)) + .unwrap() + else { + panic!("expected status") + }; + assert_eq!(incident_store, PathBuf::from("/tmp/j.cc")); + } + + /// A standing fault is traced once. A key is cleared only by an operation + /// that succeeded, so another site's short-circuit must not end it: a + /// target that is running but still exhausted closes nothing, and that + /// used to clear the reconcile pass's fault and print it again every pass. + #[test] + fn a_failing_reconcile_is_traced_once_while_a_running_target_stays_exhausted() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + std::fs::write(incident_path(&world), b"not an incident store")?; + spend_the_window(&world)?; + + for pass in 0..6 { + world.engine.supervise_one_admitted_generation()?; + let reports = world.engine.fault_reports.lock().unwrap(); + assert!( + reports.contains_key("incident-store:close:*"), + "pass {pass}: the standing reconcile fault was cleared, so it prints again" + ); + } + Ok(()) + } + + #[test] + fn a_release_that_runs_one_pass_after_each_restart_keeps_one_incident() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + + // Each cycle: the restarts are spent, the release dies, is seen running + // for one pass (its last restart took), and dies again. + for _ in 0..4 { + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1, "a flapping release opened another incident"); + assert!(records[0].is_open(), "running while still exhausted closed the incident"); + } + + // Real recovery: running, and the window no longer holds the restarts. + slide_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let records = incidents_of(&world)?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + assert_control_store_holds_no_incident(&world) + } + + #[test] + fn a_target_leaving_admission_closes_only_its_own_incident() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let service = admit_incumbent(&world)?; + let store = IncidentStore::new(&incident_path(&world)); + let condition = IncidentCondition::ContinuityExhausted; + let now = now_millis()?; + for subject in ["service", "neighbour", "gone"] { + assert!(store.open(condition, subject, now)?); + } + + // Two targets are admitted: service, and a neighbour. Only the target + // that is not admitted has its incident closed. + let mut snapshot = ControlSnapshot::read(&world.state_store)?; + let envelope = snapshot + .admitted_for("service") + .context("no admitted generation")? + .envelope + .clone(); + snapshot.admitted.push(Stored { + envelope, + value: AdmittedGeneration { + target: "neighbour".into(), + ..service + }, + }); + world.engine.reconcile_incidents(&snapshot); + + let closed = |subject: &str| -> Result> { + Ok(incidents_of(&world)? + .into_iter() + .find(|record| record.subject == subject) + .context("no such incident")? + .close_reason) + }; + assert_eq!(closed("service")?, None); + assert_eq!(closed("neighbour")?, None); + assert_eq!(closed("gone")?, Some(CloseReason::NoLongerAdmitted)); + Ok(()) + } + + #[cfg(target_os = "linux")] + #[test] + fn a_held_incident_lock_changes_nothing_continuity_decides() -> Result<()> { + use crate::incident::tests::{STUCK_AFTER, hold_lock_of, run_while_held}; + + let healthy_workload = SwitchWorkload::new(); + let healthy = EngineFixture::with_workload(healthy_workload.clone())?; + admit_incumbent(&healthy)?; + let expected = continuity_decisions(&healthy, &healthy.engine, &healthy_workload)?; + + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let path = incident_path(&world); + // Three ticks reach the open of an exhausted target, one exchange each. + let actual = run_while_held(hold_lock_of(&path), 3, || { + continuity_decisions(&world, &world.engine, &workload) + })?; + assert_eq!(actual, expected, "a held incident lock changed a continuity decision"); + assert!(!path.exists(), "a contended write landed"); + + // Every refused write is a traced fault naming no path. + { + let reports = world.engine.fault_reports.lock().unwrap(); + let faults = reports + .iter() + .filter(|(key, _)| key.starts_with("incident-store:")) + .map(|(_, report)| report.last.lock().unwrap().clone().unwrap_or_default()) + .collect::>(); + assert!(!faults.is_empty()); + let root = world.root.display().to_string(); + assert!(faults.iter().all(|fault| !fault.contains(&root)), "{faults:?}"); + } + + // The lock is free again: the next pass retries and the incident lands. + // (A child another test forked while it was held can keep it a moment.) + let deadline = std::time::Instant::now() + STUCK_AFTER; + while !incidents_of(&world)?.iter().any(|record| record.is_open()) { + assert!(std::time::Instant::now() < deadline, "the incident never landed"); + world.engine.supervise_one_admitted_generation()?; + std::thread::sleep(Duration::from_millis(20)); + } + assert_control_store_holds_no_incident(&world) + } + + /// A publish that could not land is not done, at the tick: the demotion + /// continuity needs before a restart is one exchange on the topology lock, + /// and another holder of it defers the restart instead of waiting for it. + #[cfg(target_os = "linux")] + #[test] + fn a_held_topology_lock_defers_continuity_and_mints_nothing() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + project_admitted(&world, &incumbent)?; + let store = world.engine.options.topology_store.clone(); + let projected = projected_under(&world, &incumbent.expected)?; + assert!(projected.contains(&IdunnRuntimeActivationRecord::TYPE.to_owned())); + let before = std::fs::read(&store)?; + workload.kill(); + + // One exchange: the demotion. The window is not spent, so continuity + // would restart if the projection could be demoted. + let moved = run_while_held(hold_lock_of(&store), 1, || { + world.engine.supervise_one_admitted_generation() + })?; + assert!(moved, "the deferral is a write the tick made"); + assert_eq!(std::fs::read(&store)?, before, "a contended publish changed the projection"); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!(snapshot.transactions.is_empty(), "continuity minted over a projection it could not demote"); + let recorded = snapshot.supervision_for("service").context("no deferral recorded")?.value.clone(); + assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); + let reason = recorded.continuity_deferral_reason.clone().context("no deferral reason")?; + assert!(reason.contains("locked by another holder"), "{reason}"); + assert!(!reason.contains(&world.root.display().to_string()), "{reason}"); + + // The holder is gone and the deferral has run out: the demotion lands + // and the restart proceeds. + let mut ran_out = recorded; + ran_out.continuity_deferred_until = Some(1); + set_meters(&world, &ran_out)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + + /// A deployment phase records a publication digest only after the publish + /// returned: the digest is the claim that it landed. + #[cfg(target_os = "linux")] + #[test] + fn a_contended_expected_publish_is_not_recorded_as_published() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Warming)?; + let stored = resident(&world)?; + let mut starting = seeded.clone(); + starting.phase = DeploymentPhase::Starting; + starting.expected_publication_sha256 = None; + starting.activation = None; + starting.workload = None; + starting.activation_publication_sha256 = None; + starting.warming = None; + replace_transaction(&world.state_store, &stored, &starting)?; + let store = world.engine.options.topology_store.clone(); + let stored = resident(&world)?; + + let error = run_while_held(hold_lock_of(&store), 1, || { + world.engine.advance_transaction(&stored) + }) + .expect_err("a contended publish must not advance the phase"); + assert!(format!("{error:#}").contains("locked by another holder"), "{error:#}"); + assert!(!store.exists(), "a contended publish wrote the projection"); + let after = resident(&world)?.value; + assert_eq!(after.phase, DeploymentPhase::Starting); + assert_eq!(after.expected_publication_sha256, None); + + // Free again: the same step publishes and records what it published. + world.engine.advance_transaction(&resident(&world)?)?; + let expected = seeded.expected.context("seeded transaction has no Expected")?; + assert_eq!( + resident(&world)?.value.expected_publication_sha256, + Some(expected.canonical_sha256()?) + ); + Ok(()) + } + + /// A Continuity transaction parked in Starting with nothing published yet, + /// plus the stored record the tick will see. + #[cfg(target_os = "linux")] + fn parked_in_starting(world: &EngineFixture) -> Result { + let seeded = transaction_at(world, DeploymentPhase::Warming)?; + let stored = resident(world)?; + let mut starting = seeded.clone(); + starting.phase = DeploymentPhase::Starting; + starting.expected_publication_sha256 = None; + starting.activation = None; + starting.workload = None; + starting.activation_publication_sha256 = None; + starting.warming = None; + replace_transaction(&world.state_store, &stored, &starting)?; + Ok(seeded) + } + + /// A publish that lost to a held lock is transient: before the fence the + /// transaction stays in its phase with the error recorded, no abort is + /// begun, and the next tick after the release lands the publish. A + /// continuity restart is a Continuity transaction on this same path, so + /// its attempt is minted once and a contended publish spends no second one. + #[cfg(target_os = "linux")] + #[test] + fn a_contended_starting_publish_is_retried_on_a_later_tick_not_aborted() -> Result<()> { + use crate::incident::tests::{hold_lock_of, run_while_held}; + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = parked_in_starting(&world)?; + let mut meters = TargetSupervision::new("service"); + meters.continuity_restarts = vec![now_millis()?]; + set_meters(&world, &meters)?; + let store = world.engine.options.topology_store.clone(); + + for tick in 0..2 { + run_while_held(hold_lock_of(&store), 1, || world.engine.resume_one_transaction())?; + world.engine.resume_backoff.lock().unwrap().clear(); + let held = resident(&world)?.value; + assert_eq!(held.phase, DeploymentPhase::Starting, "tick {tick}"); + assert!(held.pre_fencing_abort.is_none(), "tick {tick}: a held lock aborted the transaction"); + assert!(!held.is_terminal(), "tick {tick}"); + assert_eq!(held.expected_publication_sha256, None, "tick {tick}"); + let error = held.last_error.context("a contended publish left no last_error")?; + assert!(error.contains("locked by another holder"), "{error}"); + assert!(!store.exists(), "a contended publish wrote the projection"); + } + + // Released: the next tick lands the publish and records its digest. + world.engine.resume_one_transaction()?; + let landed = resident(&world)?.value; + assert_eq!(landed.phase, DeploymentPhase::Starting); + assert!(landed.pre_fencing_abort.is_none()); + assert_eq!( + landed.expected_publication_sha256, + Some(seeded.expected.context("seeded transaction has no Expected")?.canonical_sha256()?) + ); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + /// Only contention is transient: a publish that fails for any other reason + /// still ends the transaction before the fence, as it always did. + #[cfg(target_os = "linux")] + #[test] + fn a_non_contention_publish_error_still_aborts_before_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + parked_in_starting(&world)?; + // A directory where the store should be: the snapshot cannot be read. + std::fs::create_dir_all(&world.engine.options.topology_store)?; + world.engine.resume_one_transaction()?; + let after = resident(&world)?.value; + assert!(after.pre_fencing_abort.is_some(), "a non-contention error did not abort"); + assert!( + !after.last_error.as_deref().unwrap_or_default().contains("locked by another holder"), + "{:?}", + after.last_error + ); + Ok(()) + } + + #[test] + fn status_prints_an_open_incident_under_its_target() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + world.engine.supervise_one_admitted_generation()?; + let opened = incidents_of(&world)?[0].opened_at_unix_millis; + + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &incident_path(&world), None)?; + let printed = String::from_utf8(printed)?; + let target = printed.find("target service ").context("no target line")?; + let incident = printed + .find(&format!(" incident continuity-exhausted opened-at {opened}")) + .context("status printed no incident")?; + assert!(target < incident, "{printed}"); + Ok(()) + } + + #[test] + fn serve_ticks_until_told_to_stop() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + spend_the_window(&world)?; + let mut options = world.engine.options.clone(); + options.poll_millis = 1; + let engine = Engine::open_with_systemd_workload(options, workload.clone())?; + + let mut asked = 0; + serve_while(&engine, || { + asked += 1; + asked <= 3 + })?; + assert_eq!(asked, 4); + assert!(incidents_of(&world)?.iter().any(|record| record.is_open()), "no tick ran"); + Ok(()) + } + + #[test] + fn run_dispatches_serve_and_status() { + let temp = TempDir::new().unwrap(); + let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); + // A control store beneath a regular file: serve's boot cannot make it, + // and a serve that returned at once would say Ok instead. + let file = temp.path().join("not-a-directory"); + std::fs::write(&file, b"a file").unwrap(); + let store = file.join("control.cc"); + assert!( + run(args(&["serve", "--state-store", store.to_str().unwrap()]).into_iter()).is_err() + ); + // A control store that is not one: status reads it and refuses. + let junk = temp.path().join("junk.cc"); + std::fs::write(&junk, b"not a cultcache file").unwrap(); + assert!( + run(args(&["status", "--state-store", junk.to_str().unwrap()]).into_iter()).is_err() + ); + } + + /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment + /// waiting before Fencing. Continuity never stops and never yields a + /// deployment (budget-never-stop, handover-phase-split), so the window + /// sliding must not turn the deployment into a casualty. + #[test] + fn a_deployment_before_fencing_over_a_dead_incumbent_that_paged_is_never_yielded_and_lands() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + set_meters( + &world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + world.engine.supervise_one_admitted_generation()?; + // The oldest leaves the window: base would now yield the deployment. + set_meters( + &world, + &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + for _ in 0..3 { + let _ = world.engine.supervise_one_admitted_generation()?; + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + assert!( + after.pre_fencing_abort.is_none(), + "the deployment was yielded: {:?}", + after.pre_fencing_abort + ); + assert_eq!( + continuity_mints(&world)?, + 0, + "continuity minted beside a pre-fencing deployment" + ); + } + workload.revive(); + odin_reports_ready(&world, &redeploy, seeded_sequence(Some(&incumbent)) + 1)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + let landed = latest(&world)?; + assert_eq!(landed.transaction_id, redeploy.transaction_id); + assert_eq!(landed.phase, DeploymentPhase::Complete); + assert!(landed.pre_fencing_abort.is_none()); + Ok(()) + } + + /// The incumbent was admitted this long ago, so an anchor written "in the + /// past" is not older than its generation. + fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { + edit_incumbent(world, |generation| generation.admitted_at_unix_millis -= millis)?; + Ok(()) + } + + /// Put the target's backoff anchor `ago` milliseconds back. + fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { + let mut meters = meters_of(world, "service")?; + meters.continuity_backoff_from_unix_millis = Some(now_millis()? - ago); + set_meters(world, &meters) + } + + /// Continuity transactions minted over an incumbent (the one that admitted + /// it was seeded over nothing). + fn continuity_mints(world: &EngineFixture) -> Result { + Ok(every_transaction(world)? + .iter() + .filter(|transaction| { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.is_some() + }) + .count()) + } + + #[test] + fn continuity_keeps_restarting_past_the_paging_threshold() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + // Six charged attempts in the hour: the incident pages, nothing stops. + set_meters( + &world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + let opened = incidents_of(&world)?; + assert_eq!(opened.len(), 1); + assert!(opened[0].is_open()); + + // The streak-6 wait is 160 s. + anchor_ago(&world, 159_000)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + anchor_ago(&world, 160_001)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let after = meters_of(&world, "service")?; + assert_eq!(after.continuity_restarts.len(), 6); + assert!(after.continuity_restarts.last().is_some_and(|&last| now_millis().unwrap() - last < 60_000)); + assert_eq!(after.continuity_attempt_streak, 7); + assert_eq!(after.continuity_backoff_from_unix_millis, None); + assert_eq!(incidents_of(&world)?, opened); + Ok(()) + } + + #[test] + fn continuity_backoff_is_a_capped_doubling_of_the_attempt_streak() { + let from = 10_000_000_u64; + for (streak, seconds) in [ + (1_u32, 5_u64), + (2, 10), + (3, 20), + (4, 40), + (5, 80), + (6, 160), + (7, 300), + (8, 300), + (9, 300), + ] { + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = streak; + meters.continuity_backoff_from_unix_millis = Some(from); + // The paging log disagrees with the streak on purpose: the wait + // is a function of the streak, not of the window's count. + meters.continuity_restarts = vec![from - 10; if streak == 1 { 6 } else { 2 }]; + let due = from + seconds * 1_000; + assert_eq!(meters.backoff_due(0), Some(due), "streak {streak}"); + assert_eq!( + meters.continuity_backoff(due - 1, 0), + ContinuityBackoff::Waiting, + "streak {streak}" + ); + assert_eq!( + meters.continuity_backoff(due, 0), + ContinuityBackoff::Due { streak }, + "streak {streak}" + ); + } + // No streak, no wait. + let mut fresh = TargetSupervision::new("service"); + fresh.continuity_backoff_from_unix_millis = Some(from); + assert_eq!(fresh.continuity_backoff(from, 0), ContinuityBackoff::Due { streak: 0 }); + } + + #[test] + fn a_sustained_run_resets_the_streak_and_only_a_sustained_run() { + let now = 100_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 7; + meters.continuity_restarts = vec![1_000]; + // Lived the sustained time, and no attempt since admission. + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), + ContinuityBackoff::Due { streak: 0 } + ); + // A moment short of it. + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + 10_000), + ContinuityBackoff::Begin + ); + // Attempts after admission failed on this generation: it has not run well. + meters.continuity_restarts = vec![now - 100_000]; + assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); + // Never attempted over this log, long lived: reset. + meters.continuity_restarts = Vec::new(); + assert_eq!( + meters.continuity_backoff(now, now - 700_000), + ContinuityBackoff::Due { streak: 0 } + ); + assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); + // An anchor older than its generation decides nothing. + meters.continuity_backoff_from_unix_millis = Some(now - 800_000); + assert_eq!(meters.backoff_due(now - 700_000), None); + assert_eq!( + meters.continuity_backoff(now, now - 700_000), + ContinuityBackoff::Due { streak: 0 } + ); + assert_eq!(meters.continuity_backoff(now, now - 590_000), ContinuityBackoff::Begin); + // Streak zero with no history is simply due. + let fresh = TargetSupervision::new("service"); + assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); + } + + #[test] + fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + + // The last attempt was long ago, but the wait starts now: a wait + // anchored on the mint would already be over. + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // Streak 3 waits 20 s. + anchor_ago(&world, 19_000)?; + assert!(!world.engine.supervise_one_admitted_generation()?); + anchor_ago(&world, 20_001)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 4); + Ok(()) + } + + #[test] + fn a_queued_deployment_freezes_in_the_gap_after_a_continuity_attempt() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[3_000_000])?; + meters.continuity_attempt_streak = 2; + set_meters(&world, &meters)?; + queue_declaring_redeploy(&world, "up-service-queued")?; + + for _ in 0..4 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-queued"), + "the queued deployment never froze" + ); + assert_eq!(continuity_mints(&world)?, 0); + for _ in 0..3 { + let _ = world.engine.run_scheduler_tick(); + assert_eq!(continuity_mints(&world)?, 0); + let aborts = every_transaction(&world)? + .into_iter() + .filter_map(|transaction| transaction.pre_fencing_abort) + .collect::>(); + // The deployment may fail on its own account (a root test runner + // cannot resolve its source); continuity must never be the reason. + assert!( + aborts.iter().all(|abort| !abort.error.contains("continuity")), + "the deployment was yielded: {aborts:?}" + ); + } + Ok(()) + } + + #[test] + fn a_unit_lost_to_a_host_reboot_is_rematerialized_once_with_no_backoff_step() -> Result<()> { + let fresh_world = |workload: &Arc| -> Result { + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + meters.continuity_backoff_from_unix_millis = Some(now_millis()?); + set_meters(&world, &meters)?; + Ok(world) + }; + let workload = SwitchWorkload::new(); + let world = fresh_world(&workload)?; + let before = meters_of(&world, "service")?; + + // Control: dead but not rebooted, the ordinary 20 s wait holds. + workload.kill(); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // The host rebooted under the unit: one mint, charged to nothing. + workload.reboot(); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let after = meters_of(&world, "service")?; + assert_eq!(after.continuity_restarts, before.continuity_restarts); + assert_eq!(after.continuity_attempt_streak, 3); + assert!(after.continuity_rematerialized_at_unix_millis.is_some()); + assert_eq!(after.continuity_backoff_from_unix_millis, None); + + // The bound is spent: the next death of this generation waits like any other. + clear_live_transactions(&world)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + + // Control: a deferred projection spaces a re-materialization too. + let workload = SwitchWorkload::new(); + let world = fresh_world(&workload)?; + let mut deferred = meters_of(&world, "service")?; + deferred.defer_continuity(now_millis()?, "demotion failed"); + set_meters(&world, &deferred)?; + workload.reboot(); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 0); + Ok(()) + } + + fn supervision_envelope_raw(payload: Vec, schema: &str) -> Result { + Ok(CultCacheEnvelope { + key: "service".into(), + r#type: TargetSupervision::TYPE.into(), + payload, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(schema.into()), + }) + } + + #[test] + fn a_v1_target_supervision_record_is_lifted_and_migrated_once() -> Result<()> { + let world = EngineFixture::new()?; + let meters = restarted_ago(&[200_000])?; + // A record as every one on a running host was written: seven keys, v1. + let payload = rmp_serde::to_vec(&( + TARGET_SUPERVISION_SCHEMA_V1, + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent( + supervision_envelope_raw(payload, TARGET_SUPERVISION_SCHEMA_V1)?, + )?; + let read = meters_of(&world, "service")?; + assert_eq!(read.continuity_restarts, meters.continuity_restarts); + assert_eq!(read.continuity_attempt_streak, 0); + assert_eq!(read.continuity_backoff_from_unix_millis, None); + assert_eq!(read.continuity_rematerialized_at_unix_millis, None); + + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&world.state_store)?, 0); + let stored = ControlSnapshot::read(&world.state_store)?; + let envelope = &stored.supervision_for("service").unwrap().envelope; + assert_eq!(envelope.schema_id.as_deref(), Some(TARGET_SUPERVISION_SCHEMA)); + assert_eq!(meters_of(&world, "service")?.continuity_restarts, meters.continuity_restarts); + + let mut anchored = TargetSupervision::new("service"); + anchored.continuity_backoff_from_unix_millis = Some(0); + assert!(anchored.validate().is_err()); + anchored.continuity_backoff_from_unix_millis = None; + anchored.continuity_rematerialized_at_unix_millis = Some(0); + assert!(anchored.validate().is_err()); + Ok(()) + } + + #[test] + fn one_target_supervision_schema_id_names_one_shape() -> Result<()> { + let meters = restarted_ago(&[200_000])?; + let ten = rmp_serde::to_vec(&meters)?; + let seven = rmp_serde::to_vec(&( + TARGET_SUPERVISION_SCHEMA, + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + // The current id refuses the old shape and the old id refuses the new one. + assert!( + read_supervision_record(&supervision_envelope_raw(seven, TARGET_SUPERVISION_SCHEMA)?) + .is_err() + ); + assert!( + read_supervision_record(&supervision_envelope_raw( + ten.clone(), + TARGET_SUPERVISION_SCHEMA_V1 + )?) + .is_err() + ); + assert_eq!( + read_supervision_record(&supervision_envelope_raw(ten, TARGET_SUPERVISION_SCHEMA)?)?, + meters + ); + Ok(()) + } + + #[test] + fn continuity_keeps_minting_at_every_streak_and_log_length() -> Result<()> { + // No stop at any streak, however long, with the paging log full or not. + let full = [3_000_000_u64, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000]; + for streak in 1..=12_u32 { + for log in [&[1_000_000_u64][..], &full[..]] { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(log)?; + meters.continuity_attempt_streak = streak; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the wait begins"); + anchor_ago(&world, 300_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1, "streak {streak} log {} stopped", log.len()); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, streak + 1); + } + } + Ok(()) + } + + #[test] + fn a_mint_after_a_sustained_run_stores_the_reset_streak() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + workload.kill(); + let mut meters = restarted_ago(&[3_000_000])?; + meters.continuity_attempt_streak = 7; + set_meters(&world, &meters)?; + // Ran 700 s since the last attempt: no wait, and the streak restarts at 1. + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + + #[test] + fn a_re_materialization_counts_as_the_last_attempt_for_the_sustained_reset() { + let admitted_at = 10_000_000_u64; + let now = admitted_at + 700_000; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 5; + meters.continuity_restarts = vec![admitted_at - 1_000]; + // Before admission: a sustained run, reset. + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at - 1); + assert_eq!( + meters.continuity_backoff(now, admitted_at), + ContinuityBackoff::Due { streak: 0 } + ); + // After admission: that attempt failed on this generation. + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at + 1); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); + // Even with no streak to step: the re-materialization is an attempt, + // and the next one waits the base step from its anchor. + meters.continuity_attempt_streak = 0; + meters.continuity_restarts = Vec::new(); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Begin); + meters.continuity_backoff_from_unix_millis = Some(now); + assert_eq!(meters.backoff_due(admitted_at), Some(now + CONTINUITY_RESTART_BACKOFF_MILLIS)); + assert_eq!(meters.continuity_backoff(now, admitted_at), ContinuityBackoff::Waiting); + assert_eq!( + meters.continuity_backoff(now + CONTINUITY_RESTART_BACKOFF_MILLIS, admitted_at), + ContinuityBackoff::Due { streak: 0 } + ); + meters.continuity_rematerialized_at_unix_millis = None; + assert_eq!(meters.backoff_due(admitted_at), Some(now)); + } + + mod after_a_reboot { + use super::*; + + fn resident_continuity(world: &EngineFixture) -> Result { + Ok(ControlSnapshot::read(&world.state_store)? + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .count()) + } + + /// The attempt failed and was retired: drop continuity transactions + /// and their commands only. + fn drop_continuity(world: &EngineFixture) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let ids = snapshot + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .map(|s| s.value.command_id.clone()) + .collect::>(); + let envelopes = snapshot + .transactions + .iter() + .filter(|s| s.value.command_kind == CommandKind::Continuity) + .map(|s| s.envelope.clone()) + .chain( + snapshot + .commands + .iter() + .filter(|s| ids.contains(&s.value.command_id)) + .map(|s| s.envelope.clone()), + ) + .collect::>(); + if !envelopes.is_empty() { + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&envelopes)? + ); + } + Ok(()) + } + + /// (attempts minted on tick 1, on tick 2, whether the queued deploy froze by then). + fn gap_run(reboot: bool) -> Result<(usize, usize, bool)> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + if reboot { + workload.reboot(); + } else { + workload.kill(); + } + queue_declaring_redeploy(&world, "up-gap")?; + // One tick, in the tick's order: supervise, then freeze. + world.engine.supervise_one_admitted_generation()?; + let first = resident_continuity(&world)?; + world.engine.freeze_one_queued_command()?; + // That attempt failed and was retired. + drop_continuity(&world)?; + world.engine.supervise_one_admitted_generation()?; + let second = resident_continuity(&world)?; + world.engine.freeze_one_queued_command()?; + let frozen = every_transaction(&world)?.iter().any(|t| t.command_id == "up-gap"); + Ok((first, second, frozen)) + } + + #[test] + fn a_failed_re_materialization_is_followed_by_a_gap_a_queued_deployment_can_take( + ) -> Result<()> { + // Control: an ordinary attempt after a sustained run leaves a gap. + assert_eq!(gap_run(false)?, (1, 0, true)); + // The re-materialization is an attempt too: the next pass waits. + assert_eq!(gap_run(true)?, (1, 0, true)); + Ok(()) + } + + #[test] + fn a_running_pass_ends_the_wait_so_a_later_death_after_a_sustained_run_resets( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis.is_some()); + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(meters_of(&world, "service")?.continuity_backoff_from_unix_millis, None); + + // It runs on for days, then dies: no wait, and the streak restarts. + age_incumbent(&world, 200_000)?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + } + + /// A source that resolves one fixed recipe and never freezes anything. + struct FixedSource { + recipe: String, + } + + impl SourcePort for FixedSource { + fn resolve( + &self, + _binding: &OperatorBinding, + _resolution_id: &str, + _selected_at_unix_millis: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { + bail!("this source freezes nothing") + } + + fn observe_frozen( + &self, + _: &CompiledDeploymentPlan, + _: &FrozenSourceReceipt, + ) -> Result { + bail!("this source freezes nothing") + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// A Deploy transaction for an unrouted target, still Sealing, whose + /// source resolves `recipe`. One scheduler tick admits its plan or refuses it. + fn sealing_world(recipe: &str) -> Result { + use crate::deployment_plan::tests::BINDING; + let mut world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (binding_head, binding_tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + world.engine.source = Arc::new(FixedSource { + recipe: recipe.to_owned(), + }); + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )? + ); + Ok(world) + } + + fn unrouted_recipe() -> String { + crate::deployment_plan::tests::RECIPE + .split("[[dependencies]]") + .next() + .expect("the recipe has a body") + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ) + } + + #[test] + fn admission_refuses_a_recipe_that_declares_no_way_to_prove_readiness() -> Result<()> { + let recipe = unrouted_recipe(); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let refused = latest(&world)?; + assert!(refused.plan.is_none(), "the refused recipe was planned"); + assert!(refused.frozen_source.is_none(), "the refused recipe was frozen"); + let abort = refused + .pre_fencing_abort + .context("a recipe with no declared proof of readiness must be refused")?; + assert!( + abort.error.contains("target service declares no way to prove readiness"), + "{}", + abort.error + ); + Ok(()) + } + + #[test] + fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let planned = latest(&world)?; + assert!(planned.pre_fencing_abort.is_none(), "{:?}", planned.pre_fencing_abort); + assert_eq!( + planned.plan.as_ref().map(CompiledDeploymentPlan::readiness_class).transpose()?, + Some(ReadinessClass::OdinSelf) + ); + Ok(()) + } + + // --------------------------------------------------------------------- + // A held record is reported and never decided for. Supervision mints no + // continuity over a held generation, and nothing yields to a held record. + // --------------------------------------------------------------------- + + /// Rewrite the world's admitted generation into what a pre-B3 binary left + /// behind: no Odin declaration, admitted on Odin receipts. + fn undeclare_incumbent(world: &EngineFixture) -> Result { + let now = now_millis()?; + let mut failure = None; + let next = edit_incumbent(world, |generation| { + generation.expected.dependencies.clear(); + match IdunnRuntimeActivationLaunch::issue( + &generation.expected, + generation.activation.runtime_instance_id.clone(), + now, + &world.engine.idunn_signer, + ) { + Ok(launch) => generation.activation = launch.activation().clone(), + Err(error) => failure = Some(error), + } + })?; + if let Some(error) = failure { + return Err(error); + } + ControlSnapshot::read(&world.state_store).context("store with the undeclared generation")?; + assert!(matches!(next.readiness(), Err(ReadinessDisagreement::Undeclared(_)))); + Ok(next) + } + + /// Queue the operator's declaring redeploy of `service`. + fn queue_declaring_redeploy(world: &EngineFixture, command_id: &str) -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: command_id.into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now_millis()?, + }; + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, command.requested_at_unix_millis)?], + )?); + Ok(()) + } + + /// Every transaction the store knows: live, then archived. + fn every_transaction(world: &EngineFixture) -> Result> { + let mut all = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .map(|stored| stored.value) + .collect::>(); + all.extend(read_history_transactions(&world.state_store)); + Ok(all) + } + + fn is_continuity_over(transaction: &DeploymentTransaction, held: &AdmittedGeneration) -> bool { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.as_deref() == Some(held.generation_id.as_str()) + } + + #[test] + fn a_held_generation_that_dies_mints_no_continuity_and_leaves_its_target_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + world.engine.validate_durable_authority(&ControlSnapshot::read(&world.state_store)?)?; + + workload.kill(); + for _ in 0..4 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + assert!( + ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), + "supervision minted a transaction over a held generation" + ); + // Reported, once per state change: the same text is not offered again. + let reports = world.engine.fault_reports.lock().unwrap(); + let down = reports.get("generation-down:service").context("the dead hold is not reported")?; + let last = down.last.lock().unwrap().clone(); + assert!(last.is_some()); + assert!(down.offer(last).is_none()); + drop(reports); + + // The target is free: the declaring redeploy freezes and is not + // displaced by a continuity. + queue_declaring_redeploy(&world, "up-service-declared")?; + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held generation" + ); + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-declared"), + "the declaring redeploy never froze" + ); + Ok(()) + } + + #[test] + fn a_declaring_redeploy_is_not_yielded_when_the_held_incumbent_dies() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; + assert_eq!(redeploy.held_disagreement(), None); + workload.kill(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held incumbent" + ); + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + assert!( + after.pre_fencing_abort.is_none(), + "the redeploy yielded: {:?}", + after.pre_fencing_abort + ); + Ok(()) + } + + /// `seeded` with its own Expected undeclared, so Idunn holds it. + fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { + let mut bare = seeded.clone(); + // Undeclare the Expected and re-derive what is bound to its digest. + let expected = bare.expected.as_mut().context("no expected")?; + expected.dependencies.clear(); + bare.expected_publication_sha256 = Some(expected.canonical_sha256()?); + let activation = IdunnRuntimeActivationLaunch::issue( + expected, + seeded.activation.as_ref().context("no activation")?.runtime_instance_id.clone(), + now_millis()?, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); + bare.activation = Some(activation); + bare.validate()?; + assert!(bare.held_disagreement().is_some()); + Ok(bare) + } + + /// A pre-fence Deploy whose own Expected is undeclared, stored in place. + fn held_predeploy( + world: &EngineFixture, + incumbent: &AdmittedGeneration, + ) -> Result { + let seeded = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(incumbent))?; + let bare = undeclared(world, &seeded)?; + replace_transaction(&world.state_store, &resident(world)?, &bare)?; + Ok(bare) + } + + #[test] + fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + // The incumbent is declared; the Deploy in front of it is the held one. + let held = held_predeploy(&world, &incumbent)?; + + workload.kill(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; + } + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, held.transaction_id); + assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); + assert!(after.completion.is_none()); + Ok(()) + } + + #[test] + fn cancel_withdraws_a_held_predeploy_and_leaves_the_incumbent_untouched() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let held = held_predeploy(&world, &incumbent)?; + let before = incumbent_envelope(&world)?; + + cancel(&world.state_store, &held.command_id, "operator")?; + let cancelling = resident(&world)?.value; + assert_eq!( + cancelling.pre_fencing_abort.as_ref().map(|abort| abort.error.as_str()), + Some("cancelled by operator") + ); + assert!(cancelling.post_fencing_abort.is_none()); + + drive(&world, |transaction| transaction.completion.is_some())?; + // History also holds the incumbent's own admission: find ours by id. + let finished = every_transaction(&world)? + .into_iter() + .find(|transaction| transaction.transaction_id == held.transaction_id) + .context("the cancelled redeploy is nowhere")?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + // The target is released and nothing of the incumbent was touched. + assert!(!finished.blocks_new_target_mutation()); + assert_eq!(incumbent_envelope(&world)?, before); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| !is_continuity_over(transaction, &incumbent)), + "cancelling the redeploy minted a continuity" + ); + Ok(()) + } + + #[test] + fn cancel_refuses_a_predeploy_that_is_not_held_and_a_held_one_past_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + assert!(seeded.held_disagreement().is_none()); + assert!(cancel(&world.state_store, &seeded.command_id, "operator").is_err()); + assert!(resident(&world)?.value.pre_fencing_abort.is_none()); + + let mut past_the_fence = undeclared(&world, &seeded)?; + past_the_fence.enter_phase(DeploymentPhase::Fencing, now_millis()?); + assert!(past_the_fence.held_disagreement().is_some()); + assert!(!cancel_is_safe_for_held_prefence(&past_the_fence)); + Ok(()) + } + + // --------------------------------------------------------------------- + // B2: the meters, the v4 lift, and what `status` shows. + // --------------------------------------------------------------------- + + #[test] + fn the_actuation_log_slides() -> Result<()> { + let window = ROUTE_ACTUATION_WINDOW_MILLIS; + let t = 1_000_000_000_u64; + let mut meters = TargetSupervision::new("service"); + for offset in 0..12 { + meters.charge_route(t + offset, RouteActuation::Forward)?; + } + let refused = meters.charge_route(t + window - 1, RouteActuation::Forward).unwrap_err(); + assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + window)); + // Only the first entry has left the window: exactly one slot reopens. + meters.charge_route(t + window, RouteActuation::Forward)?; + let refused = meters.charge_route(t + window, RouteActuation::Forward).unwrap_err(); + assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + 1 + window)); + meters.validate()?; + Ok(()) + } + + #[test] + fn a_survival_actuation_is_recorded_but_never_counted_or_refused() -> Result<()> { + let t = 1_000_000_000_u64; + let mut meters = TargetSupervision::new("service"); + // Survivals first, past the ceiling: every one is admitted and + // recorded, and none takes a Forward slot. + for offset in 0..20 { + meters.charge_route(t + offset, RouteActuation::Survival)?; + } + assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); + assert_eq!(meters.route_survivals.last(), Some(&(t + 19))); + assert!(meters.route_actuations.is_empty()); + // Twelve Forward changes interleaved with more Survivals: the twelve + // are admitted, and the thirteenth is refused with a reopen time that + // only the oldest Forward entry decides. + for offset in 0..12 { + meters.charge_route(t + 100 + 2 * offset, RouteActuation::Forward)?; + meters.charge_route(t + 101 + 2 * offset, RouteActuation::Survival)?; + } + let refused = meters.charge_route(t + 200, RouteActuation::Forward).unwrap_err(); + assert_eq!( + (refused.used, refused.reopens_at_unix_millis), + (ROUTE_ACTUATION_CEILING, t + 100 + ROUTE_ACTUATION_WINDOW_MILLIS) + ); + // A full Forward ceiling still admits and records a Survival. + meters.charge_route(t + 201, RouteActuation::Survival)?; + assert_eq!(meters.route_survivals.last(), Some(&(t + 201))); + meters.validate()?; + Ok(()) + } + + #[test] + fn a_clock_stepped_back_stalls_nothing_past_one_window() -> Result<()> { + let t = 5_000_000_000_u64; + let now = t - 86_400_000; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = vec![t; ROUTE_ACTUATION_CEILING]; + meters.continuity_restarts = vec![t; CONTINUITY_PAGING_RESTARTS]; + meters.continuity_deferred_until = Some(t); + meters.continuity_deferral_reason = Some("demotion failed".into()); + meters.validate()?; + + // Entries from the future count as now: the meters hold for one window. + let read_only = meters.clone(); + assert!(read_only.restarts_page(now)); + let refused = meters.charge_route(now, RouteActuation::Forward).unwrap_err(); + assert!(refused.reopens_at_unix_millis <= now + ROUTE_ACTUATION_WINDOW_MILLIS); + // The clamp is written, so the window runs from the step and no longer. + assert_eq!(meters.route_actuations, vec![now; ROUTE_ACTUATION_CEILING]); + assert_eq!(meters.continuity_restarts, vec![now; CONTINUITY_PAGING_RESTARTS]); + assert!(!meters.restarts_page(now + CONTINUITY_RESTART_WINDOW_MILLIS)); + meters.charge_route(now + ROUTE_ACTUATION_WINDOW_MILLIS, RouteActuation::Forward)?; + + // Waits set for the future are due, not stalled for the length of the step. + assert!(!read_only.continuity_deferred(now)); + let mut anchored = TargetSupervision::new("service"); + anchored.continuity_attempt_streak = 7; + anchored.continuity_backoff_from_unix_millis = Some(t); + assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Due { streak: 7 }); + anchored.continuity_attempt_streak = 1; + anchored.continuity_backoff_from_unix_millis = Some(now + 10_000); + assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Waiting); + let route = RouteSupervisionState { + next_challenge_at_unix_millis: Some(t), + ..RouteSupervisionState::default() + }; + assert!(!route.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); + assert!(!is_waiting(now, t, RESUME_BACKOFF_CEILING_MILLIS)); + // The same waits, set for a moment ahead, still wait. + let ahead = RouteSupervisionState { + next_challenge_at_unix_millis: Some(now + 5_000), + ..RouteSupervisionState::default() + }; + assert!(ahead.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); + Ok(()) + } + + #[test] + fn a_route_challenge_wait_widens_to_a_cap_and_a_proof_clears_it() { + let age = DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS; + let mut state = RouteSupervisionState::default(); + let mut waits = Vec::new(); + for _ in 0..8 { + state.record_failed_challenge(1_000, age); + waits.push(state.next_challenge_at_unix_millis.unwrap() - 1_000); + } + assert_eq!(&waits[..3], &[age, age * 2, age * 4]); + assert_eq!(waits[7], ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); + assert_eq!(state.degraded_since_unix_millis, Some(1_000)); + state.record_proved_challenge(9_000); + assert_eq!(state.consecutive_failures, 0); + assert_eq!(state.next_challenge_at_unix_millis, None); + assert_eq!(state.degraded_since_unix_millis, None); + } + + #[test] + fn a_v3_generation_lifts_to_v4_once_and_reencodes_canonically() -> Result<()> { + for text in [FIXTURE_GENERATION_V3_ODIN, FIXTURE_GENERATION_V3_ROUTE_PROOF] { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; + let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; + SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(envelope.clone())?; + // The v3 bytes are not v4 bytes: only the typed lift may read them. + assert!(decode_record::(&envelope).is_err()); + + assert_eq!(migrate_control_store_to_current_schema(&store)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&store)?, 0); + let snapshot = ControlSnapshot::read(&store)?; + let migrated = snapshot.admitted.first().context("the lift lost the record")?; + assert_eq!(migrated.envelope.schema_id.as_deref(), Some(ADMITTED_GENERATION_SCHEMA)); + // After the lift the canonical re-encode accepts its own bytes. + let lifted: AdmittedGeneration = decode_record(&migrated.envelope)?; + assert_eq!(lifted, migrated.value); + + // The decisions are the ones the v3 record encoded; only the retired + // fields are gone, and no meters were invented. + assert_eq!(lifted.target, legacy.target); + assert_eq!(lifted.generation_id, legacy.generation_id); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.activation, legacy.activation); + assert_eq!(lifted.workload, legacy.workload); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.ready, legacy.ready); + assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.odin_authority, legacy.odin_authority); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!( + lifted.route_supervision, + legacy.route_supervision.map(|state| RouteSupervisionState { + last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, + consecutive_failures: state.consecutive_failures, + next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, + degraded_since_unix_millis: state.degraded_since_unix_millis, + }) + ); + assert_eq!(lifted.last_error, None); + assert!(snapshot.targets.is_empty()); + } + Ok(()) + } + + #[test] + fn a_v3_generation_that_shows_written_meters_is_refused_not_dropped() -> Result<()> { + let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; + let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; + assert!(legacy.route_supervision.is_some(), "the fixture is a routed generation"); + let refused = |change: fn(&mut LegacyAdmittedGenerationV3)| -> Result { + let mut edited = legacy.clone(); + change(&mut edited); + let mut tampered = envelope.clone(); + tampered.payload = rmp_serde::to_vec(&edited)?; + Ok(error_text(read_generation_record(&tampered).map(|_| ()))) + }; + let restarts = refused(|edited| edited.continuity_backoff.attempts = 2)?; + assert!(restarts.contains("refusing to drop"), "{restarts}"); + assert!(restarts.contains(&legacy.target), "the error names the record: {restarts}"); + let actuations = + refused(|edited| edited.route_supervision.as_mut().unwrap().actuations.count = 1)?; + assert!(actuations.contains("refusing to drop"), "{actuations}"); + + // A whole store carrying one refuses to boot, naming the record. + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let mut edited = legacy.clone(); + edited.continuity_backoff.attempts = 2; + let mut tampered = envelope.clone(); + tampered.payload = rmp_serde::to_vec(&edited)?; + SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(tampered)?; + assert!(migrate_control_store_to_current_schema(&store).is_err()); + Ok(()) + } + + #[test] + fn a_v3_generation_lifts_each_route_supervision_field_to_its_own_place() -> Result<()> { + let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; + let mut legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; + // Four distinct non-default values, so a swap cannot land on itself. + legacy.route_supervision = Some(LegacyRouteSupervisionStateV3 { + last_challenge_at_unix_millis: Some(1_111), + consecutive_failures: 3, + next_challenge_at_unix_millis: Some(2_222), + degraded_since_unix_millis: Some(3_333), + actuations: LegacyActuationWindowV3::default(), + }); + let mut edited = envelope.clone(); + edited.payload = rmp_serde::to_vec(&legacy)?; + let lifted = read_generation_record(&edited)?; + assert_eq!( + lifted.route_supervision, + Some(RouteSupervisionState { + last_challenge_at_unix_millis: Some(1_111), + consecutive_failures: 3, + next_challenge_at_unix_millis: Some(2_222), + degraded_since_unix_millis: Some(3_333), + }) + ); + Ok(()) + } + + #[test] + fn status_covers_the_admitted_targets_and_the_metered_ones() -> Result<()> { + // "service" is admitted with no meters at all; "metered" has meters and + // no generation. Neither set contains the other. + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + admit_incumbent(&world)?; + assert!(ControlSnapshot::read(&world.state_store)?.targets.is_empty()); + set_meters(&world, &TargetSupervision::new("metered"))?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert_eq!( + supervised_targets(&snapshot).into_iter().collect::>(), + ["metered", "service"] + ); + Ok(()) + } + + #[test] + fn target_supervision_reads_back_and_its_key_must_be_its_target() -> Result<()> { + let world = EngineFixture::new()?; + let meters = restarted_ago(&[5_000])?; + set_meters(&world, &meters)?; + assert_eq!(meters_of(&world, "service")?, meters); + assert_eq!(ControlSnapshot::read(&world.state_store)?.targets.len(), 1); + + let mut elsewhere = target_supervision_envelope(&meters, 5)?; + elsewhere.key = "other".into(); + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent(elsewhere)?; + let error = error_text(ControlSnapshot::read(&world.state_store).map(|_| ())); + assert!(error.contains("not its target"), "{error}"); + Ok(()) + } + + #[test] + fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { + let now = 10_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = (0..12).map(|offset| now - 5_000 + offset).collect(); + meters.continuity_restarts = vec![now - 9_000, now - 4_000]; + meters.continuity_deferred_until = Some(now + 500); + meters.continuity_deferral_reason = Some("projection would not demote".into()); + meters.continuity_attempt_streak = 2; + meters.continuity_backoff_from_unix_millis = Some(now - 4_000); + + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.last_error = Some("held".into()); + generation.admitted_at_unix_millis = 1; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains(&format!("target service {}", generation.generation_id)), "{lines}"); + assert!(lines.contains("held: held"), "{lines}"); + assert!(lines.contains(&format!( + "continuity attempts-in-hour 2 pages-at 6 streak 2 next-attempt-at {}", + now - 4_000 + 10_000 + )), "{lines}"); + assert!(!lines.contains("rematerialized-at"), "{lines}"); + meters.continuity_rematerialized_at_unix_millis = Some(now - 1_000); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" +"); + assert!(lines.contains(&format!("continuity rematerialized-at {}", now - 1_000)), "{lines}"); + // An anchor older than the generation is not shown as a pending wait. + generation.admitted_at_unix_millis = now; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join(" +"); + assert!(lines.contains("streak 2 next-attempt-at none"), "{lines}"); + generation.admitted_at_unix_millis = 1; + assert!(lines.contains("continuity deferred until 10000500: projection would not demote"), "{lines}"); + assert!(lines.contains(&format!( + "route actuations 12/12 in window, reopens-at {}", + now - 5_000 + ROUTE_ACTUATION_WINDOW_MILLIS + )), "{lines}"); + assert!(lines.contains("route healthy consecutive-failures 0"), "{lines}"); + assert!( + lines.contains("route survival actuations 0 in window (uncounted), last-at none"), + "{lines}" + ); + meters.route_survivals = vec![now - 7_000, now - 3_000]; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains(&format!( + "route survival actuations 2 in window (uncounted), last-at {}", + now - 3_000 + )), "{lines}"); + meters.route_survivals = (0..ROUTE_SURVIVAL_RECORD as u64).map(|n| now - 100 + n).collect(); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains("route survival actuations at least 12 in window"), "{lines}"); + // Survivals never move the ceiling's count. + assert!(lines.contains("route actuations 12/12 in window"), "{lines}"); + + // A first deployment has meters and no generation: it shows them alone. + let mut first = TargetSupervision::new("fresh"); + first.route_actuations = vec![now - 1]; + let lines = render_supervision("fresh", None, Some(&first), now).join("\n"); + assert!(lines.contains("target fresh no-admitted-generation"), "{lines}"); + assert!(lines.contains("route actuations 1/12 in window, reopens-at none"), "{lines}"); + // An unrouted, unmetered target shows no route line at all. + let lines = render_supervision("plain", Some(&unrouted_generation()?), None, now).join("\n"); + assert!(!lines.contains("route actuations"), "{lines}"); + // A generation that no longer routes, whose meters hold only survival + // actuations, still shows the record. + let mut survived = TargetSupervision::new("plain"); + survived.route_survivals = vec![now - 1]; + let lines = + render_supervision("plain", Some(&unrouted_generation()?), Some(&survived), now).join("\n"); + assert!(lines.contains("route survival actuations 1 in window"), "{lines}"); + Ok(()) + } + + fn unrouted_generation() -> Result { + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.route_supervision = None; + Ok(generation) + } + + // --------------------------------------------------------------------- + // Route-proof readiness: a routed target that declares no Odin dependency + // is admitted by Idunn's own challenges. Odin is never read. + // --------------------------------------------------------------------- + mod route_proof { + use super::*; + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + use std::time::Duration; + + /// The runtime behind both endpoints, answering Idunn's snapshot + /// challenge the way the TypeScript signer does: locally, in the + /// reported health, with `route-observation:` as the detail. + struct RuntimeStub { + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + provider: cultnet_rs::ServiceIdentitySigner, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, + state: Mutex<&'static str>, + capacity: AtomicU32, + sequence: AtomicU64, + candidate_hits: AtomicUsize, + stable_hits: AtomicUsize, + stop: AtomicBool, + /// Accept the connection and close it without a word: a process + /// that is up but not answering. Unlike a closed port, no other + /// test's listener can be handed the address. + hang_up: AtomicBool, + /// The write lease its presence claims to hold: what a candidate + /// that picked up Idunn's grant reports. + lease: Mutex>, + reply: Mutex, + /// Runs when the stable listener accepts, before it answers. + on_stable_hit: Mutex>>, + } + + /// How the runtime answers a well-formed challenge. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Reply { + Honest, + /// An answer to some other challenge. + ForeignChallengeId, + /// An HTTP error where the snapshot should be. + HttpFailure, + /// A first Odin candidate answering its Warming challenge: the + /// detail names the challenge as `idunn-warming:`. + FirstOdinWarming, + } + + impl RuntimeStub { + fn answer(&self, message_id: &str) -> Result> { + let expected = &self.expected; + let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { + schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.activation.expected_projection_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| cultnet_rs::GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: self.capacity.load(Ordering::SeqCst), + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: (*self.state.lock().unwrap()).into(), + detail: match *self.reply.lock().unwrap() { + Reply::FirstOdinWarming => format!("idunn-warming:{message_id}"), + _ => format!("route-observation:{message_id}"), + }, + write_lease_sha256: self.lease.lock().unwrap().clone(), + signer_identity_id: self.provider.entry().identity_id.clone(), + publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, + observed_at_unix_millis: now_millis()?, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation_signer.identity_id(), + activation_signature: Vec::new(), + }; + let proof = presence.canonical_proof_payload()?; + presence.signature = self + .provider + .sign::(&proof) + .signature; + presence.activation_signature = + self.activation_signer.sign_presence_proof(&presence)?; + Ok(rmp_serde::to_vec(&presence)?) + } + + fn set_state(&self, state: &'static str) { + *self.state.lock().unwrap() = state; + } + + fn serve_one(&self, mut stream: TcpStream) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(3)))?; + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let (header_end, length) = loop { + let read = stream.read(&mut chunk)?; + ensure!(read > 0, "stub client closed early"); + request.extend_from_slice(&chunk[..read]); + if let Some(index) = request.windows(4).position(|w| w == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&request[..index]).to_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .context("stub request has no Content-Length")? + .trim() + .parse::()?; + if request.len() >= index + 4 + length { + break (index + 4, length); + } + } + }; + let message = cultnet_rs::decode_cultnet_message_from_slice( + &request[header_end..header_end + length], + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + let cultnet_rs::CultNetMessage::SnapshotRequest { message_id, .. } = message + else { + bail!("stub received something other than a snapshot request"); + }; + let reply = *self.reply.lock().unwrap(); + if reply == Reply::HttpFailure { + stream.write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + )?; + return Ok(()); + } + let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id: if reply == Reply::ForeignChallengeId { + "candidate-foreign".to_owned() + } else { + message_id.clone() + }, + documents: vec![cultnet_rs::CultNetRawDocumentRecord { + schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: self.expected.target.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload: self.answer(&message_id)?, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }], + }; + let body = cultnet_rs::encode_cultnet_message_to_vec( + &response, + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + stream.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + )?; + stream.write_all(&body)?; + Ok(()) + } + + fn listen(self: &Arc, listener: TcpListener, candidate: bool) { + listener.set_nonblocking(true).expect("nonblocking listener"); + let stub = Arc::clone(self); + std::thread::spawn(move || { + while !stub.stop.load(Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + stream.set_nonblocking(false).ok(); + let hits = if candidate { + &stub.candidate_hits + } else { + &stub.stable_hits + }; + hits.fetch_add(1, Ordering::SeqCst); + if !candidate + && let Some(hook) = stub.on_stable_hit.lock().unwrap().as_ref() + { + hook(); + } + if !stub.hang_up.load(Ordering::SeqCst) { + let _ = stub.serve_one(stream); + } + } + Err(_) => std::thread::sleep(Duration::from_millis(5)), + } + } + }); + } + } + + impl Drop for RuntimeStub { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + } + } + + /// Odin, as the world it runs in sees it. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Odin { + /// Every read of the correlation store fails. + Unreachable, + /// Odin publishes a correlation that says the target is not Ready. + NotReady, + } + + struct RoutedWorld { + world: EngineFixture, + workload: Arc, + stub: Arc, + odin: Odin, + poisoned: Vec, + } + + /// A stateless, routed target that declares no Odin dependency, over + /// `minimum_capacity`, seeded at `phase` (Warming or Fencing): its + /// Warming evidence, when the phase needs it, is the candidate's own + /// answer to a real challenge. Listeners stand at both endpoints. + fn routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + provides_odin: bool, + ) -> Result { + build_routed_world( + odin, + minimum_capacity, + phase, + provides_odin, + false, + CommandKind::Continuity, + ) + } + + /// The same world owned by a deployment command instead of a + /// continuity: its route actuations are `Forward` and can be refused. + fn routed_deploy_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + ) -> Result { + build_routed_world(odin, minimum_capacity, phase, false, false, CommandKind::Deploy) + } + + /// The same world for a target with process-writable state: its + /// candidate is granted a write lease and Ready must be the candidate + /// reporting exactly that lease. + fn stateful_routed_world(odin: Odin, phase: DeploymentPhase) -> Result { + build_routed_world(odin, 1, phase, false, true, CommandKind::Continuity) + } + + fn build_routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + provides_odin: bool, + stateful: bool, + kind: CommandKind, + ) -> Result { + use crate::deployment_plan::tests::{ + BINDING, RECIPE, artifact_receipt, external_input_receipt, source, + }; + let switch = SwitchWorkload::new(); + let world = EngineFixture::with_workload(switch.clone())?; + let candidate_listener = TcpListener::bind("127.0.0.1:0")?; + let stable_listener = TcpListener::bind("127.0.0.1:0")?; + let candidate_port = candidate_listener.local_addr()?.port(); + let stable_port = stable_listener.local_addr()?.port(); + + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + let config = world.root.join("service.conf"); + let binding = BINDING + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id) + .replace("127.0.0.1:17999", &format!("127.0.0.1:{stable_port}")) + .replace( + "private_port_start = 18000", + &format!("private_port_start = {candidate_port}"), + ) + .replace( + "private_port_end = 18009", + &format!("private_port_end = {}", candidate_port + 1), + ) + .replace( + "/etc/nginx/idunn-stream-routes/service.conf", + &config.display().to_string().replace('\\', "/"), + ); + let (binding, recipe_source) = if stateful { + std::fs::create_dir_all(world.root.join("lease"))?; + ( + binding.replace( + "[brakes]", + &format!( + "[process_write_lease]\nrecord_path = \"{}\"\n\n[brakes]", + world.root.join("lease/process-write-lease.cc").display() + ), + ), + RECIPE + .replace( + "writer = \"none\"\nrecovery = \"rebuildable\"\nstartup = \"open-at-start\"", + "writer = \"process-bound-single-writer\"\nrecovery = \"preserve\"\nstartup = \"create-or-open-after-write-lease\"", + ) + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_PROCESS_WRITE_LEASE\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + ), + ) + } else { + (binding, RECIPE.to_owned()) + }; + let recipe = recipe_source + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace( + "capability = \"service.runtime\"", + &format!( + "capability = \"{}\"\ncapacity = {minimum_capacity}", + if provides_odin { + ODIN_RENDEZVOUS_CAPABILITY + } else { + "service.runtime" + } + ), + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(candidate_port), + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let expected = release.expected_projection(&plan)?; + assert!(expected.route.is_some() && expected.dependencies.is_empty()); + assert_eq!(expected.write_lease_required, stateful); + assert_eq!( + ReadinessClass::of(&expected), + Ok(if provides_odin { + ReadinessClass::OdinSelf + } else { + ReadinessClass::RouteProof + }) + ); + + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: match kind { + CommandKind::Deploy => "up-service", + CommandKind::Continuity => "continuity-service", + } + .into(), + kind, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = + DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + match kind { + CommandKind::Continuity => { + transaction.lifecycle_authorized_at_unix_millis = Some(now); + } + CommandKind::Deploy => { + transaction.frozen_source = Some(FrozenSourceReceipt { + transaction_id: transaction.transaction_id.clone(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + transaction.deployment_authorization = Some(deployment_authorization( + &world, + &expected, + &transaction.transaction_id, + now, + )?); + } + } + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&transaction.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; + let mut workload = recorded.workload.clone().context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + transaction.isolation = recorded.isolation; + transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); + transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); + transaction.sealed_release = Some(release); + transaction.expected = Some(expected.clone()); + transaction.activation = Some(activation.clone()); + transaction.workload = Some(workload); + transaction.plan = Some(plan); + + let stub = Arc::new(RuntimeStub { + expected, + activation, + provider: open_service_identity_at::( + &provider_path, + )?, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( + &seed[..], + )?, + state: Mutex::new("warming"), + capacity: AtomicU32::new(minimum_capacity), + sequence: AtomicU64::new(0), + candidate_hits: AtomicUsize::new(0), + stable_hits: AtomicUsize::new(0), + stop: AtomicBool::new(false), + hang_up: AtomicBool::new(false), + lease: Mutex::new(None), + reply: Mutex::new(Reply::Honest), + on_stable_hit: Mutex::new(None), + }); + stub.listen(candidate_listener, true); + stub.listen(stable_listener, false); + + if phase >= DeploymentPhase::Fencing && provides_odin { + // Odin's own first word about the candidate, as for any + // Odin-correlated target. + // A stateful candidate is warming, not active, until it holds + // the lease Idunn grants after the fence. + let warming = signed_correlation_as( + &world, + &transaction, + 4, + false, + if stateful { "warming" } else { "active" }, + None, + b"presence", + )?; + let authenticated = world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&world.state_store)?, + &transaction, + &warming, + None, + now, + )?; + transaction.warming = Some(WarmingEvidence::OdinTopology { + evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, + }); + transaction.odin_publisher_sequence_cursor = 4; + } else if phase >= DeploymentPhase::Fencing { + let CandidateAnswer::Answered { evidence, .. } = + world + .engine + .challenge_candidate(&transaction, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + } + if phase >= DeploymentPhase::Fencing { + transaction.route_preflight = Some(candidate_preflight(&transaction)?); + } + if stateful { + // The lease driver reads the Expected and the observed + // activation from the topology store, as Starting left them. + let plan = transaction.plan.as_ref().unwrap(); + let anchor = world.engine.provider_anchor_for_plan(plan)?; + let topology = world.engine.topology(); + topology.publish_expected(transaction.expected.as_ref().unwrap(), &anchor)?; + topology.publish_observed_activation( + transaction.expected.as_ref().unwrap(), + transaction.activation.as_ref().unwrap(), + transaction.workload.as_ref().unwrap(), + )?; + } + transaction.enter_phase(phase, now); + transaction.validate()?; + let store = SingleFileMessagePackBackingStore::new(&world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )?); + + // Odin's world. Unreachable: the correlation store is bytes no + // reader can decode, so any read of it is an error. NotReady: a + // valid correlation that says the target is not Ready. + let store = &world.engine.options.odin_correlation_store; + let poisoned = match odin { + _ if provides_odin && stateful => { + // A new warming word after the fence: what the lease + // grant waits for. + SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( + CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &transaction.target, + &transaction.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation_as( + &world, + &transaction, + 5, + false, + "warming", + None, + b"presence-5", + )?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some( + cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + ), + }, + )?; + Vec::new() + } + _ if provides_odin => { + odin_reports_ready(&world, &transaction, 5)?; + Vec::new() + } + Odin::Unreachable => { + let garbage = b"odin is not a cache".to_vec(); + std::fs::write(store, &garbage)?; + garbage + } + Odin::NotReady => { + let wrongly = signed_correlation(&world, &transaction, 9, false)?; + SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( + CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &transaction.target, + &transaction.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: wrongly, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some( + cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + ), + }, + )?; + Vec::new() + } + }; + Ok(RoutedWorld { + world, + workload: switch, + stub, + odin, + poisoned, + }) + } + + /// The preflight receipt of a candidate whose route was never + /// installed: what Warming records before Fencing, without running the + /// host's nginx. + fn candidate_preflight( + transaction: &DeploymentTransaction, + ) -> Result { + let rendered = NginxRouteDriver::new( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ) + .render(transaction.expected.as_ref().unwrap())?; + Ok(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: transaction + .activation + .as_ref() + .unwrap() + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(&rendered), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }) + } + + impl RoutedWorld { + fn step(&self) -> Result<()> { + self.world.engine.advance_transaction(&resident(&self.world)?) + } + + fn run_to_routing(&self) -> Result<()> { + for _ in 0..12 { + if self.transaction()?.phase == DeploymentPhase::Routing { + return Ok(()); + } + self.step()?; + } + bail!("the transaction never reached Routing") + } + + fn transaction(&self) -> Result { + Ok(resident(&self.world)?.value) + } + + /// Odin was never read: the poisoned store is byte for byte what + /// it was, and no step failed on it. + fn assert_odin_untouched(&self) -> Result<()> { + if self.odin == Odin::Unreachable { + assert_eq!( + std::fs::read(&self.world.engine.options.odin_correlation_store)?, + self.poisoned + ); + } + Ok(()) + } + + /// Routing runs for real: it installs the fragment through the + /// world's stub host programs and proves the stable route against + /// the stub runtime. The world only drives the steps until the + /// transaction enters Committing. + fn promote(&self) -> Result<()> { + assert_eq!(self.transaction()?.phase, DeploymentPhase::Routing); + for _ in 0..4 { + if self.transaction()?.phase == DeploymentPhase::Committing { + return Ok(()); + } + self.step()?; + } + bail!("the transaction never reached Committing") + } + } + + /// A route-proof generation from a real admission. + pub(super) fn committed_generation() -> Result { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + routed.step()?; + admitted(&routed) + } + + fn admitted(world: &RoutedWorld) -> Result { + Ok(ControlSnapshot::read(&world.world.state_store)? + .admitted + .into_iter() + .next() + .context("nothing was admitted")? + .value) + } + + #[test] + fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + // Fencing, Leasing, AwaitingReady: the candidate is still warming. + for _ in 0..12 { + routed.step()?; + if routed.transaction()?.last_error.is_some() { + break; + } + } + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.ready.is_none()); + assert_eq!( + waiting.last_error.as_deref(), + Some("candidate is still warming") + ); + // The candidate finishes warming: Ready is its own Active answer. + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.step()?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); + routed.promote()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.odin_authority.is_none()); + assert!(generation.latest_odin_observation.is_none()); + assert!(generation.route_supervision.is_some()); + // Idunn's own challenges carried every proof: the seed's warming, + // two Ready polls; the stable endpoint proved the route twice. + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 3); + assert!(routed.stub.stable_hits.load(Ordering::SeqCst) >= 2); + routed.assert_odin_untouched()?; + + // Boot and supervision read no Odin either: the topology refresh + // returns before it opens Odin's store, and durable authority + // skips a route-proof generation's receipts. + let stored = ControlSnapshot::read(&routed.world.state_store)?; + let current = stored.admitted.first().context("no admitted generation")?; + assert!(!routed.world.engine.refresh_admitted_topology(&stored, current)?); + routed.world.engine.validate_durable_authority(&stored)?; + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { + let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + routed.step()?; + let generation = admitted(&routed)?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + // The valid, not-Ready correlation was neither admitted as a + // receipt nor consulted: the record carries no Odin cursor. + assert_eq!(generation.odin_publisher_sequence_cursor, 0); + Ok(()) + } + + #[test] + fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + let warmed = routed.transaction()?; + assert_eq!(warmed.phase, DeploymentPhase::Warming); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &warmed.warming else { + bail!("Warming evidence is not the candidate's own presence"); + }; + assert!(evidence.message_id.starts_with("candidate-")); + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + routed.assert_odin_untouched()?; + + // Nothing answers: the same step waits, and says why. + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + silent.stub.hang_up.store(true, Ordering::SeqCst); + silent.step()?; + let waiting = silent.transaction()?; + assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); + assert!( + waiting + .last_error + .as_deref() + .is_some_and(|error| error.contains("candidate endpoint did not answer")), + "{:?}", + waiting.last_error + ); + Ok(()) + } + + #[test] + fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + routed.stub.capacity.store(1, Ordering::SeqCst); + let failure = routed.step().expect_err("a shortfall must be refused"); + // The shortfall is a typed value, not a sentence to search. + let disagrees = failure + .downcast_ref::() + .with_context(|| format!("the refusal is not typed: {failure:#}"))?; + let shortfall = disagrees + .disagreements + .iter() + .find(|disagreement| disagreement.code == "expected-capability-000-capacity") + .context("the capacity shortfall is not named")?; + assert!( + shortfall.expected.as_deref().is_some_and(|text| text.ends_with("capacity>=2")), + "{shortfall:?}" + ); + assert!(shortfall.observed.is_some(), "{shortfall:?}"); + assert!(admitted(&routed).is_err()); + Ok(()) + } + + #[test] + fn odin_itself_is_never_route_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let mut expected = routed.stub.expected.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + // Odin provides the rendezvous capability and declares no dependency. + expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + assert!(expected.dependencies.is_empty()); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); + + // Route-proof evidence for it is refused: its admitted generation + // is what names the Odin authority, so it must carry one. + let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; + odin.target = "odin".into(); + odin.expected = expected; + odin.ready = route_proof_evidence(); + odin.latest_odin_observation = None; + odin.odin_authority = None; + assert!(error_text(odin.validate()).contains("not route-proof")); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn route_admission_installs_through_the_host_programs_without_reading_odin() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + // One real Routing step: the fragment goes in through the world's + // stub nginx, ufw and systemctl, and the stable listener proves it. + routed.step()?; + assert!(matches!( + routed.transaction()?.routing, + Some(RoutingEvidence::Promoted { .. }) + )); + assert_eq!(routed.world.route_stubs.count("systemctl reload"), 1); + assert_eq!(routed.world.route_stubs.count("ufw allow"), 1); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.warming, + Some(WarmingEvidence::RouteProofDirect { .. }) + )); + Ok(()) + } + + #[test] + fn the_lease_refresh_asks_the_candidate_for_a_new_warming_presence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let prior = routed + .world + .engine + .rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let (_, evidence, token) = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?)? + .context("a warming candidate must refresh")?; + assert!(matches!(evidence, WarmingEvidence::RouteProofDirect { .. })); + assert_ne!(token.signed_presence_sha256(), prior.signed_presence_sha256()); + // A candidate that went Active is not warming: nothing to bind. + routed.stub.set_state("active"); + let failure = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?) + .map(|_| ()) + .expect_err("an active candidate is not a warming one"); + assert!(format!("{failure:#}").contains("not warming"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn boot_reauthenticates_a_live_route_proof_transactions_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + + // Warming evidence that does not answer the challenge it names. + let current = resident(&routed.world)?; + let mut forged = current.value.clone(); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &mut forged.warming else { + bail!("no route-proof Warming evidence"); + }; + evidence.message_id = "candidate-another".into(); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Warming evidence for another challenge"); + assert!(format!("{failure:#}").contains("exact challenge"), "{failure:#}"); + + // A Ready label carried by a presence that was only warming. + let current = resident(&routed.world)?; + let CandidateAnswer::Answered { evidence, .. } = routed + .world + .engine + .challenge_candidate(¤t.value, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + let mut forged = current.value.clone(); + forged.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + forged.ready = Some(ReadinessEvidence::RouteProof { evidence }); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Ready carried by a warming presence"); + assert!(format!("{failure:#}").contains("not active"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let ready = routed.transaction()?; + assert!(ready.ready.as_ref().and_then(ReadinessEvidence::odin).is_some()); + routed.promote()?; + routed.step()?; + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Odin); + assert_eq!( + generation.odin_authority, + Some(routed.world.engine.bootstrap_odin_authority.clone()) + ); + assert!(generation.latest_odin_observation.is_some()); + // Boot re-authenticates every Odin receipt this generation carries. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + Ok(()) + } + + #[test] + fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + routed.step()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + assert_eq!( + routed.world.engine.current_odin_authority(&snapshot)?, + routed.world.engine.bootstrap_odin_authority + ); + Ok(()) + } + + impl RoutedWorld { + /// Boot would accept this store, and Odin was never read. + fn assert_boots(&self) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.world.state_store)?; + self.world.engine.validate_durable_authority(&snapshot)?; + self.assert_odin_untouched() + } + + /// Step until `done`, checking after every step that boot would + /// still accept the store. + fn drive_until(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..20 { + if done(&self.transaction()?) { + return Ok(()); + } + self.step()?; + self.assert_boots()?; + } + bail!("the transaction never reached the awaited state") + } + + fn answer_now(&self) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + match self.world.engine.challenge_candidate( + &self.transaction()?, + &["warming", "active"], + None, + )? { + CandidateAnswer::Answered { evidence, present } => Ok((evidence, present)), + CandidateAnswer::Silent(reason) => bail!("the candidate was silent: {reason}"), + } + } + } + + // ----------------------------------------------------------------- + // The stateful path, end to end through the Engine. + // ----------------------------------------------------------------- + + #[test] + fn a_stateful_route_proof_target_holds_its_lease_from_fencing_to_commit() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.assert_boots()?; + + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Prepared { .. })))?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Leasing); + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let lease_sha256 = routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + + // The candidate has not picked the lease up: still warming. + routed.step()?; + assert_eq!( + routed.transaction()?.last_error.as_deref(), + Some("candidate is still warming") + ); + assert!(routed.transaction()?.ready.is_none()); + // Active without the lease Idunn granted is not Ready. + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("an active candidate without the granted lease is refused"); + assert!( + format!("{refusal:#}").contains("exact current process write lease"), + "{refusal:#}" + ); + assert!(routed.transaction()?.ready.is_none()); + // Active holding exactly that lease is. + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.assert_boots()?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + routed.promote()?; + routed.assert_boots()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.expected.write_lease_required); + assert_eq!(generation.leasing.lease_sha256(), Some(lease_sha256.as_str())); + assert!(generation.odin_authority.is_none()); + routed.assert_boots()?; + Ok(()) + } + + // ----------------------------------------------------------------- + // Warming is one challenge, then the phase moves. + // ----------------------------------------------------------------- + + #[test] + fn warming_challenges_once_and_then_advances_on_every_tick() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + let recorded = routed + .transaction()? + .warming + .context("the first step recorded no Warming evidence")?; + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + + // The route preflight needs the host's nginx; its receipt is recorded + // here so the phase can move on. + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + next.route_preflight = Some(candidate_preflight(&next)?); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Fencing)?; + assert_eq!( + routed.stub.candidate_hits.load(Ordering::SeqCst), + 1, + "a recorded Warming answer is never asked for again" + ); + assert_eq!(routed.transaction()?.warming, Some(recorded)); + Ok(()) + } + + #[test] + fn a_stateful_candidate_may_not_answer_its_warming_challenge_active() { + let mut expected = fixture_generation(FIXTURE_GENERATION).unwrap().1.expected; + expected.write_lease_required = true; + assert_eq!(route_proof_warming_states(&expected), ["warming"]); + expected.write_lease_required = false; + assert_eq!(route_proof_warming_states(&expected), ["warming", "active"]); + } + + #[test] + fn a_stateful_candidate_that_answers_warming_active_is_refused() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("a stateful candidate cannot be active before it holds a lease"); + assert!(format!("{refusal:#}").contains("not warming"), "{refusal:#}"); + assert!(routed.transaction()?.warming.is_none()); + Ok(()) + } + + // ----------------------------------------------------------------- + // What a challenged presence must prove. + // ----------------------------------------------------------------- + + #[test] + fn a_presence_holds_the_write_lease_exactly_when_it_is_not_warming() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let lease = sha256_id(b"the granted lease"); + let other = sha256_id(b"some other lease"); + let judge = |state: &'static str, + held: Option<&str>, + current: Option<&str>, + states: &[&str]| + -> Result<()> { + let challenged = now_millis()?; + routed.stub.set_state(state); + *routed.stub.lease.lock().unwrap() = held.map(str::to_owned); + let bytes = routed.stub.answer("candidate-probe")?; + routed + .world + .engine + .authenticate_challenged_presence( + &authority, + states, + current, + "candidate-probe", + challenged, + now_millis()?, + &bytes, + ) + .map(|_| ()) + }; + let either = ["warming", "active"]; + judge("warming", None, None, &either)?; + judge("active", Some(&lease), Some(&lease), &either)?; + for (state, held, current, named) in [ + // Warming holds nothing, whatever Idunn has granted. + ("warming", Some(lease.as_str()), None, "write lease"), + ("warming", Some(lease.as_str()), Some(lease.as_str()), "write lease"), + // Anything else holds exactly the current grant. + ("active", None, Some(lease.as_str()), "exact current process write lease"), + ("active", Some(lease.as_str()), None, "exact current process write lease"), + ( + "active", + Some(other.as_str()), + Some(lease.as_str()), + "exact current process write lease", + ), + ] { + let error = judge(state, held, current, &either) + .expect_err("a presence with the wrong lease is refused"); + assert!( + format!("{error:#}").contains(named), + "{state} {held:?} {current:?}: {error:#}" + ); + } + Ok(()) + } + + #[test] + fn a_presence_minted_before_its_challenge_is_refused() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let bytes = routed.stub.answer("candidate-probe")?; + let minted = now_millis()?; + // Answered before the challenge that claims it: an old presence + // replayed for a new question. + let replay = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted + 1_000, + &bytes, + ) + .expect_err("a presence minted before its challenge"); + assert!( + format!("{replay:#}").contains("minted before its challenge"), + "{replay:#}" + ); + let early = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted, + &bytes, + ) + .expect_err("a receipt before its challenge"); + assert!(format!("{early:#}").contains("predates its challenge"), "{early:#}"); + Ok(()) + } + + #[test] + fn direct_warming_evidence_is_bound_to_the_bytes_it_authenticated() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let id = routed.transaction()?.transaction_id; + let direct = |evidence: RuntimePresenceEvidence| WarmingEvidence::RouteProofDirect { evidence }; + + let (evidence, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present)?; + + // The digest is the authenticated presence's, but the bytes are not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_bytes.push(0); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("bytes that are not the authenticated presence"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // The bytes are the authenticated presence's, but the digest is not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_sha256 = sha256_id(b"another presence"); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("a digest that is not the authenticated presence's"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // Evidence about another answer altogether. + let (other, _) = routed.answer_now()?; + let (_, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id, direct(other), present) + .expect_err("evidence about another answer"); + Ok(()) + } + + // ----------------------------------------------------------------- + // Validation: candidate-vouched evidence belongs to route-proof targets. + // ----------------------------------------------------------------- + + #[test] + fn candidate_vouched_evidence_is_refused_on_a_target_odin_reports_on() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let odin_reported = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!( + ReadinessClass::of(odin_reported.expected.as_ref().unwrap()), + Ok(ReadinessClass::OdinCorrelated) + ); + + let mut warming = odin_reported.clone(); + warming.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + assert!(error_text(warming.validate()).contains("not route-proof")); + + let mut ready = odin_reported.clone(); + ready.ready = Some(ReadinessEvidence::RouteProof { + evidence: evidence.clone(), + }); + assert!(error_text(ready.validate()).contains("not route-proof")); + + // Direct Odin warming is Odin's alone, by class: a route-proof + // target may not claim it, and a target that provides the + // rendezvous may, whatever it is named. + let mut direct = routed.transaction()?; + direct.warming = Some(WarmingEvidence::FirstOdinDirect { + evidence: evidence.clone(), + }); + assert!(error_text(direct.validate()).contains("reserved for Odin")); + let odin = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + let mut direct = odin.transaction()?; + assert_eq!(direct.target, "service"); + direct.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + direct.validate()?; + Ok(()) + } + + #[test] + fn a_route_proof_transaction_past_awaiting_ready_needs_the_candidates_ready_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let routing = routed.transaction()?; + routing.validate()?; + let mut bare = routing.clone(); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + routed.promote()?; + let mut bare = routed.transaction()?; + assert_eq!(bare.phase, DeploymentPhase::Committing); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + Ok(()) + } + + // ----------------------------------------------------------------- + // The class is the Expected's alone. Evidence collected under another + // class is held and reported, never stepped, never repaired. + // ----------------------------------------------------------------- + + /// Odin's Ready for the world's target, as pre-B3 code recorded it for + /// every routed target. + fn odin_receipt_for( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result { + odin_receipt_with(routed, transaction, sequence, true) + } + + fn odin_receipt_with( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, + ) -> Result { + let now = now_millis()?; + let bytes = signed_correlation(&routed.world, transaction, sequence, ready)?; + let authenticated = routed.world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&routed.world.state_store)?, + transaction, + &bytes, + transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256), + now, + )?; + TopologyEvidence::from_authenticated(&authenticated, now) + } + + fn stored(generation: &AdmittedGeneration) -> Stored { + Stored { + value: generation.clone(), + envelope: CultCacheEnvelope { + key: generation.target.clone(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + } + } + + #[test] + fn a_pre_b3_transaction_over_a_route_proof_target_is_held_and_reported_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + + // Ready as pre-B3 Idunn wrote it: an Odin receipt. + let current = resident(&routed.world)?; + let mut legacy = current.value.clone(); + let receipt = odin_receipt_for(&routed, &legacy, 9)?; + legacy.latest_odin_observation = Some(receipt.clone()); + legacy.odin_publisher_sequence_cursor = 9; + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: receipt }); + replace_transaction(&routed.world.state_store, ¤t, &legacy)?; + + let disagreement = ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }; + assert_eq!(legacy.readiness_disagreement(), Some(disagreement.clone())); + // The record is readable and boot accepts it: held, not refused. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + + let hits = ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst), + ); + for _ in 0..4 { + assert!(!routed.world.engine.run_scheduler_tick()?); + } + let held = resident(&routed.world)?; + assert_eq!(held.value.phase, DeploymentPhase::Routing); + assert!(held.value.pre_fencing_abort.is_none()); + assert!(held.value.post_fencing_abort.is_none()); + assert_eq!(held.value.ready, legacy.ready, "nothing re-derived the evidence"); + assert_eq!( + held.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()) + ); + // Reported once: later ticks leave the record alone. + routed.world.engine.run_scheduler_tick()?; + assert_eq!(resident(&routed.world)?.envelope, held.envelope); + assert_eq!( + hits, + ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst) + ), + "a held transaction is asked nothing" + ); + routed.assert_odin_untouched()?; + + // An abort already under way is not held: it never reads the class. + let mut aborting = held.value.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + replace_transaction(&routed.world.state_store, &held, &aborting)?; + let after = resident(&routed.world)?; + let _ = routed.world.engine.resume_candidate(&after); + assert_ne!( + resident(&routed.world)?.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()), + "an abort in progress runs instead of being held" + ); + Ok(()) + } + + #[test] + fn a_pre_b3_generation_over_a_route_proof_target_boots_and_is_reported_not_refreshed() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + routed.promote()?; + routed.step()?; + + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("nothing was admitted")?; + // Receipts that would not survive being re-proved: authentic, but + // not Ready. Boot re-proves against the current Odin authority and + // requires exact semantic Ready, never what a generation stores, so + // this record is only bootable if boot does not re-prove it. + let receipt = odin_receipt_with(&routed, &at_routing, 9, false)?; + let mut legacy = current.value.clone(); + legacy.ready = ReadinessEvidence::OdinCorrelated { + evidence: receipt.clone(), + }; + legacy.latest_odin_observation = Some(receipt); + // The stored authority is present because an Odin-tagged record + // carries one; it is not what boot reads. + let stranger = cultnet_rs::enroll_service_identity_at::( + &routed.world.root.join("identities/stranger.cc"), + )?; + legacy.odin_authority = Some(AdmittedOdinAuthority::from_anchor(&stranger.trust_anchor()?)?); + legacy.odin_publisher_sequence_cursor = 9; + assert!( + SingleFileMessagePackBackingStore::new(&routed.world.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: legacy.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&legacy, now_millis()?)?], + )? + ); + + assert_eq!( + legacy.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }) + ); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("the generation is gone")?; + // The class chooses the path: no Odin receipt is re-proved at boot + // and none is refreshed, though the evidence is an Odin receipt. + routed.world.engine.validate_durable_authority(&snapshot)?; + assert!(!routed.world.engine.refresh_admitted_topology(&snapshot, current)?); + routed.assert_odin_untouched()?; + + routed.world.engine.supervise_one_admitted_generation()?; + assert!( + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .contains_key("generation:service"), + "supervision reports the disagreement" + ); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn only_a_transaction_that_still_reads_its_class_is_held_for_its_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + // What pre-B3 Idunn would have collected: an Odin receipt. + let mut legacy = at_routing.clone(); + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { + evidence: odin_receipt_for(&routed, &at_routing, 9)?, + }); + let disagreement = legacy.readiness_disagreement(); + assert!(disagreement.is_some()); + assert_eq!(legacy.held_disagreement(), disagreement); + + // Finished work, and an abort under way, never read the class. + let mut completed = legacy.clone(); + completed.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-1".into(), + }); + assert_eq!(completed.held_disagreement(), None); + let mut aborting = legacy.clone(); + aborting.pre_fencing_abort = Some(PreFencingAbort { + error: "operator abort".into(), + candidate_cleanup: CleanupEvidence::Pending, + topology_reconciliation: CleanupEvidence::Pending, + source_cleanup: CleanupEvidence::Pending, + }); + assert_eq!(aborting.held_disagreement(), None); + let mut aborting = legacy; + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + assert_eq!(aborting.held_disagreement(), None); + Ok(()) + } + + #[test] + fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { + let world = EngineFixture::new()?; + // Odin's store is unreadable: any read of it is an error. + std::fs::write(&world.engine.options.odin_correlation_store, b"odin is not a cache")?; + + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + let target = generation.expected.target.clone(); + assert_eq!(generation.readiness(), Ok(ReadinessClass::OdinCorrelated)); + generation.expected.dependencies.clear(); + generation.expected.route = None; + let undeclared = UndeclaredReadiness { target: target.clone() }; + assert_eq!( + generation.readiness(), + Err(ReadinessDisagreement::Undeclared(undeclared.clone())) + ); + assert!( + !world + .engine + .refresh_admitted_topology(&ControlSnapshot::default(), &stored(&generation))? + ); + + // The mirror: Odin declared, but the receipt is the candidate's. + let (_, mut odin_class) = fixture_generation(FIXTURE_GENERATION)?; + odin_class.ready = route_proof_evidence(); + assert_eq!( + odin_class.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: target.clone(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + + // A transaction says the same about what it has collected. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let odin_world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&odin_world, DeploymentPhase::Warming)?; + assert_eq!(seeded.readiness_disagreement(), None); + let mut collected = seeded.clone(); + collected.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + assert_eq!( + collected.readiness_disagreement(), + Some(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + let mut bare = seeded.clone(); + bare.expected.as_mut().unwrap().dependencies.clear(); + assert_eq!( + bare.readiness_disagreement(), + Some(ReadinessDisagreement::Undeclared(UndeclaredReadiness { + target: "service".into() + })) + ); + let mut no_expected = seeded; + no_expected.expected = None; + assert_eq!(no_expected.readiness_disagreement(), None); + Ok(()) + } + + // ----------------------------------------------------------------- + // Odin is whichever target provides the rendezvous. + // ----------------------------------------------------------------- + + #[test] + fn odin_is_the_target_that_provides_the_rendezvous_not_the_target_named_odin() -> Result<()> { + let world = EngineFixture::new()?; + let (_, fixture) = fixture_generation(FIXTURE_GENERATION)?; + + // A target named "odin" that merely depends on Odin is not Odin. + let mut named = fixture.clone(); + named.target = "odin".into(); + named.expected.target = "odin".into(); + let mut snapshot = ControlSnapshot::default(); + snapshot.admitted.push(stored(&named)); + assert!(snapshot.admitted_odin().is_none()); + assert_eq!( + world.engine.current_odin_authority(&snapshot)?, + world.engine.bootstrap_odin_authority + ); + + // The target that provides the rendezvous is, whatever it is called. + let mut provider = fixture.clone(); + provider.target = "verse".into(); + provider.expected.target = "verse".into(); + provider.expected.dependencies.clear(); + provide_odin(&mut provider.expected); + let authority = AdmittedOdinAuthority::from_anchor(&world.odin_signer.trust_anchor()?)?; + provider.odin_authority = Some(authority.clone()); + snapshot.admitted.push(stored(&provider)); + assert_eq!( + snapshot.admitted_odin().map(|found| found.value.target.as_str()), + Some("verse") + ); + assert_eq!( + snapshot.admitted_odin().and_then(|found| found.value.odin_authority.clone()), + Some(authority) + ); + Ok(()) + } + + #[test] + fn a_target_that_provides_the_rendezvous_warms_as_odin_whatever_it_is_named() -> Result<()> { + // Nothing else has been admitted that could observe it, so Odin + // observes itself directly -- and only a stateful incarnation may. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + assert_eq!(routed.transaction()?.target, "service"); + let refusal = routed + .step() + .expect_err("a stateless Odin cannot be observed directly"); + assert!( + format!("{refusal:#}").contains("direct Odin warming must be a stateful incarnation"), + "{refusal:#}" + ); + Ok(()) + } + + #[test] + fn direct_odin_warming_is_reserved_to_odin_by_class_not_by_name() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let mut named = routed.transaction()?; + named.target = "odin".into(); + let error = routed + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a target named odin that is not Odin"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + + #[test] + fn a_stateful_target_that_provides_the_rendezvous_is_observed_directly_whatever_it_is_named() + -> Result<()> { + let provider = build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true, true, CommandKind::Continuity)?; + let transaction = provider.transaction()?; + assert_eq!(transaction.target, "service"); + assert!(transaction.expected.as_ref().unwrap().write_lease_required); + provider.stub.set_state("warming"); + *provider.stub.reply.lock().unwrap() = Reply::FirstOdinWarming; + let challenged_at = now_millis()?; + let answer = provider.stub.answer("warming-probe")?; + provider.world.engine.authenticate_first_odin_warming_presence( + &transaction, + "warming-probe", + challenged_at, + now_millis()?, + &answer, + )?; + + // The reverse: named odin, stateful, but nothing provides the + // rendezvous. + let stateful = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let mut named = stateful.transaction()?; + assert!(named.expected.as_ref().unwrap().write_lease_required); + named.target = "odin".into(); + let error = stateful + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a stateful target named odin that does not provide the rendezvous"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + + // ----------------------------------------------------------------- + // A bad answer is an error. Only silence waits. + // ----------------------------------------------------------------- + + #[test] + fn a_bad_answer_to_a_challenge_is_an_error_and_only_silence_waits() -> Result<()> { + for reply in [Reply::ForeignChallengeId, Reply::HttpFailure] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + *routed.stub.reply.lock().unwrap() = reply; + let error = match routed.world.engine.challenge_candidate( + &routed.transaction()?, + &["warming"], + None, + ) { + Err(error) => error, + Ok(_) => bail!("a bad answer was taken for silence"), + }; + assert!( + matches!( + error.downcast_ref::(), + Some(ChallengeFailure::Refused(_)) + ), + "{error:#}" + ); + // Before the fence a candidate that answers wrongly is aborted, + // not waited on. + routed.world.engine.run_scheduler_tick()?; + let after = routed.transaction()?; + assert!(after.pre_fencing_abort.is_some(), "the bad answer aborted it"); + assert!(after.warming.is_none()); + } + + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + silent.stub.hang_up.store(true, Ordering::SeqCst); + assert!(matches!( + silent.world.engine.challenge_candidate(&silent.transaction()?, &["warming"], None)?, + CandidateAnswer::Silent(_) + )); + Ok(()) + } + + // ----------------------------------------------------------------- + // B2: the route gate, the meters and the supervision pass, through + // the Engine, over the world's stub host programs. + // ----------------------------------------------------------------- + + impl RoutedWorld { + /// Run a fresh world to an admitted generation whose transaction + /// is retired: what admitted-route supervision looks at. + fn admit(&self) -> Result<()> { + self.stub.set_state("active"); + self.run_to_routing()?; + self.promote()?; + self.drive_until(|transaction| transaction.is_terminal())?; + assert!(self.world.engine.retire_one_terminal_transaction()?); + Ok(()) + } + + fn fragment_path(&self) -> PathBuf { + self.world.root.join("service.conf") + } + + /// The target's route ledger holds `entries` recent actuations. + fn ledger(&self, entries: usize) -> Result<()> { + let now = now_millis()?; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = (0..entries as u64).map(|offset| now - 1_000 + offset).collect(); + set_meters(&self.world, &meters) + } + + #[cfg(unix)] + fn reloads(&self) -> usize { + self.world.route_stubs.count("systemctl reload") + } + + /// Something else rewrote the fragment: the admitted membership drifted. + fn drift(&self) -> Result<()> { + std::fs::write(self.fragment_path(), b"drifted bytes\n")?; + Ok(()) + } + + fn tick(&self) -> Result { + self.world.engine.supervise_one_admitted_generation() + } + } + + // A store that reads but cannot be written (ENOSPC on the state volume) + // is simulated with RLIMIT_FSIZE, which is process-wide: the test + // re-runs itself in a child process so no sibling test loses its writes. + // Returns true in the child, which runs the body; the parent returns + // false after asserting the child passed and hands back its stderr. + #[cfg(unix)] + fn in_unwritable_store_child(test: &str) -> Result> { + if std::env::var_os("IDUNN_TEST_UNWRITABLE_STORE_CHILD").is_some() { + return Ok(None); + } + let path = module_path!().split_once("::").map_or("", |(_, rest)| rest); + let output = std::process::Command::new(std::env::current_exe()?) + .args(["--exact", &format!("{path}::{test}"), "--nocapture", "--test-threads=1"]) + .env("IDUNN_TEST_UNWRITABLE_STORE_CHILD", "1") + .output()?; + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + assert!( + output.status.success(), + "child failed:\n{}\n{stderr}", + String::from_utf8_lossy(&output.stdout) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "the child ran no test: {}", + String::from_utf8_lossy(&output.stdout) + ); + Ok(Some(stderr)) + } + + /// Cap file growth one byte under the store's size: reads work, every + /// write fails. Child process only. + #[cfg(unix)] + fn break_store_writes(world: &EngineFixture) { + let size = std::fs::metadata(&world.state_store).unwrap().len(); + unsafe { + libc::signal(libc::SIGXFSZ, libc::SIG_IGN); + let limit = libc::rlimit { rlim_cur: size - 1, rlim_max: libc::RLIM_INFINITY }; + assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0); + } + assert!( + set_meters(world, &TargetSupervision::new("probe")).is_err(), + "the store is still writable" + ); + assert!(ControlSnapshot::read(&world.state_store).is_ok(), "the store is unreadable"); + } + + #[cfg(unix)] + fn challenge_due(world: &EngineFixture) -> Option { + world.engine.challenge_wait.lock().unwrap().get("service").map(|wait| wait.due_unix_millis) + } + + #[cfg(unix)] + #[test] + fn an_unwritable_store_neither_spins_a_failing_challenge_nor_reports_progress() -> Result<()> { + let name = "an_unwritable_store_neither_spins_a_failing_challenge_nor_reports_progress"; + let Some(stderr) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + routed.drift()?; + let before = routed.reloads(); + let began = now_millis()?; + break_store_writes(&routed.world); + let stored_wait = |routed: &RoutedWorld| -> Result> { + Ok(ControlSnapshot::read(&routed.world.state_store)? + .admitted + .iter() + .find_map(|stored| stored.value.route_supervision.as_ref()) + .and_then(|state| state.next_challenge_at_unix_millis)) + }; + let stored = stored_wait(&routed)?; + + // Fifty laps: one attempt, nothing reported as progress. + for _ in 0..50 { + assert!(!routed.tick()?, "a write the store refused was reported as progress"); + } + assert_eq!(routed.reloads() - before, 1, "the failing reload spun"); + assert_eq!(stored_wait(&routed)?, stored, "the record changed though writes fail"); + let due = challenge_due(&routed.world).expect("no in-memory wait after a failed record"); + let age = routed.world.engine.options.topology_maximum_age_millis; + assert!(due >= began + age, "the wait is shorter than one observation age: {due}"); + + // The wait expiring is the only thing that admits another + // attempt, and each one is again a single reload with no + // progress. The failed record is said once, not per attempt. + for attempt in 2..=4 { + routed.world.engine.challenge_wait.lock().unwrap().clear(); + assert!(!routed.tick()?); + assert!(!routed.tick()?); + assert_eq!(routed.reloads() - before, attempt); + } + return Ok(()); + }; + assert_eq!( + stderr.matches("could not record the failed route challenge of service").count(), + 1, + "{stderr}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unwritable_store_challenges_a_stale_route_once_per_wait() -> Result<()> { + let name = "an_unwritable_store_challenges_a_stale_route_once_per_wait"; + let Some(stderr) = in_unwritable_store_child(name)? else { + let mut routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.engine.options.topology_maximum_age_millis = 8_000; + std::thread::sleep(Duration::from_millis(8_300)); + break_store_writes(&routed.world); + assert!(!routed.tick()?, "a write the store refused was reported as progress"); + let first = challenge_due(&routed.world).expect("the proved challenge left no wait"); + for _ in 0..5 { + std::thread::sleep(Duration::from_millis(3)); + assert!(!routed.tick()?); + } + assert_eq!(challenge_due(&routed.world), Some(first), "the route was challenged again"); + return Ok(()); + }; + assert_eq!( + stderr.matches("could not record the proved route challenge of service").count(), + 1, + "{stderr}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_landed_challenge_record_is_progress_and_leaves_no_in_memory_wait() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + routed.drift()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.challenge_wait.lock().unwrap().insert( + "service".into(), + ChallengeWait { due_unix_millis: 0, stamp: snapshot.target_stamp("service") }, + ); + assert!(routed.tick()?, "a landed record is progress"); + assert_eq!(challenge_due(&routed.world), None); + Ok(()) + } + + /// A fragment that re-drifts after every restore, with the store + /// unwritable, is reloaded once per wait, however many laps run. + #[cfg(unix)] + #[test] + fn a_re_drifting_fragment_is_reloaded_once_per_wait_when_the_store_is_unwritable() -> Result<()> { + let name = "a_re_drifting_fragment_is_reloaded_once_per_wait_when_the_store_is_unwritable"; + let Some(_) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let before = routed.reloads(); + break_store_writes(&routed.world); + for _ in 0..30 { + routed.drift()?; + assert!(!routed.tick()?, "a refused write was reported as progress"); + } + assert_eq!(routed.reloads() - before, 1, "a re-drifting fragment reloaded every lap"); + for attempt in 2..=4 { + routed.world.engine.challenge_wait.lock().unwrap().clear(); + routed.drift()?; + assert!(!routed.tick()?); + routed.drift()?; + assert!(!routed.tick()?); + assert_eq!(routed.reloads() - before, attempt); + } + return Ok(()); + }; + Ok(()) + } + + /// The wait of a proved challenge whose record failed is one + /// observation age from the attempt. + #[cfg(unix)] + #[test] + fn a_proved_challenge_that_cannot_be_recorded_waits_one_observation_age() -> Result<()> { + let name = "a_proved_challenge_that_cannot_be_recorded_waits_one_observation_age"; + let Some(_) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let before = routed.reloads(); + break_store_writes(&routed.world); + let began = now_millis()?; + routed.drift()?; + assert!(!routed.tick()?); + assert_eq!(routed.reloads() - before, 1, "the repair did not run"); + let due = challenge_due(&routed.world).expect("no wait"); + let age = routed.world.engine.options.topology_maximum_age_millis; + assert!(due >= began + age, "the wait is shorter than one age: {}", due - began); + return Ok(()); + }; + Ok(()) + } + + /// The stored schedule sets the in-memory wait: with three failures + /// recorded, the fourth failed challenge waits eight ages (capped), and + /// no repair runs inside that wait. + #[cfg(unix)] + #[test] + fn a_failed_challenge_that_cannot_be_recorded_waits_as_long_as_the_stored_schedule() -> Result<()> { + let name = "a_failed_challenge_that_cannot_be_recorded_waits_as_long_as_the_stored_schedule"; + let Some(_) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + edit_incumbent(&routed.world, |generation| { + let state = generation.route_supervision.as_mut().unwrap(); + state.consecutive_failures = 3; + state.next_challenge_at_unix_millis = Some(1); + state.degraded_since_unix_millis = Some(1); + })?; + break_store_writes(&routed.world); + let began = now_millis()?; + routed.drift()?; + assert!(!routed.tick()?); + let due = challenge_due(&routed.world).expect("no wait"); + let age = routed.world.engine.options.topology_maximum_age_millis; + let want = (age * 8).min(ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); + assert!(due >= began + want, "the wait ignores the stored doubling: {}", due - began); + let reloads = routed.reloads(); + for _ in 0..10 { + routed.drift()?; + assert!(!routed.tick()?); + } + assert_eq!(routed.reloads(), reloads, "the route was repaired inside its wait"); + return Ok(()); + }; + Ok(()) + } + + /// A lost race is progress, and the next pass is governed by the + /// winner's record, not by an in-memory wait of the loser. + #[cfg(unix)] + #[test] + fn a_lost_race_leaves_no_in_memory_wait() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + let old = ControlSnapshot::read(&routed.world.state_store)?; + let stale = old.admitted_for("service").context("no generation")?; + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().last_challenge_at_unix_millis = Some(5); + })?; + routed.drift()?; + assert!(routed.world.engine.supervise_admitted_route(&old, stale)?); + assert_eq!(challenge_due(&routed.world), None, "a lost race left an in-memory wait"); + Ok(()) + } + + /// The wait left by a full disk does not outlive it: once any write for + /// the target lands, the stored schedule owns the target again and a + /// drift is repaired without waiting. + #[cfg(unix)] + #[test] + fn a_wait_left_by_a_full_disk_ends_when_any_write_for_the_target_lands() -> Result<()> { + let name = "a_wait_left_by_a_full_disk_ends_when_any_write_for_the_target_lands"; + let Some(_) = in_unwritable_store_child(name)? else { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let before = routed.reloads(); + break_store_writes(&routed.world); + routed.drift()?; + assert!(!routed.tick()?); + assert!(challenge_due(&routed.world).is_some(), "no wait to outlive"); + assert_eq!(routed.reloads() - before, 1); + + // Space is freed. Nothing has landed yet, so the wait still holds. + unsafe { + let limit = libc::rlimit { rlim_cur: libc::RLIM_INFINITY, rlim_max: libc::RLIM_INFINITY }; + assert_eq!(libc::setrlimit(libc::RLIMIT_FSIZE, &limit), 0); + } + routed.drift()?; + assert!(!routed.tick()?); + assert_eq!(routed.reloads() - before, 1, "the wait did not hold before any write landed"); + + // Something else writes for the target: the wait is over. + routed.ledger(1)?; + routed.drift()?; + assert!(routed.tick()?, "the drift was not repaired"); + assert_eq!(routed.reloads() - before, 2, "a stale wait held the repair"); + assert_eq!(challenge_due(&routed.world), None); + return Ok(()); + }; + Ok(()) + } + + /// A wait belongs to an admitted target: once the target is retired, + /// the wait is gone. + #[cfg(unix)] + #[test] + fn a_wait_ends_when_its_target_is_retired() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.challenge_wait.lock().unwrap().insert( + "service".into(), + ChallengeWait { + due_unix_millis: now_millis()? + 500_000, + stamp: snapshot.target_stamp("service"), + }, + ); + assert!(!routed.tick()?); + assert!(challenge_due(&routed.world).is_some(), "an unchanged store dropped the wait"); + assert!( + SingleFileMessagePackBackingStore::new(&routed.world.state_store) + .delete_batch_if_unchanged(&[snapshot.admitted_for("service").unwrap().envelope.clone()])? + ); + routed.tick()?; + assert_eq!(challenge_due(&routed.world), None, "a retired target kept its wait"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_full_ledger_still_restores_the_admitted_and_incumbent_route() -> Result<()> { + let full = ROUTE_ACTUATION_CEILING; + // The newest recorded Survival, or 0; the Forward ledger stays full. + let newest = |world: &EngineFixture| -> Result { + let meters = meters_of(world, "service")?; + assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); + Ok(meters.route_survivals.last().copied().unwrap_or(0)) + }; + + // (i) A post-fence abort withdraws the candidate's membership. + let aborting = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + aborting.ledger(full)?; + let before = newest(&aborting.world)?; + aborting + .world + .engine + .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; + aborting.step()?; + assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); + assert!(newest(&aborting.world)? > before, "the withdrawal was not recorded"); + + // (ii) Supervision repairs a drifted admitted fragment. + let supervised = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + supervised.admit()?; + supervised.ledger(full)?; + let (before, reloads) = (newest(&supervised.world)?, supervised.reloads()); + supervised.drift()?; + assert!(supervised.tick()?); + assert_eq!(supervised.reloads(), reloads + 1, "the repair did not run"); + assert!(newest(&supervised.world)? > before, "the repair was not recorded"); + assert!(admitted(&supervised)?.route_supervision.unwrap().degraded_since_unix_millis.is_none()); + + // (iii) A failed proof at Routing rolls the route back, though the + // install that preceded it took the last free slot. The rollback + // is recorded and leaves the Forward ledger exactly full. + let failing = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + failing.stub.set_state("active"); + failing.run_to_routing()?; + failing.ledger(full - 1)?; + failing.stub.hang_up.store(true, Ordering::SeqCst); + let before = now_millis()?; + assert!(failing.step().is_err(), "an unanswered stable route was admitted"); + assert_eq!(failing.reloads(), 2, "install then rollback"); + assert!(newest(&failing.world)? >= before, "the rollback was not recorded"); + assert!(!failing.fragment_path().exists(), "the rollback left the candidate's route"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_continuity_restart_is_never_refused_by_a_full_ledger() -> Result<()> { + // Warming: the preflight of a continuity runs and is recorded, + // and the Forward ledger is left as it was. + let warming = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + warming.step()?; + assert!(warming.transaction()?.warming.is_some()); + warming.ledger(ROUTE_ACTUATION_CEILING)?; + let ledger = meters_of(&warming.world, "service")?.route_actuations; + warming.step()?; + assert!(warming.transaction()?.route_preflight.is_some(), "the preflight was refused"); + let after = meters_of(&warming.world, "service")?; + assert_eq!(after.route_survivals.len(), 1, "the preflight was not recorded"); + assert_eq!(after.route_actuations, ledger, "a survival was counted"); + + // Routing: its install runs and is recorded, and not counted. + let routing = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routing.stub.set_state("active"); + routing.run_to_routing()?; + routing.ledger(ROUTE_ACTUATION_CEILING)?; + let ledger = meters_of(&routing.world, "service")?.route_actuations; + routing.step()?; + assert!(routing.reloads() >= 1, "the install did not run"); + assert!(routing.fragment_path().exists()); + let after = meters_of(&routing.world, "service")?; + assert_eq!(after.route_survivals.len(), 1, "the install was not recorded"); + assert_eq!(after.route_actuations, ledger, "a survival was counted"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_survival_that_cannot_be_recorded_runs_and_a_forward_is_refused() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let transaction = routed.transaction()?; + let preflight = transaction.route_preflight.clone().context("no preflight")?; + let driver = routed.world.engine.route_driver( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ); + // The ledger cannot be read, so nothing can be recorded. + std::fs::write(&routed.world.state_store, b"unreadable")?; + let engine = &routed.world.engine; + + let refused = engine + .route_gate("service", CommandKind::Deploy) + .admit(RouteActuation::Forward) + .unwrap_err(); + assert!(refused.downcast_ref::().is_none(), "{refused:#}"); + assert!(engine.charge_route_actuation("service", RouteActuation::Forward).is_err()); + + engine.charge_route_actuation("service", RouteActuation::Survival)?; + driver.withdraw_candidate_membership( + &preflight, + &engine.route_gate("service", CommandKind::Deploy), + )?; + assert_eq!(routed.reloads(), 1, "the unrecorded withdrawal did not run"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_second_failed_challenge_waits_twice_as_long_before_the_next_repair() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + let elapse = |routed: &RoutedWorld| { + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = + Some(1); + }) + }; + + routed.drift()?; + assert!(routed.tick()?); + elapse(&routed)?; + routed.drift()?; + assert!(routed.tick()?); + let state = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(state.consecutive_failures, 2); + assert_eq!( + state.next_challenge_at_unix_millis.unwrap() - state.last_challenge_at_unix_millis.unwrap(), + 2 * DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS + ); + + // The wait is longer than one observation age: no challenge, so no + // repair and no reload, however often the pass runs. + let reloads = routed.reloads(); + for _ in 0..20 { + routed.drift()?; + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "the route was repaired inside its wait"); + assert_eq!( + admitted(&routed)?.route_supervision.unwrap().consecutive_failures, + 2 + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_proved_challenge_that_cannot_be_recorded_ends_the_pass_without_failing_it() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let old = ControlSnapshot::read(&routed.world.state_store)?; + let stale = old.admitted_for("service").context("no generation")?; + // Something else writes the generation after the pass read it. + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().last_challenge_at_unix_millis = Some(5); + })?; + routed.drift()?; + assert!(routed.world.engine.supervise_admitted_route(&old, stale)?); + assert_eq!( + admitted(&routed)?.route_supervision.unwrap().last_challenge_at_unix_millis, + Some(5), + "the stale write replaced the newer generation" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_first_routed_deploy_is_metered_and_refused_at_the_ceiling() -> Result<()> { + let metered = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + metered.stub.set_state("active"); + metered.run_to_routing()?; + assert!(ControlSnapshot::read(&metered.world.state_store)?.targets.is_empty()); + metered.step()?; + assert_eq!(meters_of(&metered.world, "service")?.route_actuations.len(), 1); + + let refused = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + refused.stub.set_state("active"); + refused.run_to_routing()?; + refused.ledger(ROUTE_ACTUATION_CEILING)?; + let error = refused.step().unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert_eq!(refused.world.route_stubs.count(""), 0, "a refused install ran a program"); + assert!(!refused.fragment_path().exists()); + + // Past the fence the refusal is resumable, not an abort: the resume + // backoff spaces it and no actuation runs while it waits. + assert!(!refused.world.engine.resume_one_transaction()?); + let waiting = refused.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::Routing); + assert!(waiting.post_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|text| text.contains("ceiling"))); + assert_eq!(refused.world.route_stubs.count(""), 0); + Ok(()) + } + + #[test] + fn a_stepped_back_clock_is_settled_where_the_decision_is_made() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let ahead = now_millis()? + 86_400_000; + + // A supervision pass that would decide from the restart log settles + // it first: the write ends the pass, and nothing is minted from + // entries that only look recent because the clock stepped back. + let mut restarts = TargetSupervision::new("service"); + restarts.continuity_restarts = vec![ahead; CONTINUITY_PAGING_RESTARTS]; + set_meters(&routed.world, &restarts)?; + routed.workload.kill(); + assert!(routed.tick()?); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let settled = meters_of(&routed.world, "service")?; + assert!(settled.continuity_restarts.iter().all(|&at| at < ahead)); + + // A refused forward actuation writes the clamp, so the window runs + // from now and not from the day the clock stood on. + let mut ledger = TargetSupervision::new("service"); + ledger.route_actuations = vec![ahead; ROUTE_ACTUATION_CEILING]; + set_meters(&routed.world, &ledger)?; + let refused = routed + .world + .engine + .charge_route_actuation("service", RouteActuation::Forward) + .unwrap_err(); + assert!(refused.downcast_ref::().is_some()); + let settled = meters_of(&routed.world, "service")?; + assert!(settled.route_actuations.iter().all(|&at| at < ahead)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_refused_preflight_starts_no_private_mount_unit_and_fails_at_once_with_the_reopen_time() + -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + routed.step()?; + assert!(routed.transaction()?.warming.is_some()); + routed.ledger(ROUTE_ACTUATION_CEILING)?; + let reopens_at = meters_of(&routed.world, "service")? + .route_reopens_at(now_millis()?) + .context("the ledger is not full")?; + + // The refusal is an error of its own phase, before the fence: the + // deployment fails now, and its error carries the reopen time. + let id = routed.transaction()?.transaction_id; + routed.world.engine.resume_one_transaction()?; + let abort = routed + .transaction()? + .pre_fencing_abort + .context("the refusal did not abort the deployment")?; + assert!(abort.error.contains(&format!("reopens at unix ms {reopens_at}")), "{}", abort.error); + for _ in 0..8 { + if record_of(&routed.world, &id)?.completion.is_some() { + break; + } + routed.step()?; + } + let Some(TransactionCompletion::FailedBeforeFencing { error }) = + record_of(&routed.world, &id)?.completion + else { + let record = record_of(&routed.world, &id)?; + bail!( + "the deployment did not fail before the fence: phase {:?} completion {:?} abort {:?} error {:?}", + record.phase, record.completion, record.pre_fencing_abort, record.last_error + ); + }; + assert!(error.contains(&format!("reopens at unix ms {reopens_at}")), "{error}"); + assert_eq!(routed.world.route_stubs.count(""), 0, "a program ran"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.workload.kill(); + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + + // Tick 1: the repair charges the target's meters and the proof + // fails. Both are writes, so they end the pass: nothing else is + // decided from the snapshot they made stale. + assert!(routed.tick()?); + let state = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(state.consecutive_failures, 1); + assert!(state.degraded_since_unix_millis.is_some()); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_survivals.len(), 2); + assert!(meters.route_actuations.is_empty()); + + // Tick 2: the route waits, and the dead unit is restarted from a + // fresh read, counted in the same write that mints it. + assert!(routed.tick()?); + assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); + assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_proved_challenge_clears_the_degradation() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + assert!(routed.tick()?); + let degraded = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(degraded.consecutive_failures, 1); + assert!(degraded.degraded_since_unix_millis.is_some()); + assert!(degraded.next_challenge_at_unix_millis.is_some()); + + routed.stub.hang_up.store(false, Ordering::SeqCst); + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); + })?; + routed.drift()?; + assert!(routed.tick()?); + let healed = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(healed.consecutive_failures, 0); + assert_eq!(healed.next_challenge_at_unix_millis, None); + assert_eq!(healed.degraded_since_unix_millis, None); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failing_reload_that_puts_the_drift_back_backs_the_route_off() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + let reloads = routed.reloads(); + routed.drift()?; + for _ in 0..200 { + routed.tick()?; + } + assert_eq!(routed.reloads() - reloads, 1, "the route was reloaded again and again"); + assert_eq!(std::fs::read(routed.fragment_path())?, b"drifted bytes\n"); + assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_candidate_whose_route_proof_fails_reloads_at_most_twice_the_ceiling() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + let mut last = None; + for _ in 0..40 { + last = Some(routed.step().unwrap_err()); + } + // Every install is a counted Forward and is followed by one + // recorded rollback, so the rollbacks are bounded by the installs: + // twelve installs spend the window, each reloads twice, and every + // later attempt is refused before it runs anything. + assert_eq!(routed.reloads(), 2 * ROUTE_ACTUATION_CEILING); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_used(now_millis()?), ROUTE_ACTUATION_CEILING); + assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); + let last = last.unwrap(); + assert!(last.downcast_ref::().is_some(), "{last:#}"); + Ok(()) + } + + /// The rescue (ruling 3d A): a target whose continuity spent every + /// restart, each charging its preflight and install as Survival, is + /// still deployed: its route changes are the ceiling's first Forward. + #[cfg(unix)] + #[test] + fn a_target_that_spent_its_restarts_can_still_be_deployed() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(routed.transaction()?.warming.is_some()); + set_meters( + &routed.world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + for _ in 0..2 * CONTINUITY_PAGING_RESTARTS { + routed.world.engine.charge_route_actuation("service", RouteActuation::Survival)?; + } + let spent = meters_of(&routed.world, "service")?; + assert!(spent.restarts_page(now_millis()?)); + assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_PAGING_RESTARTS); + + // Warming's preflight and Routing's install are both admitted. + routed.step()?; + assert!(routed.transaction()?.route_preflight.is_some(), "the preflight was refused"); + routed.run_to_routing()?; + routed.promote()?; + assert!(routed.fragment_path().exists(), "the install was refused"); + assert!(routed.reloads() >= 1); + let after = meters_of(&routed.world, "service")?; + assert_eq!(after.route_actuations.len(), 2, "preflight and install are the Forward"); + Ok(()) + } + + // ----------------------------------------------------------------- + // B5: every post-fencing phase ends. + // ----------------------------------------------------------------- + + impl RoutedWorld { + /// One scheduler pass over the transaction, as the daemon runs it: + /// through the deadline resolver, with the resume backoff spent. + fn resume(&self) -> Result<()> { + self.world.engine.resume_backoff.lock().unwrap().clear(); + self.world.engine.resume_one_transaction()?; + Ok(()) + } + + fn resume_until_terminal(&self) -> Result { + for _ in 0..12 { + if latest(&self.world)?.completion.is_some() { + break; + } + self.resume()?; + } + let finished = latest(&self.world)?; + ensure!(finished.completion.is_some(), "never finished: {:?}", finished.last_error); + Ok(finished) + } + } + + /// The live transaction's phase deadline is already behind it. + fn pass_deadline(world: &EngineFixture) -> Result<()> { + let current = resident(world)?; + let mut next = current.value.clone(); + let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, ¤t, &next) + } + + /// Soul's F1 probe, committed. A continuity's route changes are + /// Survival, so no ceiling refuses its install and rollback while its + /// route proof keeps failing. The Routing deadline ends it: no attempt + /// runs after the deadline. + #[cfg(unix)] + #[test] + fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..3 { + routed.resume()?; + } + // Before the deadline each attempt installs and rolls back, and no + // resume moves the deadline. + assert_eq!(routed.reloads(), 6); + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::Routing); + assert!(waiting.post_fencing_abort.is_none(), "aborted before its deadline"); + assert!(waiting.last_error.is_some()); + assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); + + pass_deadline(&routed.world)?; + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the passed deadline did not end the phase")?; + assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + let finished = routed.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + ), + "{:?}", + finished.completion + ); + // After the deadline no attempt runs; the withdrawal writes and + // reloads once, whatever the rolled-back disk shows. + assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); + Ok(()) + } + + /// The live shape of F1. A continuity reuses the admitted Expected, so + /// its membership is the admitted fragment already on disk: while its + /// proof fails, the install and the rollback write nothing and reload + /// nothing, and neither does the withdrawal after its deadline. + #[cfg(unix)] + #[test] + fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + let binding = next.plan.as_ref().unwrap().parsed_inputs()?.1; + let admitted = NginxRouteDriver::new(binding.route.context("no route binding")?) + .render(next.expected.as_ref().context("no Expected")?)?; + let preflight = next.route_preflight.as_mut().context("no preflight")?; + preflight.incumbent_runtime_instance_id = Some(sha256_id(b"admitted-instance")); + preflight.incumbent_membership_sha256 = Some(sha256_id(&admitted)); + preflight.incumbent_configuration = Some(admitted.clone()); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + std::fs::write(routed.fragment_path(), &admitted)?; + + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..5 { + routed.resume()?; + } + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); + assert!(routed.transaction()?.last_error.is_some(), "the proof did not fail"); + assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); + + pass_deadline(&routed.world)?; + routed.resume_until_terminal()?; + assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); + assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); + Ok(()) + } + + /// The abort's withdrawal wrote the baseline back and Idunn died before + /// the reload: the disk already shows the baseline, and the retry still + /// reloads, so nginx does not keep proxying to the stopped candidate. + #[cfg(unix)] + #[test] + fn an_abort_whose_reload_a_crash_cut_off_reloads_on_retry() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + let installed = routed.reloads(); + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; + std::fs::remove_file(routed.fragment_path())?; + routed.resume()?; + assert_eq!( + routed.transaction()?.post_fencing_abort.context("no abort")?.route_restoration, + CleanupEvidence::Complete + ); + assert_eq!(routed.reloads(), installed + 1, "the withdrawal trusted unloaded bytes"); + Ok(()) + } + + /// Soul's F2 probe, committed: a post-fence withdrawal whose + /// `ufw delete` keeps failing withdraws the firewall allow first, so its + /// retries reload nginx not at all. + #[cfg(unix)] + #[test] + fn a_withdrawal_whose_firewall_step_keeps_failing_never_reloads() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + let installed = routed.reloads(); + routed.world.route_stubs.refuse_ufw_deletes()?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test"))?; + for _ in 0..20 { + routed.resume()?; + } + assert_eq!(routed.reloads(), installed, "a failing withdrawal reloaded nginx"); + assert_eq!(routed.world.route_stubs.count("ufw delete"), 20); + assert!(routed.fragment_path().exists(), "the route went before its firewall rule"); + Ok(()) + } + + /// A record lifted from before deadlines existed is timed once, from + /// the first time the resolver sees it, and never again. + #[test] + fn a_post_fence_record_without_a_deadline_is_timed_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let mut lifted = current.value.clone(); + lifted.phase_deadline = None; + replace_transaction(&routed.world.state_store, ¤t, &lifted)?; + let engine = &routed.world.engine; + + let before = now_millis()?; + assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, Some(true)); + let stamped = routed.transaction()?.phase_deadline.context("the record was not timed")?; + assert_eq!(stamped.phase, DeploymentPhase::Fencing); + assert!(stamped.entered_at_unix_millis >= before); + assert_eq!( + stamped.deadline_at_unix_millis - stamped.entered_at_unix_millis, + u64::from(lifted.plan.as_ref().unwrap().phase_deadlines().fencing_seconds) * 1000 + ); + assert_eq!(engine.resolve_phase_deadline(&resident(&routed.world)?)?, None); + assert_eq!(routed.transaction()?.phase_deadline, Some(stamped)); + Ok(()) + } + + /// `idunn expire` writes a request and nothing else; the resolver ends + /// the phase from it, and the spent request is deleted. + #[test] + fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { + let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let refused = expire(&early.world.state_store, "continuity-service", "operator") + .expect_err("a transaction before the fence was expired"); + assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); + + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let untouched = resident(&routed.world)?.envelope; + expire(&routed.world.state_store, "continuity-service", "operator")?; + assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!(snapshot.expiry_requests.len(), 1); + assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); + + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the expiry did not end the phase")?; + assert!(abort.error.contains("Fencing phase was expired by the operator"), "{}", abort.error); + routed.resume()?; + assert!( + ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), + "the spent request stayed" + ); + Ok(()) + } + + /// The `expire` verb as the CLI parses and runs it: the options land + /// where they say, and it asks only about its own command's live + /// transaction. + #[test] + fn idunn_expire_parses_its_options_and_asks_only_about_its_own_live_transaction() + -> Result<()> { + let args = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect::>(); + assert_eq!( + parse(args(&["expire", "up-service", "--requested-by", "alice", "--state-store", "/s/c.cc"]).into_iter())?, + Command::Expire { + state_store: PathBuf::from("/s/c.cc"), + command_id: "up-service".into(), + requested_by: "alice".into(), + } + ); + let help = parse(args(&["expire", "up-service", "--help"]).into_iter()).unwrap_err(); + assert!(format!("{help}").contains("idunn expire "), "{help}"); + assert!(parse(args(&["expire", "up-service", "--force"]).into_iter()).is_err()); + assert!(parse(args(&["expire"]).into_iter()).is_err()); + + // The verb runs: a store with nothing live refuses. + let empty = TempDir::new()?; + let store = empty.path().join("control.cc").display().to_string(); + let refused = run(args(&["expire", "up-service", "--state-store", &store]).into_iter()) + .expect_err("expired a command with no transaction"); + assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); + + // Another target's deployment in flight is not this command's. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let now = now_millis()?; + let other = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-other".into(), + kind: CommandKind::Deploy, + selector: "other".into(), + requested_by: "test".into(), + requested_at_unix_millis: now, + }; + let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; + assert!(SingleFileMessagePackBackingStore::new(&routed.world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: other.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: sealing.transaction_id.clone(), + current: None, + }, + ], + &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], + )?); + expire(&routed.world.state_store, "continuity-service", "operator")?; + let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].value.command_id, "continuity-service"); + Ok(()) + } + + /// The resolver reads a request as its own transaction's only: with two + /// live transactions, a request for the other leaves this one running. + #[test] + fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let now = now_millis()?; + let other = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-other".into(), + kind: CommandKind::Deploy, + selector: "other".into(), + requested_by: "test".into(), + requested_at_unix_millis: now, + }; + let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; + // The request names a command whose id has this transaction's + // length and first bytes, and differs only in its last. + let near = "continuity-servicx"; + assert_eq!(near.len(), "continuity-service".len()); + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: near.into(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + // And commands whose ids are a strict prefix of this one's, and + // extend it: a match is the whole id, in neither direction a part. + let prefix = ExpiryRequest { + command_id: "continuity-servic".into(), + ..request.clone() + }; + let extension = ExpiryRequest { + command_id: "continuity-service-2".into(), + ..request.clone() + }; + let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); + let absent = |r#type: &str, key: &str| CultCacheExpectedEnvelope { + r#type: r#type.into(), + key: key.into(), + current: None, + }; + assert!(store.compare_exchange( + &[ + absent(DeploymentCommand::TYPE, &other.command_id), + absent(DeploymentTransaction::TYPE, &sealing.transaction_id), + absent(ExpiryRequest::TYPE, &request.command_id), + absent(ExpiryRequest::TYPE, &prefix.command_id), + absent(ExpiryRequest::TYPE, &extension.command_id), + ], + &[ + command_envelope(&other, now)?, + transaction_envelope(&sealing, now)?, + expiry_request_envelope(&request, now)?, + expiry_request_envelope(&prefix, now)?, + expiry_request_envelope(&extension, now)?, + ], + )?); + assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.len(), 3); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!(snapshot.transactions.iter().filter(|t| !t.value.is_terminal()).count(), 2); + let mine = snapshot + .transactions + .iter() + .find(|t| t.value.command_id == "continuity-service") + .context("no continuity")?; + assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); + let after = ControlSnapshot::read(&routed.world.state_store)?; + let unchanged = after + .transactions + .iter() + .find(|t| t.value.command_id == "continuity-service") + .context("no continuity")?; + assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); + Ok(()) + } + + /// Q3 at the deadline, through the Engine. A continuity whose candidate + /// was granted a lease and never proved it held it fails as + /// `lease-not-adopted`; one whose Ready proved it restarts the admitted + /// release. A deployment whose candidate holds a lease is left to the + /// operator: nothing is aborted and its step still runs. + #[test] + fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { + let granted = |routed: &RoutedWorld| -> Result { + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + Ok(routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned()) + }; + + let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + granted(&unadopted)?; + pass_deadline(&unadopted.world)?; + unadopted.resume()?; + let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; + assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); + assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); + let finished = unadopted.resume_until_terminal()?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + )); + + let adopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + let lease = granted(&adopted)?; + adopted.stub.set_state("active"); + *adopted.stub.lease.lock().unwrap() = Some(lease.clone()); + adopted.step()?; + let ready = adopted.transaction()?; + let Some(ReadinessEvidence::RouteProof { evidence }) = &ready.ready else { + bail!("the candidate was not Ready"); + }; + let adoption = ready.lease_adoption.clone().context("Ready recorded no adoption")?; + assert_eq!(adoption.write_lease_sha256, lease); + assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); + assert_eq!(adoption.source, AdoptionSource::Direct); + adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + pass_deadline(&adopted.world)?; + adopted.resume()?; + let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; + assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + let finished = adopted.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestartAdmitted, + .. + }) + ), + "{:?}", + finished.completion + ); + + let deploy = + build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; + granted(&deploy)?; + pass_deadline(&deploy.world)?; + for _ in 0..3 { + deploy.resume()?; + } + let held = deploy.transaction()?; + assert!(held.post_fencing_abort.is_none(), "a leased deployment was aborted"); + assert_eq!(held.phase, DeploymentPhase::AwaitingReady); + assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); + Ok(()) + } + + /// B5 Soul A. The physical grant lands before `Granted` is durable. A + /// continuity that dies in that window and passes its deadline is + /// aborted from `Prepared`; the abort still withdraws the lease on + /// disk, so the next fencing of the target is not refused. + #[test] + fn an_abort_from_a_prepared_lease_withdraws_a_grant_already_on_disk() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + let current = resident(&routed.world)?; + let mut crashed = current.value.clone(); + let Some(LeasingEvidence::Granted { lease, lease_sha256 }) = crashed.leasing.clone() else { + bail!("no granted lease"); + }; + crashed.leasing = Some(LeasingEvidence::Prepared { lease: lease.clone(), lease_sha256 }); + replace_transaction(&routed.world.state_store, ¤t, &crashed)?; + let driver = CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ); + assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); + + pass_deadline(&routed.world)?; + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + routed.resume_until_terminal()?; + assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); + // What the next fencing does to a candidate's lease path. + driver.revoke_exact(None)?; + Ok(()) + } + + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word + /// carries the granted lease, and that Ready records the adoption, + /// sourced from Odin's topology and naming the presence Odin observed. + #[test] + fn an_odin_ready_word_records_a_stateful_candidates_lease_adoption() -> Result<()> { + let routed = build_routed_world( + Odin::Unreachable, + 1, + DeploymentPhase::Fencing, + true, + true, + CommandKind::Continuity, + )?; + for _ in 0..8 { + if routed.transaction()?.phase == DeploymentPhase::AwaitingReady { + break; + } + routed.step()?; + } + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady, "{:?}", waiting.last_error); + let lease = waiting + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + assert_eq!(waiting.lease_adoption, None); + + let ready = CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &waiting.target, + &waiting.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation_as( + &routed.world, + &waiting, + waiting.odin_publisher_sequence_cursor + 1, + true, + "active", + Some(&lease), + b"presence-ready", + )?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }; + replace_odin_entry(&routed.world.engine.options.odin_correlation_store, ready)?; + // One step admits the new sequence, the next records Ready from it. + for _ in 0..3 { + if routed.transaction()?.ready.is_some() { + break; + } + routed.step()?; + } + let adopted = routed.transaction()?; + assert!( + matches!(adopted.ready, Some(ReadinessEvidence::OdinCorrelated { .. })), + "{:?}", + adopted.last_error + ); + let adoption = adopted.lease_adoption.context("Odin's Ready recorded no adoption")?; + assert_eq!(adoption.source, AdoptionSource::OdinTopology); + assert_eq!(adoption.write_lease_sha256, lease); + assert_eq!(adoption.signed_presence_sha256, sha256_id(b"presence-ready")); + Ok(()) + } + + /// F16: Commit goes on from the topology it admitted, so Odin + /// re-stamping before the step and again during the final proof does + /// not send it round again. + #[test] + fn commit_completes_while_odin_restamps_every_tick() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote()?; + let transaction = routed.transaction()?; + let cursor = transaction.odin_publisher_sequence_cursor; + let store = routed.world.engine.options.odin_correlation_store.clone(); + replace_odin_entry(&store, odin_ready_envelope(&routed.world, &transaction, cursor + 1)?)?; + let later = odin_ready_envelope(&routed.world, &transaction, cursor + 2)?; + *routed.stub.on_stable_hit.lock().unwrap() = Some(Box::new(move || { + replace_odin_entry(&store, later.clone()).expect("Odin re-stamps"); + })); + routed.step()?; + let generation = admitted(&routed)?; + assert_eq!( + generation.latest_odin_observation.context("no Odin receipt")?.publisher_sequence, + cursor + 2, + "commit did not go on from the re-stamped topology" + ); + Ok(()) + } + + fn odin_ready_envelope( + world: &EngineFixture, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result { + let expected = transaction.expected.as_ref().context("no Expected")?; + Ok(CultCacheEnvelope { + key: crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(world, transaction, sequence, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }) + } + + fn replace_odin_entry(store: &Path, envelope: CultCacheEnvelope) -> Result<()> { + let backing = SingleFileMessagePackBackingStore::new(store); + let current = backing + .pull_all_read_only_snapshot()? + .into_iter() + .find(|entry| entry.key == envelope.key && entry.r#type == envelope.r#type); + ensure!( + backing.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: envelope.r#type.clone(), + key: envelope.key.clone(), + current, + }], + &[envelope], + )?, + "the Odin store changed under the re-stamp" + ); + Ok(()) + } + } +} From da9d2469567e3f76ae4302074ecfdc0d87d2db67 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 02:19:33 +0200 Subject: [PATCH 066/196] Continuity: the sustained-run reset is decided before the anchor; pin the streak guards continuity_backoff decides the sustained reset first, so a standing anchor (a run no pass observed) cannot hide it. The running-pass end_backoff stays: it makes the next death wait from its own discovery, not from a stale anchor. Behavioural tests: a brief run does not reset the streak, streaks 13..u32::MAX still mint and store saturating, the unobserved sustained run resets, a running pass keeps the re-materialization bound, owed_steps admission bound, the v1 lift's inner schema check. The startup message names the supervision schema; ControlSnapshot::read drops its repeat of the shared reader's key check. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 191 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 176 insertions(+), 15 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 162e5fd..b4dcd86 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1243,10 +1243,20 @@ impl TargetSupervision { /// admitted at `admitted_at`. The streak resets only after a sustained /// run: the generation has lived the sustained time and no attempt, charged /// or the one re-materialization, has been minted since it was admitted. - /// A standing anchor belongs to a death the supervision pass has seen - /// through; an observed-running pass clears it, so it never hides a reset. + /// That rule decides first: a sustained death never began a wait, so a + /// standing anchor, however it came to stand, has no say in it. fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { let streak = self.continuity_attempt_streak; + let sustained = self + .continuity_restarts + .last() + .into_iter() + .chain(&self.continuity_rematerialized_at_unix_millis) + .all(|&last| admitted_at >= last) + && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; + if sustained { + return ContinuityBackoff::Due { streak: 0 }; + } if let Some(due) = self.backoff_due(admitted_at) { return if self.owed_steps(admitted_at).is_some() && is_waiting(now, due, CONTINUITY_BACKOFF_CEILING_MILLIS) @@ -1256,14 +1266,7 @@ impl TargetSupervision { ContinuityBackoff::Due { streak } }; } - let sustained = self - .continuity_restarts - .last() - .into_iter() - .chain(&self.continuity_rematerialized_at_unix_millis) - .all(|&last| admitted_at >= last) - && now.saturating_sub(admitted_at) >= CONTINUITY_SUSTAINED_RUN_MILLIS; - if sustained || self.owed_steps(admitted_at).is_none() { + if self.owed_steps(admitted_at).is_none() { ContinuityBackoff::Due { streak: 0 } } else { ContinuityBackoff::Begin @@ -3169,10 +3172,6 @@ impl ControlSnapshot { } TargetSupervision::TYPE => { let value = read_supervision_record(&envelope)?; - ensure!( - envelope.key == value.target, - "target supervision key is not its target" - ); snapshot.targets.push(Stored { envelope, value }); } ExpiryRequest::TYPE => { @@ -5517,7 +5516,7 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { .context("migrating Idunn control records to the current schema")?; if migrated > 0 { println!( - "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA}" + "migrated {migrated} control record(s) to {DEPLOYMENT_TRANSACTION_SCHEMA} / {ADMITTED_GENERATION_SCHEMA} / {TARGET_SUPERVISION_SCHEMA}" ); } ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; @@ -13537,6 +13536,7 @@ mod tests { struct SwitchWorkload { alive: std::sync::atomic::AtomicBool, rebooted: std::sync::atomic::AtomicBool, + silent: std::sync::atomic::AtomicBool, } impl SwitchWorkload { @@ -13544,9 +13544,17 @@ mod tests { Arc::new(Self { alive: std::sync::atomic::AtomicBool::new(true), rebooted: std::sync::atomic::AtomicBool::new(false), + silent: std::sync::atomic::AtomicBool::new(false), }) } + /// The host cannot be asked: observing fails as `HostUnobservable` + /// whatever the workload is doing. + fn silence(&self, silent: bool) { + self.silent + .store(silent, std::sync::atomic::Ordering::SeqCst); + } + fn revive(&self) { self.alive .store(true, std::sync::atomic::Ordering::SeqCst); @@ -13607,6 +13615,13 @@ mod tests { _: &IdunnRuntimeActivationRecord, prior: &WorkloadObservation, ) -> Result { + if self.silent.load(std::sync::atomic::Ordering::SeqCst) { + return Err(HostUnobservable { + host: "test-host".into(), + reason: "the actuator is between sessions".into(), + } + .into()); + } ensure!( self.alive.load(std::sync::atomic::Ordering::SeqCst), "the workload is gone" @@ -16481,6 +16496,128 @@ mod tests { assert_eq!(meters.backoff_due(admitted_at), Some(now)); } + #[test] + fn continuity_mints_at_the_largest_streaks_and_stores_them_saturating() -> Result<()> { + // One test at u32::MAX kills every monotone stop gate (`streak >= N`). + let full = [3_000_000_u64, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000]; + for streak in [13_u32, 64, u32::MAX - 1, u32::MAX] { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + let mut meters = restarted_ago(&full)?; + meters.continuity_attempt_streak = streak; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the wait begins"); + anchor_ago(&world, 299_000)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak {streak}: the cap is 300 s"); + anchor_ago(&world, 300_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1, "streak {streak} stopped"); + assert_eq!( + meters_of(&world, "service")?.continuity_attempt_streak, + streak.saturating_add(1) + ); + } + Ok(()) + } + + #[test] + fn a_brief_run_does_not_reset_the_streak() -> Result<()> { + // A crash loop seen running for one poll after each start still waits + // by streak; only a sustained run resets. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 100_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000, 900_000, 800_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0); + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "a brief run reset the wait to nothing"); + anchor_ago(&world, 19_000)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "streak 3 waits 20 s"); + anchor_ago(&world, 20_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 4); + Ok(()) + } + + #[test] + fn a_sustained_run_no_pass_observed_still_resets_past_a_standing_anchor() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 100_000)?; + workload.kill(); + let mut meters = restarted_ago(&[1_000_000, 900_000, 800_000])?; + meters.continuity_attempt_streak = 3; + set_meters(&world, &meters)?; + world.engine.supervise_one_admitted_generation()?; + let anchor = meters_of(&world, "service")?.continuity_backoff_from_unix_millis; + assert!(anchor.is_some()); + // It comes back on its own and runs 700 s while every pass is silence. + workload.revive(); + workload.silence(true); + age_incumbent(&world, 700_000)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!( + meters_of(&world, "service")?.continuity_backoff_from_unix_millis, + anchor, + "a silent pass wrote the record" + ); + // Then it dies: the old anchor has no say, no wait, the streak restarts. + workload.kill(); + workload.silence(false); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); + Ok(()) + } + + #[test] + fn owed_steps_counts_the_re_materialization_only_for_its_own_generation() { + let admitted_at = 10_000_000_u64; + let mut meters = TargetSupervision::new("service"); + assert_eq!(meters.owed_steps(admitted_at), None); + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at - 1); + assert_eq!(meters.owed_steps(admitted_at), None, "an earlier generation's remat is spent"); + meters.continuity_rematerialized_at_unix_millis = Some(admitted_at); + assert_eq!(meters.owed_steps(admitted_at), Some(0)); + meters.continuity_attempt_streak = 3; + assert_eq!(meters.owed_steps(admitted_at), Some(2)); + } + + #[test] + fn a_v1_supervision_record_whose_inner_schema_differs_is_refused() -> Result<()> { + let meters = restarted_ago(&[200_000])?; + let payload = rmp_serde::to_vec(&( + "idunn.some_other_schema", + &meters.target, + &meters.route_actuations, + &meters.continuity_restarts, + &meters.continuity_deferred_until, + &meters.continuity_deferral_reason, + &meters.route_survivals, + ))?; + assert!( + read_supervision_record(&supervision_envelope_raw(payload, TARGET_SUPERVISION_SCHEMA_V1)?) + .is_err() + ); + Ok(()) + } + mod after_a_reboot { use super::*; @@ -16584,6 +16721,30 @@ mod tests { assert_eq!(meters_of(&world, "service")?.continuity_attempt_streak, 1); Ok(()) } + + #[test] + fn a_running_pass_keeps_the_re_materialization_so_the_next_death_still_waits() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 700_000)?; + workload.reboot(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(resident_continuity(&world)?, 1); + drop_continuity(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(resident_continuity(&world)?, 0, "the gap"); + // Seen running, then dead again: the failed attempt still counts. + workload.revive(); + world.engine.supervise_one_admitted_generation()?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + assert_eq!(resident_continuity(&world)?, 0, "no gap after the failed attempt"); + anchor_ago(&world, 5_001)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(resident_continuity(&world)?, 1); + Ok(()) + } } /// A source that resolves one fixed recipe and never freezes anything. From ffb42936ca762036ebe15f3d3922c5513bd39603 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 02:21:22 +0200 Subject: [PATCH 067/196] Continuity tests: admitted_at that does not make the fixture a sustained run The clock-step test anchored a target at admitted_at 0, which the sustained-first rule rightly reads as a sustained run; the unobserved-run test logged restarts after its generation's admission. Both now describe the cases they name. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index b4dcd86..e077373 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16561,7 +16561,7 @@ mod tests { admit_incumbent(&world)?; age_incumbent(&world, 100_000)?; workload.kill(); - let mut meters = restarted_ago(&[1_000_000, 900_000, 800_000])?; + let mut meters = restarted_ago(&[1_500_000, 1_400_000, 1_300_000])?; meters.continuity_attempt_streak = 3; set_meters(&world, &meters)?; world.engine.supervise_one_admitted_generation()?; @@ -17217,10 +17217,10 @@ mod tests { let mut anchored = TargetSupervision::new("service"); anchored.continuity_attempt_streak = 7; anchored.continuity_backoff_from_unix_millis = Some(t); - assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Due { streak: 7 }); + assert_eq!(anchored.continuity_backoff(now, now), ContinuityBackoff::Due { streak: 7 }); anchored.continuity_attempt_streak = 1; anchored.continuity_backoff_from_unix_millis = Some(now + 10_000); - assert_eq!(anchored.continuity_backoff(now, 0), ContinuityBackoff::Waiting); + assert_eq!(anchored.continuity_backoff(now, now), ContinuityBackoff::Waiting); let route = RouteSupervisionState { next_challenge_at_unix_millis: Some(t), ..RouteSupervisionState::default() From 9a0a6a2ce965eff4d83f3c7f07673bb7f9abe3c0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 10:36:21 +0200 Subject: [PATCH 068/196] Body map after reincarnation cut 1; cut 1 installed on Yggdrasil Co-Authored-By: Claude Opus 5.5 --- docs/authority-map.md | 19 +++++++++++++++++++ docs/deployment-authority.md | 13 +++++++++++++ docs/host-actuator.md | 5 +++-- docs/reincarnation-map.md | 13 +++++++++++++ docs/watchdog-reporting-map.md | 2 +- docs/watchdog-reporting-postmortem.md | 3 ++- docs/watchdog-reporting-target.md | 7 ++++--- 7 files changed, 55 insertions(+), 7 deletions(-) diff --git a/docs/authority-map.md b/docs/authority-map.md index dc918fd..1cbf3ca 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -173,6 +173,25 @@ health plus Ready correlation. It changes and observes route membership only afterward. Only after the proxy has adopted the candidate may Idunn clear the promotion fence and drain the incumbent. +Ownership of a target is exclusive at each instant. A live +`DeploymentTransaction` owns it from Fencing on (a stop-first rollout, from +Starting); before Fencing a deployment owns only its private candidate and +continuity owns the incumbent slot. Continuity owns the admitted generation's +survival and never stops. There is no stopped state: six charged attempts in +an hour (`restarts_page`) open the ContinuityExhausted incident and gate +nothing, and no deployment path yields to continuity. When continuity next acts +is derived only from its own record in `TargetSupervision` (schema v2, a v1 +record lifts at store open): the attempt streak, the backoff anchor (written +by the dead-and-free pass, the only writer), and the once-per-admission +re-materialization bound. The workload driver alone answers whether a lost unit +was a host reboot (`WorkloadPort::lost_to_host_reboot`). Crash restart, restart +after a sustained run and reboot re-materialization all mint through one CAS; +continuity supervision never aborts a transaction. Operator-facing incidents +and `idunn status` are projections that decide nothing. Cut line: cuts 2 to 4 +of `reincarnation-map.md` still owe continuity restarting alongside a +pre-fencing build, dependency failures not growing the streak, and the +dead-generation route refresh. + The deployment brake gates changes to source, release, configuration, schema, unit, or authority binding. The lifecycle brake separately gates restart of an already-admitted incarnation. Neither target brake may gate Idunn itself or an diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index da7f028..6d464e5 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -305,6 +305,19 @@ continuity restart does not fetch a branch, rebuild, rerun a migration, alter the launch contract, or consume the deployment brake. An explicit lifecycle brake may suspend this actuation without changing deployment authority. +Continuity never stops. Six charged attempts in an hour only open the +ContinuityExhausted incident (a page; the wire name stays, nothing is +exhausted) and gate no mint, wait or yield. When the next attempt is due is a +function of continuity's own record: a stored attempt streak, a backoff anchor +written when the pass first finds the admitted generation dead and the target +free, and a capped doubling wait from that anchor (5 s base, 300 s cap). A run +of 600 s or more by the admitted body resets the streak. A generation lost to a +host reboot (the workload driver's evidence, `lost_to_host_reboot`) is +re-materialized once per admission without charge or wait. The target is free +for a deployment's candidate between attempts, and a deployment never yields to +continuity before Fencing; from Fencing on (or from Starting for a stop-first +rollout) the live transaction alone owns the target. + An admitted generation keeps its Ready and latest Odin receipts as durable proof of the exact activation at their recorded admission times. Dependency selection reauthenticates those receipts at those times; continuity and route diff --git a/docs/host-actuator.md b/docs/host-actuator.md index f2fb1c7..cdb8cde 100644 --- a/docs/host-actuator.md +++ b/docs/host-actuator.md @@ -82,8 +82,9 @@ running a script, and a dead `muninn serve` is seen the tick it dies. 2. No path on yggdrasil can start, stop or observe a host workload except through the actuator session bound to that host's anchor. 3. A host with no attached actuator is a failed observation, which - continuity counts and gives up on after six restarts inside one hour, - exactly like a unit that will not start. + continuity counts as a charged attempt and retries on its capped backoff + (it never gives up; six in an hour only pages), exactly like a unit that + will not start. 4. The actuator cannot be told to run a program the recipe did not declare; `allowed_programs` on the host-native runner is enforced on the host. diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md index 84fdaac..7a7da85 100644 --- a/docs/reincarnation-map.md +++ b/docs/reincarnation-map.md @@ -371,3 +371,16 @@ fences the absent incumbent as nothing and admits the new release. Ops, out of scope: nginx boot order (`gamecult-ops`), and retention of the roughly 130 dead transient units. + +## Installed + +Cut 1 (`1da150a`, merge of `idunn/reincarnation` at `ffb4293`) was installed on +Yggdrasil on 2026-10-19 at 08:34:45 UTC with the operator's OK. The binary is +`/usr/local/bin/idunn` sha256 `f0e58308…`, built in `/srv/build/idunn-1da150a` +(`cargo test --lib`: 414 passed). The pre-install store backup is +`/root/idunn-store-backup-*-pre-reincarnation-c1`. Restoring it means restoring +its stores and `idunn.running` together, because the cb60965 binary cannot read +`idunn.target_supervision.v2`. At first open Idunn logged +"migrated 5 control record(s) … idunn.target_supervision.v2". streampixels-service +kept running through the restart (unit 73ca40ba, PID 968752, NRestarts 0). The +catalog returned 200, and 4443/udp was listening. diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index 5d98c4d..4c9ada7 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -18,7 +18,7 @@ probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC | # | What | Where | Result | |---|---|---|---| | I1 | Alert path | `src/control_plane.rs:3798-3812` (`ReportOnce`), `:4599-4610` (`report_once`), `:5114-5138` (`note_fault`, `clear_fault`) | The only operator signal is `eprintln!`. De-dup is `fault_reports: Mutex>` (`:4392`), in memory: an Idunn restart forgets it and reprints. `note_fault` also writes the transaction's `last_error`. | -| I2 | Continuity exhaustion | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | +| I2 | Continuity exhaustion (since `idunn-reincarnation` cut 1: `restarts_page`, a paging threshold only; nothing stops, and the line numbers below are pre-cut) | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | | I3 | Route degradation | `RouteSupervisionState` `:890-942`; `record_failed_challenge` sets `degraded_since_unix_millis` on the **first** failed challenge, `record_proved_challenge` clears it; reported at `:5975` under `route:` | Durable per incarnation (inside `admitted_generation.v4`); a new generation starts from `default()`. Backoff doubles from the topology max age to a cap. | | I4 | Other operator-facing faults | `:5645-5651` (`generation-down:`, "holds a dead admitted generation down"), `:6016` (`route-record:`), `note_fault` per transaction | Also stderr plus `last_error`. Not named by the target; listed so the paging-set question sees them. | | I5 | `OperatorRequired` on main | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | diff --git a/docs/watchdog-reporting-postmortem.md b/docs/watchdog-reporting-postmortem.md index 3249d2d..964698d 100644 --- a/docs/watchdog-reporting-postmortem.md +++ b/docs/watchdog-reporting-postmortem.md @@ -10,7 +10,8 @@ Read them with `view`; this page does not restate them. Idunn is now a watchdog that reaches the operator on Discord. - **Idunn** opens a durable incident at the site where a continuity budget is - exhausted. The incident is written to `incidents.cc`, a world-readable file + exhausted (since `idunn-reincarnation` cut 1 this is a paging threshold + only; continuity keeps restarting). The incident is written to `incidents.cc`, a world-readable file in its projection directory with a private 0600 lock, and closes when the target runs again and its restart window is no longer exhausted. - **Bifrost's watchdog-notice reader** runs from a one-minute timer. It sends diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index 381565d..5d21c75 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -23,9 +23,10 @@ Today nobody would hear about it. Idunn already detects several conditions an operator must act on, and for each one it writes a single `eprintln!` line to journald (`report_once`, `control_plane.rs:4599`): -- A continuity restart budget is exhausted (6 per hour; `restarts_exhausted`, - decided at `control_plane.rs:5709`). Idunn stops restarting the target, and - the only trace is one log line. +- A continuity paging threshold is reached (6 charged attempts per hour; + `restarts_page`, formerly `restarts_exhausted`). Continuity never stops: + it keeps restarting on a capped backoff, and this threshold only pages. When + this target was written the only trace was one log line. - An admitted route is degraded. It fails its stable-route challenge (`supervise_admitted_route`, ~5973). The degraded mark is durable, but no one is told. From 61383297b2ca2b06ef668c2502ad3a0184263eed Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 10:59:51 +0200 Subject: [PATCH 069/196] Reincarnation map: cut 2 phase-split mapped (two slots, one handover CAS) Co-Authored-By: Claude Opus 5.5 --- docs/reincarnation-map.md | 223 +++++++++++++++++++++++++++++++++++--- 1 file changed, 207 insertions(+), 16 deletions(-) diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md index 7a7da85..d63a6b3 100644 --- a/docs/reincarnation-map.md +++ b/docs/reincarnation-map.md @@ -8,6 +8,11 @@ no kind: body facts, the model page, the authority map's rationale and the cut sequence. Line numbers are at `74a7008`; `src/` is byte-identical to `c2477d4`, so the earlier anchors still hold. +Cut 2 pass, Imagination, 2026-10-19 (same campaign and session): body facts +B13-B18, the cut-2 rows of the model page and the cut-2 parts of sections 3-5 +are anchored at `9a0a6a2`, whose `src/` is byte-identical to `ffb4293` (cut 1's +head). The B1-B12 anchors above stay at `74a7008`. + Read the in-force set with the rulings-in-force and open-follow-ups recipes for root `idunn-reincarnation`; the cut in flight is the specs-with-no-report recipe. The first spec (`cut-c1-durable-stop.r1`) was withdrawn when the @@ -210,6 +215,82 @@ of the same rule (`5_000 << (live.len() - 1)`, `:1169`). The cut gives `TargetSupervision` keys use it, because every record on Yggdrasil predates them. +**B13. Two live transactions for one target make the control store +unreadable.** Source read at `9a0a6a2`. `ControlSnapshot::validate` refuses +"multiple transactions claim current-incarnation authority for one target" +over `owns_target_authority` (`:3245-3254`), and that predicate is just +`phase != Complete` (`:1896-1898`). Every reader of the store goes through it: +the scheduler tick, `idunn status`, cancel, expire. So if supervision minted a +continuity beside a pre-fencing deployment today, the next read would fail +the whole engine. Coexistence needs the identity rule changed first, and that +is the cut's first commit. + +**B14. One predicate answers four questions, and the phase rule lives in +one caller.** `blocks_new_target_mutation` (`:1903-1909`, live or post-commit +cleanup pending) is read by the freeze busy check (`:5934-5942`), the +supervision blocker (`:5998-6001`, a single `.find`), candidate-port +reservation (`:7997`) and status (`:4613`). `owns_target_authority` is read by +validation (`:3249`) and by sibling ordering inside one command (`:3384`). +Neither looks at the phase. The only phase-aware ownership rule is inline in +supervision (`:6002-6012`, owns from Fencing, or from Starting for +stop-then-start). The freeze refusal is why a `profile:` command needs every +target free in one tick (follow-up `profile-deploy-joint-gap`). + +**B15. What a pre-fencing deployment reads about its incumbent.** Warming's +semantic warming check uses the incumbent's lease digest +(`:6904-6905`, `is_semantic_warming` `:9530-9560`: Odin reports the candidate's +one disagreement as the lease the incumbent holds). Route preflight uses the +incumbent's route observation (`:7026-7047`). Isolation is proven against the +incumbent's workload (`:7050-7076`). Each goes through `exact_incumbent` +(`:7768-7781`), which bails "target incumbent changed after transaction +creation" once a continuity commit replaced the generation. Sealing, and +Starting for candidate-then-promote, read no incumbent. Stop-then-start's +Starting stops it (`:6857-6863`), which is already the owner's act. The +Sealing-to-Starting write is a `replace_transaction` at `:6806-6808`, not +`transition` (`:8914-8927`), so the two edges at which a transaction can take +a target are written by two different functions today. + +**B16. The commit already guards the incumbent; nothing guards the +handover.** `advance_committing` expects the admitted envelope it read +(`:7701-7702`, CAS `:7743-7764`), so a deployment can never commit over a +generation that moved under it. Entering Fencing is an unguarded +`replace_transaction` of the transaction alone. Continuity's mint CAS +(`:6371-6400`) guards its command, transaction and supervision record, and no +other transaction. The store accepts read-only expectations: an expected +entry with no replacement only guards (CultLib +`packages/cultcache-rs/src/lib.rs:1327-1368`). One engine holds the store +(`ProcessLock::acquire`, `:5514`), but the CLI writes transactions too: +`cancel` sets a pre-fencing abort on a live deployment (`:3014-3083`). So a +handover decided by one CAS over the target's records is exact, and a tick +order is not. + +**B17. What coexisting candidates share, and what they do not.** Continuity +reuses the admitted plan, so its candidate port is the admitted generation's, +already reserved (`:7988-7992`); a deployment picks another. Topology is keyed +per incarnation (`:8495-8498`): a continuity publishes under the admitted key, +a deployment under its own. No candidate holds a write lease before its own +Leasing, which follows Fencing, so two pre-fencing candidates cannot both +write. Route actuation by a continuity is always `Survival`, never refused +(`:4870-4876`); a deployment's `Forward` cannot starve it. The Odin replay +floor `max_odin_sequence` (`:3399-3435`) spans every transaction of the target +and the admitted cursor; incumbent refresh and candidate warming already share +it during every deployment today, so the replay check (`:9498-9501`) gains no +new exposure. Stop-then-start is declared only by route-less host-actuator +bindings (`deployment.rs:756-764`, `:1245-1249`), and no control-plane test +builds one (grep `StopThenStart` in `src/control_plane.rs`: the definition +site only). It is live on Yggdrasil: raven-muninn declares it +(`git grep stop-then-start` in `gamecult-ops`: +`idunn/yggdrasil/bindings/raven-muninn.toml.in`). So the Starting edge of the +handover carries production traffic, and its engine-level proof is owed +before the closing install (follow-up `stop-first-handover-fixture`). + +**B18. A pre-fencing error is an abort.** `resume_candidate` turns any `Err` +from a pre-fencing step into `begin_pre_fencing_abort` unless it is +`LockContended` (`:5768-5783`). So anything the cut makes a deployment wait on +must return `Ok` (a gate wait, `record_gate_wait` `:8170-8182`), or the wait +becomes the yield that ruling `handover-phase-split` forbids. The same path is +how a superseded continuity ends itself, which is wanted. + ## 2. Model page One row per persistent kind the campaign touches. @@ -223,12 +304,17 @@ One row per persistent kind the campaign touches. | Projection deferral (keys 4-5) | target | Unchanged: a failed demotion defers the next mint 30 s. It spaces a re-materialization too | Continuity supervision | | Admitted generation (`idunn.admitted_generation`) | key = target, `generation_id` = `generation-` | Replaced atomically by every committed deployment or continuity. A dead incarnation keeps its record until replaced | The committing transaction | | Incident `ContinuityExhausted` (incidents.cc) | condition + subject (target) | Opened at 6 charged attempts in the hour while the target is dead and free. Closed when the target runs and the window no longer holds 6 (`exhaustion-close-rule-op`). Retired after retention. The name is the wire name and stays, though nothing is exhausted any more | Idunn's incident writer, a notification projection: never read by ownership | -| Transaction and pre-fencing abort (`idunn.deployment_transaction.v4`, history.cc once terminal) | `tx-` and its `command_id` (`up-…` or `continuity-…`) | Sealing … Complete. A pre-fencing abort completes `FailedBeforeFencing`, a post-fencing abort `FailedAfterFencing` with a named recovery. Retired to history | The scheduler advances it. After cut 1, continuity supervision never aborts a transaction; the transaction's own steps do | +| Transaction and pre-fencing abort (`idunn.deployment_transaction.v4`, history.cc once terminal) | `tx-` and its `command_id` (`up-…` or `continuity-…`). Keyed by transaction, so a continuity and a deployment for one target are two records with no shared key | Sealing … Complete. A pre-fencing abort completes `FailedBeforeFencing`, a post-fencing abort `FailedAfterFencing` with a named recovery. Retired to history. After cut 2, per target at any instant: at most one live Deploy, at most one live Continuity, at most one owner (validated by `ControlSnapshot::validate`, B13) | The scheduler advances it. After cut 1, continuity supervision never aborts a transaction; the transaction's own steps do. After cut 2, a continuity whose target was taken, or whose source generation was replaced, aborts itself (superseded) | +| Target ownership (cut 2: derived, no record) | target | A transaction owns its target from `takes_target_at()` (Fencing, or Starting for stop-then-start) until Complete; a Complete transaction with pending post-commit cleanup drains it. Entered only through the handover CAS (`Engine::take_target`) | Derived from the phase. The handover CAS is the one writer of a phase entry into `takes_target_at()` | +| `incumbent_generation_id` on a transaction (key 9, unchanged format) | the generation's id | Deploy: set at freeze; before the handover it names the generation its incumbent-relative evidence was proven against and is rebased when the admitted generation moves (clearing that evidence); pinned by the handover CAS, exact from then on. Continuity: its source generation, never rebased | Deploy: its own pre-handover rebase step and the handover CAS. Continuity: the mint | | Unit materialization (systemd transient unit `idunn--`) | the activation's hash | Created by Starting. Lost on a host reboot (`/run/systemd/transient`). A failed unit stays loaded; a cleanly stopped one is collected | The workload driver actuates. Whether a loss was a reboot is the driver's evidence (`WorkloadPort::lost_to_host_reboot`, cut 1) | No cell is open. Both forks are ruled (`budget-never-stop`, `handover-phase-split`). The numbers that `budget-never-stop` left open are -the defaulted ruling `continuity-backoff-defaults`. +the defaulted ruling `continuity-backoff-defaults`. Cut 2 opens one question, +`fencing-over-owning-continuity` (does a deployment ready to fence wait for a +continuity that already owns the target, or preempt it); the spec is written +for the recommended answer, wait. ## 3. Authority map: continuity and deployment @@ -269,10 +355,69 @@ campaign-level map, and cut 1 is one step of it. advances through `advance_transaction`. - **Deletion line.** Cut 1 deletes the yield (`:6034-6059`), the exhaustion `continue` and its "free for a deployment" text (`:6062-6085`), - and the mint-anchored wait (`:1160-1184`). Cut 2 deletes the blocker - `continue` for a pre-fencing deployment (`:6033`), so continuity restarts - alongside the build, and the "incumbent changed after transaction - creation" refusal at Fencing (`:7556-7569`). + and the mint-anchored wait (`:1160-1184`). Cut 2 (anchors at `9a0a6a2`) + deletes `owns_target_authority` and `blocks_new_target_mutation` + (`:1894-1909`), the inline phase rule and the single-blocker `.find` in + supervision (`:5998-6012`), the blocker `continue` (`:6229-6233`), the + freeze busy check (`:5934-5942`), the one-live-transaction-per-target + validation (`:3245-3254`), and the "target incumbent changed after + transaction creation" text (`:7779`). + +### Cut 2: the two slots and the handover + +Each target has one **incumbent slot**: its `AdmittedGeneration`, the body +running it, its write lease and its stable-route membership. Each live +transaction has its own **candidate slot**: its incarnation key, activation, +candidate port and unit. + +- **Owner.** Before any transaction takes the target, continuity supervision + owns the incumbent slot: it alone decides to mint a restart. From the + handover CAS until Complete, the one transaction that took the target owns + it, fencing, leasing, routing and committing. Its commit hands the slot + back to supervision as a new generation. A transaction owns its own + candidate slot from mint or freeze until it is terminal. A pre-handover + deployment owns nothing else. +- **The handover.** `Engine::take_target` is the only writer of a phase entry + into `takes_target_at()` (Fencing, or Starting for stop-then-start), for + both kinds. It is one CAS. It expects the transaction's own envelope, the + target's admitted envelope (the one `incumbent_generation_id` names, or + absent), and every other non-terminal transaction of the target, each + unchanged, and it writes only the transaction. Of two transactions + entering from one view, exactly one CAS succeeds. The loser's next step + re-reads: a deployment gate-waits for the owner, and a continuity aborts + itself as superseded. +- **Inputs.** Continuity: as cut 1, plus whether any transaction owns or + drains the target, or a continuity is live for it (the mint gate). A + pre-handover deployment: its plan, its brake, and the admitted generation as + it is now, which it rebases onto. The handover: the target's records above. +- **Outputs.** The handover writes the new phase. A rebase rewrites + `incumbent_generation_id` and clears the evidence proven against the old + incumbent: `warming`, `route_preflight`, `isolation`. A superseded continuity + writes its own pre-fencing abort. +- **Derived state.** `owns_target()` is derived from the phase and + `takes_target_at()`. `drains_incumbent()` is derived from the post-commit + cleanup. The freeze refusal is derived from "a live Deploy for this + target". `incumbent_generation_id` on a pre-handover deployment is the + generation its evidence was proven against, not a pin. +- **Forbidden writers.** No path enters `takes_target_at()` except + `take_target`. Continuity supervision aborts nothing, and mints nothing + while another transaction owns or drains the target or a continuity is + live. A deployment never aborts because another transaction owns the + target: it gate-waits (B18). A continuity is never rebased. Freeze reads no + continuity state. No new persistent record decides ownership: it is the + phase. +- **Shared paths.** Candidate-then-promote's Warming-to-Fencing and + stop-then-start's Sealing-to-Starting go through one phase-entry seam, + which calls `take_target` exactly when the entered phase is + `takes_target_at()`. Deployments and continuities use the same CAS. A + single-target and a `profile:` command use the same freeze rule. + +Demotions, named: `incumbent_generation_id` is no longer a pin before the +handover; it is derived from the admitted generation by the rebase. +`blocks_new_target_mutation` is no longer an owner; it splits into +`is_live`, `owns_target` and `drains_incumbent`, each read by the consumer +that asks that question. The freeze busy check is no longer a +target-ownership gate; it guards only one live deployment per target. Demotions, named: `restarts_exhausted` is no longer an owner of anything. It is renamed `restarts_page`, and it opens and holds the incident. The @@ -341,19 +486,64 @@ operator is paged during the outage, not at 20:24. The `idunn up` at 20:24 freezes in the next gap. Continuity defers to it, nothing aborts it, and it fences the absent incumbent as nothing and admits the new release. +Why cut 2 is shaped this way. The handover is the phase, guarded by one CAS +over the target's records, rather than a new per-target "owner" record. A +record would be a second spelling of what the phase already says, and it +would need its own lifecycle, crash repair and migration. A CAS that reads +every other live transaction of the target is enough because a newly minted +or frozen transaction starts at Sealing and cannot own anything (B16). +The rebase clears exactly the evidence that names an incumbent (B15): warming +names its lease, preflight its route and isolation its workload. Keeping any +of them would let the deployment fence one generation with proofs about +another. A deployment waits in Warming while another transaction owns the +target, because Warming's steps read the incumbent that owner is fencing, and +a failed pre-fencing step is an abort (B18). Sealing and Starting keep +running, so the long part, the build, still overlaps. A continuity that loses +the target is aborted, not parked: it restarts a generation that is about to +be replaced, and while it lives it blocks continuity of the generation that +replaces it. + +Prior art on question `fencing-over-owning-continuity` (wait for a +continuity that already owns the target, or preempt it), from +`docs/research/restart-budget-prior-art.md` at `74a7008`. It leans the other +way from the recommendation, and the recommendation says why. Kubernetes' +rolling update scales old ReplicaSets down unhealthy pods first, so that +"unhealthy replicas will block deployment" does not happen (§1.4, +`rolling.go`). That is preemption of a failing old replica. Nomad separates +the two by layer: the client restarts old allocations under their own policy, +and the server reconciler places the new ones (§2.3). Neither waits on an +old replica's restart. But deleting a pod is the kubelet's ordinary teardown. +In Idunn, preempting a continuity past its own Fencing means a deployment +driving another transaction's post-fencing abort. That abort revokes a lease +and restores a projection, and it is the path that wedged Ghostlight (B7). +Waiting is bounded by the owner's own phase deadlines (`PhaseDeadline`, +`:938-960`) and adds no new abort path. Recommendation: wait, at medium +confidence. Preempt is the better answer once post-fencing aborts are known +not to wedge, which is follow-up `ghostlight-abort-lease-before-stop`. + +Residuals, recorded and not fixed. A wedged post-fencing abort of a +continuity (B7, follow-up `ghostlight-abort-lease-before-stop`) holds a +waiting deployment as long as it holds the target. A continuity superseded +after its mint stays charged; that lengthens one wait for a generation that +is being replaced. Stop-then-start has no control-plane fixture (B17), so in +cut 2 its edge is pinned structurally (one seam, one call) and by a unit test +of the phase mapping, not by an engine run. raven-muninn uses it, so the +engine test is owed before cut 5. + ## 5. Cut sequence 1. **c1-never-stop**: spec in the mind (specs-with-no-report recipe). Depends on nothing. -2. **Handover before Fencing** (`handover-phase-split`): continuity - restarts the incumbent while a pre-fencing deployment builds. This deletes - the blocker `continue` for a pre-fencing Deploy, and lets one continuity and - one pre-fencing deployment coexist per target, which touches - `blocks_new_target_mutation`, `freeze_command` and candidate ports. The - deployment takes the target at Fencing over whatever is admitted - (`exact_incumbent`, `:7556`). Entering Fencing needs a hand-off with a - live continuity transaction, decided as data rather than by tick order. - Waits on cut 1, because it rewrites the same block. +2. **c2-phase-split** (`handover-phase-split`): spec + `cut-c2-phase-split.r1` in the mind, branch `idunn/reincarnation-c2` from + `9a0a6a2`. Continuity restarts the incumbent while a pre-fencing + deployment builds. The ownership predicates split (B13, B14). The handover + is one CAS at `takes_target_at()`. A pre-handover deployment rebases onto + whatever is admitted, and a superseded continuity aborts itself. Freeze + refuses only a second live deployment for a target, which dissolves + follow-up `profile-deploy-joint-gap`. It folds in the test-only follow-up + `c1-sustained-reset-test-reach`. Open question: + `fencing-over-owning-continuity`. Depends on cut 1 (merged at `1da150a`). 3. **Dependency failures do not grow the backoff** (`budget-never-stop`): an attempt that failed because of a declared dependency (Odin's stable route, the target's own stable route @@ -367,7 +557,8 @@ fences the absent incumbent as nothing and admits the new release. not every tick. Waits on cut 1, because it rewrites the same function. 5. **Closing proof on Yggdrasil**: an operator-approved install of the merged cuts. streampixels-service survives the install, and `idunn status` is - checked before and after. Waits on 1-4. + checked before and after. Waits on 1-4 and on follow-up + `stop-first-handover-fixture` (raven-muninn is stop-then-start, B17). Ops, out of scope: nginx boot order (`gamecult-ops`), and retention of the roughly 130 dead transient units. From d91ecba759b898114332c0db02cfd60a00fd90ea Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:07:13 +0200 Subject: [PATCH 070/196] Cut 2 negative tests, failing at 6138329: restart beside a building deployment, profile freeze over a crash loop a_dead_incumbent_is_restarted_alongside_a_deployment_that_is_building and a_profile_deploy_freezes_over_a_crash_looping_target use only base API and pin the states cut 2 (handover-phase-split) ends. They fail until the predicate split and the open gates land. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 100 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e077373..d67d9d3 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16091,6 +16091,99 @@ mod tests { Ok(()) } + /// The 2026-10-18 shape at the engine: a dead incumbent with a deployment + /// building over it. Continuity restarts the incumbent beside the build + /// and neither touches the other (handover-phase-split). + #[test] + fn a_dead_incumbent_is_restarted_alongside_a_deployment_that_is_building() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + let stored_deploy = |world: &EngineFixture| -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == deploy.transaction_id) + .context("the deployment left the live set") + }; + let before = stored_deploy(&world)?; + + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1, "no restart beside the building deployment"); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 2); + assert_eq!(stored_deploy(&world)?.envelope, before.envelope); + + for _ in 0..2 { + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + let after = stored_deploy(&world)?; + assert_eq!(after.envelope, before.envelope); + assert!(after.value.pre_fencing_abort.is_none()); + } + Ok(()) + } + + /// A `profile:` command over two targets, one of them crash-looping with a + /// live continuity, freezes at once: freeze refuses only a second live + /// deployment of a target (follow-up profile-deploy-joint-gap). + #[test] + fn a_profile_deploy_freezes_over_a_crash_looping_target() -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let continuity = ControlSnapshot::read(&world.state_store)?.transactions.remove(0); + + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + for target in ["service", "other"] { + let paired = binding.replacen( + "target = \"service\"\n", + &format!("target = \"{target}\"\nprofiles = [\"pair\"]\n"), + 1, + ); + std::fs::write(world.root.join(format!("bindings/{target}.toml")), paired)?; + } + queue_deploy_of(&world, "up-pair", "profile:pair")?; + assert!(world.engine.freeze_one_queued_command()?, "the profile command stayed queued"); + + let live = ControlSnapshot::read(&world.state_store)?.transactions; + let mut frozen = live + .iter() + .filter(|stored| stored.value.command_id == "up-pair") + .map(|stored| (stored.value.ordinal, stored.value.target.as_str())) + .collect::>(); + frozen.sort(); + assert_eq!(frozen, [(0, "other"), (1, "service")]); + let kept = live + .iter() + .find(|stored| stored.value.transaction_id == continuity.value.transaction_id) + .context("the continuity left the live set")?; + assert_eq!(kept.envelope, continuity.envelope); + + // A live deployment of `service` still refuses a second command for it. + queue_deploy_of(&world, "up-service-second", "service")?; + assert!(!world.engine.freeze_one_queued_command()?); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| transaction.command_id != "up-service-second") + ); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -16923,11 +17016,16 @@ mod tests { ); std::fs::create_dir_all(world.root.join("bindings"))?; std::fs::write(world.root.join("bindings/service.toml"), binding)?; + queue_deploy_of(world, command_id, "service") + } + + /// Queue a Deploy command for `selector`; the bindings are the caller's. + fn queue_deploy_of(world: &EngineFixture, command_id: &str, selector: &str) -> Result<()> { let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), command_id: command_id.into(), kind: CommandKind::Deploy, - selector: "service".into(), + selector: selector.into(), requested_by: "operator".into(), requested_at_unix_millis: now_millis()?, }; From b4bb158707bd20ff56b4200910adfcf26a65724b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:11:36 +0200 Subject: [PATCH 071/196] Cut 2, commit 1: split the ownership predicate and the one-transaction-per-target validation owns_target_authority and blocks_new_target_mutation answered four questions with one phase-blind rule. They become is_live, owns_target (derived from the phase and takes_target_at) and drains_incumbent, each read by the consumer that asks that question: sibling order, validation, status, ports, freeze and the supervision gates. Validation now holds, per target, at most one owner, one live Deploy and one live Continuity, each refused with its own message, so a continuity beside a pre-handover deployment is readable. Freeze and the mint gate are still conservative (any live or draining transaction refuses), so no intermediate commit lets two transactions coexist unguarded. seeded_transaction mints a fresh command id when its default is taken, so a world can hold two of one kind. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 216 +++++++++++++++++++++++++++++++++---------- 1 file changed, 165 insertions(+), 51 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d67d9d3..16ea8d7 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,18 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// The phase whose entry is the handover of a target: Starting when the rollout +/// stops the incumbent first (it is stopped there by design), else Fencing. The +/// one spelling of the rule; a free function so the stop-then-start arm is +/// testable without a host-actuator fixture. +fn handover_phase(stops_incumbent_first: bool) -> DeploymentPhase { + if stops_incumbent_first { + DeploymentPhase::Starting + } else { + DeploymentPhase::Fencing + } +} + impl DeploymentTransaction { /// The one writer of `phase`. It stamps the update time and the phase /// deadline together, so a record outside Fencing..=Committing carries no @@ -1891,21 +1903,36 @@ impl DeploymentTransaction { } } - /// Complete transactions retain retryable cleanup work, but admission has - /// already transferred current-incarnation authority to AdmittedGeneration. - fn owns_target_authority(&self) -> bool { + /// The phase whose entry is the handover: the one transaction that makes + /// it takes the target's incumbent slot from continuity supervision. + fn takes_target_at(&self) -> DeploymentPhase { + handover_phase(self.rollout_stops_incumbent_first()) + } + + /// Not Complete: the transaction still holds its own candidate slot. + /// Read by sibling order inside one command, freeze (a live Deploy + /// refuses a second), the mint gate (a live Continuity refuses a second), + /// candidate-port reservation and validation. + fn is_live(&self) -> bool { self.phase != DeploymentPhase::Complete } + /// From the handover until Complete the transaction owns the target's + /// incumbent slot: it fences, leases, routes and commits. Derived from the + /// phase, never stored. Read by validation (one owner), supervision (which + /// defers to an owner), the handover CAS, a waiting deployment and status. + fn owns_target(&self) -> bool { + (self.takes_target_at()..DeploymentPhase::Complete).contains(&self.phase) + } + /// Admission has moved to the new generation at Complete, but an exact /// draining incumbent still reserves its process and candidate endpoint. - /// No later mutation for the same target may overlap that cleanup. - fn blocks_new_target_mutation(&self) -> bool { - self.owns_target_authority() - || self - .post_commit_cleanup - .as_ref() - .is_some_and(|cleanup| !cleanup.is_complete()) + /// Read by the mint gate, the handover, candidate-port reservation and + /// status: no later mutation of the target may overlap that cleanup. + fn drains_incumbent(&self) -> bool { + self.post_commit_cleanup + .as_ref() + .is_some_and(|cleanup| !cleanup.is_complete()) } /// Where what this transaction has collected disagrees with what its own @@ -3242,16 +3269,27 @@ impl ControlSnapshot { "transaction kind differs from its immutable command" ); } - let mut live_targets = BTreeSet::new(); + // Per target: one owner of the incumbent slot, one live Deploy, one + // live Continuity. A continuity may sit beside a deployment that has + // not yet taken the target (handover-phase-split). + let mut owners = BTreeSet::new(); + let mut live_deploys = BTreeSet::new(); + let mut live_continuities = BTreeSet::new(); for transaction in self .transactions .iter() - .filter(|stored| stored.value.owns_target_authority()) + .filter(|stored| stored.value.is_live()) { + let target = transaction.value.target.as_str(); ensure!( - live_targets.insert(transaction.value.target.as_str()), - "multiple transactions claim current-incarnation authority for one target" + !transaction.value.owns_target() || owners.insert(target), + "multiple transactions own the incumbent slot of one target" ); + let (seen, kind) = match transaction.value.command_kind { + CommandKind::Deploy => (&mut live_deploys, "deployments"), + CommandKind::Continuity => (&mut live_continuities, "continuity transactions"), + }; + ensure!(seen.insert(target), "multiple live {kind} for one target"); if let Some(fencing) = transaction .value .fencing @@ -3381,7 +3419,7 @@ impl ControlSnapshot { fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool { self.transactions.iter().any(|stored| { - stored.value.owns_target_authority() + stored.value.is_live() && stored.value.command_id == transaction.command_id && stored.value.ordinal < transaction.ordinal }) @@ -4608,9 +4646,10 @@ fn status( } writeln!( out, - " terminal {} owns-target {}", + " terminal {} owns-target {} drains {}", transaction.is_terminal(), - transaction.blocks_new_target_mutation() + transaction.owns_target(), + transaction.drains_incumbent() )?; } } @@ -5934,7 +5973,7 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -5995,23 +6034,19 @@ impl Engine { &anyhow::Error::new(disagreement), ); } - let blocker = snapshot.transactions.iter().find(|stored| { - stored.value.target == current.value.target - && stored.value.blocks_new_target_mutation() - }); - if blocker.is_some_and(|stored| { - let owns_from = if stored.value.rollout_stops_incumbent_first() { - // The incumbent is stopped at Starting by design; a - // continuity restart in that window would fight the - // candidate for the same host resources. - DeploymentPhase::Starting - } else { - DeploymentPhase::Fencing - }; - stored.value.phase >= owns_from && stored.value.phase < DeploymentPhase::Complete - }) { + let on_target = || { + snapshot + .transactions + .iter() + .filter(|stored| stored.value.target == current.value.target) + }; + // A transaction that owns the incumbent slot fences, stops or + // replaces this generation: nothing here touches it meanwhile. + if on_target().any(|stored| stored.value.owns_target()) { continue; } + let blocked = on_target() + .any(|stored| stored.value.is_live() || stored.value.drains_incumbent()); match self.supervise_admitted_route(&snapshot, current) { Ok(true) => { progressed = true; @@ -6056,9 +6091,7 @@ impl Engine { // the admitted one leaves the projection naming an incarnation that // no longer exists -- which the workload rightly refuses to adopt. // That record is published from observation once the process is up. - if blocker.is_none() - || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) - { + if !on_target().any(|stored| stored.value.is_live()) { let repair = (|| -> Result<()> { let topology = self.topology(); let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; @@ -6228,7 +6261,7 @@ impl Engine { // Continuity defers to any live transaction for its target and // never aborts one (handover-phase-split). - if blocker.is_some() { + if blocked { continue; } if supervision.restarts_page(now) { @@ -7994,7 +8027,7 @@ impl Engine { snapshot .transactions .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) .filter_map(|stored| stored.value.plan.as_ref()), ) { @@ -11595,8 +11628,9 @@ mod tests { }); assert!(!transaction.is_terminal()); - assert!(!transaction.owns_target_authority()); - assert!(transaction.blocks_new_target_mutation()); + assert!(!transaction.is_live()); + assert!(!transaction.owns_target()); + assert!(transaction.drains_incumbent()); Ok(()) } @@ -11835,7 +11869,7 @@ mod tests { ); assert_eq!(lifted.phase_deadline, None); assert_eq!(lifted.lease_adoption, None); - assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); + assert_eq!(lifted.is_live(), legacy.phase != DeploymentPhase::Complete); } assert_eq!( phases.iter().map(|(_, phase)| *phase).collect::>(), @@ -12457,16 +12491,23 @@ mod tests { declare_odin(&mut expected); let now = now_millis()?; + // A second seeding of one kind into one world is a second command. + let base_id = format!( + "{}-service{}", + match kind { + CommandKind::Deploy => "up", + CommandKind::Continuity => "continuity", + }, + if incumbent.is_some() { "-again" } else { "" } + ); + let known = ControlSnapshot::read(&world.state_store)?.commands; + let command_id = (1..) + .map(|n| if n == 1 { base_id.clone() } else { format!("{base_id}-{n}") }) + .find(|id| known.iter().all(|stored| stored.value.command_id != *id)) + .context("no free command id")?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!( - "{}-service{}", - match kind { - CommandKind::Deploy => "up", - CommandKind::Continuity => "continuity", - }, - if incumbent.is_some() { "-again" } else { "" } - ), + command_id, kind, selector: "service".into(), requested_by: "test".into(), @@ -14220,7 +14261,7 @@ mod tests { assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); assert_eq!(lifted.phase, archived.phase, "{name}"); assert_eq!(lifted.completion, archived.completion, "{name}"); - assert!(!lifted.owns_target_authority(), "{name}"); + assert!(!lifted.is_live(), "{name}"); } else { assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); assert!(!lifted.is_terminal(), "{name}"); @@ -16184,6 +16225,79 @@ mod tests { Ok(()) } + /// The stored transaction `id`, moved into `phase` in place. + fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { + let stored = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == id) + .context("no such live transaction")?; + let mut next = stored.value.clone(); + next.enter_phase(phase, now_millis()?); + replace_transaction(&world.state_store, &stored, &next) + } + + #[test] + fn the_handover_phase_is_starting_only_for_stop_then_start() -> Result<()> { + assert_eq!(handover_phase(true), DeploymentPhase::Starting); + assert_eq!(handover_phase(false), DeploymentPhase::Fencing); + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let mut seeded = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!(seeded.takes_target_at(), DeploymentPhase::Fencing); + // The target is owned from the handover to Complete, and not after; + // it is live until Complete. + for (phase, owns) in [ + (DeploymentPhase::Sealing, false), + (DeploymentPhase::Starting, false), + (DeploymentPhase::Warming, false), + (DeploymentPhase::Fencing, true), + (DeploymentPhase::Leasing, true), + (DeploymentPhase::AwaitingReady, true), + (DeploymentPhase::Routing, true), + (DeploymentPhase::Committing, true), + (DeploymentPhase::Complete, false), + ] { + seeded.phase = phase; + assert_eq!(seeded.owns_target(), owns, "{phase:?}"); + assert_eq!(seeded.is_live(), phase != DeploymentPhase::Complete, "{phase:?}"); + } + Ok(()) + } + + #[test] + fn the_store_holds_one_deploy_one_continuity_and_one_owner_per_target() -> Result<()> { + let seed = |first: (CommandKind, DeploymentPhase), second: (CommandKind, DeploymentPhase)| { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + for (kind, phase) in [first, second] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + if phase != DeploymentPhase::Warming { + rephase(&world, &seeded.transaction_id, phase)?; + } + } + Ok::<_, anyhow::Error>(world) + }; + use CommandKind::{Continuity, Deploy}; + use DeploymentPhase::{Fencing, Warming}; + + // A pre-handover deployment beside an owning continuity, and the + // reverse: one owner, one of each kind. + for (first, second) in [((Deploy, Warming), (Continuity, Fencing)), ((Continuity, Warming), (Deploy, Fencing))] { + let world = seed(first, second)?; + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 2); + } + let refused = |first, second| -> Result { + let world = seed(first, second)?; + Ok(format!("{:#}", ControlSnapshot::read(&world.state_store).err().context("two were accepted")?)) + }; + assert!(refused((Deploy, Fencing), (Continuity, Fencing))?.contains("own the incumbent slot")); + assert!(refused((Deploy, Warming), (Deploy, Warming))?.contains("multiple live deployments")); + assert!(refused((Continuity, Warming), (Continuity, Warming))?.contains("multiple live continuity transactions")); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -17207,7 +17321,7 @@ mod tests { )); assert!(finished.is_terminal()); // The target is released and nothing of the incumbent was touched. - assert!(!finished.blocks_new_target_mutation()); + assert!(!finished.is_live() && !finished.drains_incumbent()); assert_eq!(incumbent_envelope(&world)?, before); assert!( every_transaction(&world)? From 755306d654c3c4151f9eb3ad16d137e37d411b54 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:13:38 +0200 Subject: [PATCH 072/196] Cut 2, commit 2: one phase-entry seam and one handover CAS Engine::enter is the only writer of a move into the next phase; transition and the Sealing tail both call it. Entering the phase where a transaction takes its target (takes_target_at: Fencing, or Starting for stop-then-start) goes through Engine::take_target, one CAS that expects the transaction's own envelope, the target's admitted envelope as read (or its absence) and every other non-terminal transaction of the target, and writes only the transaction. Of two entrants from one view exactly one wins; a losing deployment waits with the owner named, a losing continuity is superseded. exact_incumbent now says what a mismatch is: a fault, not a race. Test helper: the brake release is per transaction (a world can hold two deployments). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 332 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 314 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 16ea8d7..5abe6ef 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,20 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// What `Engine::take_target` decided about one transaction's entry into the +/// phase where it takes its target. +#[derive(Debug, PartialEq, Eq)] +enum Handover { + /// The CAS wrote the entry: the transaction owns the target. + Took, + /// The CAS lost to a concurrent write; nothing was written. + Lost, + /// Another transaction owns or drains the target; nothing was written. + OwnedBy(String), + /// The admitted generation is no longer the one the transaction names. + Moved, +} + /// The phase whose entry is the handover of a target: Starting when the rollout /// stops the incumbent first (it is stopped there by design), else Fencing. The /// one spelling of the rule; a free function so the stop-then-start arm is @@ -3383,6 +3397,17 @@ impl ControlSnapshot { .find(|stored| stored.value.target == target) } + /// The transaction, other than `except`, that holds the target's + /// incumbent slot or is still draining it. The one lookup supervision, a + /// waiting deployment and the handover share. + fn target_owner(&self, target: &str, except: Option<&str>) -> Option<&Stored> { + self.transactions.iter().find(|stored| { + stored.value.target == target + && Some(stored.value.transaction_id.as_str()) != except + && (stored.value.owns_target() || stored.value.drains_incumbent()) + }) + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -6838,7 +6863,9 @@ impl Engine { })?; next.enter_phase(DeploymentPhase::Starting, now); next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) + // Starting is the handover for stop-then-start, so this write goes + // through the seam that decides it. + self.enter(current, next) } fn advance_starting(&self, current: &Stored) -> Result<()> { @@ -7798,6 +7825,10 @@ impl Engine { Ok(()) } + /// The admitted generation this transaction was proven against. A + /// pre-handover deployment is rebased onto the admitted generation before + /// any step reads its incumbent, and the handover pins it after, so a + /// mismatch here is a fault, not a race. fn exact_incumbent<'a>( &self, snapshot: &'a ControlSnapshot, @@ -7809,7 +7840,9 @@ impl Engine { (Some(expected), Some(current)) if current.value.generation_id == *expected => { Ok(Some(current)) } - _ => bail!("target incumbent changed after transaction creation"), + _ => bail!( + "admitted generation changed under a transaction that owns the target or names it as its source" + ), } } @@ -8949,14 +8982,108 @@ impl Engine { current: &Stored, next_phase: DeploymentPhase, ) -> Result<()> { - ensure!( - next_phase as u8 == current.value.phase as u8 + 1, - "deployment phase transition is not adjacent" - ); let mut next = current.value.clone(); next.enter_phase(next_phase, now_millis()?); next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) + self.enter(current, next) + } + + /// The one phase-entry seam. Every move into the next phase is written + /// here, and entering the phase where the transaction takes its target + /// (`takes_target_at`) is decided by `take_target`: that CAS is the + /// handover, and nothing else writes that entry. + fn enter( + &self, + current: &Stored, + next: DeploymentTransaction, + ) -> Result<()> { + ensure!( + next.phase as u8 == current.value.phase as u8 + 1, + "deployment phase transition is not adjacent" + ); + if next.phase != next.takes_target_at() { + return replace_transaction(&self.options.state_store, current, &next); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let target = ¤t.value.target; + match (self.take_target(&snapshot, current, &next)?, current.value.command_kind) { + // Lost to a concurrent writer: the next step reads the new world. + (Handover::Took | Handover::Lost, _) => Ok(()), + // Wait, never abort: a pre-fencing error would end the deployment. + (Handover::OwnedBy(owner), CommandKind::Deploy) => self.record_gate_wait( + current, + &format!( + "target {target} is owned by {owner}; this deployment fences what is admitted when it completes" + ), + ), + // The next step rebases onto the generation now admitted. + (Handover::Moved, CommandKind::Deploy) => Ok(()), + (Handover::OwnedBy(owner), CommandKind::Continuity) => { + bail!("continuity superseded: target {target} is owned by {owner}") + } + (Handover::Moved, CommandKind::Continuity) => { + bail!("continuity superseded: the generation it restarts is no longer admitted") + } + } + } + + /// The handover: one CAS that writes `next` (the entry into + /// `takes_target_at`) only if nothing else moved. It expects the + /// transaction's own envelope, the target's admitted envelope as read + /// (or its absence), and every other non-terminal transaction of the + /// target as read, so of two transactions entering from one view exactly + /// one succeeds. + fn take_target( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + next: &DeploymentTransaction, + ) -> Result { + let target = current.value.target.as_str(); + let id = current.value.transaction_id.as_str(); + if let Some(owner) = snapshot.target_owner(target, Some(id)) { + return Ok(Handover::OwnedBy(owner.value.transaction_id.clone())); + } + let admitted = snapshot.admitted_for(target); + if admitted.map(|stored| stored.value.generation_id.as_str()) + != current.value.incumbent_generation_id.as_deref() + { + return Ok(Handover::Moved); + } + next.validate()?; + let mut expected = vec![ + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: id.into(), + current: Some(current.envelope.clone()), + }, + CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: target.into(), + current: admitted.map(|stored| stored.envelope.clone()), + }, + ]; + expected.extend( + snapshot + .transactions + .iter() + .filter(|stored| { + stored.value.target == target + && stored.value.transaction_id != id + && !stored.value.is_terminal() + }) + .map(|stored| CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: stored.value.transaction_id.clone(), + current: Some(stored.envelope.clone()), + }), + ); + let took = SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &expected, + &[transaction_envelope(next, next.updated_at_unix_millis)?], + )?; + Ok(if took { Handover::Took } else { Handover::Lost }) } /// Every post-fencing phase ends. Before a live transaction past the @@ -12329,10 +12456,17 @@ mod tests { IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, }; - let signer = enroll_service_identity_at::( - &world.root.join("identities/brake-operator.cc"), - )?; - export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + // One operator per world, however many deployments it releases. + let signer_path = world.root.join("identities/brake-operator.cc"); + let signer = if signer_path.exists() { + open_service_identity_at::(&signer_path)? + } else { + let signer = + enroll_service_identity_at::(&signer_path)?; + export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + signer + }; + let authorization_id = format!("authorization-{transaction_id}"); let mut record = IdunnDeploymentBrakeRecord { schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), @@ -12343,7 +12477,7 @@ mod tests { reason: "test rollout".into(), observed_at_unix_millis: now, expires_at_unix_millis: Some(now + 600_000), - authorization_id: Some("authorization-test".into()), + authorization_id: Some(authorization_id.clone()), authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), authorized_release_id: Some(expected.sealed_release_id.clone()), authorized_deployment_id: Some(transaction_id.to_owned()), @@ -12363,7 +12497,7 @@ mod tests { record.validate()?; let canonical_brake_bytes = rmp_serde::to_vec(&record)?; Ok(DeploymentAuthorization { - authorization_id: "authorization-test".into(), + authorization_id, brake_sha256: sha256_id(&canonical_brake_bytes), canonical_brake_bytes, authorized_at_unix_millis: now, @@ -16225,16 +16359,178 @@ mod tests { Ok(()) } - /// The stored transaction `id`, moved into `phase` in place. - fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { - let stored = ControlSnapshot::read(&world.state_store)? + /// The live transaction `id`. + fn resident_id(world: &EngineFixture, id: &str) -> Result> { + ControlSnapshot::read(&world.state_store)? .transactions .into_iter() .find(|stored| stored.value.transaction_id == id) - .context("no such live transaction")?; + .context("no such live transaction") + } + + /// `stored` one phase on, as the seam would be asked to write it. + fn entering(stored: &Stored, phase: DeploymentPhase) -> Result { let mut next = stored.value.clone(); next.enter_phase(phase, now_millis()?); - replace_transaction(&world.state_store, &stored, &next) + Ok(next) + } + + /// Drop every live transaction but `keep`. + fn clear_other_live_transactions(world: &EngineFixture, keep: &str) -> Result<()> { + let others = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .filter(|stored| stored.value.transaction_id != keep) + .map(|stored| stored.envelope) + .collect::>(); + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged(&others)?); + Ok(()) + } + + /// The stored transaction `id`, moved into `phase` in place. + fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { + let stored = resident_id(world, id)?; + replace_transaction(&world.state_store, &stored, &entering(&stored, phase)?) + } + + /// A Deploy at `deploy_phase` and a Continuity at `continuity_phase`, both + /// over one incumbent of `service`, and that incumbent. + fn deploy_beside_continuity( + world: &EngineFixture, + deploy_phase: DeploymentPhase, + continuity_phase: DeploymentPhase, + ) -> Result<(DeploymentTransaction, DeploymentTransaction, AdmittedGeneration)> { + let incumbent = admit_incumbent(world)?; + let deploy = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let continuity = seeded_transaction( + world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + for (seeded, phase) in [(&deploy, deploy_phase), (&continuity, continuity_phase)] { + if phase != DeploymentPhase::Warming { + rephase(world, &seeded.transaction_id, phase)?; + } + } + Ok((deploy, continuity, incumbent)) + } + + #[test] + fn exactly_one_transaction_takes_a_target_from_one_view() -> Result<()> { + let take = |snapshot: &ControlSnapshot, world: &EngineFixture, id: &str| -> Result { + let stored = resident_id(world, id)?; + world.engine.take_target(snapshot, &stored, &entering(&stored, DeploymentPhase::Fencing)?) + }; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let view = ControlSnapshot::read(&world.state_store)?; + + assert_eq!(take(&view, &world, &continuity.transaction_id)?, Handover::Took); + // The second claimant read the same world: its CAS loses. + assert_eq!(take(&view, &world, &deploy.transaction_id)?, Handover::Lost); + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!(after.transactions.iter().filter(|stored| stored.value.owns_target()).count(), 1); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Warming); + // A fresh view names the owner instead of racing it. + assert_eq!( + take(&after, &world, &deploy.transaction_id)?, + Handover::OwnedBy(continuity.transaction_id.clone()) + ); + + // The reverse world: the deployment wins and the continuity loses. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let view = ControlSnapshot::read(&world.state_store)?; + assert_eq!(take(&view, &world, &deploy.transaction_id)?, Handover::Took); + assert_eq!(take(&view, &world, &continuity.transaction_id)?, Handover::Lost); + Ok(()) + } + + #[test] + fn the_handover_loses_when_the_admitted_generation_moved_after_it_was_read() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, _, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + // Only the continuity is a rival here; take it out of the guards' way. + clear_other_live_transactions(&world, &deploy.transaction_id)?; + let view = ControlSnapshot::read(&world.state_store)?; + let stored = resident_id(&world, &deploy.transaction_id)?; + + // Same generation, new envelope: a refresh landed after the read. + edit_incumbent(&world, |generation| generation.odin_publisher_sequence_cursor += 1)?; + assert_eq!( + world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, + Handover::Lost + ); + // A view that already shows another generation is Moved, not raced. + let mut other = stored.clone(); + other.value.incumbent_generation_id = Some("generation-elsewhere".into()); + assert_eq!( + world.engine.take_target( + &ControlSnapshot::read(&world.state_store)?, + &other, + &entering(&other, DeploymentPhase::Fencing)? + )?, + Handover::Moved + ); + Ok(()) + } + + #[test] + fn the_seam_makes_a_deployment_wait_and_a_continuity_step_aside() -> Result<()> { + // A deployment ready to fence while a continuity owns the target waits + // where it is, with the owner named and no abort. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let stored = resident_id(&world, &deploy.transaction_id)?; + world.engine.enter(&stored, entering(&stored, DeploymentPhase::Fencing)?)?; + let waiting = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.pre_fencing_abort.is_none()); + assert!( + waiting.last_error.as_deref().is_some_and(|note| note.contains(&continuity.transaction_id)), + "{:?}", + waiting.last_error + ); + // A deployment whose generation moved is left for its rebase. + let world_two = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy_two, _, _) = + deploy_beside_continuity(&world_two, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + clear_other_live_transactions(&world_two, &deploy_two.transaction_id)?; + let mut stored_two = resident_id(&world_two, &deploy_two.transaction_id)?; + stored_two.value.incumbent_generation_id = Some("generation-elsewhere".into()); + let before = ControlSnapshot::read(&world_two.state_store)?.transactions[0].envelope.clone(); + world_two.engine.enter(&stored_two, entering(&stored_two, DeploymentPhase::Fencing)?)?; + assert_eq!(ControlSnapshot::read(&world_two.state_store)?.transactions[0].envelope, before); + + // A continuity in the same position is superseded, by either cause. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Fencing, DeploymentPhase::Warming)?; + let stored = resident_id(&world, &continuity.transaction_id)?; + let error = world + .engine + .enter(&stored, entering(&stored, DeploymentPhase::Fencing)?) + .expect_err("a continuity entered a target another transaction owns"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + assert!(format!("{error:#}").contains(&deploy.transaction_id), "{error:#}"); + clear_other_live_transactions(&world, &continuity.transaction_id)?; + let mut stored = resident_id(&world, &continuity.transaction_id)?; + stored.value.incumbent_generation_id = Some("generation-elsewhere".into()); + let error = world + .engine + .enter(&stored, entering(&stored, DeploymentPhase::Fencing)?) + .expect_err("a continuity entered over a generation that moved"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + // Phases are entered one at a time. + let stored = resident_id(&world, &continuity.transaction_id)?; + assert!(world.engine.enter(&stored, entering(&stored, DeploymentPhase::Routing)?).is_err()); + Ok(()) } #[test] From 9139d7edb8964827ee9910f2f164bf76b93eecda Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:16:05 +0200 Subject: [PATCH 073/196] Cut 2, commit 3: rebase before the handover, wait in Warming, supersede a continuity settle_before_handover runs after the two abort dispatches and before any pre-handover step. A Deploy rebases onto the admitted generation when it differs from the one its evidence was proven against: incumbent_generation_id is rewritten, warming, route_preflight and isolation are dropped and the Odin cursor is raised, nothing else. In Warming it waits (a gate note, no error, so no abort) while another transaction owns or drains the target. A Continuity is never rebased: when its target is owned or its generation replaced it ends itself through the ordinary pre-fencing abort. Tests drive a real continuity restart to Complete beside a deployment and check the deployment rebases onto it and fences it. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 306 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 294 insertions(+), 12 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5abe6ef..1229bdd 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,11 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// The note a deployment keeps while another transaction owns its target. +fn waiting_for_owner(target: &str, owner: &str) -> String { + format!("target {target} is owned by {owner}; this deployment fences what is admitted when it completes") +} + /// What `Engine::take_target` decided about one transaction's entry into the /// phase where it takes its target. #[derive(Debug, PartialEq, Eq)] @@ -6738,6 +6743,11 @@ impl Engine { if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() { return self.advance_pre_fencing_abort(current); } + if current.value.phase < current.value.takes_target_at() + && self.settle_before_handover(current)? + { + return Ok(()); + } match current.value.phase { DeploymentPhase::Sealing => self.advance_sealing(current), DeploymentPhase::Starting => self.advance_starting(current), @@ -8988,6 +8998,92 @@ impl Engine { self.enter(current, next) } + /// Before its handover a Deploy is proven against the generation that was + /// admitted when the evidence was gathered. When another generation has + /// been admitted since (continuity restarted the incumbent), rewrite the + /// incumbent and drop exactly the evidence that names the old one: the + /// warming (its lease), the route preflight (its route) and the isolation + /// (its workload). A continuity is never rebased. Returns whether it wrote. + fn rebase_onto_admitted(&self, current: &Stored) -> Result { + let held = ¤t.value; + if held.command_kind != CommandKind::Deploy + || held.pre_fencing_abort.is_some() + || held.phase >= held.takes_target_at() + { + return Ok(false); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let admitted = snapshot.admitted_for(&held.target); + let admitted_id = admitted.map(|stored| stored.value.generation_id.as_str()); + if admitted_id == held.incumbent_generation_id.as_deref() { + return Ok(false); + } + self.persist_same_phase(current, |next| { + next.incumbent_generation_id = admitted_id.map(str::to_owned); + next.warming = None; + next.route_preflight = None; + next.isolation = None; + next.odin_publisher_sequence_cursor = next + .odin_publisher_sequence_cursor + .max(admitted.map_or(0, |stored| stored.value.odin_publisher_sequence_cursor)); + Ok(()) + })?; + Ok(true) + } + + /// What a transaction that has not yet taken its target does first, when + /// the world moved under it. `true` means this decided the step. + /// + /// A Deploy rebases onto the generation now admitted; in Warming, whose + /// steps read the incumbent an owner is fencing, it waits while another + /// transaction owns or drains the target. Both return Ok: a pre-fencing + /// error is an abort, and no deployment is aborted for continuity. A + /// Continuity restarts one generation and no other, so it ends itself + /// (through the ordinary pre-fencing abort) when its target is taken or + /// its generation replaced. + fn settle_before_handover(&self, current: &Stored) -> Result { + let held = ¤t.value; + match held.command_kind { + CommandKind::Deploy => { + if self.rebase_onto_admitted(current)? { + return Ok(true); + } + if held.phase != DeploymentPhase::Warming { + return Ok(false); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id)) + else { + return Ok(false); + }; + self.record_gate_wait( + current, + &waiting_for_owner(&held.target, &owner.value.transaction_id), + )?; + Ok(true) + } + CommandKind::Continuity => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id)) + { + bail!( + "continuity superseded: target {} is owned by {}", + held.target, + owner.value.transaction_id + ); + } + ensure!( + snapshot + .admitted_for(&held.target) + .map(|stored| stored.value.generation_id.as_str()) + == held.incumbent_generation_id.as_deref(), + "continuity superseded: the generation it restarts is no longer admitted" + ); + Ok(false) + } + } + } + /// The one phase-entry seam. Every move into the next phase is written /// here, and entering the phase where the transaction takes its target /// (`takes_target_at`) is decided by `take_target`: that CAS is the @@ -9010,12 +9106,9 @@ impl Engine { // Lost to a concurrent writer: the next step reads the new world. (Handover::Took | Handover::Lost, _) => Ok(()), // Wait, never abort: a pre-fencing error would end the deployment. - (Handover::OwnedBy(owner), CommandKind::Deploy) => self.record_gate_wait( - current, - &format!( - "target {target} is owned by {owner}; this deployment fences what is admitted when it completes" - ), - ), + (Handover::OwnedBy(owner), CommandKind::Deploy) => { + self.record_gate_wait(current, &waiting_for_owner(target, &owner)) + } // The next step rebases onto the generation now admitted. (Handover::Moved, CommandKind::Deploy) => Ok(()), (Handover::OwnedBy(owner), CommandKind::Continuity) => { @@ -12863,23 +12956,47 @@ mod tests { } } - /// One engine step at a time until `stop` holds, recording each phase - /// entered and checking after every step that the deadline on the record - /// is exactly the current post-fencing phase's, and absent outside one. + /// The transaction wherever it now lives: live, or archived once terminal. + fn latest_id(world: &EngineFixture, id: &str) -> Result { + match resident_id(world, id) { + Ok(stored) => Ok(stored.value), + Err(_) => read_history_transactions(&world.state_store) + .into_iter() + .find(|transaction| transaction.transaction_id == id) + .context("the transaction is in neither the live set nor history"), + } + } + + /// `drive_id` for the world's only transaction. fn drive( world: &EngineFixture, stop: impl Fn(&DeploymentTransaction) -> bool, + ) -> Result> { + match resident(world) { + Ok(sole) => drive_id(world, &sole.value.transaction_id, stop), + Err(_) => Ok(Vec::new()), + } + } + + /// One engine step of transaction `id` at a time until `stop` holds, + /// recording each phase entered and checking after every step that the + /// deadline on the record is exactly the current post-fencing phase's, and + /// absent outside one. + fn drive_id( + world: &EngineFixture, + id: &str, + stop: impl Fn(&DeploymentTransaction) -> bool, ) -> Result> { let mut phases = Vec::new(); for _ in 0..40 { - let Ok(current) = resident(world) else { + let Ok(current) = resident_id(world, id) else { return Ok(phases); }; if stop(¤t.value) { return Ok(phases); } world.engine.advance_transaction(¤t)?; - let after = latest(world)?; + let after = latest_id(world, id)?; match &after.phase_deadline { Some(deadline) => assert_eq!(deadline.phase, after.phase), None => assert!( @@ -16461,7 +16578,7 @@ mod tests { let stored = resident_id(&world, &deploy.transaction_id)?; // Same generation, new envelope: a refresh landed after the read. - edit_incumbent(&world, |generation| generation.odin_publisher_sequence_cursor += 1)?; + edit_incumbent(&world, |generation| generation.admitted_at_unix_millis += 1)?; assert_eq!( world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, Handover::Lost @@ -16594,6 +16711,171 @@ mod tests { Ok(()) } + /// The continuity transaction minted over `service`, whatever its state. + fn the_continuity(world: &EngineFixture) -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.command_kind == CommandKind::Continuity) + .context("no live continuity") + } + + #[test] + fn a_deployment_rebases_onto_the_generation_continuity_admitted_mid_build_and_fences_it() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let deploy_id = deploy.transaction_id.as_str(); + workload.kill(); + assert!(world.engine.supervise_one_admitted_generation()?); + let continuity_id = the_continuity(&world)?.value.transaction_id; + + // Continuity restarts the incumbent beside the build, to Complete. + workload.revive(); + drive_id(&world, &continuity_id, |t| t.phase == DeploymentPhase::Warming)?; + odin_republishes(&world, &latest_id(&world, &continuity_id)?, seeded_sequence(Some(&incumbent)) + 10)?; + drive_id(&world, &continuity_id, |t| t.is_terminal())?; + let restarted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the restart admitted nothing")? + .value + .clone(); + assert_ne!(restarted.generation_id, incumbent.generation_id); + assert_eq!(restarted.transaction_id, continuity_id); + let untouched = resident_id(&world, deploy_id)?; + assert_eq!(untouched.value.incumbent_generation_id, Some(incumbent.generation_id.clone())); + + // The deployment's next step is the rebase and nothing else. + world.engine.advance_transaction(&untouched)?; + let rebased = resident_id(&world, deploy_id)?.value; + assert_eq!(rebased.incumbent_generation_id, Some(restarted.generation_id.clone())); + assert_eq!(rebased.phase, DeploymentPhase::Warming); + assert!(rebased.warming.is_none() && rebased.route_preflight.is_none() && rebased.isolation.is_none()); + assert!(rebased.pre_fencing_abort.is_none()); + assert!(rebased.odin_publisher_sequence_cursor >= restarted.odin_publisher_sequence_cursor); + assert_eq!(rebased.expected, untouched.value.expected); + + // It then fences the generation continuity admitted, and replaces it. + odin_reports_ready(&world, &deploy, restarted.odin_publisher_sequence_cursor + 10)?; + drive_id(&world, deploy_id, |t| t.is_terminal())?; + let landed = latest_id(&world, deploy_id)?; + assert!(matches!(landed.completion, Some(TransactionCompletion::Admitted { .. })), "{landed:?}"); + assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); + assert!(matches!( + landed.post_commit_cleanup.map(|cleanup| cleanup.incumbent), + Some( + IncumbentCleanupEvidence::Pending { generation_id, .. } + | IncumbentCleanupEvidence::Complete { generation_id } + ) if generation_id == restarted.generation_id + )); + let now_admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the deployment admitted nothing")? + .value + .transaction_id + .clone(); + assert_eq!(now_admitted, deploy_id); + Ok(()) + } + + /// Kill the Warming step with the workload: if the gate did not hold the + /// deployment, running the step would be a pre-fencing error and an abort. + #[test] + fn a_deployment_waits_in_warming_while_a_continuity_owns_the_target() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Leasing)?; + workload.kill(); + let deploy_id = deploy.transaction_id.as_str(); + let before = resident_id(&world, deploy_id)?; + + for _ in 0..3 { + let stored = resident_id(&world, deploy_id)?; + world.engine.advance_transaction(&stored)?; + } + let waiting = resident_id(&world, deploy_id)?; + assert_eq!(waiting.value.phase, DeploymentPhase::Warming); + assert!(waiting.value.pre_fencing_abort.is_none()); + assert_eq!(waiting.value.warming, before.value.warming); + assert_eq!(waiting.value.isolation, before.value.isolation); + assert!( + waiting.value.last_error.as_deref().is_some_and(|note| note.contains(&continuity.transaction_id)), + "{:?}", + waiting.value.last_error + ); + + // The build overlaps: a deployment still in Sealing is not gated. + rephase(&world, deploy_id, DeploymentPhase::Sealing)?; + let sealing = resident_id(&world, deploy_id)?; + world.engine.advance_transaction(&sealing)?; + let after = resident_id(&world, deploy_id)?.value; + assert!( + after.last_error.as_deref().is_some_and(|note| !note.contains(&continuity.transaction_id)), + "Sealing was held by the owner: {:?}", + after.last_error + ); + + // The owner completes having moved the generation: the next step is the + // rebase, and with the target free the deployment goes on. + rephase(&world, deploy_id, DeploymentPhase::Warming)?; + workload.revive(); + clear_other_live_transactions(&world, deploy_id)?; + edit_incumbent(&world, |generation| generation.generation_id = "generation-restarted".into())?; + world.engine.advance_transaction(&resident_id(&world, deploy_id)?)?; + let rebased = resident_id(&world, deploy_id)?.value; + assert_eq!(rebased.incumbent_generation_id.as_deref(), Some("generation-restarted")); + assert!(rebased.warming.is_none()); + Ok(()) + } + + #[test] + fn a_continuity_whose_target_was_taken_or_generation_replaced_aborts_itself_and_is_not_rebased() -> Result<()> { + let resume = |world: &EngineFixture, id: &str| -> Result<()> { + for _ in 0..12 { + let stored = resident_id(world, id); + let Ok(stored) = stored else { break }; + world.engine.resume_candidate(&stored)?; + world.engine.resume_backoff.lock().unwrap().clear(); + } + Ok(()) + }; + // Taken: the deployment owns the target; the continuity ends itself and + // the deployment goes on, untouched. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Fencing, DeploymentPhase::Warming)?; + let owner = resident_id(&world, &deploy.transaction_id)?; + resume(&world, &continuity.transaction_id)?; + let ended = latest_id(&world, &continuity.transaction_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.envelope, owner.envelope); + + // Replaced: the generation it restarts is gone. It is not rebased. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, incumbent) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + clear_other_live_transactions(&world, &continuity.transaction_id)?; + edit_incumbent(&world, |generation| generation.generation_id = "generation-elsewhere".into())?; + resume(&world, &continuity.transaction_id)?; + let ended = latest_id(&world, &continuity.transaction_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert_eq!(ended.incumbent_generation_id, Some(incumbent.generation_id)); + let _ = deploy; + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; From 278cc5c408ed3a7ac9b1f066eec65e044c0f3826 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:20:30 +0200 Subject: [PATCH 074/196] Cut 2, commit 4: open the gates; fold in the sustained-reset test reach Freeze refuses only a second live Deploy of a target, so a profile command freezes over crash-looping targets. Supervision defers to an owner, a drain or its own live continuity, never to a deployment that has not taken the target, and the admitted Expected repair runs beside one. Cut 1's interim test (no continuity beside a pre-fencing deployment) goes with the state it asserted. Fold-in c1-sustained-reset-test-reach: the lower edge to the millisecond, the newest attempt (not the oldest) deciding the reset, and the mid-wait crossing recorded in continuity_backoff's doc and pinned. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 245 +++++++++++++++++++++++++++---------------- 1 file changed, 155 insertions(+), 90 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 1229bdd..01005b1 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1244,7 +1244,9 @@ impl TargetSupervision { /// run: the generation has lived the sustained time and no attempt, charged /// or the one re-materialization, has been minted since it was admitted. /// That rule decides first: a sustained death never began a wait, so a - /// standing anchor, however it came to stand, has no say in it. + /// standing anchor, however it came to stand, has no say in it. A wait + /// already running when the generation reaches the sustained time ends + /// there instead: an accepted shortening, always under the cap. fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { let streak = self.continuity_attempt_streak; let sustained = self @@ -5680,8 +5682,8 @@ impl Engine { let continuity_progress = self.supervise_one_admitted_generation()?; // Freezing is not gated on the others' progress: a queued command // whose target is free must not wait behind an unrelated target's - // wedge. `freeze_command` already refuses a busy target, so this is - // still at most one freeze per tick and never a second claimant. + // wedge. `freeze_command` refuses only a second live deployment of a + // target, and this freezes at most one command per tick. let froze = self.freeze_one_queued_command()?; Ok(transaction_progress || continuity_progress || froze) } @@ -5963,8 +5965,10 @@ impl Engine { Ok(false) } - /// Freeze one queued command into its transactions. `Ok(false)` means its - /// target is busy and nothing was written. + /// Freeze one queued command into its transactions. `Ok(false)` means a + /// live deployment already exists for one of its targets and nothing was + /// written. Continuity, owners and drains never refuse a freeze: a + /// deployment that has not taken its target owns nothing. fn freeze_command( &self, snapshot: &ControlSnapshot, @@ -6003,7 +6007,9 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) + .filter(|stored| { + stored.value.command_kind == CommandKind::Deploy && stored.value.is_live() + }) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -6075,8 +6081,13 @@ impl Engine { if on_target().any(|stored| stored.value.owns_target()) { continue; } - let blocked = on_target() - .any(|stored| stored.value.is_live() || stored.value.drains_incumbent()); + // Continuity defers to an owner, a drain or its own live attempt, + // never to a deployment that has not taken the target, and aborts + // nothing (handover-phase-split). + let mint_blocked = snapshot.target_owner(¤t.value.target, None).is_some() + || on_target().any(|stored| { + stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity + }); match self.supervise_admitted_route(&snapshot, current) { Ok(true) => { progressed = true; @@ -6121,28 +6132,26 @@ impl Engine { // the admitted one leaves the projection naming an incarnation that // no longer exists -- which the workload rightly refuses to adopt. // That record is published from observation once the process is up. - if !on_target().any(|stored| stored.value.is_live()) { - let repair = (|| -> Result<()> { - let topology = self.topology(); - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - if !topology.admitted_expected_projection_is_exact( - ¤t.value.expected, - &provider_anchor, - )? { - ensure!( - topology.publish_expected(¤t.value.expected, &provider_anchor)? - == current.value.expected.canonical_sha256()?, - "admitted Expected projection repair differs" - ); - } - Ok(()) - })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target + let repair = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.admitted_expected_projection_is_exact( + ¤t.value.expected, + &provider_anchor, + )? { + ensure!( + topology.publish_expected(¤t.value.expected, &provider_anchor)? + == current.value.expected.canonical_sha256()?, + "admitted Expected projection repair differs" ); } + Ok(()) + })(); + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -6289,9 +6298,7 @@ impl Engine { } let continuity_key = format!("continuity:{}", current.value.target); - // Continuity defers to any live transaction for its target and - // never aborts one (handover-phase-split). - if blocked { + if mint_blocked { continue; } if supervision.restarts_page(now) { @@ -13829,6 +13836,9 @@ mod tests { alive: std::sync::atomic::AtomicBool, rebooted: std::sync::atomic::AtomicBool, silent: std::sync::atomic::AtomicBool, + /// Runtime instances that stay alive when the rest are killed: a + /// candidate building beside a dead incumbent. + spared: std::sync::Mutex>, } impl SwitchWorkload { @@ -13837,9 +13847,15 @@ mod tests { alive: std::sync::atomic::AtomicBool::new(true), rebooted: std::sync::atomic::AtomicBool::new(false), silent: std::sync::atomic::AtomicBool::new(false), + spared: std::sync::Mutex::new(Vec::new()), }) } + /// `kill` leaves this runtime instance running. + fn spare(&self, runtime_instance_id: &str) { + self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); + } + /// The host cannot be asked: observing fails as `HostUnobservable` /// whatever the workload is doing. fn silence(&self, silent: bool) { @@ -13914,8 +13930,14 @@ mod tests { } .into()); } + let spared = match prior { + WorkloadObservation::Systemd(observed) => { + self.spared.lock().unwrap().contains(&observed.runtime_instance_id) + } + WorkloadObservation::Host(_) => false, + }; ensure!( - self.alive.load(std::sync::atomic::Ordering::SeqCst), + spared || self.alive.load(std::sync::atomic::Ordering::SeqCst), "the workload is gone" ); Ok(prior.clone()) @@ -16329,53 +16351,6 @@ mod tests { ); } - /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment - /// waiting before Fencing. Continuity never stops and never yields a - /// deployment (budget-never-stop, handover-phase-split), so the window - /// sliding must not turn the deployment into a casualty. - #[test] - fn a_deployment_before_fencing_over_a_dead_incumbent_that_paged_is_never_yielded_and_lands() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - workload.kill(); - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - world.engine.supervise_one_admitted_generation()?; - // The oldest leaves the window: base would now yield the deployment. - set_meters( - &world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - for _ in 0..3 { - let _ = world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!( - after.pre_fencing_abort.is_none(), - "the deployment was yielded: {:?}", - after.pre_fencing_abort - ); - assert_eq!( - continuity_mints(&world)?, - 0, - "continuity minted beside a pre-fencing deployment" - ); - } - workload.revive(); - odin_reports_ready(&world, &redeploy, seeded_sequence(Some(&incumbent)) + 1)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - let landed = latest(&world)?; - assert_eq!(landed.transaction_id, redeploy.transaction_id); - assert_eq!(landed.phase, DeploymentPhase::Complete); - assert!(landed.pre_fencing_abort.is_none()); - Ok(()) - } - /// The incumbent was admitted this long ago, so an anchor written "in the /// past" is not older than its generation. fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { @@ -16788,7 +16763,7 @@ mod tests { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; let (deploy, continuity, _) = - deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Leasing)?; + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; workload.kill(); let deploy_id = deploy.transaction_id.as_str(); let before = resident_id(&world, deploy_id)?; @@ -16838,7 +16813,9 @@ mod tests { for _ in 0..12 { let stored = resident_id(world, id); let Ok(stored) = stored else { break }; - world.engine.resume_candidate(&stored)?; + // The last step archives the record, which the scheduler's own + // progress read reports as an error; the record is what counts. + let _ = world.engine.resume_candidate(&stored); world.engine.resume_backoff.lock().unwrap().clear(); } Ok(()) @@ -16876,6 +16853,62 @@ mod tests { Ok(()) } + /// A dead incumbent with a deployment building beside it (its candidate + /// alive) whose Warming evidence is complete, and that deployment. + fn dead_incumbent_with_a_deployment_ready_to_fence( + workload: &SwitchWorkload, + world: &EngineFixture, + ) -> Result { + let incumbent = admit_incumbent(world)?; + age_incumbent(world, 500_000)?; + let deploy = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let candidate = deploy.activation.as_ref().context("no activation")?; + workload.spare(&candidate.runtime_instance_id); + workload.kill(); + Ok(deploy) + } + + #[test] + fn a_continuity_minted_as_a_deployment_takes_the_target_is_resolved_next_pass() -> Result<()> { + // (a) The deployment fences first in the tick, so supervision reads its + // ownership and mints nothing. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + assert!(world.engine.run_scheduler_tick()?); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + assert_eq!(continuity_mints(&world)?, 0); + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0); + + // (b) Supervision minted from the older view: the continuity starts at + // Sealing and owns nothing, the deployment takes the target, and the + // continuity is superseded at its next step, never the deployment. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + assert!(world.engine.supervise_one_admitted_generation()?); + let continuity_id = the_continuity(&world)?.value.transaction_id; + assert_eq!(resident_id(&world, &continuity_id)?.value.phase, DeploymentPhase::Sealing); + world.engine.advance_transaction(&resident_id(&world, &deploy.transaction_id)?)?; + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + world.engine.resume_backoff.lock().unwrap().clear(); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!(snapshot.transactions.iter().filter(|stored| stored.value.owns_target()).count() <= 1); + } + let ended = latest_id(&world, &continuity_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert!(latest_id(&world, &deploy.transaction_id)?.pre_fencing_abort.is_none()); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -16980,11 +17013,19 @@ mod tests { meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), ContinuityBackoff::Due { streak: 0 } ); - // A moment short of it. - assert_eq!( - meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + 10_000), - ContinuityBackoff::Begin - ); + // A moment short of it, down to the millisecond. + for short in [10_000, 1_000, 1] { + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + short), + ContinuityBackoff::Begin, + "{short} ms short" + ); + } + // The newest attempt decides, not the oldest: one before admission and + // one after it, on a generation that has lived long enough. + meters.continuity_restarts = vec![now - 800_000, now - 100_000]; + assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); + meters.continuity_restarts = vec![1_000]; // Attempts after admission failed on this generation: it has not run well. meters.continuity_restarts = vec![now - 100_000]; assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); @@ -17008,6 +17049,29 @@ mod tests { assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); } + #[test] + fn a_wait_that_crosses_the_sustained_run_ends_early_and_only_by_less_than_the_cap() { + let admitted = 50_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 7; + meters.continuity_restarts = vec![admitted - 1_000]; + // The wait began when the generation was 400 s old and runs 300 s. + meters.continuity_backoff_from_unix_millis = Some(admitted + 400_000); + assert_eq!(meters.backoff_due(admitted), Some(admitted + 700_000)); + assert_eq!( + meters.continuity_backoff(admitted + CONTINUITY_SUSTAINED_RUN_MILLIS - 1, admitted), + ContinuityBackoff::Waiting + ); + assert_eq!( + meters.continuity_backoff(admitted + CONTINUITY_SUSTAINED_RUN_MILLIS, admitted), + ContinuityBackoff::Due { streak: 0 } + ); + assert!( + CONTINUITY_SUSTAINED_RUN_MILLIS - 400_000 < CONTINUITY_BACKOFF_CEILING_MILLIS, + "the shortening is under the cap" + ); + } + #[test] fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { let workload = SwitchWorkload::new(); @@ -17057,10 +17121,10 @@ mod tests { all.iter().any(|transaction| transaction.command_id == "up-service-queued"), "the queued deployment never froze" ); - assert_eq!(continuity_mints(&world)?, 0); + // Freeze needs no gap (a deployment before its handover owns nothing), + // and continuity may restart the incumbent beside it after its wait. for _ in 0..3 { let _ = world.engine.run_scheduler_tick(); - assert_eq!(continuity_mints(&world)?, 0); let aborts = every_transaction(&world)? .into_iter() .filter_map(|transaction| transaction.pre_fencing_abort) @@ -17864,10 +17928,11 @@ mod tests { for _ in 0..4 { let _ = world.engine.supervise_one_admitted_generation()?; } - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, held.transaction_id); + let after = resident_id(&world, &held.transaction_id)?.value; assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); assert!(after.completion.is_none()); + // Continuity restarts the declared incumbent beside it, once. + assert_eq!(continuity_mints(&world)?, 1); Ok(()) } From a7bfbc2296ac5a5a42ebf9d0e599d113ead9b5cf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:22:58 +0200 Subject: [PATCH 075/196] Cut 2 tests: a launching switch workload, so a real continuity restart runs beside a deployment Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 01005b1..e2518f4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13839,6 +13839,9 @@ mod tests { /// Runtime instances that stay alive when the rest are killed: a /// candidate building beside a dead incumbent. spared: std::sync::Mutex>, + /// Whether a start launches a process (a fresh private identity) + /// instead of refusing like `StillWorkload`. + launches: std::sync::atomic::AtomicBool, } impl SwitchWorkload { @@ -13848,9 +13851,15 @@ mod tests { rebooted: std::sync::atomic::AtomicBool::new(false), silent: std::sync::atomic::AtomicBool::new(false), spared: std::sync::Mutex::new(Vec::new()), + launches: std::sync::atomic::AtomicBool::new(false), }) } + /// From now on a start launches a process distinct from every recorded one. + fn launching(&self) { + self.launches.store(true, std::sync::atomic::Ordering::SeqCst); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -13897,6 +13906,9 @@ mod tests { expected: &IdunnExpectedIncarnationRecord, launch: IdunnRuntimeActivationLaunch, ) -> Result { + if self.launches.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(launch.activation().clone()); + } StillWorkload.prepare_activation(plan, expected, launch) } fn start_prepared( @@ -13907,7 +13919,25 @@ mod tests { expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) + if !self.launches.load(std::sync::atomic::Ordering::SeqCst) { + return StillWorkload.start_prepared(plan, release, installed, expected, activation); + } + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("recorded transaction has no workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + // Another UID and other namespaces than any recorded unit. + observed.process_uids[0] += 1_000_000; + observed.pid_namespace_id += 1_000_000; + observed.mount_namespace_id += 1_000_000; + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + Ok(workload) } fn discard_prepared( &self, @@ -16704,6 +16734,7 @@ mod tests { let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; let deploy_id = deploy.transaction_id.as_str(); + workload.launching(); workload.kill(); assert!(world.engine.supervise_one_admitted_generation()?); let continuity_id = the_continuity(&world)?.value.transaction_id; From 1953bf233b3aa366eb555e113029ae73a2bec4cc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:57:31 +0200 Subject: [PATCH 076/196] Cut 2: one per-target lookup, and tests that kill the survivors it exposed ControlSnapshot::on_target is the one place a transaction is filed under a target; target_owner, the supervision gates and the handover read it. Tests: ownership is per target and names only a holder; a draining target is not restarted until its cleanup ends; the admitted Expected is repaired beside a building deployment; a live continuity alone holds back a second mint. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 109 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e2518f4..63bfe73 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3408,13 +3408,23 @@ impl ControlSnapshot { /// incumbent slot or is still draining it. The one lookup supervision, a /// waiting deployment and the handover share. fn target_owner(&self, target: &str, except: Option<&str>) -> Option<&Stored> { - self.transactions.iter().find(|stored| { - stored.value.target == target - && Some(stored.value.transaction_id.as_str()) != except + self.on_target(target).find(|stored| { + Some(stored.value.transaction_id.as_str()) != except && (stored.value.owns_target() || stored.value.drains_incumbent()) }) } + /// Every transaction of the target, in whatever state. + fn on_target<'a>( + &'a self, + target: &str, + ) -> impl Iterator> + use<'a> { + let target = target.to_owned(); + self.transactions + .iter() + .filter(move |stored| stored.value.target == target) + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -6070,22 +6080,19 @@ impl Engine { &anyhow::Error::new(disagreement), ); } - let on_target = || { - snapshot - .transactions - .iter() - .filter(|stored| stored.value.target == current.value.target) - }; // A transaction that owns the incumbent slot fences, stops or // replaces this generation: nothing here touches it meanwhile. - if on_target().any(|stored| stored.value.owns_target()) { + if snapshot + .on_target(¤t.value.target) + .any(|stored| stored.value.owns_target()) + { continue; } // Continuity defers to an owner, a drain or its own live attempt, // never to a deployment that has not taken the target, and aborts // nothing (handover-phase-split). let mint_blocked = snapshot.target_owner(¤t.value.target, None).is_some() - || on_target().any(|stored| { + || snapshot.on_target(¤t.value.target).any(|stored| { stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity }); match self.supervise_admitted_route(&snapshot, current) { @@ -9165,13 +9172,8 @@ impl Engine { ]; expected.extend( snapshot - .transactions - .iter() - .filter(|stored| { - stored.value.target == target - && stored.value.transaction_id != id - && !stored.value.is_terminal() - }) + .on_target(target) + .filter(|stored| stored.value.transaction_id != id && !stored.value.is_terminal()) .map(|stored| CultCacheExpectedEnvelope { r#type: DeploymentTransaction::TYPE.into(), key: stored.value.transaction_id.clone(), @@ -16415,6 +16417,8 @@ mod tests { assert_eq!(stored_deploy(&world)?.envelope, before.envelope); for _ in 0..2 { + // Due at once again (streak 0), so only the live attempt holds it back. + set_meters(&world, &TargetSupervision::new("service"))?; let _ = world.engine.supervise_one_admitted_generation()?; assert_eq!(continuity_mints(&world)?, 1); let after = stored_deploy(&world)?; @@ -16884,6 +16888,75 @@ mod tests { Ok(()) } + #[test] + fn ownership_is_read_per_target_and_names_only_a_transaction_that_holds_it() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let mut view = ControlSnapshot::read(&world.state_store)?; + assert_eq!(view.on_target("service").count(), 2); + assert_eq!(view.on_target("elsewhere").count(), 0); + // The pre-handover deployment holds nothing; the continuity holds the slot. + let owner = view.target_owner("service", None).context("no owner")?; + assert_eq!(owner.value.transaction_id, continuity.transaction_id); + assert!(view.target_owner("service", Some(&continuity.transaction_id)).is_none()); + assert!(view.target_owner("elsewhere", None).is_none()); + // The same records filed under another target are that target's. + for stored in &mut view.transactions { + stored.value.target = "elsewhere".into(); + } + assert_eq!(view.on_target("service").count(), 0); + assert!(view.target_owner("service", None).is_none()); + assert_eq!(view.on_target("elsewhere").count(), 2); + assert!(view.target_owner("elsewhere", None).is_some()); + let _ = deploy; + Ok(()) + } + + /// A deployment that has committed still drains the old incumbent; the + /// target is not restarted over that cleanup, and is once it ends. + #[test] + fn a_target_draining_its_old_incumbent_is_not_restarted_until_the_drain_ends() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &deploy, seeded_sequence(Some(&incumbent)) + 1)?; + drive_id(&world, &deploy.transaction_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, &deploy.transaction_id)?.value; + assert!(draining.drains_incumbent() && !draining.owns_target() && !draining.is_live()); + + age_incumbent(&world, 500_000)?; + workload.kill(); + for _ in 0..3 { + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "restarted over a draining incumbent"); + } + drive_id(&world, &deploy.transaction_id, |t| t.is_terminal())?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + Ok(()) + } + + /// A deployment that has not taken the target no longer suppresses the + /// repair of the admitted Expected (only the Expected, only when inexact). + #[test] + fn the_admitted_expected_is_repaired_beside_a_deployment_that_is_building() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let store = world.engine.options.topology_store.clone(); + if store.exists() { + std::fs::remove_file(&store)?; + } + assert_eq!(projected_under(&world, &incumbent.expected)?, Vec::::new()); + + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From dd8707b5a26755e76b39d5bbaa5d9c1f962dcac9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:14:46 +0200 Subject: [PATCH 077/196] Cut 2: pin the candidate-port reservation and the rebase guards directly candidate_ports_in_use is the one reading of which ports admitted, live and draining transactions hold. Tests pin it, select_candidate_port's routed and unrouted answers, and the three conditions under which a transaction is not rebased. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 115 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 63bfe73..0e1683e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3425,6 +3425,22 @@ impl ControlSnapshot { .filter(move |stored| stored.value.target == target) } + /// The candidate ports taken by admitted generations and by transactions + /// that are live or still draining; a finished one has released its port. + fn candidate_ports_in_use(&self) -> BTreeSet { + self.admitted + .iter() + .map(|stored| &stored.value.plan) + .chain( + self.transactions + .iter() + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) + .filter_map(|stored| stored.value.plan.as_ref()), + ) + .filter_map(|plan| plan.candidate_port) + .collect() + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -8074,24 +8090,7 @@ impl Engine { let Some(route) = &binding.route else { return Ok(None); }; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut used = BTreeSet::new(); - for plan in snapshot - .admitted - .iter() - .map(|stored| &stored.value.plan) - .chain( - snapshot - .transactions - .iter() - .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) - .filter_map(|stored| stored.value.plan.as_ref()), - ) - { - if let Some(port) = plan.candidate_port { - used.insert(port); - } - } + let used = ControlSnapshot::read(&self.options.state_store)?.candidate_ports_in_use(); (route.private_port_start..=route.private_port_end) .find(|port| !used.contains(port)) .map(Some) @@ -16957,6 +16956,86 @@ mod tests { Ok(()) } + #[test] + fn a_candidate_port_is_held_by_the_admitted_a_live_and_a_draining_transaction_only() -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let mut view = ControlSnapshot::read(&world.state_store)?; + assert!(view.candidate_ports_in_use().is_empty()); + + // Ports are set in memory: the fixture plans are unrouted. + view.admitted[0].value.plan.candidate_port = Some(18000); + for stored in &mut view.transactions { + let port = if stored.value.transaction_id == deploy.transaction_id { 18001 } else { 18002 }; + stored.value.plan.as_mut().context("no plan")?.candidate_port = Some(port); + } + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001, 18002])); + // Complete with cleanup still pending drains: its port stays held. + let finished = view + .transactions + .iter_mut() + .find(|stored| stored.value.transaction_id == continuity.transaction_id) + .context("no continuity")?; + finished.value.phase = DeploymentPhase::Complete; + finished.value.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Pending, + }); + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001, 18002])); + let finished = view + .transactions + .iter_mut() + .find(|stored| stored.value.transaction_id == continuity.transaction_id) + .context("no continuity")?; + finished.value.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Complete, + }); + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001])); + + // A routed binding gets the first free port of its range; an unrouted one none. + let routed = OperatorBinding::parse(BINDING)?; + assert_eq!(world.engine.select_candidate_port(&routed)?, Some(routed.route.as_ref().context("route")?.private_port_start)); + let (head, tail) = BINDING.split_once("[route]").context("route")?; + let unrouted = OperatorBinding::parse(&format!("{head}[brakes]{}", tail.split_once("[brakes]").context("brakes")?.1))?; + assert_eq!(world.engine.select_candidate_port(&unrouted)?, None); + Ok(()) + } + + #[test] + fn only_a_deployment_before_its_handover_is_rebased() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + // Another generation is admitted under both. + edit_incumbent(&world, |generation| generation.generation_id = "generation-restarted".into())?; + let rebase = |id: &str| -> Result { + world.engine.rebase_onto_admitted(&resident_id(&world, id)?) + }; + // A continuity restarts one generation and no other. + assert!(!rebase(&continuity.transaction_id)?); + assert_ne!( + resident_id(&world, &continuity.transaction_id)?.value.incumbent_generation_id.as_deref(), + Some("generation-restarted") + ); + // A deployment already ending is left to end. + let ending = resident_id(&world, &deploy.transaction_id)?; + let mut aborting = ending.value.clone(); + aborting.pre_fencing_abort = Some(pre_fencing_abort_intent(&aborting, "cancelled by operator")); + replace_transaction(&world.state_store, &ending, &aborting)?; + assert!(!rebase(&deploy.transaction_id)?); + let ending = resident_id(&world, &deploy.transaction_id)?; + let mut live = ending.value.clone(); + live.pre_fencing_abort = None; + replace_transaction(&world.state_store, &ending, &live)?; + // Past its handover the incumbent is pinned, whatever is admitted. + rephase(&world, &deploy.transaction_id, DeploymentPhase::Fencing)?; + assert!(!rebase(&deploy.transaction_id)?); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From 6ea0d127d8ea2242625922f083fe29e32f66d461 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:15:19 +0200 Subject: [PATCH 078/196] Cut 2 test: a finished transaction retired mid-handover is not a lost race Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0e1683e..0b20e33 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17036,6 +17036,33 @@ mod tests { Ok(()) } + #[test] + fn the_handover_does_not_wait_on_a_finished_transaction_being_retired() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + // A committed transaction left resident, finished and not yet retired. + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, seeded_sequence(None) + 1)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + let incumbent = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let view = ControlSnapshot::read(&world.state_store)?; + assert!(view.transactions.iter().any(|stored| stored.value.is_terminal())); + + // It is retired between the read and the handover; that is no race. + assert!(world.engine.retire_one_terminal_transaction()?); + let stored = resident_id(&world, &deploy.transaction_id)?; + assert_eq!( + world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, + Handover::Took + ); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From 5b421c3cc7885265b32948e1ec202b0624296ecc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 13:04:27 +0200 Subject: [PATCH 079/196] Deploy provenance target: no per-deploy operator gate, typed provenance on the transaction Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-target.md | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 docs/deploy-provenance-target.md diff --git a/docs/deploy-provenance-target.md b/docs/deploy-provenance-target.md new file mode 100644 index 0000000..74015c7 --- /dev/null +++ b/docs/deploy-provenance-target.md @@ -0,0 +1,61 @@ +# Deployment provenance instead of an approval gate: target + +Status: target, Self, 2026-10-19 (campaign `idunn-deploy-provenance` in +Eureka's mind). It states the ends. The map will live in +`deploy-provenance-map.md`. + +## Why + +Operator, 2026-10-19: + +> The rule about the deployment brake was instituted by a cautious agent, but +> it doesn't actually suit how GameCult operates. The whole point of Eureka is +> to give the Mind, Persona and Self agency over its own Body. It's requiring +> my approval for every deployment that violates our principles. Yes, we +> certainly want to know which agent approved the deployment and why (this is +> the authority provenance which we might want to add to the transaction), but +> that's not the same as gating every deploy on my approval. + +On the same day a StreamPixels deploy (tx-b40c7240) waited on the brake +because only the root-held operator identity could release it. The agent +running the deploy could neither proceed nor record why it should. + +`F:\Projects\CLAUDE.md` says a deployment brake *may* gate artifact, revision, +configuration, schema, unit or authority changes. It does not say that +operator approval is the gate. + +## Invariants + +1. **No deployment waits on a per-transaction operator approval.** An agent + acting for the Mind, Persona or Self may admit a deployment of a target it + is authorized for. +2. **Every transaction carries typed authority provenance.** That is: who + approved it (agent or instance, and faculty), under which session or + ruling, and why. It is written before Fencing, it is immutable, and + `idunn status` and history show it. It is part of the transaction + document, not a log line. +3. **A brake that survives is a stop, not a gate.** The operator or an + authorized agent can halt deployment of a target, and the halt is itself + provenance-bearing. The deployment brake and the continuity lifecycle brake + stay separate (`F:\Projects\CLAUDE.md` Infrastructure). Neither gates Idunn + itself, and the deployment brake never gates continuity. +4. **Existing invariants hold.** These are `idunn-reincarnation` target r2, + `route-continuity-target.md` 1-7 and `authority-map.md`. + +## Scope + +In: + +- The deployment-brake release path in Idunn and `idunn-provision`. +- The provenance record on the transaction. +- `idunn status` and history projection of the provenance. +- The gamecult-ops runbooks that describe a per-transaction operator release + (`heimdall-discord-launch.md`, `idunn-host-raven.md`, the StreamPixels + runbook). + +Out: + +- `idunn-reincarnation`. This campaign is mapped after that campaign's cut 2 + merges, and coordinates with it rather than bundling. +- Harness permission rules on the workstation, which are the operator's. +- Heimdall's onboarding failures. From 94d3b53b55007063c60f8f5e184f1043ed6a622e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:00:49 +0200 Subject: [PATCH 080/196] Cut 2 fixes: stop-then-start handover fixture, supervision owner skip, rebase preflight, archive-as-progress - Add an engine-level stop-then-start fixture (route-less host-actuator plan) and a test that two such transactions never both enter Starting, in either order; it kills enter taking the CAS only at Fencing and takes_target_at ignoring the rollout. Discharges follow-up stop-first-handover-fixture. - Pin supervision's skip of an owned target: a fenced generation is not refreshed from Odin, and a stop-first owner's stopped incumbent is not restarted. - The rebase test now carries a route preflight, so keeping it dies. - The step that ends a transaction archives it inside its write; the progress read now treats a terminal archived record as progress, not a fault. - enter's doc no longer claims every phase move goes through it. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 364 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 333 insertions(+), 31 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0b20e33..108d5f7 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5894,11 +5894,22 @@ impl Engine { } } let after = ControlSnapshot::read(&self.options.state_store)?; - let live = after + let Some(live) = after .transactions .iter() .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while checking scheduler progress")?; + else { + // The step that ends a transaction archives it inside its own + // write, so it is gone from the live set because it finished: that + // is the step's progress. Gone from both is a fault. + ensure!( + read_history_transactions(&self.options.state_store) + .iter() + .any(|archived| archived.transaction_id == id && archived.is_terminal()), + "transaction disappeared while checking scheduler progress" + ); + return Ok(true); + }; Ok(state_advanced(¤t.value, &live.value)) } @@ -9097,10 +9108,12 @@ impl Engine { } } - /// The one phase-entry seam. Every move into the next phase is written - /// here, and entering the phase where the transaction takes its target - /// (`takes_target_at`) is decided by `take_target`: that CAS is the - /// handover, and nothing else writes that entry. + /// The seam for the forward moves that can be the handover (`transition` + /// and the end of Sealing). Entering the phase where the transaction takes + /// its target (`takes_target_at`) is decided by `take_target`: that CAS is + /// the handover, and nothing else writes that entry. The other writes + /// (Starting to Warming, completions, aborts, same-phase evidence) go to + /// `replace_transaction` directly because none of them enters that phase. fn enter( &self, current: &Stored, @@ -12605,6 +12618,27 @@ mod tests { }) } + /// `binding` with its provider trust anchor and signer pointed at this + /// world's provider identity, enrolled on first use. + fn bind_provider(world: &EngineFixture, binding: &str) -> Result { + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = if provider_path.exists() { + open_service_identity_at::(&provider_path)? + } else { + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + provider + }; + Ok(binding + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id)) + } + /// What makes a target Odin: it provides the rendezvous. fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { @@ -12657,32 +12691,19 @@ mod tests { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, }; - let provider_path = world.root.join("identities/provider.cc"); - let provider_anchor = world.root.join("identities/provider-anchor.cc"); - let provider = if provider_path.exists() { - open_service_identity_at::(&provider_path)? - } else { - let provider = - enroll_service_identity_at::(&provider_path)?; - export_service_identity_trust_anchor(&provider, &provider_anchor)?; - provider - }; - let (binding_head, binding_tail) = BINDING .split_once("[route]") .context("binding has no route table")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail - .split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ) - .replace( - "/etc/gamecult/trust/service.cc", - &provider_anchor.display().to_string(), - ) - .replace("service-runtime-signer", &provider.entry().identity_id); + let binding = bind_provider( + world, + &format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ), + )?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -13987,17 +14008,25 @@ mod tests { /// Admit a generation for "service" through the real phase machine and /// retire its transaction, leaving the world with an incumbent. fn admit_incumbent(world: &EngineFixture) -> Result { + Ok(admit_seeded_incumbent(world)?.0) + } + + /// `admit_incumbent`, and the transaction that was admitted. + fn admit_seeded_incumbent( + world: &EngineFixture, + ) -> Result<(AdmittedGeneration, DeploymentTransaction)> { let seeded = transaction_at(world, DeploymentPhase::Fencing)?; odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; drive(world, |transaction| { transaction.phase == DeploymentPhase::Complete })?; assert!(world.engine.retire_one_terminal_transaction()?); - Ok(ControlSnapshot::read(&world.state_store)? + let admitted = ControlSnapshot::read(&world.state_store)? .admitted_for("service") .context("commit wrote no admitted generation")? .value - .clone()) + .clone(); + Ok((admitted, seeded)) } #[test] @@ -16686,6 +16715,260 @@ mod tests { Ok(()) } + /// A route-less host-actuator target that rolls out stop-then-start: the + /// seeded systemd binding with its runner, workload and rollout swapped. + fn stop_first_binding(world: &EngineFixture) -> Result { + use crate::deployment_plan::tests::BINDING; + const HOST: &str = r#"[runners.rust] +driver = "host-native" +affordances = ["source-read", "artifact-write"] +allowed_programs = ["cargo"] + +[workload] +driver = "host-actuator" +host = "yggdrasil" +host_trust_anchor_store = "/etc/gamecult/trust/host-actuator.cc" +release_root = "/srv/service/releases" +runtime_root = "/run/gamecult/idunn/service" +state_root = "/var/lib/gamecult/service" + +[workload.argument_bindings] +state_root = "/var/lib/gamecult/service" + +[workload.secret_files] +GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" + +"#; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; + let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; + let (identity, rest) = rest.split_once("[route]").context("no route")?; + let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; + bind_provider( + world, + &format!("{head}{HOST}[runtime_identity]{identity}[brakes]{brakes}") + .replace("candidate-then-promote", "stop-then-start"), + ) + } + + /// The seeded transaction `id`, rewritten as the Sealing transaction of a + /// stop-then-start target: the host-actuator plan, release and Expected, + /// and none of the evidence that follows Sealing. + fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + use crate::deployment_plan::tests::{ + RECIPE, artifact_receipt, external_input_receipt, source, + }; + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let binding = stop_first_binding(world)?; + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", "GAMECULT_IDUNN_RUNTIME_BUNDLE", "GAMECULT_RUNTIME_PRESENCE_IDENTITY"]"#, + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + &format!("service-{id}"), + None, + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let mut expected = release.expected_projection(&plan)?; + declare_odin(&mut expected); + let now = now_millis()?; + if next.command_kind == CommandKind::Deploy { + next.frozen_source = Some(FrozenSourceReceipt { + transaction_id: id.into(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + next.deployment_authorization = + Some(deployment_authorization(world, &expected, id, now)?); + } + next.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + next.sealed_release = Some(release); + next.expected = Some(expected); + next.plan = Some(plan); + next.expected_publication_sha256 = None; + next.activation_publication_sha256 = None; + next.activation = None; + next.workload = None; + next.warming = None; + next.isolation = None; + next.enter_phase(DeploymentPhase::Sealing, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// An admitted incumbent of `service`, and beside it a Deploy and a + /// Continuity (their ids, in that order), both stop-then-start at Sealing. + fn stop_first_world( + workload: Arc, + ) -> Result<(EngineFixture, String, String)> { + let world = EngineFixture::with_workload(workload)?; + let incumbent = admit_incumbent(&world)?; + let mut ids = Vec::new(); + for kind in [CommandKind::Deploy, CommandKind::Continuity] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + make_stop_first(&world, &seeded.transaction_id)?; + ids.push(seeded.transaction_id); + } + let continuity = ids.pop().context("no continuity")?; + let deploy = ids.pop().context("no deploy")?; + Ok((world, deploy, continuity)) + } + + /// Stop-then-start takes the target at Starting, by the same CAS as + /// Fencing: of two such transactions at Sealing, whichever enters Starting + /// first owns the target, and the other neither enters nor aborts. + #[test] + fn two_stop_then_start_transactions_never_both_enter_starting() -> Result<()> { + use DeploymentPhase::{Sealing, Starting}; + for deploy_first in [true, false] { + let (world, deploy, continuity) = stop_first_world(Arc::new(StillWorkload))?; + let (first, second) = if deploy_first { + (deploy.clone(), continuity.clone()) + } else { + (continuity.clone(), deploy.clone()) + }; + let stored = resident_id(&world, &first)?; + assert_eq!(stored.value.takes_target_at(), Starting); + assert!(!stored.value.owns_target()); + world.engine.enter(&stored, entering(&stored, Starting)?)?; + let owner = resident_id(&world, &first)?.value; + assert_eq!(owner.phase, Starting); + assert!(owner.owns_target(), "a stop-first transaction at Starting does not own the target"); + + let rival = resident_id(&world, &second)?; + let outcome = world.engine.enter(&rival, entering(&rival, Starting)?); + if deploy_first { + let error = outcome.expect_err("a continuity entered Starting over an owner"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + assert!(format!("{error:#}").contains(&deploy), "{error:#}"); + } else { + outcome?; + let waiting = resident_id(&world, &second)?.value; + assert!( + waiting.last_error.as_deref().is_some_and(|note| note.contains(&continuity)), + "{:?}", + waiting.last_error + ); + } + // The store still reads: one owner, and the rival held at Sealing. + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!(after.transactions.iter().filter(|stored| stored.value.owns_target()).count(), 1); + let held = resident_id(&world, &second)?.value; + assert_eq!(held.phase, Sealing); + assert!(held.pre_fencing_abort.is_none()); + } + Ok(()) + } + + /// A stop-first deployment that has entered Starting has stopped the + /// incumbent by design; supervision must read it as the owner and not + /// restart what it stopped. + #[test] + fn supervision_does_not_restart_the_incumbent_a_stop_first_owner_stopped() -> Result<()> { + let workload = SwitchWorkload::new(); + let (world, deploy, _) = stop_first_world(workload.clone())?; + clear_other_live_transactions(&world, &deploy)?; + age_incumbent(&world, 500_000)?; + let stored = resident_id(&world, &deploy)?; + world.engine.enter(&stored, entering(&stored, DeploymentPhase::Starting)?)?; + workload.kill(); + + let before = incumbent_envelope(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "supervision restarted what the owner stopped"); + assert_eq!(incumbent_envelope(&world)?, before); + + // Released, the same dead incumbent is restarted. + clear_other_live_transactions(&world, "")?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + Ok(()) + } + + /// An owner fencing the incumbent is the only writer of it: supervision + /// skips the generation, so it neither refreshes it from Odin nor repairs + /// or retires around it. + #[test] + fn supervision_leaves_a_generation_its_owner_is_fencing_alone() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (incumbent, admitted) = admit_seeded_incumbent(&world)?; + let owner = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + rephase(&world, &owner.transaction_id, DeploymentPhase::Fencing)?; + // Odin has a newer reading of the incumbent than the one it holds. + odin_republishes(&world, &admitted, seeded_sequence(None) + 5)?; + + let before = incumbent_envelope(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(incumbent_envelope(&world)?, before, "supervision refreshed a generation an owner is fencing"); + + // The same world with no owner takes the reading, so the skip alone held it. + clear_other_live_transactions(&world, "")?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_ne!(incumbent_envelope(&world)?, before); + Ok(()) + } + + /// The step that ends a transaction archives it inside its own write. It + /// left the live set because it finished: progress, not a fault. + #[test] + fn the_step_that_ends_a_transaction_reports_progress_and_no_fault() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let ended = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let id = ended.transaction_id; + let mut last = None; + for _ in 0..20 { + let Ok(current) = resident_id(&world, &id) else { + break; + }; + let moved = world.engine.resume_candidate(¤t)?; + if resident_id(&world, &id).is_err() { + last = Some(moved); + break; + } + } + assert_eq!(last, Some(true), "the ending step reported no progress"); + assert!(latest_id(&world, &id)?.is_terminal()); + assert!(world.engine.fault_reports.lock().unwrap().is_empty()); + + // Through the tick, the way the scheduler runs it: no fault is kept. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let ended = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + for _ in 0..20 { + if resident_id(&world, &ended.transaction_id).is_err() { + break; + } + world.engine.resume_one_transaction()?; + } + assert!(latest_id(&world, &ended.transaction_id)?.is_terminal()); + assert!(world.engine.fault_reports.lock().unwrap().is_empty()); + Ok(()) + } + #[test] fn the_store_holds_one_deploy_one_continuity_and_one_owner_per_target() -> Result<()> { let seed = |first: (CommandKind, DeploymentPhase), second: (CommandKind, DeploymentPhase)| { @@ -16737,6 +17020,24 @@ mod tests { let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; let deploy_id = deploy.transaction_id.as_str(); + // The build recorded a route preflight against the incumbent it saw. + let building = resident_id(&world, deploy_id)?; + let mut preflighted = building.value.clone(); + preflighted.route_preflight = Some(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: building + .value + .activation + .as_ref() + .context("no activation")? + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(b"membership"), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }); + replace_transaction(&world.state_store, &building, &preflighted)?; workload.launching(); workload.kill(); assert!(world.engine.supervise_one_admitted_generation()?); @@ -16758,6 +17059,7 @@ mod tests { assert_eq!(untouched.value.incumbent_generation_id, Some(incumbent.generation_id.clone())); // The deployment's next step is the rebase and nothing else. + assert!(untouched.value.route_preflight.is_some() && untouched.value.warming.is_some()); world.engine.advance_transaction(&untouched)?; let rebased = resident_id(&world, deploy_id)?.value; assert_eq!(rebased.incumbent_generation_id, Some(restarted.generation_id.clone())); From 9ea6c719217a21cc7b7c7f9d344807b1e2031a71 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:01:59 +0200 Subject: [PATCH 081/196] Cut 2 test: the stop-first fixture builds without a network input A host-native runner has no network profile, so the host-actuator recipe drops the external input the systemd recipe fetches. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 108d5f7..3932388 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16755,7 +16755,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// and none of the evidence that follows Sealing. fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { use crate::deployment_plan::tests::{ - RECIPE, artifact_receipt, external_input_receipt, source, + RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); @@ -16769,6 +16769,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, r#"["GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", "GAMECULT_IDUNN_RUNTIME_BUNDLE", "GAMECULT_RUNTIME_PRESENCE_IDENTITY"]"#, ); + // A host-native runner has no network profile to fetch an input with. + let (before_inputs, inputs) = recipe + .split_once("[[external_inputs]]") + .context("recipe has no external inputs")?; + let recipe = format!( + "{before_inputs}[[artifacts]]{}", + inputs.split_once("[[artifacts]]").context("recipe has no artifacts")?.1 + ); let plan = compile_deployment_plan( recipe.as_bytes(), binding.as_bytes(), @@ -16781,7 +16789,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let release = SealedRelease::new( &plan, vec![artifact_receipt()], - vec![external_input_receipt()], + Vec::new(), 120, )?; let mut expected = release.expected_projection(&plan)?; From 3265ccb575eec95293c020e074ed3970c73efe48 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:15:07 +0200 Subject: [PATCH 082/196] Cut c2b test: an abort stops a candidate that holds its lease before revoking it Negative test for ghostlight-abort-lease-before-stop: the candidate holds the lease lock until stopped. Recorded failing at this commit; the fix follows. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3932388..947a031 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13864,6 +13864,11 @@ mod tests { /// Whether a start launches a process (a fresh private identity) /// instead of refusing like `StillWorkload`. launches: std::sync::atomic::AtomicBool, + /// Files whose advisory locks stand for a process that holds a lease + /// for its lifetime: `stop` is what releases them. + holds: std::sync::Mutex>, + /// How many times the process was stopped. + stops: std::sync::atomic::AtomicUsize, } impl SwitchWorkload { @@ -13874,6 +13879,8 @@ mod tests { silent: std::sync::atomic::AtomicBool::new(false), spared: std::sync::Mutex::new(Vec::new()), launches: std::sync::atomic::AtomicBool::new(false), + holds: std::sync::Mutex::new(Vec::new()), + stops: std::sync::atomic::AtomicUsize::new(0), }) } @@ -13882,6 +13889,24 @@ mod tests { self.launches.store(true, std::sync::atomic::Ordering::SeqCst); } + /// The process takes a shared lock on `store`'s authority lock, as a + /// candidate holding its lifetime write lease does, until stopped. + #[cfg(unix)] + fn hold_lease_lock(&self, store: &std::path::Path) { + use std::os::unix::io::AsRawFd; + let mut lock = store.as_os_str().to_os_string(); + lock.push(".lock"); + let file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(lock) + .unwrap(); + // SAFETY: flock on a descriptor this function owns. + assert_eq!(unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) }, 0); + self.holds.lock().unwrap().push(file); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -13995,6 +14020,8 @@ mod tests { Ok(prior.clone()) } fn stop(&self, _: &WorkloadObservation) -> Result<()> { + self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.holds.lock().unwrap().clear(); Ok(()) } fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { @@ -21881,6 +21908,34 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// ghostlight-abort-lease-before-stop. A candidate that holds its + /// lifetime write lease keeps the lease's shared lock until it is + /// stopped, so the abort stops it before revoking; revoking first + /// fails on the lock on every attempt and the abort never ends. + #[cfg(unix)] + #[test] + fn an_abort_stops_a_candidate_that_holds_its_lease_before_revoking_it() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + let lease_path = routed.world.root.join("lease/process-write-lease.cc"); + routed.workload.hold_lease_lock(&lease_path); + let driver = CultCacheWriteLeaseDriver::new("service", &lease_path); + + pass_deadline(&routed.world)?; + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + let stops = routed.workload.stops.load(Ordering::SeqCst); + let finished = routed.resume_until_terminal()?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.last_error + ); + assert!(driver.observe_empty()?, "the abort left the lease on disk"); + assert!(routed.workload.stops.load(Ordering::SeqCst) > stops, "the candidate was never stopped"); + Ok(()) + } + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word /// carries the granted lease, and that Ready records the adoption, /// sourced from Odin's topology and naming the presence Odin observed. From ffb22e368b5cfd1ad74760184387b2e52b9f2d30 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:15:47 +0200 Subject: [PATCH 083/196] Cut c2b: the post-fencing abort stops its candidate before revoking the lifetime-held lease ruling c2b-abort-stops-before-revoke; fixes follow-up ghostlight-abort-lease-before-stop. Mirrors advance_fencing. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 947a031..e745693 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -9355,8 +9355,9 @@ impl Engine { } /// Withdraw the candidate in the one order that never leaves two writers: - /// take away its route, then its write lease, then the process itself, and - /// only then reconcile the projection. + /// take away its route, then stop the process and withdraw its write lease + /// (the lease is lifetime-held, so the holder stops first), then clean up, + /// and only then reconcile the projection. fn advance_post_fencing_abort(&self, current: &Stored) -> Result<()> { let abort = required( ¤t.value.post_fencing_abort, @@ -9396,6 +9397,15 @@ impl Engine { .process_write_lease .context("leased target has no write-lease binding")? .record_path; + // The candidate holds its lifetime write lease; revoking it under + // a live holder fails on the shared lock forever. A stopped + // process writes nothing, so stop first (as advance_fencing does). + if let Some(workload) = ¤t.value.workload { + let plan = required(¤t.value.plan, "abandoned candidate plan")?; + self.workload_for(plan)?.stop(workload).context( + "stopping the abandoned candidate before revoking its lifetime-held write lease", + )?; + } let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); driver .revoke_exact(Some(lease)) From 86926c4e426f4473f5557f93e216ce94f9b7c266 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:23:59 +0200 Subject: [PATCH 084/196] Cut c2b: one derivation of every deadline; Warming, aborts and drains are timed WARMING_DEADLINE_SECONDS and PRE_FENCING_ABORT_DEADLINE_SECONDS are Idunn constants (no PhaseDeadlines field: plan ids stay). DEADLINE_PHASES is the one spelling of the deadline range. deadline_for_current_activity derives the duration of a phase, an abort from its beginning, or a drain to its retire; retime is the only non-test writer. leaves_to_operator is derived from it. Validation asks for an activity, not a post-fencing phase. Records written by this cut are forward-only (ruling c2b-forward-only-records). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 310 ++++++++++++++++++++++++++++++++++++----- src/deployment_plan.rs | 7 + 2 files changed, 283 insertions(+), 34 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e745693..c9179c6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -35,7 +35,8 @@ use crate::deployment::{ capability_compatible, }; use crate::deployment_plan::{ - CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, + CompiledDeploymentPlan, DependencyProviderAuthority, PRE_FENCING_ABORT_DEADLINE_SECONDS, + SealedRelease, WARMING_DEADLINE_SECONDS, compile_deployment_plan, }; use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, @@ -925,8 +926,15 @@ impl LeaseAdoptionEvidence { } } -/// When the transaction entered its current post-fencing phase and when that -/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. +/// The phases whose entry starts a deadline: the one spelling of it. +const DEADLINE_PHASES: std::ops::RangeInclusive = + DeploymentPhase::Warming..=DeploymentPhase::Committing; + +/// What a live record is doing now and when that must end: a deadline phase +/// from its entry, an abort from its beginning, a drain to its retire. The +/// post-fencing durations come from the plan's frozen `PhaseDeadlines`; the +/// rest are Idunn's. `DeploymentTransaction::deadline_for_current_activity` +/// is the one derivation. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct PhaseDeadline { @@ -936,7 +944,8 @@ struct PhaseDeadline { } impl PhaseDeadline { - /// `None` for a phase that has no deadline (pre-fencing and Complete). + /// `None` for a phase that has no deadline of its own (Sealing, Starting + /// and Complete). fn entering( phase: DeploymentPhase, plan: &CompiledDeploymentPlan, @@ -944,6 +953,7 @@ impl PhaseDeadline { ) -> Option { let deadlines = plan.phase_deadlines(); let seconds = match phase { + DeploymentPhase::Warming => WARMING_DEADLINE_SECONDS, DeploymentPhase::Fencing => deadlines.fencing_seconds, DeploymentPhase::Leasing => deadlines.leasing_seconds, DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, @@ -959,9 +969,12 @@ impl PhaseDeadline { } } -/// How the deadline resolver ends a post-fencing phase. +/// How the deadline resolver ends a deadline phase. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum PhaseEnd { + /// Before the fence the transaction owns only its candidate: it ends + /// through its own pre-fencing abort. + AbortBeforeFencing, /// Abort through the post-fencing path. `lease_not_adopted` names a /// granted lease with no adoption evidence. Abort { lease_not_adopted: bool }, @@ -1720,25 +1733,112 @@ fn handover_phase(stops_incumbent_first: bool) -> DeploymentPhase { } impl DeploymentTransaction { - /// The one writer of `phase`. It stamps the update time and the phase - /// deadline together, so a record outside Fencing..=Committing carries no - /// deadline by construction. No other code assigns `phase`. + /// The one writer of `phase`. It stamps the update time and, through + /// `retime`, the deadline of what the record now does. No other code + /// assigns `phase`. fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { self.phase = phase; self.updated_at_unix_millis = now; - self.phase_deadline = self - .plan - .as_ref() - .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); + self.retime(now); } - /// A live transaction past the fence that is not already being aborted: - /// the only kind whose phase a deadline, or an operator's expiry, ends. + /// Whether the abort of the pre-fence path still has cleanup to do. + fn pre_fencing_abort_pending(&self) -> bool { + self.completion.is_none() + && self + .pre_fencing_abort + .as_ref() + .is_some_and(|abort| !abort.is_complete()) + } + + /// The one derivation of every duration the record is held to. (1) A + /// pending pre-fencing abort is timed from now by Idunn's constant. (2) A + /// draining Complete is timed from its incumbent's retire time (the route + /// promotion plus the drain, or now when nothing was promoted) for the + /// plan's committing duration. (3) Any other Complete has none. (4) The + /// rest take their phase's duration from now, so a post-fencing abort gets + /// the frozen duration of the phase it aborts, counted from its beginning. + fn deadline_for_current_activity(&self, now: u64) -> Option { + if self.pre_fencing_abort_pending() { + return Some(PhaseDeadline { + phase: self.phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: now.saturating_add( + u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000, + ), + }); + } + if self.phase == DeploymentPhase::Complete { + if !matches!(self.completion, Some(TransactionCompletion::Admitted { .. })) + || !self.drains_incumbent() + { + return None; + } + let plan = self.plan.as_ref()?; + let from = match self + .routing + .as_ref() + .and_then(RoutingEvidence::promoted_at_unix_millis) + { + Some(promoted_at) => { + route_drain_deadline(promoted_at, plan.parsed_inputs().ok()?.1.rollout.drain_seconds) + .ok()? + } + None => now, + }; + return Some(PhaseDeadline { + phase: self.phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: from + .max(now) + .saturating_add(u64::from(plan.phase_deadlines().committing_seconds) * 1000), + }); + } + PhaseDeadline::entering(self.phase, self.plan.as_ref()?, now) + } + + /// Stamp the deadline of what the record does now. The only non-test + /// writer that sets one: phase entry, an abort's beginning, the commit to a + /// draining Complete and the resolver's lift call it. + fn retime(&mut self, now: u64) { + self.phase_deadline = self.deadline_for_current_activity(now); + } + + /// Whether a deadline can describe the record's current activity: a + /// deadline phase, a pending pre-fencing abort, or an admitted Complete. + fn carries_a_deadline(&self) -> bool { + DEADLINE_PHASES.contains(&self.phase) + || self.pre_fencing_abort_pending() + || (self.phase == DeploymentPhase::Complete + && matches!(self.completion, Some(TransactionCompletion::Admitted { .. }))) + } + + /// A live transaction in a deadline phase that is not already being + /// aborted: the only kind whose phase a deadline, or an operator's expiry, + /// ends. fn resolves_by_deadline(&self) -> bool { self.completion.is_none() && self.pre_fencing_abort.is_none() && self.post_fencing_abort.is_none() - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) + && DEADLINE_PHASES.contains(&self.phase) + } + + /// Whether Idunn has run out of ways to end this record and leaves it to + /// the operator: its deadline has passed in an abort that cannot finish, a + /// drain whose stop keeps failing, or a stateful deploy. Derived, never + /// stored. The step goes on retrying and the target stays owned. + fn leaves_to_operator(&self, now: u64) -> bool { + if self.is_terminal() { + return false; + } + let Some(deadline) = &self.phase_deadline else { + return false; + }; + now >= deadline.deadline_at_unix_millis + && (self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some() + || self.drains_incumbent() + || (self.resolves_by_deadline() && self.phase_end() == PhaseEnd::OperatorRequired)) } /// What an ended post-fencing phase resolves to (Q3). A candidate that was @@ -1750,6 +1850,9 @@ impl DeploymentTransaction { /// new state contract, and restoring the incumbent over that is not /// Idunn's call: it is left for the operator. fn phase_end(&self) -> PhaseEnd { + if self.phase < DeploymentPhase::Fencing { + return PhaseEnd::AbortBeforeFencing; + } let leasing = self.leasing.as_ref(); let granted = leasing.and_then(LeasingEvidence::lease).is_some(); let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); @@ -2150,11 +2253,10 @@ impl DeploymentTransaction { if let Some(deadline) = &self.phase_deadline { ensure!( deadline.phase == self.phase - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&deadline.phase) + && self.carries_a_deadline() && deadline.entered_at_unix_millis > 0 && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, - "phase deadline does not describe the current post-fencing phase" + "phase deadline does not describe the record's current activity" ); } if let Some(adoption) = &self.lease_adoption { @@ -3012,7 +3114,7 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> ); ensure!( current.value.resolves_by_deadline(), - "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" ); let now = now_millis()?; let request = ExpiryRequest { @@ -7849,6 +7951,9 @@ impl Engine { generation_id: generation.generation_id.clone(), }); complete.post_commit_cleanup = Some(post_commit_cleanup); + // enter_phase ran before the cleanup was set: a draining record + // carries its drain's deadline. + complete.retime(now); complete.validate()?; let admitted_expected = CultCacheExpectedEnvelope { @@ -9205,9 +9310,10 @@ impl Engine { /// command an operator asked to expire, is resolved as `phase_end` says. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// - /// The deadline is written only by `enter_phase`, so nothing here or in a - /// resume path extends it. A record lifted from before deadlines existed - /// has none; its phase is timed once, from the first time this sees it. + /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the + /// commit and this lift), so nothing in a step or a resume path extends one. + /// A record lifted from before deadlines existed has none; what it does is + /// timed once, from the first time this sees it. fn resolve_phase_deadline( &self, current: &Stored, @@ -9250,6 +9356,10 @@ impl Engine { self.begin_post_fencing_abort(current, error)?; Ok(Some(true)) } + PhaseEnd::AbortBeforeFencing => { + self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; + Ok(Some(true)) + } PhaseEnd::OperatorRequired => { self.report_once( &format!("deadline:{}", transaction.transaction_id), @@ -9313,8 +9423,10 @@ impl Engine { "pre-fencing abort intent is already durable" ); let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); + let now = now_millis()?; self.persist_same_phase(current, |next| { next.pre_fencing_abort = Some(abort); + next.retime(now); Ok(()) }) } @@ -9348,8 +9460,10 @@ impl Engine { "post-fencing abort intent is already durable" ); let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); + let now = now_millis()?; self.persist_same_phase(current, |next| { next.post_fencing_abort = Some(abort); + next.retime(now); Ok(()) }) } @@ -12391,11 +12505,12 @@ mod tests { } #[test] - fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { + fn phase_deadlines_follow_the_plan_and_cover_warming_through_committing() -> Result<()> { let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; let plan = &generation.plan; let defaults = plan.phase_deadlines(); for (phase, seconds) in [ + (DeploymentPhase::Warming, WARMING_DEADLINE_SECONDS), (DeploymentPhase::Fencing, defaults.fencing_seconds), (DeploymentPhase::Leasing, defaults.leasing_seconds), (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), @@ -12410,7 +12525,6 @@ mod tests { for phase in [ DeploymentPhase::Sealing, DeploymentPhase::Starting, - DeploymentPhase::Warming, DeploymentPhase::Complete, ] { assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); @@ -12467,7 +12581,7 @@ mod tests { } #[test] - fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { + fn a_deadline_must_describe_the_records_current_activity() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; let plan = transaction.plan.clone().unwrap(); let now = transaction.updated_at_unix_millis; @@ -12482,10 +12596,25 @@ mod tests { backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; assert!(backwards.validate().is_err()); // Entry is not ordered against the wall-clock stamp of later writes. - let mut entered_after_the_stamp = transaction; + let mut entered_after_the_stamp = transaction.clone(); entered_after_the_stamp.phase_deadline = PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); entered_after_the_stamp.validate()?; + + // The activity rule: Warming carries a deadline; Sealing only while + // its abort is pending; Complete only once admitted. + let mut other = transaction.clone(); + other.phase = DeploymentPhase::Warming; + assert!(other.carries_a_deadline()); + other.phase = DeploymentPhase::Sealing; + assert!(!other.carries_a_deadline(), "Sealing has no activity to time"); + other.pre_fencing_abort = Some(pre_fencing_abort_intent(&other, "refused")); + assert!(other.carries_a_deadline()); + other.pre_fencing_abort = None; + other.phase = DeploymentPhase::Complete; + assert!(!other.carries_a_deadline()); + other.completion = Some(TransactionCompletion::Admitted { generation_id: "g".into() }); + assert!(other.carries_a_deadline()); Ok(()) } @@ -13019,8 +13148,8 @@ mod tests { /// One engine step of transaction `id` at a time until `stop` holds, /// recording each phase entered and checking after every step that the - /// deadline on the record is exactly the current post-fencing phase's, and - /// absent outside one. + /// deadline on the record is exactly the current phase's, and absent + /// outside a deadline phase. fn drive_id( world: &EngineFixture, id: &str, @@ -13038,10 +13167,12 @@ mod tests { let after = latest_id(world, id)?; match &after.phase_deadline { Some(deadline) => assert_eq!(deadline.phase, after.phase), + // A waiter's Warming deadline is disarmed while another + // transaction owns the target. None => assert!( - !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&after.phase), - "a post-fencing record lost its deadline" + after.phase == DeploymentPhase::Warming + || !DEADLINE_PHASES.contains(&after.phase), + "a deadline-phase record lost its deadline" ), } if phases.last() != Some(&after.phase) { @@ -13087,15 +13218,28 @@ mod tests { } #[test] - fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { + fn a_pre_fence_abort_is_timed_from_its_beginning_and_ends_without_a_deadline() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; transaction_at(&world, DeploymentPhase::Warming)?; let warming = resident(&world)?; - assert_eq!(warming.value.phase_deadline, None); + let warming_deadline = warming.value.phase_deadline.context("Warming has no deadline")?; + assert_eq!( + warming_deadline.deadline_at_unix_millis - warming_deadline.entered_at_unix_millis, + u64::from(WARMING_DEADLINE_SECONDS) * 1000 + ); + let before = now_millis()?; world .engine .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; + let aborting = latest(&world)?; + let deadline = aborting.phase_deadline.context("the abort has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Warming); + assert!(deadline.entered_at_unix_millis >= before); + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000 + ); drive(&world, |transaction| transaction.completion.is_some())?; let finished = latest(&world)?; assert_eq!(finished.phase, DeploymentPhase::Complete); @@ -13139,6 +13283,96 @@ mod tests { Ok(()) } + #[test] + fn an_abort_is_timed_from_its_beginning_with_its_phases_duration() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Routing + })?; + // The phase's own deadline is long past when the abort begins. + let routing = resident(&world)?; + let mut stale = routing.value.clone(); + let deadline = stale.phase_deadline.as_mut().context("Routing has no deadline")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, &routing, &stale)?; + + let before = now_millis()?; + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let aborting = latest(&world)?; + let deadline = aborting.phase_deadline.context("the abort has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Routing); + assert!(deadline.entered_at_unix_millis >= before); + let routing_millis = + u64::from(aborting.plan.as_ref().unwrap().phase_deadlines().routing_seconds) * 1000; + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + routing_millis + ); + assert!(!aborting.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(aborting.leaves_to_operator(deadline.deadline_at_unix_millis)); + Ok(()) + } + + #[test] + fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + let (_, mut draining) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + let plan = draining.plan.clone().unwrap(); + let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; + let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; + assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); + draining.phase = DeploymentPhase::Complete; + draining.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-new".into(), + }); + draining.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Pending { + generation_id: "generation-old".into(), + workload: workload(1001, 40, 50), + }, + source: SourceCleanupEvidence::Complete, + }); + let promoted_at = 10_000_000_u64; + match draining.routing.as_mut() { + Some(RoutingEvidence::Promoted { promoted_at_unix_millis, .. }) => { + *promoted_at_unix_millis = promoted_at; + } + other => bail!("the fixture is not promoted: {other:?}"), + } + + // Before the retire time: the retire time plus the committing duration. + draining.retime(promoted_at + 1); + let deadline = draining.phase_deadline.context("a drain has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Complete); + assert_eq!(deadline.entered_at_unix_millis, promoted_at + 1); + assert_eq!(deadline.deadline_at_unix_millis, promoted_at + drain + committing); + assert!(!draining.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(draining.leaves_to_operator(deadline.deadline_at_unix_millis)); + + // After it: the committing duration from now. + let late = promoted_at + drain + 5_000; + draining.retime(late); + assert_eq!(draining.phase_deadline.unwrap().deadline_at_unix_millis, late + committing); + + // Nothing promoted: from now. + draining.routing = Some(RoutingEvidence::SkippedUnrouted); + draining.retime(promoted_at); + assert_eq!(draining.phase_deadline.unwrap().deadline_at_unix_millis, promoted_at + committing); + + // A finished drain, and any other Complete, carry none. + draining.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Complete { generation_id: "generation-old".into() }, + source: SourceCleanupEvidence::Complete, + }); + draining.retime(promoted_at); + assert_eq!(draining.phase_deadline, None); + Ok(()) + } + #[test] fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; @@ -16277,6 +16511,7 @@ mod tests { let stored = resident(&world)?; let mut starting = seeded.clone(); starting.phase = DeploymentPhase::Starting; + starting.phase_deadline = None; starting.expected_publication_sha256 = None; starting.activation = None; starting.workload = None; @@ -16314,6 +16549,7 @@ mod tests { let stored = resident(world)?; let mut starting = seeded.clone(); starting.phase = DeploymentPhase::Starting; + starting.phase_deadline = None; starting.expected_publication_sha256 = None; starting.activation = None; starting.workload = None; @@ -21645,8 +21881,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[test] fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + // Warming is a deadline phase: the operator can expire it, and + // the pre-fencing abort then ends it. Once aborting it cannot be + // expired again. + expire(&early.world.state_store, "continuity-service", "operator")?; + early.resume()?; + assert!(early.transaction()?.pre_fencing_abort.is_some(), "the expiry did not end Warming"); let refused = expire(&early.world.state_store, "continuity-service", "operator") - .expect_err("a transaction before the fence was expired"); + .expect_err("an aborting transaction was expired"); assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 603c9d2..1363c48 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -475,6 +475,13 @@ pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300; pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120; pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120; +/// Idunn's own deadlines, not binding-declared: a new `PhaseDeadlines` field +/// would change every v3 plan id (`recomputed_plan_id` re-encodes the plan). +/// Warming is timed for every transaction, owner or not. +pub const WARMING_DEADLINE_SECONDS: u32 = 600; +/// A pre-fencing abort is timed from its beginning. +pub const PRE_FENCING_ABORT_DEADLINE_SECONDS: u32 = 300; + /// The resolved per-phase deadline durations a plan carries. Entering a /// post-fencing phase stamps its deadline from them, and the deadline /// resolver ends the phase once it passes. From a14055115cbf0f06dbcd653ebbc71383b6ecd044 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:37:21 +0200 Subject: [PATCH 085/196] Cut c2b: owner deadlines; a waiter never waits on an owner Idunn will not finish wait_for_owner disarms a waiter Warming deadline (the owner bounds the wait) and ends the waiter through its own pre-fencing abort when the owner leaves to the operator (owner-left-to-operator); it never aborts the owner. The resolver times Warming for every transaction, lifts a deadline onto a record that needs one, skips an owned waiter, and leaves an abort or drain past its deadline to the operator (report only; the incident follows). Rulings c2b-waiter-ends-on- operator-owner, c2b-warming-deadline, c2b-abort-and-drain-deadlines. Negative tests recorded failing at 9ea6c71. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 408 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 383 insertions(+), 25 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c9179c6..d408ee6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -8372,6 +8372,49 @@ impl Engine { } } + /// A deployment waits on the transaction that owns or drains its target. + /// The owner bounds the wait, so the waiter's own Warming deadline is + /// disarmed (the resolver re-arms it when the target is free). When the + /// owner's deadline has passed in a state Idunn will not end by itself, + /// the waiter ends through its own pre-fencing abort (the caller's Err + /// path): it never aborts the owner. + fn wait_for_owner( + &self, + current: &Stored, + owner_id: &str, + ) -> Result<()> { + let held = ¤t.value; + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if snapshot.transactions.iter().any(|stored| { + stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) + }) { + bail!( + "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", + held.target + ); + } + let detail = waiting_for_owner(&held.target, owner_id); + if held.last_error.as_deref() == Some(detail.as_str()) && held.phase_deadline.is_none() { + return Ok(()); + } + let mut next = held.clone(); + next.last_error = Some(detail); + next.phase_deadline = None; + next.updated_at_unix_millis = now; + replace_transaction(&self.options.state_store, current, &next) + } + + /// Idunn has run out of ways to end this record. It stays owned and its + /// step keeps retrying; the operator is told. + fn leave_to_operator(&self, transaction: &DeploymentTransaction, cause: &str) { + let id = &transaction.transaction_id; + self.report_once( + &format!("deadline:{id}"), + format!("Idunn leaves {id} to the operator: {cause}; it keeps retrying the step"), + ); + } + fn record_gate_wait( &self, current: &Stored, @@ -9185,11 +9228,8 @@ impl Engine { else { return Ok(false); }; - self.record_gate_wait( - current, - &waiting_for_owner(&held.target, &owner.value.transaction_id), - )?; - Ok(true) + self.wait_for_owner(current, &owner.value.transaction_id) + .map(|()| true) } CommandKind::Continuity => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; @@ -9238,7 +9278,7 @@ impl Engine { (Handover::Took | Handover::Lost, _) => Ok(()), // Wait, never abort: a pre-fencing error would end the deployment. (Handover::OwnedBy(owner), CommandKind::Deploy) => { - self.record_gate_wait(current, &waiting_for_owner(target, &owner)) + self.wait_for_owner(current, &owner) } // The next step rebases onto the generation now admitted. (Handover::Moved, CommandKind::Deploy) => Ok(()), @@ -9305,33 +9345,61 @@ impl Engine { Ok(if took { Handover::Took } else { Handover::Lost }) } - /// Every post-fencing phase ends. Before a live transaction past the - /// fence runs its step, a phase whose deadline has passed, or whose - /// command an operator asked to expire, is resolved as `phase_end` says. + /// Every live record is held to a deadline. Before a transaction runs its + /// step, a deadline phase whose deadline has passed, or whose command an + /// operator asked to expire, is resolved as `phase_end` says; an abort or + /// a drain past its deadline is left to the operator and its step retries. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the - /// commit and this lift), so nothing in a step or a resume path extends one. - /// A record lifted from before deadlines existed has none; what it does is - /// timed once, from the first time this sees it. + /// commit and this lift), so nothing in a step or a resume path extends + /// one. A record with none whose activity needs one is timed once, from + /// the first time this sees it: that is also how a waiter is re-armed once + /// the target is free. fn resolve_phase_deadline( &self, current: &Stored, ) -> Result> { let transaction = ¤t.value; - if !transaction.resolves_by_deadline() { + if transaction.is_terminal() { + return Ok(None); + } + // An owned waiter is timed by its owner. + if transaction.phase < DeploymentPhase::Fencing + && transaction.pre_fencing_abort.is_none() + && ControlSnapshot::read(&self.options.state_store)? + .target_owner(&transaction.target, Some(&transaction.transaction_id)) + .is_some() + { return Ok(None); } let now = now_millis()?; let Some(deadline) = transaction.phase_deadline else { let mut next = transaction.clone(); - next.enter_phase(transaction.phase, now); + next.retime(now); if next.phase_deadline.is_none() { return Ok(None); } replace_transaction(&self.options.state_store, current, &next)?; return Ok(Some(true)); }; + if !transaction.resolves_by_deadline() { + if transaction.leaves_to_operator(now) { + let what = if transaction.drains_incumbent() { + "draining its incumbent" + } else { + "its abort" + }; + self.leave_to_operator( + transaction, + &format!( + "{what} in the {:?} phase passed its deadline at unix ms {}", + transaction.phase, deadline.deadline_at_unix_millis + ), + ); + } + return Ok(None); + } let requested = ControlSnapshot::read(&self.options.state_store)? .expiry_requests .iter() @@ -9347,6 +9415,10 @@ impl Engine { return Ok(None); }; match transaction.phase_end() { + PhaseEnd::AbortBeforeFencing => { + self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; + Ok(Some(true)) + } PhaseEnd::Abort { lease_not_adopted } => { let error = if lease_not_adopted { anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") @@ -9356,17 +9428,10 @@ impl Engine { self.begin_post_fencing_abort(current, error)?; Ok(Some(true)) } - PhaseEnd::AbortBeforeFencing => { - self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; - Ok(Some(true)) - } PhaseEnd::OperatorRequired => { - self.report_once( - &format!("deadline:{}", transaction.transaction_id), - format!( - "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", - transaction.transaction_id - ), + self.leave_to_operator( + transaction, + &format!("{cause}, and its stateful candidate was issued a write lease"), ); Ok(None) } @@ -14113,6 +14178,8 @@ mod tests { holds: std::sync::Mutex>, /// How many times the process was stopped. stops: std::sync::atomic::AtomicUsize, + /// Runtime instances whose stop fails, whatever is asked. + failing_stops: std::sync::Mutex>, } impl SwitchWorkload { @@ -14125,6 +14192,7 @@ mod tests { launches: std::sync::atomic::AtomicBool::new(false), holds: std::sync::Mutex::new(Vec::new()), stops: std::sync::atomic::AtomicUsize::new(0), + failing_stops: std::sync::Mutex::new(Vec::new()), }) } @@ -14151,6 +14219,11 @@ mod tests { self.holds.lock().unwrap().push(file); } + /// From now on stopping this runtime instance fails. + fn fail_stop_of(&self, runtime_instance_id: &str) { + self.failing_stops.lock().unwrap().push(runtime_instance_id.to_owned()); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -14263,8 +14336,14 @@ mod tests { ); Ok(prior.clone()) } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { + fn stop(&self, stopping: &WorkloadObservation) -> Result<()> { self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if let WorkloadObservation::Systemd(observed) = stopping { + ensure!( + !self.failing_stops.lock().unwrap().contains(&observed.runtime_instance_id), + "the unit would not stop" + ); + } self.holds.lock().unwrap().clear(); Ok(()) } @@ -16844,6 +16923,254 @@ mod tests { Ok((deploy, continuity, incumbent)) } + /// One scheduler pass over every live transaction, with no resume backoff + /// standing between it and the step. + fn resume_pass(world: &EngineFixture) -> Result { + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_one_transaction() + } + + /// The runtime instance whose workload `id` runs. + fn runtime_instance_of(world: &EngineFixture, id: &str) -> Result { + match resident_id(world, id)?.value.workload { + Some(WorkloadObservation::Systemd(observed)) => Ok(observed.runtime_instance_id), + other => bail!("transaction {id} has no systemd workload: {other:?}"), + } + } + + /// The deadline of transaction `id` is already behind it. + fn pass_deadline_of(world: &EngineFixture, id: &str) -> Result<()> { + let current = resident_id(world, id)?; + let mut next = current.value.clone(); + let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, ¤t, &next) + } + + /// NEGATIVE (a). A continuity that owns the target is aborting and its + /// abort step fails on every attempt. Its deadline passes: Idunn leaves it + /// to the operator and keeps it owned, and the deployment waiting behind it + /// ends itself before fencing instead of waiting forever. + #[test] + fn a_waiting_deployment_ends_when_its_owners_abort_is_left_to_the_operator() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + workload.fail_stop_of(&runtime_instance_of(&world, owner_id)?); + world + .engine + .begin_post_fencing_abort(&resident_id(&world, owner_id)?, anyhow!("candidate refused"))?; + + // Before the deadline the deployment waits on a bounded owner. + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, deploy_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); + assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); + + pass_deadline_of(&world, owner_id)?; + for _ in 0..10 { + resume_pass(&world)?; + } + let owner = resident_id(&world, owner_id)?.value; + assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); + assert!(owner.owns_target()); + let ended = latest_id(&world, deploy_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); + assert!(ended.fencing.is_none()); + Ok(()) + } + + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop + /// keeps failing. A deployment waits behind it until the drain's deadline, + /// then ends itself; the drain stays and the target stays owned. + #[test] + fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; + let drainer_id = drainer.transaction_id.as_str(); + drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, drainer_id)?.value; + assert!(draining.drains_incumbent(), "the incumbent was already retired"); + assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); + match &draining.workload { + Some(WorkloadObservation::Systemd(_)) => {} + other => bail!("no workload: {other:?}"), + } + let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = + &draining.post_commit_cleanup + else { + bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); + }; + let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; + workload.fail_stop_of(&old.runtime_instance_id); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; + let waiter_id = waiter.transaction_id.as_str(); + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, waiter_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); + assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); + + pass_deadline_of(&world, drainer_id)?; + for _ in 0..10 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + let ended = latest_id(&world, waiter_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(drainer_id), "{error}"); + Ok(()) + } + + /// A stop-then-start transaction held in Warming with a started unit, as + /// the Starting step leaves it (this world serves no host actuator to + /// start one). + fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let expected = next.expected.clone().context("no Expected")?; + let now = now_millis()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&next.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + next.expected_publication_sha256 = Some(expected.canonical_sha256()?); + next.activation_publication_sha256 = Some(activation.canonical_sha256()?); + next.activation = Some(activation); + next.workload = Some(workload); + next.enter_phase(DeploymentPhase::Warming, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting + /// and waits in Warming for an Odin word that never comes. Its Warming + /// deadline ends it, and the deployment waiting at its Sealing tail takes + /// the target. + #[test] + fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( + ) -> Result<()> { + use DeploymentPhase::{Starting, Warming}; + let (world, deploy, continuity) = stop_first_world(SwitchWorkload::new())?; + warm_stop_first(&world, &continuity)?; + let owner = resident_id(&world, &continuity)?; + assert_eq!(owner.value.phase, Warming); + assert!(owner.value.owns_target() && owner.value.phase_deadline.is_some()); + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, Starting)?)?; + let held = resident_id(&world, &deploy)?.value; + assert_eq!(held.phase, DeploymentPhase::Sealing); + assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); + + pass_deadline_of(&world, &continuity)?; + world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; + let aborting = resident_id(&world, &continuity)?.value; + let abort = aborting.pre_fencing_abort.clone().context("the deadline did not end Warming")?; + assert!(abort.error.contains("Warming phase passed its deadline"), "{}", abort.error); + // This world serves no host actuator, so the abandoned unit cannot be + // stopped here; the abort's cleanup is recorded done. + let mut cleaned = aborting.clone(); + let intent = cleaned.pre_fencing_abort.as_mut().unwrap(); + intent.candidate_cleanup = CleanupEvidence::Complete; + intent.topology_reconciliation = CleanupEvidence::Complete; + replace_transaction(&world.state_store, &resident_id(&world, &continuity)?, &cleaned)?; + for _ in 0..12 { + if resident_id(&world, &continuity).is_err() { + break; + } + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; + } + let ended = latest_id(&world, &continuity)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("Warming phase passed its deadline"), "{error}"); + + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, Starting)?)?; + let took = resident_id(&world, &deploy)?.value; + assert_eq!(took.phase, Starting); + assert!(took.owns_target() && took.pre_fencing_abort.is_none()); + Ok(()) + } + + #[test] + fn a_waiters_warming_deadline_is_disarmed_while_an_owner_holds_the_target_and_rearmed_when_it_ends( + ) -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + rephase(&world, deploy_id, DeploymentPhase::Warming)?; + assert!(resident_id(&world, deploy_id)?.value.phase_deadline.is_some()); + + resume_pass(&world)?; + let waiting = resident_id(&world, deploy_id)?.value; + assert_eq!(waiting.phase_deadline, None, "the owner does not bound the waiter"); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); + // Held for as long as the owner stands: a deadline that fell due + // meanwhile would otherwise abort it. + for _ in 0..3 { + resume_pass(&world)?; + } + assert!(resident_id(&world, deploy_id)?.value.pre_fencing_abort.is_none()); + + // The owner retires without moving the generation. The waiter's own + // entry is long past, but it is timed from now. + clear_other_live_transactions(&world, deploy_id)?; + let stored = resident_id(&world, deploy_id)?; + let mut aged = stored.value.clone(); + aged.created_at_unix_millis = 1; + aged.updated_at_unix_millis = 1; + replace_transaction(&world.state_store, &stored, &aged)?; + let retired = now_millis()?; + resume_pass(&world)?; + let rearmed = resident_id(&world, deploy_id)?.value; + let deadline = rearmed.phase_deadline.context("the waiter was not re-armed")?; + assert_eq!(deadline.phase, DeploymentPhase::Warming); + assert!(deadline.entered_at_unix_millis >= retired); + assert!(rearmed.pre_fencing_abort.is_none()); + Ok(()) + } + #[test] fn exactly_one_transaction_takes_a_target_from_one_view() -> Result<()> { let take = |snapshot: &ControlSnapshot, world: &EngineFixture, id: &str| -> Result { @@ -21876,6 +22203,37 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// A candidate no one ever answers for is ended by its Warming + /// deadline, or by the operator's expiry: it owns only itself. + #[test] + fn a_candidate_that_never_correlates_aborts_before_fencing_at_its_warming_deadline() -> Result<()> { + for by_operator in [false, true] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..3 { + routed.resume()?; + } + let waiting = routed.transaction()?; + assert!(waiting.pre_fencing_abort.is_none(), "it ended before its deadline"); + assert!(waiting.phase_deadline.is_some()); + assert_eq!(waiting.phase, DeploymentPhase::Warming); + + let cause = if by_operator { + expire(&routed.world.state_store, "continuity-service", "operator")?; + "expired by the operator" + } else { + pass_deadline(&routed.world)?; + "Warming phase passed its deadline" + }; + routed.resume()?; + let abort = routed.transaction()?.pre_fencing_abort.context("it was not aborted")?; + assert!(abort.error.contains(cause), "{}", abort.error); + let ended = routed.resume_until_terminal()?; + assert!(matches!(ended.completion, Some(TransactionCompletion::FailedBeforeFencing { .. }))); + } + Ok(()) + } + /// `idunn expire` writes a request and nothing else; the resolver ends /// the phase from it, and the spent request is deleted. #[test] From 126d710de3edb997e1a40ce2415e6ba80a920af1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:39:42 +0200 Subject: [PATCH 086/196] Cut c2b: operator-required incident, opened at the decision site and closed when spent IncidentCondition::OperatorRequired (subject: transaction id) opens in leave_to_operator, only for a record that leaves_to_operator, and closes as recovered in close_spent_operator_incidents once no transaction in the snapshot is left to the operator. Incident writes are never progress. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 90 +++++++++++++++++++++++++++++++++++++++++++- src/incident.rs | 27 ++++++++++--- 2 files changed, 110 insertions(+), 7 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d408ee6..8732e4c 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5880,6 +5880,8 @@ impl Engine { fn resume_one_transaction(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; + // Incident writes are never progress. + self.close_spent_operator_incidents(&snapshot, now_millis()?); let mut candidates = snapshot .transactions .iter() @@ -8406,9 +8408,19 @@ impl Engine { } /// Idunn has run out of ways to end this record. It stays owned and its - /// step keeps retrying; the operator is told. + /// step keeps retrying; the operator is told. The incident comes first and + /// the trace second, and only for a record that is left to the operator + /// (an operator's expiry of a stateful deploy is reported, not paged). fn leave_to_operator(&self, transaction: &DeploymentTransaction, cause: &str) { let id = &transaction.transaction_id; + let now = now_millis(); + if now.as_ref().is_ok_and(|now| transaction.leaves_to_operator(*now)) { + self.record_incident( + "open", + id, + now.and_then(|now| self.incidents().open(IncidentCondition::OperatorRequired, id, now)), + ); + } self.report_once( &format!("deadline:{id}"), format!("Idunn leaves {id} to the operator: {cause}; it keeps retrying the step"), @@ -9438,6 +9450,24 @@ impl Engine { } } + /// Close the operator-required incident of every transaction that is no + /// longer left to the operator: it ended, or its step went on. A + /// transaction that is still left keeps its incident open. + fn close_spent_operator_incidents(&self, snapshot: &ControlSnapshot, now: u64) { + let store = self.incidents(); + let close = || -> Result<()> { + for subject in store.open_subjects(IncidentCondition::OperatorRequired)? { + if !snapshot.transactions.iter().any(|stored| { + stored.value.transaction_id == subject && stored.value.leaves_to_operator(now) + }) { + store.close(IncidentCondition::OperatorRequired, &subject, CloseReason::Recovered, now)?; + } + } + Ok(()) + }; + self.record_incident("close", "*", close()); + } + /// Delete expiry requests whose command has no phase left to end: it was /// resolved, it finished, or its command is gone. fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { @@ -14219,6 +14249,11 @@ mod tests { self.holds.lock().unwrap().push(file); } + /// Every stop works again. + fn mend_stops(&self) { + self.failing_stops.lock().unwrap().clear(); + } + /// From now on stopping this runtime instance fails. fn fail_stop_of(&self, runtime_instance_id: &str) { self.failing_stops.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -16964,7 +16999,9 @@ mod tests { .engine .begin_post_fencing_abort(&resident_id(&world, owner_id)?, anyhow!("candidate refused"))?; - // Before the deadline the deployment waits on a bounded owner. + // Before the deadline the deployment waits on a bounded owner, and no + // one is paged. + assert!(world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); for _ in 0..3 { resume_pass(&world)?; } @@ -16980,6 +17017,11 @@ mod tests { let owner = resident_id(&world, owner_id)?.value; assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); assert!(owner.owns_target()); + // Exactly one incident, however many passes ran. + assert_eq!( + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [owner_id] + ); let ended = latest_id(&world, deploy_id)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); @@ -16989,6 +17031,46 @@ mod tests { Ok(()) } + #[test] + fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let id = seeded.transaction_id.as_str(); + workload.fail_stop_of(&runtime_instance_of(&world, id)?); + world.engine.begin_post_fencing_abort(&resident_id(&world, id)?, anyhow!("candidate refused"))?; + pass_deadline_of(&world, id)?; + let open = || world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired); + for _ in 0..4 { + resume_pass(&world)?; + } + assert_eq!(open()?, [id]); + let records = world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + + // Still left to the operator: still open, and not written again. + resume_pass(&world)?; + assert_eq!(world.engine.incidents().read()?, records); + + // The step succeeds and the transaction ends; the next pass closes it. + workload.mend_stops(); + for _ in 0..12 { + if resident_id(&world, id).is_err() { + break; + } + resume_pass(&world)?; + } + assert!(latest_id(&world, id)?.is_terminal()); + resume_pass(&world)?; + assert!(open()?.is_empty()); + let closed = world.engine.incidents().read()?; + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + resume_pass(&world)?; + assert_eq!(world.engine.incidents().read()?, closed, "a spent incident was written again"); + Ok(()) + } + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop /// keeps failing. A deployment waits behind it until the drain's deadline, /// then ends itself; the drain stays and the target stays owned. @@ -17037,6 +17119,10 @@ mod tests { resume_pass(&world)?; } assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + assert_eq!( + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [drainer_id] + ); let ended = latest_id(&world, waiter_id)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); diff --git a/src/incident.rs b/src/incident.rs index 1d7f095..a0941ff 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -32,6 +32,9 @@ pub(crate) const INCIDENT_RETENTION_MILLIS: u64 = 7 * 24 * 60 * 60 * 1000; #[serde(rename_all = "kebab-case")] pub(crate) enum IncidentCondition { ContinuityExhausted, + /// A transaction past its deadline in a state Idunn will not end by + /// itself. Subject: the transaction id. + OperatorRequired, } impl IncidentCondition { @@ -40,6 +43,7 @@ impl IncidentCondition { pub(crate) const fn name(self) -> &'static str { match self { Self::ContinuityExhausted => "continuity-exhausted", + Self::OperatorRequired => "operator-required", } } @@ -49,6 +53,7 @@ impl IncidentCondition { pub(crate) const fn ends_with_admission(self) -> bool { match self { Self::ContinuityExhausted => true, + Self::OperatorRequired => false, } } } @@ -271,6 +276,17 @@ impl IncidentStore { Ok(true) } + /// The subjects of every open incident of `condition`. + pub(crate) fn open_subjects(&self, condition: IncidentCondition) -> Result> { + Ok(self + .snapshot()? + .records + .into_iter() + .filter(|record| record.is_open() && record.condition == condition) + .map(|record| record.subject) + .collect()) + } + /// Close the open incident for `(condition, subject)`, if there is one. /// `Ok(true)` means this call wrote the closure. pub(crate) fn close( @@ -468,11 +484,12 @@ pub(crate) mod tests { ); // The wire spelling readers match on is the spelling the key and // status use. - let payload = rmp_serde::to_vec(&IncidentCondition::ContinuityExhausted).unwrap(); - assert_eq!( - rmp_serde::from_slice::(&payload).unwrap(), - IncidentCondition::ContinuityExhausted.name() - ); + for condition in [IncidentCondition::ContinuityExhausted, IncidentCondition::OperatorRequired] { + let payload = rmp_serde::to_vec(&condition).unwrap(); + assert_eq!(rmp_serde::from_slice::(&payload).unwrap(), condition.name()); + } + assert_eq!(IncidentCondition::OperatorRequired.name(), "operator-required"); + assert!(!IncidentCondition::OperatorRequired.ends_with_admission()); let payload = rmp_serde::to_vec(&CloseReason::NoLongerAdmitted).unwrap(); assert_eq!( rmp_serde::from_slice::(&payload).unwrap(), From 0bfadcfc717908be51933255c75b0cae0c5dd9e2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 16:10:11 +0200 Subject: [PATCH 087/196] Cut c2b test: pin the stateful-deploy arm of leaves_to_operator and the incident close predicate Survivors of cargo mutants --in-diff on 9ea6c71..126d710: the OperatorRequired clause of leaves_to_operator (now a table beside phase_end) and the close predicate (a vanished subject closes while another transaction is still left). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8732e4c..3169542 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13948,6 +13948,15 @@ mod tests { transaction.leasing = leasing.clone(); transaction.lease_adoption = lease_adoption.clone(); assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); + // Only a stateful deploy is left to the operator when its phase ends. + transaction.retime(1_000); + let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; + assert_eq!( + transaction.leaves_to_operator(due), + expected == PhaseEnd::OperatorRequired, + "{kind:?} {leasing:?} {lease_adoption:?}" + ); + assert!(!transaction.leaves_to_operator(due - 1)); let recovery = if kind == Continuity && lease_adoption.is_some() { TerminalRecovery::RestartAdmitted } else { @@ -17048,6 +17057,13 @@ mod tests { let records = world.engine.incidents().read()?; assert_eq!(records.len(), 1); + // An incident whose transaction is gone closes even while another is + // still left to the operator. + world.engine.incidents().open(IncidentCondition::OperatorRequired, "tx-vanished", now_millis()?)?; + resume_pass(&world)?; + assert_eq!(open()?, [id]); + let records = world.engine.incidents().read()?; + // Still left to the operator: still open, and not written again. resume_pass(&world)?; assert_eq!(world.engine.incidents().read()?, records); @@ -17064,8 +17080,8 @@ mod tests { resume_pass(&world)?; assert!(open()?.is_empty()); let closed = world.engine.incidents().read()?; - assert_eq!(closed.len(), 1); - assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + assert!(closed.iter().all(|record| record.close_reason == Some(CloseReason::Recovered))); + assert_eq!(closed.iter().filter(|record| record.subject == id).count(), 1); resume_pass(&world)?; assert_eq!(world.engine.incidents().read()?, closed, "a spent incident was written again"); Ok(()) From 21849eb09753cccf0286105a4ff1e91ce30f533f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:48:03 +0200 Subject: [PATCH 088/196] Cut c2b fix: a drain whose stored plan no longer parses is bounded, not unbounded deadline_for_current_activity's drain arm returned None when parsed_inputs failed (the running binary re-parses and re-admits the frozen binding), which left the drain with no deadline, unpaged, and disarmed every waiter behind it. The retire time is now the promotion plus the drain when the binding parses, and otherwise now; the frozen committing duration from it bounds the drain. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3169542..0f4385b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1775,17 +1775,20 @@ impl DeploymentTransaction { return None; } let plan = self.plan.as_ref()?; - let from = match self + // The retire time needs the binding's drain, which the running + // binary re-parses and re-admits: a plan frozen by an older binary + // may not parse now. That must not leave a drain with no deadline, + // so a retire time that cannot be derived is now, and the + // committing duration from it bounds the drain. + let from = self .routing .as_ref() .and_then(RoutingEvidence::promoted_at_unix_millis) - { - Some(promoted_at) => { - route_drain_deadline(promoted_at, plan.parsed_inputs().ok()?.1.rollout.drain_seconds) - .ok()? - } - None => now, - }; + .and_then(|promoted_at| { + let (_, binding) = plan.parsed_inputs().ok()?; + route_drain_deadline(promoted_at, binding.rollout.drain_seconds).ok() + }) + .unwrap_or(now); return Some(PhaseDeadline { phase: self.phase, entered_at_unix_millis: now, From a254a3e6a5f767201f07ee7d9892d821076cfe24 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:49:14 +0200 Subject: [PATCH 089/196] Cut c2b fix: pin the forever-waits at the waiter, the pre-fence abort deadline, the stateful incident lifecycle and the archive vanish Negative tests (a)(b)(c) are one marked harness that ages a deadline only when one exists and ends on the waiter's own record, so at the base they fail at the waiter assertion (still waiting), not at setup. (c) now runs the Odin gate-wait it names: a stop-then-start continuity on a systemd unit observes and waits, where the host-actuator step of a world with no actuator errs at once. New: a stop-first owner's wedged pre-fence abort pages and ends its waiter; a waiter's own wedged pre-fence abort pages beside a standing owner; a stateful deploy's incident is silent under an early expiry, one per episode, and closes while the record goes on; an unparsable drain plan is bounded; a transaction gone from both live set and history stays a fault. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 635 +++++++++++++++++++++++++++++++------------ 1 file changed, 467 insertions(+), 168 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0f4385b..43810c4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13417,12 +13417,24 @@ mod tests { } #[test] - fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + fn a_drain_whose_stored_plan_no_longer_parses_is_still_bounded() -> Result<()> { + let (mut draining, promoted_at) = promoted_draining_record()?; + let committing = u64::from(draining.plan.as_ref().unwrap().phase_deadlines().committing_seconds) * 1000; + // A binding the running binary refuses: the drain seconds cannot be read. + draining.plan.as_mut().unwrap().binding_blob = b"not an operator binding".to_vec(); + assert!(draining.plan.as_ref().unwrap().parsed_inputs().is_err()); + draining.retime(promoted_at + 1); + let deadline = draining.phase_deadline.context("an unparsable plan left the drain with no deadline")?; + assert_eq!(deadline.deadline_at_unix_millis, promoted_at + 1 + committing); + assert!(!draining.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(draining.leaves_to_operator(deadline.deadline_at_unix_millis)); + Ok(()) + } + + /// A Complete record draining its incumbent, whose route was promoted at + /// the returned time. + fn promoted_draining_record() -> Result<(DeploymentTransaction, u64)> { let (_, mut draining) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; - let plan = draining.plan.clone().unwrap(); - let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; - let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; - assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); draining.phase = DeploymentPhase::Complete; draining.completion = Some(TransactionCompletion::Admitted { generation_id: "generation-new".into(), @@ -13441,6 +13453,16 @@ mod tests { } other => bail!("the fixture is not promoted: {other:?}"), } + Ok((draining, promoted_at)) + } + + #[test] + fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + let (mut draining, promoted_at) = promoted_draining_record()?; + let plan = draining.plan.clone().unwrap(); + let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; + let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; + assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); // Before the retire time: the retire time plus the committing duration. draining.retime(promoted_at + 1); @@ -15789,6 +15811,104 @@ mod tests { Ok(()) } + /// A workload port whose observation deletes every live transaction from + /// the control store once armed: the step ends, and the record is simply + /// gone, archived nowhere. + struct VanishingWorkload { + store: Mutex>, + armed: std::sync::atomic::AtomicBool, + } + + impl WorkloadPort for VanishingWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + if self.armed.load(std::sync::atomic::Ordering::SeqCst) { + let path = self.store.lock().unwrap().clone().context("no store")?; + let gone = ControlSnapshot::read(&path)? + .transactions + .into_iter() + .map(|stored| stored.envelope) + .collect::>(); + assert!(SingleFileMessagePackBackingStore::new(&path).delete_batch_if_unchanged(&gone)?); + } + StillWorkload.observe(expected, activation, prior) + } + fn stop(&self, observation: &WorkloadObservation) -> Result<()> { + StillWorkload.stop(observation) + } + fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result { + StillWorkload.is_permanently_stopped(observation) + } + fn lost_to_host_reboot(&self, observation: &WorkloadObservation, at: u64) -> Result { + StillWorkload.lost_to_host_reboot(observation, at) + } + } + + /// The step that ends a transaction archives it in its own write, so a + /// record gone from the live set is finished only if history holds it. One + /// that vanished from both is a fault, never progress. + #[test] + fn a_transaction_gone_from_both_the_live_set_and_history_is_a_fault() -> Result<()> { + let workload = Arc::new(VanishingWorkload { + store: Mutex::new(None), + armed: std::sync::atomic::AtomicBool::new(false), + }); + let world = EngineFixture::with_workload(workload.clone())?; + *workload.store.lock().unwrap() = Some(world.state_store.clone()); + let live = transaction_at(&world, DeploymentPhase::Warming)?; + let id = live.transaction_id.as_str(); + // The Warming step waits for Odin; waiting again rewrites nothing, so + // the step succeeds without touching the record it will find missing. + world.engine.resume_candidate(&resident_id(&world, id)?)?; + assert!(resident_id(&world, id)?.value.last_error.is_some(), "the Warming step did not wait"); + workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); + world.engine.resume_backoff.lock().unwrap().clear(); + let error = world + .engine + .resume_candidate(&resident_id(&world, id)?) + .expect_err("a record gone from both stores was read as a finished one"); + assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); + assert!(read_history_transactions(&world.state_store).is_empty()); + Ok(()) + } + // ---- fix 2: supervision owns the Expected-only precondition ---- #[test] @@ -16970,6 +17090,13 @@ mod tests { Ok((deploy, continuity, incumbent)) } + // BEGIN c2b-forever-wait harness + // The forever-waits of cut c2b, driven through the API that exists at + // 9ea6c71 (the base): each test ends on the waiter's own record, so at the + // base it fails at the waiter assertion (still waiting), never at setup. + // Lines ending in `head-only` read what only the cut adds (the incident + // store, `leaves_to_operator`); the base probe strips them. + /// One scheduler pass over every live transaction, with no resume backoff /// standing between it and the step. fn resume_pass(world: &EngineFixture) -> Result { @@ -16995,6 +17122,32 @@ mod tests { replace_transaction(&world.state_store, ¤t, &next) } + /// The deadline of `id`, when it has one, is already behind it. A record + /// that has none is left alone: at the base nothing times it, and the + /// scenario must still run to the waiter's assertion. + fn age_deadline_of(world: &EngineFixture, id: &str) -> Result<()> { + if resident_id(world, id)?.value.phase_deadline.is_some() { + pass_deadline_of(world, id)?; + } + Ok(()) + } + + /// The deployment `waiter_id` ended before fencing, with a cause naming + /// the owner it will not wait for. + fn assert_ended_for_owner(world: &EngineFixture, waiter_id: &str, owner_id: &str) -> Result<()> { + let ended = latest_id(world, waiter_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); + assert!(ended.fencing.is_none()); + Ok(()) + } + + fn operator_required(world: &EngineFixture) -> Result> { + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) // head-only + } + /// NEGATIVE (a). A continuity that owns the target is aborting and its /// abort step fails on every attempt. Its deadline passes: Idunn leaves it /// to the operator and keeps it owned, and the deployment waiting behind it @@ -17013,36 +17166,266 @@ mod tests { // Before the deadline the deployment waits on a bounded owner, and no // one is paged. - assert!(world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); + assert!(operator_required(&world)?.is_empty()); // head-only for _ in 0..3 { resume_pass(&world)?; } let waiting = resident_id(&world, deploy_id)?.value; assert!(waiting.pre_fencing_abort.is_none()); assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); - assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); + assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); // head-only - pass_deadline_of(&world, owner_id)?; - for _ in 0..10 { + age_deadline_of(&world, owner_id)?; + for _ in 0..20 { resume_pass(&world)?; } let owner = resident_id(&world, owner_id)?.value; assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); assert!(owner.owns_target()); // Exactly one incident, however many passes ran. - assert_eq!( - world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, - [owner_id] - ); - let ended = latest_id(&world, deploy_id)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + assert_eq!(operator_required(&world)?, [owner_id]); // head-only + assert_ended_for_owner(&world, deploy_id, owner_id) + } + + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop + /// keeps failing. A deployment waits behind it until the drain's deadline, + /// then ends itself; the drain stays and the target stays owned. + #[test] + fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; + let drainer_id = drainer.transaction_id.as_str(); + drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, drainer_id)?.value; + assert!(draining.drains_incumbent(), "the incumbent was already retired"); + assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); // head-only + let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = + &draining.post_commit_cleanup + else { + bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); }; - assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); - assert!(ended.fencing.is_none()); + let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; + workload.fail_stop_of(&old.runtime_instance_id); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; + let waiter_id = waiter.transaction_id.as_str(); + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, waiter_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); + assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); + assert!(operator_required(&world)?.is_empty()); // head-only + + age_deadline_of(&world, drainer_id)?; + for _ in 0..20 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + assert_eq!(operator_required(&world)?, [drainer_id]); // head-only + assert_ended_for_owner(&world, waiter_id, drainer_id) + } + + /// A stop-then-start transaction held in Warming with a started unit, as + /// the Starting step leaves it. + fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let expected = next.expected.clone().context("no Expected")?; + let now = now_millis()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&next.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + next.expected_publication_sha256 = Some(expected.canonical_sha256()?); + next.activation_publication_sha256 = Some(activation.canonical_sha256()?); + next.activation = Some(activation); + next.workload = Some(workload); + next.enter_phase(DeploymentPhase::Warming, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// The stop-then-start binding of `stop_first_binding`, on a systemd unit + /// instead of a host actuator: its Warming step observes the unit through + /// the world's workload port and waits for Odin's word, where the + /// host-actuator step of a world that serves no actuator errs at once. + fn stop_first_systemd_binding(world: &EngineFixture) -> Result { + use crate::deployment_plan::tests::BINDING; + let host_runner = "[runners.rust]\ndriver = \"host-native\"\naffordances = [\"source-read\", \"artifact-write\"]\nallowed_programs = [\"cargo\"]\n\n"; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; + let (_, rest) = rest.split_once("[workload]").context("no workload")?; + let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; + let (identity, rest) = rest.split_once("[route]").context("no route")?; + let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; + bind_provider( + world, + &format!("{head}{host_runner}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") + .replace("candidate-then-promote", "stop-then-start"), + ) + } + + /// A stop-then-start deployment and continuity over one incumbent, on a + /// systemd unit. The continuity is held in Warming with a started unit and + /// the deployment, one `enter` later, waits at its Sealing tail on it. + fn warm_stop_first_owner( + workload: Arc, + ) -> Result<(EngineFixture, String, String)> { + let world = EngineFixture::with_workload(workload)?; + let incumbent = admit_incumbent(&world)?; + let binding = stop_first_systemd_binding(&world)?; + let mut ids = Vec::new(); + for kind in [CommandKind::Deploy, CommandKind::Continuity] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + make_stop_first(&world, &seeded.transaction_id, &binding)?; + ids.push(seeded.transaction_id); + } + let continuity = ids.pop().context("no continuity")?; + let deploy = ids.pop().context("no deploy")?; + warm_stop_first(&world, &continuity)?; + let owner = resident_id(&world, &continuity)?; + assert_eq!(owner.value.phase, DeploymentPhase::Warming); + assert!(owner.value.owns_target()); + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?)?; + let held = resident_id(&world, &deploy)?.value; + assert_eq!(held.phase, DeploymentPhase::Sealing); + assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); + Ok((world, deploy, continuity)) + } + + /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting + /// and waits in Warming for an Odin word that never comes: its step runs, + /// observes the unit and gate-waits. Its Warming deadline ends it, and the + /// deployment waiting at its Sealing tail takes the target. + #[test] + fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( + ) -> Result<()> { + let (world, deploy, continuity) = warm_stop_first_owner(SwitchWorkload::new())?; + // The gate-wait is the Warming step itself, not an error of it. + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, &continuity)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.pre_fencing_abort.is_none(), "the Warming step failed: {:?}", waiting.last_error); + assert!(waiting.last_error.is_some(), "the Warming step did not wait for Odin"); + assert!(resident_id(&world, &deploy)?.value.last_error.as_deref().is_some_and(|n| n.contains(&continuity))); + + age_deadline_of(&world, &continuity)?; + for _ in 0..12 { + if resident_id(&world, &continuity).is_err() { + break; + } + resume_pass(&world)?; + } + let ended = latest_id(&world, &continuity)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("Warming phase passed its deadline"), "{error}"); + assert!(operator_required(&world)?.is_empty()); // head-only + + // The pass that retired the owner, or the next, moves the waiter to + // Starting; stop there, before its Starting step runs. + for _ in 0..3 { + if resident_id(&world, &deploy)?.value.phase >= DeploymentPhase::Starting { + break; + } + resume_pass(&world)?; + } + let took = resident_id(&world, &deploy)?.value; + assert_eq!(took.phase, DeploymentPhase::Starting); + assert!(took.owns_target() && took.pre_fencing_abort.is_none()); Ok(()) } + /// The wedged pre-fence abort of a stop-then-start owner is paged too: the + /// waiter behind it ends, and one incident opens. + #[test] + fn a_waiting_deployment_ends_when_a_stop_first_owners_prefence_abort_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let (world, deploy, owner) = warm_stop_first_owner(workload.clone())?; + workload.fail_stop_of(&runtime_instance_of(&world, &owner)?); + world + .engine + .begin_pre_fencing_abort(&resident_id(&world, &owner)?, anyhow!("candidate refused"))?; + for _ in 0..3 { + resume_pass(&world)?; + } + // Before the abort's deadline: it waits, unpaged. + let aborting = resident_id(&world, &owner)?.value; + assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); + assert!(operator_required(&world)?.is_empty()); // head-only + assert!(resident_id(&world, &deploy)?.value.pre_fencing_abort.is_none()); + + age_deadline_of(&world, &owner)?; + for _ in 0..20 { + resume_pass(&world)?; + } + let aborting = resident_id(&world, &owner)?.value; + assert!(!aborting.is_terminal() && aborting.owns_target(), "the owner was ended"); + assert_eq!(operator_required(&world)?, [owner.as_str()]); // head-only + assert_ended_for_owner(&world, &deploy, &owner) + } + + /// A waiter whose own pre-fence abort is wedged, beside an owner it will + /// not outlast, is paged by its own deadline: the owner's standing does not + /// excuse it. + #[test] + fn a_waiters_wedged_prefence_abort_is_paged_beside_a_standing_owner() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + workload.fail_stop_of(&runtime_instance_of(&world, deploy_id)?); + world + .engine + .begin_pre_fencing_abort(&resident_id(&world, deploy_id)?, anyhow!("candidate refused"))?; + for _ in 0..3 { + resume_pass(&world)?; + } + let aborting = resident_id(&world, deploy_id)?.value; + assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); + assert!(!operator_required(&world)?.iter().any(|subject| subject == deploy_id)); // head-only + + age_deadline_of(&world, deploy_id)?; + for _ in 0..3 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, deploy_id)?.value.is_terminal()); + assert!(operator_required(&world)?.iter().any(|subject| subject == deploy_id), "{owner_id}"); // head-only + Ok(()) + } + // END c2b-forever-wait harness + #[test] fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { let workload = SwitchWorkload::new(); @@ -17090,153 +17473,6 @@ mod tests { Ok(()) } - /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop - /// keeps failing. A deployment waits behind it until the drain's deadline, - /// then ends itself; the drain stays and the target stays owned. - #[test] - fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( - ) -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; - odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; - let drainer_id = drainer.transaction_id.as_str(); - drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; - let draining = resident_id(&world, drainer_id)?.value; - assert!(draining.drains_incumbent(), "the incumbent was already retired"); - assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); - match &draining.workload { - Some(WorkloadObservation::Systemd(_)) => {} - other => bail!("no workload: {other:?}"), - } - let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = - &draining.post_commit_cleanup - else { - bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); - }; - let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; - workload.fail_stop_of(&old.runtime_instance_id); - - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("nothing admitted")? - .value - .clone(); - let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; - let waiter_id = waiter.transaction_id.as_str(); - for _ in 0..3 { - resume_pass(&world)?; - } - let waiting = resident_id(&world, waiter_id)?.value; - assert!(waiting.pre_fencing_abort.is_none()); - assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); - assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); - - pass_deadline_of(&world, drainer_id)?; - for _ in 0..10 { - resume_pass(&world)?; - } - assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); - assert_eq!( - world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, - [drainer_id] - ); - let ended = latest_id(&world, waiter_id)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("owner-left-to-operator") && error.contains(drainer_id), "{error}"); - Ok(()) - } - - /// A stop-then-start transaction held in Warming with a started unit, as - /// the Starting step leaves it (this world serves no host actuator to - /// start one). - fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { - let stored = resident_id(world, id)?; - let mut next = stored.value.clone(); - let expected = next.expected.clone().context("no Expected")?; - let now = now_millis()?; - let launch = IdunnRuntimeActivationLaunch::issue( - &expected, - runtime_instance_id(&next.transaction_id)?, - now, - &world.engine.idunn_signer, - )?; - let mut seed = Vec::new(); - let activation = launch.write_credential(&mut seed)?; - let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? - .1 - .workload - .context("no recorded workload")?; - match &mut workload { - WorkloadObservation::Systemd(observed) => { - observed.runtime_instance_id = activation.runtime_instance_id.clone(); - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), - } - next.expected_publication_sha256 = Some(expected.canonical_sha256()?); - next.activation_publication_sha256 = Some(activation.canonical_sha256()?); - next.activation = Some(activation); - next.workload = Some(workload); - next.enter_phase(DeploymentPhase::Warming, now); - replace_transaction(&world.state_store, &stored, &next) - } - - /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting - /// and waits in Warming for an Odin word that never comes. Its Warming - /// deadline ends it, and the deployment waiting at its Sealing tail takes - /// the target. - #[test] - fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( - ) -> Result<()> { - use DeploymentPhase::{Starting, Warming}; - let (world, deploy, continuity) = stop_first_world(SwitchWorkload::new())?; - warm_stop_first(&world, &continuity)?; - let owner = resident_id(&world, &continuity)?; - assert_eq!(owner.value.phase, Warming); - assert!(owner.value.owns_target() && owner.value.phase_deadline.is_some()); - let waiting = resident_id(&world, &deploy)?; - world.engine.enter(&waiting, entering(&waiting, Starting)?)?; - let held = resident_id(&world, &deploy)?.value; - assert_eq!(held.phase, DeploymentPhase::Sealing); - assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); - - pass_deadline_of(&world, &continuity)?; - world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; - let aborting = resident_id(&world, &continuity)?.value; - let abort = aborting.pre_fencing_abort.clone().context("the deadline did not end Warming")?; - assert!(abort.error.contains("Warming phase passed its deadline"), "{}", abort.error); - // This world serves no host actuator, so the abandoned unit cannot be - // stopped here; the abort's cleanup is recorded done. - let mut cleaned = aborting.clone(); - let intent = cleaned.pre_fencing_abort.as_mut().unwrap(); - intent.candidate_cleanup = CleanupEvidence::Complete; - intent.topology_reconciliation = CleanupEvidence::Complete; - replace_transaction(&world.state_store, &resident_id(&world, &continuity)?, &cleaned)?; - for _ in 0..12 { - if resident_id(&world, &continuity).is_err() { - break; - } - world.engine.resume_backoff.lock().unwrap().clear(); - world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; - } - let ended = latest_id(&world, &continuity)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("Warming phase passed its deadline"), "{error}"); - - let waiting = resident_id(&world, &deploy)?; - world.engine.enter(&waiting, entering(&waiting, Starting)?)?; - let took = resident_id(&world, &deploy)?.value; - assert_eq!(took.phase, Starting); - assert!(took.owns_target() && took.pre_fencing_abort.is_none()); - Ok(()) - } - #[test] fn a_waiters_warming_deadline_is_disarmed_while_an_owner_holds_the_target_and_rearmed_when_it_ends( ) -> Result<()> { @@ -17458,13 +17694,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// The seeded transaction `id`, rewritten as the Sealing transaction of a /// stop-then-start target: the host-actuator plan, release and Expected, /// and none of the evidence that follows Sealing. - fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + fn make_stop_first(world: &EngineFixture, id: &str, binding: &str) -> Result<()> { use crate::deployment_plan::tests::{ RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); - let binding = stop_first_binding(world)?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -17533,11 +17768,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ) -> Result<(EngineFixture, String, String)> { let world = EngineFixture::with_workload(workload)?; let incumbent = admit_incumbent(&world)?; + let binding = stop_first_binding(&world)?; let mut ids = Vec::new(); for kind in [CommandKind::Deploy, CommandKind::Continuity] { let seeded = seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id)?; + make_stop_first(&world, &seeded.transaction_id, &binding)?; ids.push(seeded.transaction_id); } let continuity = ids.pop().context("no continuity")?; @@ -22592,6 +22828,69 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// The incident of a stateful deployment left to the operator is one + /// per episode. An operator's expiry before the deadline reports the + /// hold and pages no one; the deadline opens one incident; and the + /// incident closes as soon as the record is no longer left, while the + /// transaction goes on. + #[test] + fn a_stateful_deploys_incident_is_one_per_episode_and_follows_its_deadline() -> Result<()> { + let deploy = + build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; + deploy.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let held = deploy.transaction()?; + let id = held.transaction_id.as_str(); + let open = || deploy.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired); + let now = now_millis()?; + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: held.command_id.clone(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + assert!(SingleFileMessagePackBackingStore::new(&deploy.world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: request.command_id.clone(), + current: None, + }], + &[expiry_request_envelope(&request, now)?], + )?); + + // The operator asked, the deadline has not passed: nobody is paged, + // however many passes see the request. + for _ in 0..4 { + deploy.resume()?; + } + assert!(open()?.is_empty(), "an expiry before the deadline paged the operator"); + assert!(deploy.world.engine.incidents().read()?.is_empty()); + assert!(deploy.transaction()?.post_fencing_abort.is_none(), "a leased deployment was aborted"); + + // The deadline passes: one incident, not one per pass. + pass_deadline(&deploy.world)?; + for _ in 0..4 { + deploy.resume()?; + } + assert_eq!(open()?, [id]); + assert_eq!(deploy.world.engine.incidents().read()?.len(), 1); + + // The record goes on with a deadline in the future (its step + // advanced): the incident closes while the transaction lives. + let current = resident(&deploy.world)?; + let mut next = current.value.clone(); + let due = next.phase_deadline.as_mut().context("no deadline")?; + due.entered_at_unix_millis = now_millis()?; + due.deadline_at_unix_millis = due.entered_at_unix_millis + 3_600_000; + replace_transaction(&deploy.world.state_store, ¤t, &next)?; + deploy.resume()?; + assert!(!deploy.transaction()?.is_terminal()); + assert!(open()?.is_empty(), "the incident outlived the episode"); + let records = deploy.world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + Ok(()) + } + /// B5 Soul A. The physical grant lands before `Granted` is durable. A /// continuity that dies in that window and passes its deadline is /// aborted from `Prepared`; the abort still withdraws the lease on From 8b6c539a6a1ba89e2714bb347633077e0bbaa568 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:51:02 +0200 Subject: [PATCH 090/196] Cut c2b fix: the stop-first harness owner runs on a systemd unit --- src/control_plane.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 43810c4..c8b0cd6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15897,7 +15897,8 @@ mod tests { // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. world.engine.resume_candidate(&resident_id(&world, id)?)?; - assert!(resident_id(&world, id)?.value.last_error.is_some(), "the Warming step did not wait"); + let waited = resident_id(&world, id)?.value; + assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?} {:?}", waited.phase, waited.warming); workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); world.engine.resume_backoff.lock().unwrap().clear(); let error = world @@ -17276,15 +17277,13 @@ mod tests { /// host-actuator step of a world that serves no actuator errs at once. fn stop_first_systemd_binding(world: &EngineFixture) -> Result { use crate::deployment_plan::tests::BINDING; - let host_runner = "[runners.rust]\ndriver = \"host-native\"\naffordances = [\"source-read\", \"artifact-write\"]\nallowed_programs = [\"cargo\"]\n\n"; - let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; - let (_, rest) = rest.split_once("[workload]").context("no workload")?; + let (head, rest) = BINDING.split_once("[workload]").context("no workload")?; let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; bind_provider( world, - &format!("{head}{host_runner}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") + &format!("{head}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") .replace("candidate-then-promote", "stop-then-start"), ) } @@ -17680,7 +17679,6 @@ state_root = "/var/lib/gamecult/service" GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" "#; - let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; From c6805d48a79252ed2493ac62b260e86da43c2f6a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:51:38 +0200 Subject: [PATCH 091/196] Cut c2b fix: restore stop_first_binding's runner split --- src/control_plane.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index c8b0cd6..8e43d81 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17679,6 +17679,7 @@ state_root = "/var/lib/gamecult/service" GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" "#; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; From 2ff57533b112f36e1486cae801f1dc6b82cdf564 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:53:58 +0200 Subject: [PATCH 092/196] Cut c2b fix: a test world runs a host-actuator target's workload through its own port --- src/control_plane.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8e43d81..e7a8009 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5039,6 +5039,10 @@ struct Engine { docker_runner: DockerRunnerDriver, systemd_workload: Arc, host_actuators: Option, + /// A test world that serves no host actuator runs a host-actuator + /// binding's workload through this port instead. + #[cfg(test)] + host_workload: Mutex>>, history_report: ReportOnce, /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. @@ -5140,6 +5144,8 @@ impl Engine { docker_runner: DockerRunnerDriver::default(), systemd_workload, host_actuators, + #[cfg(test)] + host_workload: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), resume_backoff: Mutex::default(), @@ -5375,9 +5381,15 @@ impl Engine { let (_, binding) = plan.parsed_inputs()?; Ok(match &binding.workload { WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), - WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { - access: self.host_access()?, - }), + WorkloadBinding::HostActuator(_) => { + #[cfg(test)] + if let Some(port) = self.host_workload.lock().expect("host workload mutex").clone() { + return Ok(port); + } + Arc::new(HostActuatorWorkloadDriver { + access: self.host_access()?, + }) + } }) } @@ -15892,7 +15904,8 @@ mod tests { }); let world = EngineFixture::with_workload(workload.clone())?; *workload.store.lock().unwrap() = Some(world.state_store.clone()); - let live = transaction_at(&world, DeploymentPhase::Warming)?; + let incumbent = admit_incumbent(&world)?; + let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Continuity, Some(&incumbent))?; let id = live.transaction_id.as_str(); // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. From 95e807661867d3f3afb4975e2fd8012246a8c768 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:55:04 +0200 Subject: [PATCH 093/196] Cut c2b fix: the stop-first owner harness uses the host-actuator world with its workload port --- src/control_plane.rs | 39 +++++++-------------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e7a8009..dacf5d9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17284,41 +17284,16 @@ mod tests { replace_transaction(&world.state_store, &stored, &next) } - /// The stop-then-start binding of `stop_first_binding`, on a systemd unit - /// instead of a host actuator: its Warming step observes the unit through - /// the world's workload port and waits for Odin's word, where the - /// host-actuator step of a world that serves no actuator errs at once. - fn stop_first_systemd_binding(world: &EngineFixture) -> Result { - use crate::deployment_plan::tests::BINDING; - let (head, rest) = BINDING.split_once("[workload]").context("no workload")?; - let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; - let (identity, rest) = rest.split_once("[route]").context("no route")?; - let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; - bind_provider( - world, - &format!("{head}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") - .replace("candidate-then-promote", "stop-then-start"), - ) - } - /// A stop-then-start deployment and continuity over one incumbent, on a /// systemd unit. The continuity is held in Warming with a started unit and /// the deployment, one `enter` later, waits at its Sealing tail on it. fn warm_stop_first_owner( workload: Arc, ) -> Result<(EngineFixture, String, String)> { - let world = EngineFixture::with_workload(workload)?; - let incumbent = admit_incumbent(&world)?; - let binding = stop_first_systemd_binding(&world)?; - let mut ids = Vec::new(); - for kind in [CommandKind::Deploy, CommandKind::Continuity] { - let seeded = - seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id, &binding)?; - ids.push(seeded.transaction_id); - } - let continuity = ids.pop().context("no continuity")?; - let deploy = ids.pop().context("no deploy")?; + let (world, deploy, continuity) = stop_first_world(workload.clone())?; + // This world serves no host actuator: the target's host workload runs + // through the same port as everything else here. + *world.engine.host_workload.lock().unwrap() = Some(workload); // seam warm_stop_first(&world, &continuity)?; let owner = resident_id(&world, &continuity)?; assert_eq!(owner.value.phase, DeploymentPhase::Warming); @@ -17706,12 +17681,13 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// The seeded transaction `id`, rewritten as the Sealing transaction of a /// stop-then-start target: the host-actuator plan, release and Expected, /// and none of the evidence that follows Sealing. - fn make_stop_first(world: &EngineFixture, id: &str, binding: &str) -> Result<()> { + fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { use crate::deployment_plan::tests::{ RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); + let binding = stop_first_binding(world)?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -17780,12 +17756,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ) -> Result<(EngineFixture, String, String)> { let world = EngineFixture::with_workload(workload)?; let incumbent = admit_incumbent(&world)?; - let binding = stop_first_binding(&world)?; let mut ids = Vec::new(); for kind in [CommandKind::Deploy, CommandKind::Continuity] { let seeded = seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id, &binding)?; + make_stop_first(&world, &seeded.transaction_id)?; ids.push(seeded.transaction_id); } let continuity = ids.pop().context("no continuity")?; From 9844806eb125159bcca9d99fbc2963134c5915cb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:56:28 +0200 Subject: [PATCH 094/196] Cut c2b fix: the waiter's own entry attempt drives the owner harness --- src/control_plane.rs | 62 +++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index dacf5d9..c5a51a8 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15902,21 +15902,20 @@ mod tests { store: Mutex::new(None), armed: std::sync::atomic::AtomicBool::new(false), }); - let world = EngineFixture::with_workload(workload.clone())?; + let (world, _deploy, owner) = stop_first_world(workload.clone())?; + *world.engine.host_workload.lock().unwrap() = Some(workload.clone()); *workload.store.lock().unwrap() = Some(world.state_store.clone()); - let incumbent = admit_incumbent(&world)?; - let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Continuity, Some(&incumbent))?; - let id = live.transaction_id.as_str(); + warm_stop_first(&world, &owner)?; // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. - world.engine.resume_candidate(&resident_id(&world, id)?)?; - let waited = resident_id(&world, id)?.value; - assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?} {:?}", waited.phase, waited.warming); + world.engine.resume_candidate(&resident_id(&world, &owner)?)?; + let waited = resident_id(&world, &owner)?.value; + assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?}", waited.phase); workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); world.engine.resume_backoff.lock().unwrap().clear(); let error = world .engine - .resume_candidate(&resident_id(&world, id)?) + .resume_candidate(&resident_id(&world, &owner)?) .expect_err("a record gone from both stores was read as a finished one"); assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); assert!(read_history_transactions(&world.state_store).is_empty()); @@ -17158,6 +17157,32 @@ mod tests { Ok(()) } + /// What the scheduler does for a deployment waiting at its Sealing tail: it + /// asks to enter Starting, and a refusal ends it before fencing. (A resume + /// pass would stop first at the deployment brake this world never releases.) + fn step_waiter(world: &EngineFixture, id: &str) -> Result<()> { + let waiting = resident_id(world, id)?; + if waiting.value.pre_fencing_abort.is_some() { + return Ok(()); + } + if let Err(error) = world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?) { + world.engine.begin_pre_fencing_abort(&resident_id(world, id)?, error)?; + } + Ok(()) + } + + /// Passes of the scheduler over the owner, each followed by the waiter's + /// own attempt to enter Starting. + fn run_owner_and_waiter(world: &EngineFixture, waiter: &str, passes: usize) -> Result<()> { + for _ in 0..passes { + resume_pass(world)?; + if resident_id(world, waiter).is_ok() { + step_waiter(world, waiter)?; + } + } + Ok(()) + } + fn operator_required(world: &EngineFixture) -> Result> { world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) // head-only } @@ -17315,9 +17340,7 @@ mod tests { ) -> Result<()> { let (world, deploy, continuity) = warm_stop_first_owner(SwitchWorkload::new())?; // The gate-wait is the Warming step itself, not an error of it. - for _ in 0..3 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 3)?; let waiting = resident_id(&world, &continuity)?.value; assert_eq!(waiting.phase, DeploymentPhase::Warming); assert!(waiting.pre_fencing_abort.is_none(), "the Warming step failed: {:?}", waiting.last_error); @@ -17329,7 +17352,7 @@ mod tests { if resident_id(&world, &continuity).is_err() { break; } - resume_pass(&world)?; + run_owner_and_waiter(&world, &deploy, 1)?; } let ended = latest_id(&world, &continuity)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { @@ -17340,12 +17363,7 @@ mod tests { // The pass that retired the owner, or the next, moves the waiter to // Starting; stop there, before its Starting step runs. - for _ in 0..3 { - if resident_id(&world, &deploy)?.value.phase >= DeploymentPhase::Starting { - break; - } - resume_pass(&world)?; - } + step_waiter(&world, &deploy)?; let took = resident_id(&world, &deploy)?.value; assert_eq!(took.phase, DeploymentPhase::Starting); assert!(took.owns_target() && took.pre_fencing_abort.is_none()); @@ -17363,9 +17381,7 @@ mod tests { world .engine .begin_pre_fencing_abort(&resident_id(&world, &owner)?, anyhow!("candidate refused"))?; - for _ in 0..3 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 3)?; // Before the abort's deadline: it waits, unpaged. let aborting = resident_id(&world, &owner)?.value; assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); @@ -17373,9 +17389,7 @@ mod tests { assert!(resident_id(&world, &deploy)?.value.pre_fencing_abort.is_none()); age_deadline_of(&world, &owner)?; - for _ in 0..20 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 20)?; let aborting = resident_id(&world, &owner)?.value; assert!(!aborting.is_terminal() && aborting.owns_target(), "the owner was ended"); assert_eq!(operator_required(&world)?, [owner.as_str()]); // head-only From 8230c544acf3a45f0874c3906e368a94fce63c59 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:58:13 +0200 Subject: [PATCH 095/196] Cut c2b fix: a pre-fencing abort keeps its deadline through its last cleanup write (a Sealing waiter's abort could not finish) --- src/control_plane.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c5a51a8..52842ea 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1808,10 +1808,13 @@ impl DeploymentTransaction { } /// Whether a deadline can describe the record's current activity: a - /// deadline phase, a pending pre-fencing abort, or an admitted Complete. + /// deadline phase, a pre-fencing abort not yet ended, or an admitted Complete. fn carries_a_deadline(&self) -> bool { + // An abort carries its deadline until the record ends, including the + // write that completes its last cleanup: a Sealing or Starting record + // has no phase duration to fall back on, and that write is validated. DEADLINE_PHASES.contains(&self.phase) - || self.pre_fencing_abort_pending() + || (self.completion.is_none() && self.pre_fencing_abort.is_some()) || (self.phase == DeploymentPhase::Complete && matches!(self.completion, Some(TransactionCompletion::Admitted { .. }))) } @@ -15918,7 +15921,7 @@ mod tests { .resume_candidate(&resident_id(&world, &owner)?) .expect_err("a record gone from both stores was read as a finished one"); assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); - assert!(read_history_transactions(&world.state_store).is_empty()); + assert!(read_history_transactions(&world.state_store).iter().all(|archived| archived.transaction_id != owner)); Ok(()) } @@ -17162,7 +17165,7 @@ mod tests { /// pass would stop first at the deployment brake this world never releases.) fn step_waiter(world: &EngineFixture, id: &str) -> Result<()> { let waiting = resident_id(world, id)?; - if waiting.value.pre_fencing_abort.is_some() { + if waiting.value.pre_fencing_abort.is_some() || waiting.value.phase >= DeploymentPhase::Starting { return Ok(()); } if let Err(error) = world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?) { From 58f213fe1525c9f3acacbe40c705e55a03a4d791 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:24:22 +0200 Subject: [PATCH 096/196] Cut c2c: boot lift returns a wedged continuity abort to forward retry A pre-c2c Idunn aborted a continuity at its phase deadline, which stops the only running copy of the admitted body. At boot, a continuity abort that has not stopped its running candidate (candidate_cleanup Pending, lease_withdrawal not Complete, workload observed, issued lease exact on disk) is withdrawn and the record retimed, the exact inverse of begin_post_fencing_abort. Negative tests carry the live shapes of tx-4eaed669 (Routing, adopted lease) and tx-039b91e9 (AwaitingReady, granted lease); the lift's refusals are pinned per predicate input. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 301 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 52842ea..092324d 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5626,6 +5626,79 @@ struct ProjectionReconciliation { } impl Engine { + /// Once at boot: withdraw the abort of a continuity that has not stopped + /// its running candidate, and retime the record so its phase retries + /// forward. A pre-c2c Idunn aborted a continuity at its deadline; that + /// abort stops the only running copy of the admitted body (ruling + /// c2c-boot-returns-wedged-aborts-forward). The result is the exact inverse + /// of `begin_post_fencing_abort`. + /// + /// A continuity abort now begins only over a permanently stopped + /// candidate, so this predicate cannot match a new record: it is a + /// migration with a deletion line (follow-up c2c-delete-boot-lift). A + /// record it cannot decide is reported and left aborting. + fn return_wedged_continuity_aborts_forward(&self) -> Result> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut returned = Vec::new(); + for stored in &snapshot.transactions { + let transaction = &stored.value; + let Some(abort) = &transaction.post_fencing_abort else { + continue; + }; + if transaction.command_kind != CommandKind::Continuity + || transaction.completion.is_some() + || abort.candidate_cleanup != CleanupEvidence::Pending + || abort.lease_withdrawal == CleanupEvidence::Complete + { + continue; + } + let id = &transaction.transaction_id; + let lifted = (|| -> Result { + let plan = required(&transaction.plan, "transaction plan")?; + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(plan)? + .observe(expected, activation, workload)?; + if let Some(lease) = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + { + let path = plan + .parsed_inputs()? + .1 + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + if !CultCacheWriteLeaseDriver::new(&transaction.target, path) + .observe_exact(lease)? + { + return Ok(false); + } + } + let now = now_millis()?; + let mut next = transaction.clone(); + next.post_fencing_abort = None; + next.last_error = Some(abort.error.clone()); + next.updated_at_unix_millis = now; + next.retime(now); + replace_transaction(&self.options.state_store, stored, &next)?; + Ok(true) + })(); + match lifted { + Ok(true) => returned.push(id.clone()), + Ok(false) => {} + Err(error) => eprintln!( + "Idunn could not decide whether continuity {id} may retry forward; its abort is left running: {error:#}" + ), + } + } + Ok(returned) + } + /// Once at boot: demote the activation a failed continuity left standing, /// where the projection still names exactly the activation that failed /// transaction issued. A continuity abort before the single resolution @@ -5739,6 +5812,16 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; + match engine.return_wedged_continuity_aborts_forward() { + Ok(returned) => { + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + } + Err(error) => eprintln!("Idunn boot could not lift wedged continuity aborts: {error:#}"), + } match engine.reconcile_failed_continuity_projections() { Ok(outcome) => { for transaction_id in &outcome.demoted { @@ -22362,6 +22445,18 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// Scheduler passes until `done`, at most twelve. + #[cfg(unix)] + fn drive_until_resuming(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..12 { + if done(&self.transaction()?) { + return Ok(()); + } + self.resume()?; + } + bail!("the transaction never reached the awaited state: {:?}", self.transaction()?.last_error) + } + fn resume_until_terminal(&self) -> Result { for _ in 0..12 { if latest(&self.world)?.completion.is_some() { @@ -22954,6 +23049,212 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// What a pre-c2c Idunn left when a continuity's phase passed its + /// deadline: the abort begun, route restoration done, the lease + /// withdrawal and candidate cleanup still owed, and the deadline stale. + #[cfg(unix)] + fn wedge_abort(routed: &RoutedWorld, error: &str) -> Result<()> { + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("{error}"))?; + routed.resume()?; + let aborting = routed.transaction()?.post_fencing_abort.context("no abort")?; + assert_eq!(aborting.route_restoration, CleanupEvidence::Complete); + assert_eq!(aborting.candidate_cleanup, CleanupEvidence::Pending); + pass_deadline(&routed.world) + } + + #[cfg(unix)] + fn lease_of(routed: &RoutedWorld) -> Result<(IdunnProcessWriteLeaseRecord, String)> { + let leasing = routed.transaction()?.leasing.context("no leasing evidence")?; + Ok(( + leasing.issued_lease().context("no issued lease")?.clone(), + leasing.lease_sha256().context("no granted lease")?.to_owned(), + )) + } + + #[cfg(unix)] + fn lease_driver(routed: &RoutedWorld) -> (CultCacheWriteLeaseDriver, PathBuf) { + let path = routed.world.root.join("lease/process-write-lease.cc"); + (CultCacheWriteLeaseDriver::new("service", &path), path) + } + + /// The live shape of tx-4eaed669 (Odin): Routing, lease granted and + /// adopted, the proof failing, the abort begun at the Routing deadline. + /// A pre-c2c Idunn went on to stop the candidate and the target lost + /// its only running copy. The boot lift returns the record to forward + /// retry; nothing is stopped, and the proof, once it passes, commits. + #[cfg(unix)] + #[test] + fn a_wedged_odin_shaped_continuity_abort_is_returned_to_forward_retry_at_boot() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + assert!(routed.transaction()?.lease_adoption.is_some()); + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + wedge_abort( + &routed, + "the Routing phase passed its deadline at unix ms 1792324133638", + )?; + let stops = routed.workload.stops.load(Ordering::SeqCst); + + let lifted_at = now_millis()?; + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + for _ in 0..10 { + routed.resume()?; + } + assert_eq!( + routed.workload.stops.load(Ordering::SeqCst), + stops, + "the candidate was stopped" + ); + let retrying = routed.transaction()?; + assert_eq!(returned, [retrying.transaction_id.clone()]); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + assert!(retrying.post_fencing_abort.is_none(), "the abort came back"); + assert!(retrying.completion.is_none()); + assert_eq!(retrying.phase, DeploymentPhase::Routing); + let deadline = retrying.phase_deadline.context("no deadline")?; + assert!(deadline.entered_at_unix_millis >= lifted_at, "the stale deadline stayed"); + assert!(routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); + + routed.stub.hang_up.store(false, Ordering::SeqCst); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert!(routed.fragment_path().exists(), "the route membership was not re-installed"); + Ok(()) + } + + /// The live shape of tx-039b91e9 (ghostlight): AwaitingReady with the + /// lease granted and not adopted, aborted at the readiness deadline. + /// Returned to forward retry, the candidate that later answers is + /// adopted and the record enters Routing. + #[cfg(unix)] + #[test] + fn a_wedged_ghostlight_shaped_continuity_abort_is_returned_to_forward_retry_at_boot() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + wedge_abort( + &routed, + "lease-not-adopted: the AwaitingReady phase passed its deadline at unix ms 1792324133638", + )?; + let stops = routed.workload.stops.load(Ordering::SeqCst); + + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + for _ in 0..5 { + routed.resume()?; + } + let waiting = routed.transaction()?; + assert_eq!(returned, [waiting.transaction_id.clone()]); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + assert!(waiting.post_fencing_abort.is_none() && waiting.completion.is_none()); + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.lease_adoption.is_none()); + + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Routing)?; + let adopted = routed.transaction()?; + assert_eq!(adopted.lease_adoption.context("no adoption recorded")?.write_lease_sha256, lease_sha256); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + Ok(()) + } + + /// The lift is a function of its inputs: each case lacks one thing the + /// lift requires, and its abort stays and runs to its end. + #[cfg(unix)] + #[test] + fn the_boot_lift_leaves_every_abort_it_cannot_prove_harmless() -> Result<()> { + #[derive(Clone, Copy, Debug)] + enum Case { + CandidateNotRunning, + LeaseWithdrawn, + CandidateCleaned, + LeaseRevokedUnderIt, + DeployWithRunningCandidate, + CompletedRecord, + } + for case in [ + Case::CandidateNotRunning, + Case::LeaseWithdrawn, + Case::CandidateCleaned, + Case::LeaseRevokedUnderIt, + Case::DeployWithRunningCandidate, + Case::CompletedRecord, + ] { + let deploy = matches!(case, Case::DeployWithRunningCandidate); + // The cleanup comes after the lease withdrawal, so a case with + // cleanup done and the withdrawal not is a stateless target's. + let stateful = !matches!(case, Case::CandidateCleaned); + let routed = build_routed_world( + Odin::Unreachable, + 1, + DeploymentPhase::Fencing, + false, + stateful, + if deploy { CommandKind::Deploy } else { CommandKind::Continuity }, + )?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + routed.world.engine.begin_post_fencing_abort( + &resident(&routed.world)?, + anyhow!("the AwaitingReady phase passed its deadline"), + )?; + // Route restoration; then each case takes the abort one real + // step, or one input, away from the lift's predicate. + routed.resume()?; + match case { + Case::CandidateNotRunning => routed.workload.kill(), + Case::LeaseWithdrawn | Case::CandidateCleaned => routed.resume()?, + Case::LeaseRevokedUnderIt => { + let (lease, _) = lease_of(&routed)?; + lease_driver(&routed).0.revoke_exact(Some(&lease))?; + } + Case::DeployWithRunningCandidate => {} + Case::CompletedRecord => { + routed.resume_until_terminal()?; + } + } + let abort = routed.transaction().ok().and_then(|t| t.post_fencing_abort); + match case { + Case::LeaseWithdrawn => assert_eq!( + abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), + Some((CleanupEvidence::Complete, CleanupEvidence::Pending)) + ), + Case::CandidateCleaned => assert_eq!( + abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), + Some((CleanupEvidence::Skipped, CleanupEvidence::Complete)) + ), + _ => {} + } + let before = ControlSnapshot::read(&routed.world.state_store)?; + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty(), "lifted {returned:?}"); + let after = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!( + before.transactions.iter().map(|t| &t.envelope).collect::>(), + after.transactions.iter().map(|t| &t.envelope).collect::>(), + "the lift wrote a record it left" + ); + let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.completion + ); + } + Ok(()) + } + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word /// carries the granted lease, and that Ready records the adoption, /// sourced from Odin's topology and naming the presence Odin observed. From 063409489e95682117ad5afb29e4441d74bde2e8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:33:29 +0200 Subject: [PATCH 097/196] Cut c2c: a continuity past Fencing retries forward and pages, never aborts PhaseEnd::RetryForward: phase_end answers it for every continuity at or past Fencing, because aborting restores the body its candidate already runs (ruling c2c-retry-decided-by-record). The resolver leaves it to the operator (operator-required incident, report_once) and the phase step keeps running. leaves_to_operator covers it. DeploymentTransaction::expirable is the one spelling the expire CLI, the spent-request retirement and the resolver's requested flag read: a continuity past Fencing cannot be expired. Deploy keeps abort-to-incumbent and the stateful operator-required arm. Tests: the PhaseEnd table, the Routing deadline forward retry and its incident episode, the stateful deadline, expiry refusal and spent-request retirement, and a deployment still aborting to its incumbent. Abort-mechanics tests begin their aborts directly. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 318 ++++++++++++++++++++++++++++++++----------- 1 file changed, 236 insertions(+), 82 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 092324d..3e36787 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -981,6 +981,10 @@ enum PhaseEnd { /// A stateful deployment whose candidate holds an issued lease: the phase /// is reported and not resolved. OperatorRequired, + /// A continuity at or past Fencing: its candidate is the only running copy + /// of the admitted body, so aborting it restores what it already runs. The + /// resolver leaves it to the operator and its phase step keeps running. + RetryForward, } /// How a post-fencing failure recovers. `RestoreIncumbent` is what a @@ -1844,17 +1848,29 @@ impl DeploymentTransaction { && (self.pre_fencing_abort.is_some() || self.post_fencing_abort.is_some() || self.drains_incumbent() - || (self.resolves_by_deadline() && self.phase_end() == PhaseEnd::OperatorRequired)) + || (self.resolves_by_deadline() + && matches!(self.phase_end(), PhaseEnd::OperatorRequired | PhaseEnd::RetryForward))) } - /// What an ended post-fencing phase resolves to (Q3). A candidate that was - /// never issued a write lease cannot have written, so the transaction - /// aborts to the incumbent through the post-fencing path, stateless or not. - /// A continuity always aborts: its candidate is the admitted release, and - /// supervision restarts that release under the continuity backoff. A - /// deployment whose candidate was issued a lease may have written under a - /// new state contract, and restoring the incumbent over that is not - /// Idunn's call: it is left for the operator. + /// Whether an operator's expiry can end this record's phase: it resolves by + /// deadline, and is not a continuity past Fencing, whose expiry abort would + /// stop the only running copy of the admitted body (ruling + /// c2c-expire-refuses-continuity-past-fencing). + fn expirable(&self) -> bool { + self.resolves_by_deadline() + && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) + } + + /// What an ended post-fencing phase resolves to (Q3). A continuity at or + /// past Fencing retries forward: its incumbent is the admitted generation, + /// which supervision mints a continuity only after seeing dead, and from + /// Fencing the continuity owns the target, so aborting restores the body + /// its candidate already runs (ruling c2c-retry-decided-by-record). A + /// deployment whose candidate was never issued a write lease cannot have + /// written, so it aborts to the incumbent through the post-fencing path, + /// stateless or not. One whose candidate was issued a lease may have + /// written under a new state contract, and restoring the incumbent over + /// that is not Idunn's call: it is left for the operator. fn phase_end(&self) -> PhaseEnd { if self.phase < DeploymentPhase::Fencing { return PhaseEnd::AbortBeforeFencing; @@ -1863,8 +1879,9 @@ impl DeploymentTransaction { let granted = leasing.and_then(LeasingEvidence::lease).is_some(); let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); match self.command_kind { + CommandKind::Continuity => PhaseEnd::RetryForward, CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, - _ => PhaseEnd::Abort { + CommandKind::Deploy => PhaseEnd::Abort { lease_not_adopted: granted && self.lease_adoption.is_none(), }, } @@ -3119,8 +3136,13 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> "command has multiple live transactions" ); ensure!( - current.value.resolves_by_deadline(), - "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + current.value.expirable(), + "{}", + if current.value.resolves_by_deadline() { + "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" + } else { + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + } ); let now = now_millis()?; let request = ExpiryRequest { @@ -9460,8 +9482,11 @@ impl Engine { /// Every live record is held to a deadline. Before a transaction runs its /// step, a deadline phase whose deadline has passed, or whose command an - /// operator asked to expire, is resolved as `phase_end` says; an abort or - /// a drain past its deadline is left to the operator and its step retries. + /// operator asked to expire, is resolved as `phase_end` says: aborted before + /// the fence, aborted after it, or left to the operator with its step + /// retrying (a stateful deploy, or a continuity, whose candidate is the only + /// running copy of its body). An abort or a drain past its deadline is left + /// to the operator the same way. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the @@ -9513,10 +9538,11 @@ impl Engine { } return Ok(None); } - let requested = ControlSnapshot::read(&self.options.state_store)? - .expiry_requests - .iter() - .any(|request| request.value.command_id == transaction.command_id); + let requested = transaction.expirable() + && ControlSnapshot::read(&self.options.state_store)? + .expiry_requests + .iter() + .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { format!( "the {:?} phase passed its deadline at unix ms {}", @@ -9548,6 +9574,15 @@ impl Engine { ); Ok(None) } + PhaseEnd::RetryForward => { + self.leave_to_operator( + transaction, + &format!( + "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, stop its unit by hand" + ), + ); + Ok(None) + } } } @@ -9578,7 +9613,7 @@ impl Engine { .filter(|request| { !snapshot.transactions.iter().any(|stored| { stored.value.command_id == request.value.command_id - && stored.value.resolves_by_deadline() + && stored.value.expirable() }) }) .map(|request| request.envelope.clone()) @@ -14061,22 +14096,24 @@ mod tests { (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), - (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), - (Continuity, Some(prepared), None, abort(false)), - (Continuity, Some(granted.clone()), None, abort(true)), - (Continuity, Some(granted), adoption.clone(), abort(false)), + (Continuity, None, None, PhaseEnd::RetryForward), + (Continuity, Some(LeasingEvidence::SkippedStateless), None, PhaseEnd::RetryForward), + (Continuity, Some(prepared), None, PhaseEnd::RetryForward), + (Continuity, Some(granted.clone()), None, PhaseEnd::RetryForward), + (Continuity, Some(granted), adoption.clone(), PhaseEnd::RetryForward), ]; for (kind, leasing, lease_adoption, expected) in cases { transaction.command_kind = kind; transaction.leasing = leasing.clone(); transaction.lease_adoption = lease_adoption.clone(); assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); - // Only a stateful deploy is left to the operator when its phase ends. + // Only a stateful deploy and a continuity are left to the operator + // when their phase ends; the other deploys abort. transaction.retime(1_000); let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; assert_eq!( transaction.leaves_to_operator(due), - expected == PhaseEnd::OperatorRequired, + matches!(expected, PhaseEnd::OperatorRequired | PhaseEnd::RetryForward), "{kind:?} {leasing:?} {lease_adoption:?}" ); assert!(!transaction.leaves_to_operator(due - 1)); @@ -22482,11 +22519,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// Soul's F1 probe, committed. A continuity's route changes are /// Survival, so no ceiling refuses its install and rollback while its - /// route proof keeps failing. The Routing deadline ends it: no attempt - /// runs after the deadline. + /// route proof keeps failing. Its Routing deadline does not end it: the + /// continuity pages the operator and retries forward, never stopping its + /// candidate (ruling post-fence-forward-retry). #[cfg(unix)] #[test] - fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { + fn a_continuity_whose_route_proof_fails_retries_forward_past_its_routing_deadline_and_pages() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; @@ -22504,36 +22542,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(waiting.last_error.is_some()); assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); + let stops = routed.workload.stops.load(Ordering::SeqCst); pass_deadline(&routed.world)?; - routed.resume()?; - let abort = routed - .transaction()? - .post_fencing_abort - .context("the passed deadline did not end the phase")?; - assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); - let finished = routed.resume_until_terminal()?; - assert!( - matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - ), - "{:?}", - finished.completion + for _ in 0..5 { + routed.resume()?; + } + // Past the deadline the continuity is not aborted: aborting it + // would stop the only running copy of the admitted body. It is left + // to the operator, and its Routing step keeps running. + let left = routed.transaction()?; + assert!(left.post_fencing_abort.is_none(), "the passed deadline aborted a continuity"); + assert_eq!(left.phase, DeploymentPhase::Routing); + assert!(left.completion.is_none()); + assert_eq!( + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [left.transaction_id.clone()] ); - // After the deadline no attempt runs; the withdrawal writes and - // reloads once, whatever the rolled-back disk shows. - assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); + assert_eq!(routed.world.engine.incidents().read()?.len(), 1, "one page per episode"); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert!(routed.reloads() > 6, "no attempt ran after the deadline"); + + // The proof passes: the phase moves on, and the first pass after + // closes the incident. + routed.stub.hang_up.store(false, Ordering::SeqCst); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; + routed.resume()?; + assert!( + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty(), + "the incident outlived the episode" + ); + let records = routed.world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); Ok(()) } /// The live shape of F1. A continuity reuses the admitted Expected, so /// its membership is the admitted fragment already on disk: while its /// proof fails, the install and the rollback write nothing and reload - /// nothing, and neither does the withdrawal after its deadline. + /// nothing, and neither does its forward retry after its deadline. #[cfg(unix)] #[test] fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() @@ -22562,8 +22610,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); pass_deadline(&routed.world)?; - routed.resume_until_terminal()?; - assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); + for _ in 0..5 { + routed.resume()?; + } + assert!(routed.transaction()?.post_fencing_abort.is_none(), "a passed deadline aborted it"); + assert_eq!(routed.world.route_stubs.count(""), 0, "its retry ran a host program"); assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); Ok(()) } @@ -22689,10 +22740,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .expect_err("an aborting transaction was expired"); assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let untouched = resident(&routed.world)?.envelope; - expire(&routed.world.state_store, "continuity-service", "operator")?; - assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); + expire(&routed.world.state_store, "up-service", "operator")?; + assert!(expire(&routed.world.state_store, "up-service", "operator").is_err()); let snapshot = ControlSnapshot::read(&routed.world.state_store)?; assert_eq!(snapshot.expiry_requests.len(), 1); assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); @@ -22708,6 +22759,36 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), "the spent request stayed" ); + + // A continuity past Fencing retries forward: expiring it would + // abort the only running copy of its body. The CLI refuses, and a + // request a pre-c2c binary stored for one is retired as spent. + let continuity = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let refused = expire(&continuity.world.state_store, "continuity-service", "operator") + .expect_err("a continuity past Fencing was expired"); + assert!(format!("{refused:#}").contains("retries forward"), "{refused:#}"); + assert!(ControlSnapshot::read(&continuity.world.state_store)?.expiry_requests.is_empty()); + let now = now_millis()?; + let stored = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: "continuity-service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + assert!(SingleFileMessagePackBackingStore::new(&continuity.world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: stored.command_id.clone(), + current: None, + }], + &[expiry_request_envelope(&stored, now)?], + )?); + continuity.resume()?; + assert!( + ControlSnapshot::read(&continuity.world.state_store)?.expiry_requests.is_empty(), + "the request for a continuity past Fencing stayed" + ); + assert!(continuity.transaction()?.post_fencing_abort.is_none(), "the request aborted a continuity"); Ok(()) } @@ -22739,7 +22820,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); // Another target's deployment in flight is not this command's. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let now = now_millis()?; let other = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -22765,10 +22846,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ], &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], )?); - expire(&routed.world.state_store, "continuity-service", "operator")?; + expire(&routed.world.state_store, "up-service", "operator")?; let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; assert_eq!(requests.len(), 1); - assert_eq!(requests[0].value.command_id, "continuity-service"); + assert_eq!(requests[0].value.command_id, "up-service"); Ok(()) } @@ -22776,7 +22857,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// live transactions, a request for the other leaves this one running. #[test] fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let now = now_millis()?; let other = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -22789,8 +22870,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; // The request names a command whose id has this transaction's // length and first bytes, and differs only in its last. - let near = "continuity-servicx"; - assert_eq!(near.len(), "continuity-service".len()); + let near = "up-servicx"; + assert_eq!(near.len(), "up-service".len()); let request = ExpiryRequest { schema_version: EXPIRY_REQUEST_SCHEMA.into(), command_id: near.into(), @@ -22800,11 +22881,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide // And commands whose ids are a strict prefix of this one's, and // extend it: a match is the whole id, in neither direction a part. let prefix = ExpiryRequest { - command_id: "continuity-servic".into(), + command_id: "up-servic".into(), ..request.clone() }; let extension = ExpiryRequest { - command_id: "continuity-service-2".into(), + command_id: "up-service-2".into(), ..request.clone() }; let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); @@ -22835,24 +22916,26 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let mine = snapshot .transactions .iter() - .find(|t| t.value.command_id == "continuity-service") + .find(|t| t.value.command_id == "up-service") .context("no continuity")?; assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); let after = ControlSnapshot::read(&routed.world.state_store)?; let unchanged = after .transactions .iter() - .find(|t| t.value.command_id == "continuity-service") + .find(|t| t.value.command_id == "up-service") .context("no continuity")?; assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); Ok(()) } - /// Q3 at the deadline, through the Engine. A continuity whose candidate - /// was granted a lease and never proved it held it fails as - /// `lease-not-adopted`; one whose Ready proved it restarts the admitted - /// release. A deployment whose candidate holds a lease is left to the - /// operator: nothing is aborted and its step still runs. + /// The deadline of a stateful target, through the Engine. A deployment + /// whose candidate holds a lease is left to the operator: nothing is + /// aborted and its step still runs. So is a continuity, adopted or not, + /// whose candidate is the only running copy of the admitted body. An + /// abort begun on a continuity directly still names its recovery: it + /// restarts the admitted release when its Ready proved the lease, and + /// restores the incumbent when it never did. #[test] fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { let granted = |routed: &RoutedWorld| -> Result { @@ -22866,13 +22949,29 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .to_owned()) }; + // Past its deadline a continuity is not aborted, adopted or not: + // its candidate is the only running copy of the admitted body. It + // is left to the operator and its step runs. + let open = |routed: &RoutedWorld| -> Result> { + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) + }; let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; granted(&unadopted)?; pass_deadline(&unadopted.world)?; - unadopted.resume()?; - let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); - assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); + for _ in 0..3 { + unadopted.resume()?; + } + let held = unadopted.transaction()?; + assert!(held.post_fencing_abort.is_none(), "an unadopted continuity was aborted"); + assert_eq!(held.phase, DeploymentPhase::AwaitingReady); + assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); + assert_eq!(open(&unadopted)?, [held.transaction_id.clone()]); + // An abort begun on it directly (a candidate found dead) still + // restores the incumbent when the lease was never adopted. + unadopted + .world + .engine + .begin_post_fencing_abort(&resident(&unadopted.world)?, anyhow!("test abort"))?; let finished = unadopted.resume_until_terminal()?; assert!(matches!( finished.completion, @@ -22896,10 +22995,19 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); assert_eq!(adoption.source, AdoptionSource::Direct); adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + adopted.stub.hang_up.store(true, Ordering::SeqCst); pass_deadline(&adopted.world)?; - adopted.resume()?; - let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + for _ in 0..3 { + adopted.resume()?; + } + let held = adopted.transaction()?; + assert!(held.post_fencing_abort.is_none(), "an adopted continuity was aborted"); + assert_eq!(held.phase, DeploymentPhase::Routing); + assert_eq!(open(&adopted)?, [held.transaction_id.clone()]); + adopted + .world + .engine + .begin_post_fencing_abort(&resident(&adopted.world)?, anyhow!("test abort"))?; let finished = adopted.resume_until_terminal()?; assert!( matches!( @@ -22991,7 +23099,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } /// B5 Soul A. The physical grant lands before `Granted` is durable. A - /// continuity that dies in that window and passes its deadline is + /// continuity that dies in that window and whose abort begins is /// aborted from `Prepared`; the abort still withdraws the lease on /// disk, so the next fencing of the target is not refused. #[test] @@ -23011,9 +23119,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ); assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); - pass_deadline(&routed.world)?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); routed.resume_until_terminal()?; assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); // What the next fencing does to a candidate's lease path. @@ -23034,9 +23145,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.workload.hold_lease_lock(&lease_path); let driver = CultCacheWriteLeaseDriver::new("service", &lease_path); - pass_deadline(&routed.world)?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); let stops = routed.workload.stops.load(Ordering::SeqCst); let finished = routed.resume_until_terminal()?; assert!( @@ -23049,6 +23163,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// The deadline's abort is a continuity's no longer, and a deployment's + /// still: a deployment's candidate is a new release and its incumbent + /// the admitted body, which an abort restores. + #[cfg(unix)] + #[test] + fn a_deploy_with_a_live_incumbent_still_aborts_to_it_at_its_deadline() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_none(), "aborted before its deadline"); + + pass_deadline(&routed.world)?; + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the passed deadline did not end a deployment")?; + assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!( + routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), + "the boot lift took a deployment's abort" + ); + assert!(routed.transaction()?.post_fencing_abort.is_some()); + let finished = routed.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + ), + "{:?}", + finished.completion + ); + Ok(()) + } + /// What a pre-c2c Idunn left when a continuity's phase passed its /// deadline: the abort begun, route restoration done, the lease /// withdrawal and candidate cleanup still owed, and the deadline stale. From 81ab69046b44e23c8e23060bffe5a9f16b94c798 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:36:55 +0200 Subject: [PATCH 098/196] Negative test: long multibyte sealing failure must abort and free the target Fails at 5b421c3 (abort write refused by require_detail). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 57 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index e077373..752415a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16866,6 +16866,63 @@ mod tests { Ok(()) } + /// A source whose resolve step fails with a fixed message, as a build or + /// resolve step does when its output is a long log. + struct FailingSource { + message: String, + } + + impl SourcePort for FailingSource { + fn resolve( + &self, + _: &OperatorBinding, + _: &str, + _: u64, + ) -> Result { + bail!("{}", self.message) + } + + fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { + bail!("this source freezes nothing") + } + + fn observe_frozen( + &self, + _: &CompiledDeploymentPlan, + _: &FrozenSourceReceipt, + ) -> Result { + bail!("this source freezes nothing") + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + #[test] + fn a_sealing_failure_with_a_long_multibyte_error_aborts_and_frees_the_target() -> Result<()> { + // 3000 chars of 3-byte box drawing: 9000 bytes, the shape of a build log. + let log = format!("{}\nthe real error is here", "\u{2500}".repeat(3000)); + let mut world = sealing_world(&unrouted_recipe())?; + world.engine.source = Arc::new(FailingSource { message: log }); + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + } + assert!( + resident(&world).is_err(), + "the failed transaction still owns the target" + ); + let done = latest(&world)?; + let TransactionCompletion::FailedBeforeFencing { error } = + done.completion.context("the aborted transaction did not complete")? + else { + bail!("the transaction did not fail before fencing"); + }; + assert!(error.len() <= 2048, "{}", error.len()); + assert!(error.ends_with("the real error is here"), "the tail was lost"); + Ok(()) + } + #[test] fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { let recipe = unrouted_recipe().replace( From cc4a72ad43bf89f1e34c18be959fd5bbab8b6a57 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:37:03 +0200 Subject: [PATCH 099/196] Cut c2c: a record left to the operator backs off to the continuity cap ResumeBackoff stores the ceiling it was armed with and the skip check reads it. resume_ceiling is the continuity cap (300 s) for a record that leaves_to_operator and the short cap otherwise, for every such record: a forward-retrying continuity, a stateful deploy, a wedged abort, a drain. A step that returns Ok without moving a record left to the operator is backed off like a failed attempt, so a silent candidate is challenged once per backoff, not every poll. Nothing in TargetSupervision is touched. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 136 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 122 insertions(+), 14 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3e36787..f750257 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5112,11 +5112,26 @@ impl RouteActuationGate for EngineRouteGate<'_> { struct ResumeBackoff { failures: u32, not_before_unix_millis: u64, + /// The longest wait this backoff was armed with: the skip check reads it, + /// so the check and the wait cannot disagree. + ceiling_millis: u64, } /// Longest wait between attempts at a failing transaction step. const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; +/// The longest wait between attempts at one transaction's step: the continuity +/// cap for a record left to the operator, which retries on the slow cadence its +/// page promises (ruling c2c-forward-retry-cadence), and the shorter one for +/// every other. +fn resume_ceiling(transaction: &DeploymentTransaction, now: u64) -> u64 { + if transaction.leaves_to_operator(now) { + CONTINUITY_BACKOFF_CEILING_MILLIS + } else { + RESUME_BACKOFF_CEILING_MILLIS + } +} + impl Engine { fn open(options: RuntimeOptions) -> Result { Self::open_with_systemd_workload( @@ -6054,7 +6069,7 @@ impl Engine { .get(id) .is_some_and(|backoff| { now_millis().is_ok_and(|now| { - is_waiting(now, backoff.not_before_unix_millis, RESUME_BACKOFF_CEILING_MILLIS) + is_waiting(now, backoff.not_before_unix_millis, backoff.ceiling_millis) }) }) { @@ -6064,15 +6079,29 @@ impl Engine { self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); return Ok(false); } + let left = now_millis().is_ok_and(|now| current.value.leaves_to_operator(now)); let advanced = self.advance_transaction(current); if advanced.is_ok() { - // The step ran and succeeded: the fault, if any, has recovered, so - // a later failure is news. A skipped or failed attempt clears nothing. - self.resume_backoff - .lock() - .expect("resume backoff mutex") - .remove(id); - self.clear_fault(id); + // A step that returns Ok without moving a record left to the + // operator (a gate-wait on a silent candidate) is an attempt like + // a failed one: it backs off, and the fault it recorded stands. + let unmoved = left + && ControlSnapshot::read(&self.options.state_store)? + .transactions + .iter() + .find(|stored| stored.value.transaction_id == id) + .is_some_and(|live| !state_advanced(¤t.value, &live.value)); + if unmoved { + self.back_off(¤t.value)?; + } else { + // The step ran and succeeded: the fault, if any, has recovered, so + // a later failure is news. A skipped or failed attempt clears nothing. + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + self.clear_fault(id); + } } if let Err(error) = advanced { let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; @@ -6088,7 +6117,7 @@ impl Engine { .remove(id); return Ok(true); } - self.back_off(id)?; + self.back_off(&latest.value)?; if latest.value.phase < DeploymentPhase::Fencing { // Before the fence an abort that cannot finish a step is // resumable, never a post-fence abort: that path refuses a @@ -6142,16 +6171,20 @@ impl Engine { /// Wait twice as long after each consecutive failure of a transaction's /// step, from one poll interval up to a ceiling. - fn back_off(&self, transaction_id: &str) -> Result<()> { + fn back_off(&self, transaction: &DeploymentTransaction) -> Result<()> { let now = now_millis()?; + let ceiling_millis = resume_ceiling(transaction, now); let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); - let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); - let wait = backoff_wait(self.options.poll_millis, failures, RESUME_BACKOFF_CEILING_MILLIS); + let failures = all + .get(&transaction.transaction_id) + .map_or(0, |backoff| backoff.failures); + let wait = backoff_wait(self.options.poll_millis, failures, ceiling_millis); all.insert( - transaction_id.to_string(), + transaction.transaction_id.clone(), ResumeBackoff { failures: failures.saturating_add(1), not_before_unix_millis: now.saturating_add(wait), + ceiling_millis, }, ); Ok(()) @@ -14061,11 +14094,45 @@ mod tests { Ok(()) } + /// The slow cadence belongs to a record that is left to the operator, + /// whichever way it came to be: it waits up to the continuity cap between + /// attempts from its deadline on, and up to the short cap before. + #[test] + fn a_record_left_to_the_operator_backs_off_to_the_continuity_cap() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let lease = write_lease(); + let leased = LeasingEvidence::Granted { + lease_sha256: lease.canonical_sha256()?, + lease, + }; + transaction.retime(1_000); + let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; + let mut aborting = transaction.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "wedged")); + let mut deploy = transaction.clone(); + deploy.command_kind = CommandKind::Deploy; + let mut stateful = deploy.clone(); + stateful.leasing = Some(leased); + let mut continuity = transaction.clone(); + continuity.command_kind = CommandKind::Continuity; + for (what, record) in [ + ("a forward-retrying continuity", &continuity), + ("a stateful deploy", &stateful), + ("a wedged abort", &aborting), + ] { + assert_eq!(resume_ceiling(record, due - 1), RESUME_BACKOFF_CEILING_MILLIS, "{what} before its deadline"); + assert_eq!(resume_ceiling(record, due), CONTINUITY_BACKOFF_CEILING_MILLIS, "{what} at its deadline"); + } + // A deploy that aborts at its deadline is not left to anyone. + assert_eq!(resume_ceiling(&deploy, due), RESUME_BACKOFF_CEILING_MILLIS); + Ok(()) + } + /// Q3 as a table: which ended phases abort, which name the unadopted /// lease, which are left to the operator, and which recovery the terminal /// record names. #[test] - fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { + fn an_ended_phase_is_resolved_by_command_kind_and_lease() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; let lease = write_lease(); let sha256 = lease.canonical_sha256()?; @@ -15794,6 +15861,7 @@ mod tests { ResumeBackoff { failures: 3, not_before_unix_millis, + ceiling_millis: RESUME_BACKOFF_CEILING_MILLIS, }, ); }; @@ -23203,6 +23271,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// A record left to the operator whose step returns Ok without moving it + /// (a gate-wait on a silent candidate) is backed off like a failed + /// attempt, to the continuity cap: its candidate is challenged once per + /// backoff, not once per pass. Before its deadline every pass challenges. + #[test] + fn a_continuity_left_to_the_operator_challenges_a_silent_candidate_once_per_backoff() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let engine = &routed.world.engine; + let id = routed.transaction()?.transaction_id; + let hits = || routed.stub.candidate_hits.load(Ordering::SeqCst); + + let before = hits(); + for _ in 0..3 { + engine.resume_one_transaction()?; + } + assert!(hits() >= before + 3, "a pass inside the deadline did not challenge"); + assert!(engine.resume_backoff.lock().unwrap().get(&id).is_none()); + + pass_deadline(&routed.world)?; + engine.resume_one_transaction()?; + let first = hits(); + assert!(first > before + 3, "the pass past the deadline did not challenge"); + let armed = *engine.resume_backoff.lock().unwrap().get(&id).context("the unmoved step set no backoff")?; + assert_eq!(armed.ceiling_millis, CONTINUITY_BACKOFF_CEILING_MILLIS); + for _ in 0..5 { + engine.resume_one_transaction()?; + } + assert_eq!(hits(), first, "a wedged candidate was challenged inside its backoff"); + assert!(routed.transaction()?.post_fencing_abort.is_none()); + + // The skip check reads the ceiling the backoff was armed with: a + // wait longer than the short cap is still a wait. + engine.resume_backoff.lock().unwrap().get_mut(&id).unwrap().not_before_unix_millis = + now_millis()? + 200_000; + engine.resume_one_transaction()?; + assert_eq!(hits(), first, "the skip check read the short cap"); + Ok(()) + } + /// What a pre-c2c Idunn left when a continuity's phase passed its /// deadline: the abort begun, route restoration done, the lease /// withdrawal and candidate cleanup still owed, and the deadline stale. From 18c5b643a7c80b96af1f25a93726c87fa2e28426 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:37:38 +0200 Subject: [PATCH 100/196] Every persisted detail goes through bounded_detail; delete truncate require_detail judges bytes, truncate cut chars, so a long multibyte build error made the abort write invalid and wedged the record in Sealing. bounded_detail keeps head and tail within 2048 bytes, drops NUL, and never returns empty. Also bounds gate-wait notes, cancel notes, and the continuity deferral reason at their writers. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 88 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 752415a..16e45f3 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1311,7 +1311,7 @@ impl TargetSupervision { fn defer_continuity(&mut self, now: u64, reason: &str) { self.continuity_deferred_until = Some(now.saturating_add(CONTINUITY_DEFERRAL_MILLIS)); - self.continuity_deferral_reason = Some(reason.to_owned()); + self.continuity_deferral_reason = Some(bounded_detail(reason)); } fn validate(&self) -> Result<()> { @@ -1856,7 +1856,7 @@ impl DeploymentTransaction { fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; - let detail = truncate(&format!("{error:#}"), 2048); + let detail = bounded_detail(&format!("{error:#}")); transaction.enter_phase(DeploymentPhase::Complete, now); transaction.last_error = Some(detail.clone()); transaction.pre_fencing_abort = Some(PreFencingAbort { @@ -3043,7 +3043,7 @@ fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> } else { next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); } - next.last_error = Some(error); + next.last_error = Some(bounded_detail(&error)); next.updated_at_unix_millis = now_millis()?; replace_transaction(store_path, current, &next)?; println!("{command_id} cancellation requested; Idunn is performing exact candidate cleanup"); @@ -5621,7 +5621,7 @@ impl Engine { /// record's `last_error`, which status renders. Recording is best effort: /// a store too broken to take the note is already in stderr. fn note_fault(&self, what: &str, transaction_id: &str, error: &anyhow::Error) { - let detail = truncate(&format!("{error:#}"), 2048); + let detail = bounded_detail(&format!("{error:#}")); let offered = { let mut reports = self.fault_reports.lock().expect("fault report mutex"); reports @@ -5648,6 +5648,8 @@ impl Engine { /// Write `detail` as the record's `last_error`, touching nothing else and /// not archiving: a terminal record whose archive is the fault stays put. fn record_last_error(&self, transaction_id: &str, detail: &str) -> Result<()> { + let detail = bounded_detail(detail); + let detail = detail.as_str(); let snapshot = ControlSnapshot::read(&self.options.state_store)?; let Some(stored) = snapshot .transactions @@ -6330,7 +6332,7 @@ impl Engine { current.value.target ); let mut deferred = supervision.clone(); - deferred.defer_continuity(now, &truncate(&format!("{error:#}"), 2048)); + deferred.defer_continuity(now, &bounded_detail(&format!("{error:#}"))); self.write_target_supervision( snapshot.supervision_for(¤t.value.target), &deferred, @@ -8172,11 +8174,12 @@ impl Engine { current: &Stored, detail: &str, ) -> Result<()> { - if current.value.last_error.as_deref() == Some(detail) { + let detail = bounded_detail(detail); + if current.value.last_error.as_deref() == Some(detail.as_str()) { return Ok(()); } let mut next = current.value.clone(); - next.last_error = Some(detail.into()); + next.last_error = Some(detail); next.updated_at_unix_millis = now_millis()?; replace_transaction(&self.options.state_store, current, &next) } @@ -9015,7 +9018,7 @@ impl Engine { current: &Stored, error: &anyhow::Error, ) -> Result<()> { - let detail = truncate(&format!("{error:#}"), 2048); + let detail = bounded_detail(&format!("{error:#}")); if current.value.last_error.as_deref() == Some(detail.as_str()) { return Ok(()); } @@ -9073,7 +9076,7 @@ impl Engine { current.value.post_fencing_abort.is_none(), "post-fencing abort intent is already durable" ); - let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); + let abort = post_fencing_abort_intent(¤t.value, &bounded_detail(&format!("{error:#}"))); self.persist_same_phase(current, |next| { next.post_fencing_abort = Some(abort); Ok(()) @@ -9439,7 +9442,7 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { PreFencingAbort { - error: truncate(error, 2048), + error: bounded_detail(error), candidate_cleanup: transaction.candidate_cleanup_owed(), topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { @@ -9455,7 +9458,7 @@ fn post_fencing_abort_intent( error: &str, ) -> PostFencingAbort { PostFencingAbort { - error: truncate(error, 2048), + error: bounded_detail(error), route_restoration: if transaction.route_preflight.is_some() { CleanupEvidence::Pending } else { @@ -9728,8 +9731,34 @@ fn require_detail(value: &str, label: &str) -> Result<()> { Ok(()) } -fn truncate(value: &str, length: usize) -> String { - value.chars().take(length).collect() +const DETAIL_LIMIT: usize = 2048; +const DETAIL_HEAD: usize = 512; +const DETAIL_ELISION: &str = " +...[elided]... +"; +const DETAIL_EMPTY: &str = "(no detail)"; + +/// The only producer of persisted free-text details; `require_detail` is the +/// only judge, and this output always satisfies it. NUL is dropped, empty +/// input gets a fixed placeholder, and an over-long value keeps its head and +/// its tail (where a build log's error is) cut on char boundaries. +fn bounded_detail(value: &str) -> String { + let value = value.replace('', ""); + if value.is_empty() { + return DETAIL_EMPTY.to_owned(); + } + if value.len() <= DETAIL_LIMIT { + return value; + } + let mut head = DETAIL_HEAD; + while !value.is_char_boundary(head) { + head -= 1; + } + let mut tail = value.len() - (DETAIL_LIMIT - DETAIL_HEAD - DETAIL_ELISION.len()); + while !value.is_char_boundary(tail) { + tail += 1; + } + format!("{}{DETAIL_ELISION}{}", &value[..head], &value[tail..]) } fn sha256_id(bytes: &[u8]) -> String { @@ -16899,6 +16928,39 @@ mod tests { } } + #[test] + fn bounded_detail_output_always_satisfies_require_detail() -> Result<()> { + for unit in ["a", "\u{e9}", "\u{2500}", "\u{1f600}"] { + for bytes in [0, 1, 511, 512, 513, 2047, 2048, 2049, 2600, 9000] { + // A prefix of one byte offsets every multibyte char against the cuts. + for prefix in ["", "x", "xx", "xxx"] { + let input = format!("{prefix}{}", unit.repeat(bytes / unit.len())); + let output = bounded_detail(&input); + require_detail(&output, "detail")?; + assert!(output.len() <= 2048, "{} bytes", output.len()); + } + } + } + let nul = bounded_detail("a\0b\0"); + require_detail(&nul, "detail")?; + assert_eq!(nul, "ab"); + assert_eq!(bounded_detail("\0"), bounded_detail("")); + require_detail(&bounded_detail(""), "detail")?; + let exact = "a".repeat(2048); + assert_eq!(bounded_detail(&exact), exact); + Ok(()) + } + + #[test] + fn bounded_detail_keeps_the_tail_of_a_long_error() { + let input = format!("first line\n{}\nthe last line", "\u{2500}".repeat(5000)); + let output = bounded_detail(&input); + assert!(output.starts_with("first line")); + assert!(output.ends_with("the last line")); + assert!(output.contains(DETAIL_ELISION)); + assert!(output.len() > 1024, "the budget was not used: {}", output.len()); + } + #[test] fn a_sealing_failure_with_a_long_multibyte_error_aborts_and_frees_the_target() -> Result<()> { // 3000 chars of 3-byte box drawing: 9000 bytes, the shape of a build log. From bff308aea781817e44a935c38ce2d26e0ddbf47e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:39:32 +0200 Subject: [PATCH 101/196] Cut c2c: expire's refusal names its reason without reading the deadline predicate Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index f750257..ad06ea9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3138,7 +3138,7 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> ensure!( current.value.expirable(), "{}", - if current.value.resolves_by_deadline() { + if current.value.command_kind == CommandKind::Continuity && current.value.phase >= DeploymentPhase::Fencing && current.value.post_fencing_abort.is_none() { "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" } else { "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" From d9846e8847a675008f2877fa26842991fdc45f6d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:39:33 +0200 Subject: [PATCH 102/196] Cut c2c: route-continuity invariant 5 states the continuity forward retry A continuity past its post-fencing deadline is left to the operator and retries forward on a capped backoff; a stateless deployment aborts to its incumbent; a stateful deployment is left to the operator (ruling post-fence-forward-retry). Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index 77c201b..c4d3bc8 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -52,12 +52,13 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: the admitted incarnation. The proof answers an Idunn challenge and is signed with the credential Idunn issued to that launch. Invariant 3 changes where the proof travels, not what it proves. -5. **Every post-fencing phase ends.** Each phase after fencing has an admitted - deadline. When a deadline expires, Idunn resolves it: stateless targets abort - to the incumbent. Stateful targets get an explicit terminal record with a - named recovery. A granted lease that the process does not pick up within its - window is a named failure with an Idunn-owned recovery. It is never a silent - wait. Resumable-forever is gone. +5. **Every post-fencing phase has a deadline and is never a silent wait.** A + deployment's stateless phase that passes its deadline aborts to its + incumbent. A continuity past its deadline is left to the operator + (operator-required incident) and retries its phase forward on a capped + backoff, because aborting it would stop the only running copy of the admitted + body (ruling post-fence-forward-retry). A stateful deployment is left to the + operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never meets drift between the receipt and the projection. Rollback demotes the exact From 8f00df079fbc41ca42e3d05870b0ae7d6fc82004 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:40:52 +0200 Subject: [PATCH 103/196] Sealing-failure test: clear the resume backoff between ticks Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 16e45f3..6b508a6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16968,6 +16968,7 @@ mod tests { let mut world = sealing_world(&unrouted_recipe())?; world.engine.source = Arc::new(FailingSource { message: log }); for _ in 0..8 { + world.engine.resume_backoff.lock().unwrap().clear(); let _ = world.engine.run_scheduler_tick(); } assert!( From ec45906ead9f2034c14c5c2bc54addfae8cd8c1a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 19:07:23 +0200 Subject: [PATCH 104/196] bounded_detail: use std char-boundary floor/ceil instead of hand loops The hand loops hung under mutation (infinite loops); std does it. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 6b508a6..23c1962 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -9750,14 +9750,10 @@ fn bounded_detail(value: &str) -> String { if value.len() <= DETAIL_LIMIT { return value; } - let mut head = DETAIL_HEAD; - while !value.is_char_boundary(head) { - head -= 1; - } - let mut tail = value.len() - (DETAIL_LIMIT - DETAIL_HEAD - DETAIL_ELISION.len()); - while !value.is_char_boundary(tail) { - tail += 1; - } + let head = value.floor_char_boundary(DETAIL_HEAD); + let tail = value.ceil_char_boundary( + value.len() - (DETAIL_LIMIT - DETAIL_HEAD - DETAIL_ELISION.len()), + ); format!("{}{DETAIL_ELISION}{}", &value[..head], &value[tail..]) } From a7ad3de5593a086e2522eb8bafda1b9f72cd9af4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 19:58:09 +0200 Subject: [PATCH 105/196] c2d negative: a waiting Deploy must land over a forward-retrying continuity past its deadline Red at e529858: the waiter ends FailedBeforeFencing owner-left-to-operator. The yield (ruling continuity-yields-to-deploy) lands in the next commit. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 96 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 11 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index de88449..28ea709 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17643,6 +17643,80 @@ mod tests { } // END c2b-forever-wait harness + // BEGIN c2d continuity-yields + // A deployment waits at Warming beside a continuity that retries forward + // past its deadline (ruling continuity-yields-to-deploy). The pass order is + // the hostile one: the waiter always looks at the owner first. + + /// One pass over the waiter and then the owner, never clearing the resume + /// backoff. A record that left the live set is skipped. + fn waiter_then_owner(world: &EngineFixture, waiter: &str, owner: &str) -> Result<()> { + for id in [waiter, owner] { + if let Ok(stored) = resident_id(world, id) { + world.engine.resume_candidate(&stored)?; + } + } + Ok(()) + } + + /// A continuity at Fencing driven alone to AwaitingReady, where it + /// gate-waits for an Odin word that never comes, beside a Deploy waiting in + /// Warming. Returns the world, the workload, and the two ids. + fn waiter_beside_gate_waiting_continuity() + -> Result<(EngineFixture, Arc, DeploymentTransaction, String, String, AdmittedGeneration)> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, incumbent) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.clone(), continuity.transaction_id.clone()); + drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::AwaitingReady)?; + pass_deadline_of(&world, &owner_id)?; + Ok((world, workload, deploy, deploy_id, owner_id, incumbent)) + } + + /// NEGATIVE. The continuity's deadline has passed and it retries forward; + /// the deployment waiting on it must not end, the continuity must yield. + #[test] + fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { + let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); + for _ in 0..2 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + let waiter = latest_id(&world, &deploy_id)?; + assert!(waiter.completion.is_none(), "the waiter ended: {:?}", waiter.completion); + assert!(waiter.pre_fencing_abort.is_none()); + let owner = resident_id(&world, &owner_id)?.value; + let abort = owner.post_fencing_abort.as_ref().context("the continuity did not yield")?; + assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); + // Nothing stopped the candidate before the yield began. + assert_eq!(workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + + for _ in 0..8 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + world.engine.resume_backoff.lock().unwrap().clear(); + } + // The yield's abort retimed the record: its incident, if one opened, closed. + assert!(!operator_required(&world)?.contains(&owner_id)); + let ended = latest_id(&world, &owner_id)?; + assert!(matches!(ended.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), "{:?}", ended.completion); + assert_eq!(workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops + 1); + + odin_reports_ready(&world, &deploy, seeded_sequence(Some(&incumbent)) + 10)?; + drive_id(&world, &deploy_id, |t| t.is_terminal())?; + let landed = latest_id(&world, &deploy_id)?; + assert!(matches!(landed.completion, Some(TransactionCompletion::Admitted { .. })), "{:?}", landed.completion); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + assert_eq!(admitted.transaction_id, deploy_id); + assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); + Ok(()) + } + // END c2d continuity-yields + #[test] fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { let workload = SwitchWorkload::new(); @@ -22726,7 +22800,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(waiting.last_error.is_some()); assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); pass_deadline(&routed.world)?; for _ in 0..5 { routed.resume()?; @@ -22743,7 +22817,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide [left.transaction_id.clone()] ); assert_eq!(routed.world.engine.incidents().read()?.len(), 1, "one page per episode"); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped"); assert!(routed.reloads() > 6, "no attempt ran after the deadline"); // The proof passes: the phase moves on, and the first pass after @@ -22758,7 +22832,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let records = routed.world.engine.incidents().read()?; assert_eq!(records.len(), 1); assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); Ok(()) } @@ -23335,7 +23409,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let finished = routed.resume_until_terminal()?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), @@ -23343,7 +23417,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide finished.last_error ); assert!(driver.observe_empty()?, "the abort left the lease on disk"); - assert!(routed.workload.stops.load(Ordering::SeqCst) > stops, "the candidate was never stopped"); + assert!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst) > stops, "the candidate was never stopped"); Ok(()) } @@ -23480,7 +23554,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide &routed, "the Routing phase passed its deadline at unix ms 1792324133638", )?; - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let lifted_at = now_millis()?; let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; @@ -23488,7 +23562,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.resume()?; } assert_eq!( - routed.workload.stops.load(Ordering::SeqCst), + routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped" ); @@ -23504,7 +23578,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.stub.hang_up.store(false, Ordering::SeqCst); routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); assert!(routed.fragment_path().exists(), "the route membership was not re-installed"); Ok(()) } @@ -23525,7 +23599,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide &routed, "lease-not-adopted: the AwaitingReady phase passed its deadline at unix ms 1792324133638", )?; - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; for _ in 0..5 { @@ -23533,7 +23607,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } let waiting = routed.transaction()?; assert_eq!(returned, [waiting.transaction_id.clone()]); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped"); assert!(driver.observe_exact(&lease)?, "the lease left the disk"); assert!(waiting.post_fencing_abort.is_none() && waiting.completion.is_none()); assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); @@ -23544,7 +23618,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Routing)?; let adopted = routed.transaction()?; assert_eq!(adopted.lease_adoption.context("no adoption recorded")?.write_lease_sha256, lease_sha256); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); Ok(()) } From 4f68809f0b300c5a8830f51f18c5b2d061e41172 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 19:59:37 +0200 Subject: [PATCH 106/196] Continuity yields to a Deploy awaiting the handover (ruling continuity-yields-to-deploy) A continuity past its deadline at or past Fencing begins its own post-fencing abort when a Deploy on its target awaits the handover; the waiter keeps waiting on an owner that retries forward. Adds retries_forward and awaits_handover, both derived. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 53 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 28ea709..34cf549 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -983,7 +983,9 @@ enum PhaseEnd { OperatorRequired, /// A continuity at or past Fencing: its candidate is the only running copy /// of the admitted body, so aborting it restores what it already runs. The - /// resolver leaves it to the operator and its phase step keeps running. + /// resolver leaves it to the operator and its phase step keeps running, + /// until a deployment awaits the handover, when the continuity yields + /// through its own post-fencing abort (ruling continuity-yields-to-deploy). RetryForward, } @@ -1861,6 +1863,14 @@ impl DeploymentTransaction { && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) } + /// A continuity at or past Fencing that its deadline leaves to the operator + /// while its step runs. It yields to a deployment awaiting the handover + /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. + /// Derived, never stored. + fn retries_forward(&self) -> bool { + self.resolves_by_deadline() && self.phase_end() == PhaseEnd::RetryForward + } + /// What an ended post-fencing phase resolves to (Q3). A continuity at or /// past Fencing retries forward: its incumbent is the admitted generation, /// which supervision mints a continuity only after seeing dead, and from @@ -2064,6 +2074,16 @@ impl DeploymentTransaction { self.phase != DeploymentPhase::Complete } + /// A live deployment whose next move is the handover: not ended, not + /// aborting, one phase short of `takes_target_at`. Derived, never stored. + /// Read by the continuity's yield and the boot lift. + fn awaits_handover(&self) -> bool { + self.command_kind == CommandKind::Deploy + && self.completion.is_none() + && self.pre_fencing_abort.is_none() + && self.phase as u8 + 1 == self.takes_target_at() as u8 + } + /// From the handover until Complete the transaction owns the target's /// incumbent slot: it fences, leases, routes and commits. Derived from the /// phase, never stored. Read by validation (one owner), supervision (which @@ -8535,9 +8555,11 @@ impl Engine { /// A deployment waits on the transaction that owns or drains its target. /// The owner bounds the wait, so the waiter's own Warming deadline is /// disarmed (the resolver re-arms it when the target is free). When the - /// owner's deadline has passed in a state Idunn will not end by itself, - /// the waiter ends through its own pre-fencing abort (the caller's Err - /// path): it never aborts the owner. + /// owner's deadline has passed in a state Idunn will not end by itself and + /// will not yield (a wedged abort or drain), the waiter ends through its own + /// pre-fencing abort (the caller's Err path): it never aborts the owner. A + /// waiter keeps waiting on an owner that retries forward, because that + /// owner yields to it on its next pass. fn wait_for_owner( &self, current: &Stored, @@ -8547,7 +8569,9 @@ impl Engine { let now = now_millis()?; let snapshot = ControlSnapshot::read(&self.options.state_store)?; if snapshot.transactions.iter().any(|stored| { - stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) + stored.value.transaction_id == owner_id + && stored.value.leaves_to_operator(now) + && !stored.value.retries_forward() }) { bail!( "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", @@ -9522,7 +9546,9 @@ impl Engine { /// the fence, aborted after it, or left to the operator with its step /// retrying (a stateful deploy, or a continuity, whose candidate is the only /// running copy of its body). An abort or a drain past its deadline is left - /// to the operator the same way. + /// to the operator the same way. A continuity left to the operator yields, + /// through its own post-fencing abort, to a deployment awaiting the + /// handover; it never writes the deployment. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the @@ -9611,6 +9637,21 @@ impl Engine { Ok(None) } PhaseEnd::RetryForward => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(deploy) = snapshot + .on_target(&transaction.target) + .find(|other| other.value.awaits_handover()) + { + self.begin_post_fencing_abort( + current, + anyhow!( + "continuity-yields: {cause}; deployment {} awaits the handover of {}", + deploy.value.transaction_id, + transaction.target + ), + )?; + return Ok(Some(true)); + } self.leave_to_operator( transaction, &format!( From 196e0dc884f1a573a37c545f80dbb52f14f80127 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:03:48 +0200 Subject: [PATCH 107/196] c2d: a resolver write spends the resume backoff; the boot lift leaves a yield resume_candidate removes the transaction's backoff when the resolver moved its record, so a yield abort is not held behind the forward retry's backoff (ruling c2d-yield-mechanics). return_wedged_continuity_aborts_forward skips an abort whose target holds a Deploy awaiting the handover. Tests: the yield table, the unit rows, the c2b waiter-still-ends row, the backoff spend, and lift case (vii). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 197 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 189 insertions(+), 8 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 34cf549..e2494f9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5690,10 +5690,12 @@ impl Engine { /// c2c-boot-returns-wedged-aborts-forward). The result is the exact inverse /// of `begin_post_fencing_abort`. /// - /// A continuity abort now begins only over a permanently stopped - /// candidate, so this predicate cannot match a new record: it is a - /// migration with a deletion line (follow-up c2c-delete-boot-lift). A - /// record it cannot decide is reported and left aborting. + /// A continuity abort now begins only over a permanently stopped candidate + /// or to yield to a deployment awaiting the handover, so this predicate + /// matches only aborts over a running candidate with nothing awaiting the + /// target, which Idunn no longer creates: it is a migration with a deletion + /// line (follow-up c2c-delete-boot-lift). A record it cannot decide is + /// reported and left aborting. fn return_wedged_continuity_aborts_forward(&self) -> Result> { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut returned = Vec::new(); @@ -5706,6 +5708,9 @@ impl Engine { || transaction.completion.is_some() || abort.candidate_cleanup != CleanupEvidence::Pending || abort.lease_withdrawal == CleanupEvidence::Complete + || snapshot + .on_target(&transaction.target) + .any(|other| other.value.awaits_handover()) { continue; } @@ -6082,6 +6087,11 @@ impl Engine { fn resume_candidate(&self, current: &Stored) -> Result { let id = current.value.transaction_id.as_str(); if let Some(moved) = self.resolve_phase_deadline(current)? { + if moved { + // A phase-ending write is movement: the next step is not held + // behind the backoff a forward retry armed. + self.resume_backoff.lock().expect("resume backoff mutex").remove(id); + } return Ok(moved); } if self @@ -17721,9 +17731,7 @@ mod tests { fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); - for _ in 0..2 { - waiter_then_owner(&world, &deploy_id, &owner_id)?; - } + waiter_then_owner(&world, &deploy_id, &owner_id)?; let waiter = latest_id(&world, &deploy_id)?; assert!(waiter.completion.is_none(), "the waiter ended: {:?}", waiter.completion); assert!(waiter.pre_fencing_abort.is_none()); @@ -17756,6 +17764,167 @@ mod tests { assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); Ok(()) } + + fn stops_of(workload: &SwitchWorkload) -> usize { + workload.stops.load(std::sync::atomic::Ordering::SeqCst) + } + + /// The yield is a function of its inputs: the deadline, the continuity's + /// own record, and a deployment that awaits the handover. + #[test] + fn a_continuity_that_retries_forward_yields_only_past_its_deadline_and_only_to_a_deploy_awaiting_the_handover( + ) -> Result<()> { + // (i) Before the deadline nothing yields and the Deploy waits. + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let stops = stops_of(&workload); + for _ in 0..5 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + let waiting = resident_id(&world, &deploy_id)?.value; + assert!(waiting.pre_fencing_abort.is_none() && waiting.completion.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(&owner_id))); + assert_eq!(stops_of(&workload), stops); + + // (ii) Past the deadline, a Deploy still building awaits nothing. + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let stops = stops_of(&workload); + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + pass_deadline_of(&world, &owner_id)?; + for _ in 0..4 { + waiter_then_owner(&world, "", &owner_id)?; + } + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + + // (iii) A Deploy already ending before the fence awaits nothing. + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + + // (iv) Nor does a Deploy that has ended. + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; + for _ in 0..12 { + if latest_id(&world, &deploy_id)?.completion.is_some() { + break; + } + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, &deploy_id, "")?; + } + assert!(latest_id(&world, &deploy_id)?.completion.is_some()); + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + Ok(()) + } + + /// `awaits_handover` and `retries_forward` row by row. + #[test] + fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let waiting = resident_id(&world, &deploy_id)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.awaits_handover()); + for phase in [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Fencing, + DeploymentPhase::Complete, + ] { + let mut moved = waiting.clone(); + moved.phase = phase; + assert!(!moved.awaits_handover(), "{phase:?}"); + } + let mut ended = waiting.clone(); + ended.completion = Some(TransactionCompletion::FailedBeforeFencing { error: "x".into() }); + assert!(!ended.awaits_handover()); + let mut continuity = resident_id(&world, &owner_id)?.value; + let mut as_continuity = waiting.clone(); + as_continuity.command_kind = CommandKind::Continuity; + assert!(!as_continuity.awaits_handover()); + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("x"))?; + assert!(!resident_id(&world, &deploy_id)?.value.awaits_handover()); + + // The stop-then-start handover is Starting: its Sealing tail awaits it. + let (stop_first, stop_deploy, _) = stop_first_world(Arc::new(StillWorkload))?; + let mut sealing = resident_id(&stop_first, &stop_deploy)?.value; + sealing.phase = DeploymentPhase::Sealing; + assert!(sealing.awaits_handover()); + sealing.phase = DeploymentPhase::Warming; + assert!(!sealing.awaits_handover()); + + // A continuity retries forward from Fencing to Committing, until aborting. + for phase in [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + ] { + continuity.phase = phase; + assert!(continuity.retries_forward(), "{phase:?}"); + } + let mut deploy_record = waiting.clone(); + deploy_record.phase = DeploymentPhase::Fencing; + assert!(!deploy_record.retries_forward()); + continuity.post_fencing_abort = Some(post_fencing_abort_intent(&continuity, "x")); + assert!(!continuity.retries_forward()); + Ok(()) + } + + /// The c2b waiter still ends on an owner that will not yield: the yield's + /// own abort wedged past its deadline. + #[test] + fn the_c2b_waiter_still_ends_on_an_owner_that_will_not_yield() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + workload.fail_stop_of(&runtime_instance_of(&world, &owner_id)?); + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, &deploy_id, &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + pass_deadline_of(&world, &owner_id)?; + for _ in 0..20 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + assert!(!resident_id(&world, &owner_id)?.value.is_terminal()); + assert_ended_for_owner(&world, &deploy_id, &owner_id) + } + + /// The yield's abort is not held behind the backoff the forward retry + /// armed: movement spends it. + #[test] + fn a_yield_abort_is_not_held_behind_the_forward_retry_backoff() -> Result<()> { + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + // The Deploy is still building, so the pass past the deadline retries + // forward and arms the continuity backoff. + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + let armed = *world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&owner_id) + .context("the unmoved step set no backoff")?; + assert_eq!(armed.ceiling_millis, CONTINUITY_BACKOFF_CEILING_MILLIS); + // It reaches the handover: the next pass yields, and the abort runs + // with no clearing and no fake time. + rephase(&world, &deploy_id, DeploymentPhase::Warming)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + for _ in 0..3 { + waiter_then_owner(&world, "", &owner_id)?; + } + let done = latest_id(&world, &owner_id)?; + let cleaned = done.post_fencing_abort.as_ref().map(|abort| abort.candidate_cleanup); + assert_eq!(cleaned, Some(CleanupEvidence::Complete), "{:?}", done.last_error); + Ok(()) + } // END c2d continuity-yields #[test] @@ -23676,6 +23845,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide LeaseRevokedUnderIt, DeployWithRunningCandidate, CompletedRecord, + DeployAwaitsTheTarget, } for case in [ Case::CandidateNotRunning, @@ -23684,6 +23854,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Case::LeaseRevokedUnderIt, Case::DeployWithRunningCandidate, Case::CompletedRecord, + Case::DeployAwaitsTheTarget, ] { let deploy = matches!(case, Case::DeployWithRunningCandidate); // The cleanup comes after the lease withdrawal, so a case with @@ -23713,11 +23884,17 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide lease_driver(&routed).0.revoke_exact(Some(&lease))?; } Case::DeployWithRunningCandidate => {} + // The abort of the Odin-shaped negative, begun to yield: a + // Deploy awaits the handover of its target. + Case::DeployAwaitsTheTarget => { + seeded_transaction(&routed.world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + } Case::CompletedRecord => { routed.resume_until_terminal()?; } } - let abort = routed.transaction().ok().and_then(|t| t.post_fencing_abort); + let owner_id = the_continuity(&routed.world).ok().map(|c| c.value.transaction_id); + let abort = the_continuity(&routed.world).ok().and_then(|c| c.value.post_fencing_abort); match case { Case::LeaseWithdrawn => assert_eq!( abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), @@ -23738,6 +23915,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide after.transactions.iter().map(|t| &t.envelope).collect::>(), "the lift wrote a record it left" ); + if let (Case::DeployAwaitsTheTarget, Some(owner)) = (case, &owner_id) { + assert!(abort.is_some(), "the lift withdrew the yield"); + clear_other_live_transactions(&routed.world, owner)?; + } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), From a8683212598053a2bbce82ca3a454e056de82e6f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:06:00 +0200 Subject: [PATCH 108/196] c2d yield table: name the row an error comes from Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e2494f9..dc57b9d 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17780,7 +17780,7 @@ mod tests { for _ in 0..5 { waiter_then_owner(&world, &deploy_id, &owner_id)?; } - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(i)")?.value.post_fencing_abort.is_none()); let waiting = resident_id(&world, &deploy_id)?.value; assert!(waiting.pre_fencing_abort.is_none() && waiting.completion.is_none()); assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(&owner_id))); @@ -17794,7 +17794,7 @@ mod tests { for _ in 0..4 { waiter_then_owner(&world, "", &owner_id)?; } - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(ii)")?.value.post_fencing_abort.is_none()); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); @@ -17803,7 +17803,7 @@ mod tests { world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; pass_deadline_of(&world, &owner_id)?; waiter_then_owner(&world, "", &owner_id)?; - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(iii)")?.value.post_fencing_abort.is_none()); // (iv) Nor does a Deploy that has ended. let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; @@ -17818,7 +17818,7 @@ mod tests { assert!(latest_id(&world, &deploy_id)?.completion.is_some()); pass_deadline_of(&world, &owner_id)?; waiter_then_owner(&world, "", &owner_id)?; - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(iv)")?.value.post_fencing_abort.is_none()); Ok(()) } From fd6e3e65af2aa781c39b638aeb42ea773a421e62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:06:48 +0200 Subject: [PATCH 109/196] c2d yield table: show how row (i) ends --- src/control_plane.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index dc57b9d..5db88d4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17779,6 +17779,8 @@ mod tests { let stops = stops_of(&workload); for _ in 0..5 { waiter_then_owner(&world, &deploy_id, &owner_id)?; + let owner = latest_id(&world, &owner_id)?; + assert!(owner.completion.is_none(), "(i) {:?} {:?} {:?}", owner.completion, owner.phase, owner.last_error); } assert!(resident_id(&world, &owner_id).with_context(|| "(i)")?.value.post_fencing_abort.is_none()); let waiting = resident_id(&world, &deploy_id)?.value; From f79deecdb70ce6a5b845ca178850e961c01c247a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:07:30 +0200 Subject: [PATCH 110/196] c2d tests: the helper stops short of the deadline; each test passes it --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5db88d4..3b2d51e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17721,7 +17721,6 @@ mod tests { deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; let (deploy_id, owner_id) = (deploy.transaction_id.clone(), continuity.transaction_id.clone()); drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::AwaitingReady)?; - pass_deadline_of(&world, &owner_id)?; Ok((world, workload, deploy, deploy_id, owner_id, incumbent)) } @@ -17730,6 +17729,7 @@ mod tests { #[test] fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + pass_deadline_of(&world, &owner_id)?; let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); waiter_then_owner(&world, &deploy_id, &owner_id)?; let waiter = latest_id(&world, &deploy_id)?; From 09bd12764323b3dc4d023041ca16bd7c845c164f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:09:28 +0200 Subject: [PATCH 111/196] c2d: stateful yield test, an Odin-shaped continuity yields its lease to a waiting Deploy Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3b2d51e..49fe7f9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -23834,6 +23834,76 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// NEGATIVE. The live shape of tx-4eaed669 (Odin) left to the operator + /// at its Routing deadline, with a Deploy now awaiting the handover of + /// its target. The continuity yields through its own abort: the lease + /// leaves the disk only after the stop, the target is free, and the + /// Deploy, unended, fences a path that is already empty. + #[cfg(unix)] + #[test] + fn an_odin_shaped_continuity_left_to_the_operator_yields_its_lease_to_a_waiting_deploy() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + assert!(routed.transaction()?.lease_adoption.is_some()); + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + let owner_id = routed.transaction()?.transaction_id; + pass_deadline(&routed.world)?; + routed.resume()?; + let left = routed.transaction()?; + assert!(left.post_fencing_abort.is_none(), "the continuity aborted with nothing awaiting the target"); + assert_eq!(routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, [owner_id.as_str()]); + assert!(driver.observe_exact(&lease)?); + + let deploy = seeded_transaction(&routed.world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + for _ in 0..12 { + routed.resume()?; + let owner = latest_id(&routed.world, &owner_id)?; + if owner.post_fencing_abort.is_none() { + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(driver.observe_exact(&lease)?, "the lease left before the yield"); + } + if owner.completion.is_some() { + break; + } + } + let ended = latest_id(&routed.world, &owner_id)?; + assert!( + matches!( + ended.completion, + Some(TransactionCompletion::FailedAfterFencing { recovery: TerminalRecovery::RestartAdmitted, .. }) + ), + "{:?}", + ended.completion + ); + assert!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst) > stops); + assert!(!driver.observe_exact(&lease)?, "the lease stayed on disk"); + assert!(ControlSnapshot::read(&routed.world.state_store)?.target_owner("service", None).is_none()); + let waiting = resident_id(&routed.world, &deploy.transaction_id)?.value; + assert!(waiting.completion.is_none() && waiting.pre_fencing_abort.is_none()); + + // Its next move is the fence, over a lease path that is empty. + rephase(&routed.world, &deploy.transaction_id, DeploymentPhase::Fencing)?; + routed.world.engine.advance_transaction(&resident_id(&routed.world, &deploy.transaction_id)?)?; + let fenced = latest_id(&routed.world, &deploy.transaction_id)?; + assert!( + matches!( + fenced.fencing, + Some(FencingEvidence::Revoked { candidate_lease_path_verified_empty: true, .. }) + ), + "{:?} {:?}", + fenced.fencing, + fenced.last_error + ); + Ok(()) + } + /// The lift is a function of its inputs: each case lacks one thing the /// lift requires, and its abort stays and runs to its end. #[cfg(unix)] From 4e66a65758873eab4d3f41ef395a0096dc0ff9bd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:10:18 +0200 Subject: [PATCH 112/196] c2d stateful yield test: drop the Fencing half the seeded Deploy cannot honestly assert Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 49fe7f9..d872c4e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -23838,7 +23838,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// at its Routing deadline, with a Deploy now awaiting the handover of /// its target. The continuity yields through its own abort: the lease /// leaves the disk only after the stop, the target is free, and the - /// Deploy, unended, fences a path that is already empty. + /// Deploy is unended. (The Fencing half is not asserted here: the seeded + /// Deploy plans a stateless target, so its fence skips the lease path.) #[cfg(unix)] #[test] fn an_odin_shaped_continuity_left_to_the_operator_yields_its_lease_to_a_waiting_deploy() -> Result<()> { @@ -23887,20 +23888,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(ControlSnapshot::read(&routed.world.state_store)?.target_owner("service", None).is_none()); let waiting = resident_id(&routed.world, &deploy.transaction_id)?.value; assert!(waiting.completion.is_none() && waiting.pre_fencing_abort.is_none()); - - // Its next move is the fence, over a lease path that is empty. - rephase(&routed.world, &deploy.transaction_id, DeploymentPhase::Fencing)?; - routed.world.engine.advance_transaction(&resident_id(&routed.world, &deploy.transaction_id)?)?; - let fenced = latest_id(&routed.world, &deploy.transaction_id)?; - assert!( - matches!( - fenced.fencing, - Some(FencingEvidence::Revoked { candidate_lease_path_verified_empty: true, .. }) - ), - "{:?} {:?}", - fenced.fencing, - fenced.last_error - ); Ok(()) } From 81bfb037123d97287c024fdab407404253233a1a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:11:10 +0200 Subject: [PATCH 113/196] route-continuity-target invariant 5: the continuity yields to a deployment awaiting the handover Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index c4d3bc8..c014a2a 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -57,7 +57,10 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: incumbent. A continuity past its deadline is left to the operator (operator-required incident) and retries its phase forward on a capped backoff, because aborting it would stop the only running copy of the admitted - body (ruling post-fence-forward-retry). A stateful deployment is left to the + body (ruling post-fence-forward-retry), until a deployment awaits the + handover of its target: then it yields through its own post-fencing abort, + and the deployment fences what is admitted (ruling + continuity-yields-to-deploy). A stateful deployment is left to the operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never From 11f7fdbf09dc4c6447eacb86f83c525f03d828c7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:28:06 +0200 Subject: [PATCH 114/196] c2c fix 1/3: negative test, a lift that cannot observe must not let the abort stop the candidate Fails at 81bfb03: an observe error leaves the abort running and its first step stops the candidate. Also pins the lease_withdrawal==Complete conjunct (case LeaseWithdrawalRecordedComplete) and makes the refusal table's dead-candidate case provably dead (SwitchWorkload::bury). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 106 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d872c4e..e7297b0 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14522,6 +14522,10 @@ mod tests { stops: std::sync::atomic::AtomicUsize, /// Runtime instances whose stop fails, whatever is asked. failing_stops: std::sync::Mutex>, + /// The unit is absent or failed, which the driver reports as + /// permanently stopped: positive evidence of death, unlike an + /// observation that merely errs. + buried: std::sync::atomic::AtomicBool, } impl SwitchWorkload { @@ -14535,9 +14539,16 @@ mod tests { holds: std::sync::Mutex::new(Vec::new()), stops: std::sync::atomic::AtomicUsize::new(0), failing_stops: std::sync::Mutex::new(Vec::new()), + buried: std::sync::atomic::AtomicBool::new(false), }) } + /// The process is dead and the driver can prove it. + fn bury(&self) { + self.kill(); + self.buried.store(true, std::sync::atomic::Ordering::SeqCst); + } + /// From now on a start launches a process distinct from every recorded one. fn launching(&self) { self.launches.store(true, std::sync::atomic::Ordering::SeqCst); @@ -14584,6 +14595,8 @@ mod tests { } fn revive(&self) { + self.buried + .store(false, std::sync::atomic::Ordering::SeqCst); self.alive .store(true, std::sync::atomic::Ordering::SeqCst); self.rebooted @@ -14695,7 +14708,7 @@ mod tests { Ok(()) } fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(false) + Ok(self.buried.load(std::sync::atomic::Ordering::SeqCst)) } fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) @@ -23834,6 +23847,81 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// NEGATIVE. The boot lift runs before anything it needs may answer: a + /// systemctl error, a host-actuator hub no actuator has reached. The + /// abort of a candidate it could not observe is held, not run (its first + /// step stops the candidate), and the lift is retried on later passes. + /// Once observation answers Ok the record is lifted; once it answers + /// dead the abort proceeds. + #[cfg(unix)] + #[test] + fn a_boot_lift_that_cannot_observe_holds_the_abort_and_retries() -> Result<()> { + for hub_unattached in [false, true] { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + wedge_abort(&routed, "the Routing phase passed its deadline at unix ms 1792324133638")?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + let observation_fails = || { + if hub_unattached { + routed.workload.silence(true); + } else { + routed.workload.kill(); + } + }; + + // Boot: observation errs. Nothing is lifted and nothing stops. + observation_fails(); + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty(), "lifted {returned:?} it could not observe"); + for _ in 0..5 { + routed.resume()?; + assert_eq!( + routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), + stops, + "an undecided abort stopped the candidate (hub_unattached {hub_unattached})" + ); + } + let held = routed.transaction()?; + let abort = held.post_fencing_abort.context("the abort was dropped")?; + assert_eq!(abort.lease_withdrawal, CleanupEvidence::Pending); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + + // Observation answers Ok: the next pass lifts the record. + routed.workload.silence(false); + routed.workload.revive(); + routed.resume()?; + let lifted = routed.transaction()?; + assert!(lifted.post_fencing_abort.is_none(), "a later pass did not lift the record"); + assert_eq!(lifted.phase, DeploymentPhase::Routing); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + } + + // Observation answers dead: the abort proceeds to its end. + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + wedge_abort(&routed, "the AwaitingReady phase passed its deadline")?; + routed.workload.silence(true); + assert!(routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty()); + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some_and(|a| a.candidate_cleanup == CleanupEvidence::Pending)); + routed.workload.silence(false); + routed.workload.bury(); + let finished = routed.resume_until_terminal()?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.completion + ); + Ok(()) + } + /// NEGATIVE. The live shape of tx-4eaed669 (Odin) left to the operator /// at its Routing deadline, with a Deploy now awaiting the handover of /// its target. The continuity yields through its own abort: the lease @@ -23899,6 +23987,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[derive(Clone, Copy, Debug)] enum Case { CandidateNotRunning, + LeaseWithdrawalRecordedComplete, LeaseWithdrawn, CandidateCleaned, LeaseRevokedUnderIt, @@ -23908,6 +23997,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } for case in [ Case::CandidateNotRunning, + Case::LeaseWithdrawalRecordedComplete, Case::LeaseWithdrawn, Case::CandidateCleaned, Case::LeaseRevokedUnderIt, @@ -23936,13 +24026,25 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide // step, or one input, away from the lift's predicate. routed.resume()?; match case { - Case::CandidateNotRunning => routed.workload.kill(), + Case::CandidateNotRunning => routed.workload.bury(), Case::LeaseWithdrawn | Case::CandidateCleaned => routed.resume()?, Case::LeaseRevokedUnderIt => { let (lease, _) = lease_of(&routed)?; lease_driver(&routed).0.revoke_exact(Some(&lease))?; } Case::DeployWithRunningCandidate => {} + // The record says the lease was withdrawn while the candidate + // still runs and the lease is still exact on disk: only the + // record's own evidence keeps the lift off it. + Case::LeaseWithdrawalRecordedComplete => { + let stored = resident(&routed.world)?; + let mut next = stored.value.clone(); + next.post_fencing_abort + .as_mut() + .context("no abort")? + .lease_withdrawal = CleanupEvidence::Complete; + replace_transaction(&routed.world.state_store, &stored, &next)?; + } // The abort of the Odin-shaped negative, begun to yield: a // Deploy awaits the handover of its target. Case::DeployAwaitsTheTarget => { From 35d9c7fcf8b3f11be03f0ef0878714e8f70d8807 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:30:18 +0200 Subject: [PATCH 115/196] c2c fix 2/3: the lift defers on an observe error; the retry exit is worded from the driver An abort over a candidate the boot lift could not observe is held in process memory: resume_candidate runs no step of it and retries the lift each pass (retry_held_lift) until observation answers Ok (lifted) or the driver proves the candidate dead (the abort proceeds). The predicate and write move into consider_lifting_continuity_abort, which boot and the retry share. Ruling c2c-lift-defers-on-observe-error. The page and the expire refusal name the manual exit from the plan's workload binding: a unit to stop for systemd, a process to kill for a host actuator. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 233 ++++++++++++++++++++++++++++++++----------- 1 file changed, 173 insertions(+), 60 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e7297b0..786486f 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1863,6 +1863,23 @@ impl DeploymentTransaction { && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) } + /// The operator's only way to restart a forward-retrying continuity, worded + /// from the driver its plan binds: a systemd target has a unit to stop (a + /// stopped transient unit is forgotten, which is permanently stopped); a + /// host-actuator target has no unit, only a process on the managed host + /// that the actuator then reports not running. + fn manual_exit(&self) -> &'static str { + match self + .plan + .as_ref() + .and_then(|plan| plan.parsed_inputs().ok()) + .map(|(_, binding)| binding.workload) + { + Some(WorkloadBinding::HostActuator(_)) => "kill its process on the managed host by hand", + _ => "stop its unit by hand", + } + } + /// A continuity at or past Fencing that its deadline leaves to the operator /// while its step runs. It yields to a deployment awaiting the handover /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. @@ -3159,9 +3176,12 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> current.value.expirable(), "{}", if current.value.command_kind == CommandKind::Continuity && current.value.phase >= DeploymentPhase::Fencing && current.value.post_fencing_abort.is_none() { - "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" + format!( + "a continuity past Fencing retries forward and cannot be expired; to restart it, {} and Idunn aborts it as dead", + current.value.manual_exit() + ) } else { - "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel".to_owned() } ); let now = now_millis()?; @@ -5097,6 +5117,10 @@ struct Engine { /// Process-local on purpose: a restart retries at once, and no schema /// carries a clock that only pacing reads. resume_backoff: Mutex>, + /// Continuity aborts the boot lift could not decide (their candidate could + /// not be observed): no step of them runs until `retry_held_lift` decides. + /// Process-local: a restart asks again at boot. + lift_held: Mutex>, /// When each routed target's next challenge is due, held only when the /// stored `RouteSupervisionState` could not record the last one. The stored /// value owns the schedule and survives restarts; this spaces attempts @@ -5209,6 +5233,7 @@ impl Engine { history_report: ReportOnce::default(), fault_reports: Mutex::default(), resume_backoff: Mutex::default(), + lift_held: Mutex::default(), challenge_wait: Mutex::default(), }) } @@ -5682,6 +5707,17 @@ struct ProjectionReconciliation { unexplained: Vec, } +/// What the lift decided about one continuity abort. +enum LiftDecision { + /// The abort was withdrawn and the record retimed. + Lifted, + /// The abort stands and runs: not a lift candidate, or its candidate is + /// provably dead. + KeepAborting, + /// The candidate could not be observed and is not provably dead. + Undecided(anyhow::Error), +} + impl Engine { /// Once at boot: withdraw the abort of a continuity that has not stopped /// its running candidate, and retime the record so its phase retries @@ -5694,73 +5730,126 @@ impl Engine { /// or to yield to a deployment awaiting the handover, so this predicate /// matches only aborts over a running candidate with nothing awaiting the /// target, which Idunn no longer creates: it is a migration with a deletion - /// line (follow-up c2c-delete-boot-lift). A record it cannot decide is - /// reported and left aborting. + /// line (follow-up c2c-delete-boot-lift). A record whose candidate cannot + /// be observed is held, not aborted: `resume_candidate` runs no step of it + /// and asks `retry_held_lift` each pass, until observation answers (ruling + /// c2c-lift-defers-on-observe-error). fn return_wedged_continuity_aborts_forward(&self) -> Result> { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut returned = Vec::new(); for stored in &snapshot.transactions { - let transaction = &stored.value; - let Some(abort) = &transaction.post_fencing_abort else { - continue; - }; - if transaction.command_kind != CommandKind::Continuity - || transaction.completion.is_some() - || abort.candidate_cleanup != CleanupEvidence::Pending - || abort.lease_withdrawal == CleanupEvidence::Complete - || snapshot - .on_target(&transaction.target) - .any(|other| other.value.awaits_handover()) - { - continue; - } - let id = &transaction.transaction_id; - let lifted = (|| -> Result { - let plan = required(&transaction.plan, "transaction plan")?; - let expected = required(&transaction.expected, "Expected projection")?; - let activation = required(&transaction.activation, "activation")?; - let Some(workload) = &transaction.workload else { - return Ok(false); - }; - self.workload_for(plan)? - .observe(expected, activation, workload)?; - if let Some(lease) = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::issued_lease) - { - let path = plan - .parsed_inputs()? - .1 - .process_write_lease - .context("leased target has no write-lease binding")? - .record_path; - if !CultCacheWriteLeaseDriver::new(&transaction.target, path) - .observe_exact(lease)? - { - return Ok(false); - } + let id = &stored.value.transaction_id; + match self.consider_lifting_continuity_abort(&snapshot, stored) { + LiftDecision::Lifted => returned.push(id.clone()), + LiftDecision::KeepAborting => {} + LiftDecision::Undecided(error) => { + eprintln!( + "Idunn could not decide whether continuity {id} may retry forward; its abort is held, not run: {error:#}" + ); + self.lift_held.lock().expect("lift held mutex").insert(id.clone()); } - let now = now_millis()?; - let mut next = transaction.clone(); - next.post_fencing_abort = None; - next.last_error = Some(abort.error.clone()); - next.updated_at_unix_millis = now; - next.retime(now); - replace_transaction(&self.options.state_store, stored, &next)?; - Ok(true) - })(); - match lifted { - Ok(true) => returned.push(id.clone()), - Ok(false) => {} - Err(error) => eprintln!( - "Idunn could not decide whether continuity {id} may retry forward; its abort is left running: {error:#}" - ), } } Ok(returned) } + /// A held record's later pass: lift it, release it to its abort over + /// positive evidence of death, or keep holding. `Some(moved)` means the + /// pass is spent on the record; `None` lets the pass go on. + fn retry_held_lift(&self, current: &Stored) -> Result> { + let id = current.value.transaction_id.as_str(); + if !self.lift_held.lock().expect("lift held mutex").contains(id) { + return Ok(None); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + match self.consider_lifting_continuity_abort(&snapshot, current) { + LiftDecision::Undecided(error) => { + self.note_fault("holds continuity abort", id, &error); + Ok(Some(false)) + } + LiftDecision::Lifted => { + self.lift_held.lock().expect("lift held mutex").remove(id); + self.clear_fault(id); + println!( + "returned continuity {id} to forward retry: its abort had not stopped its running candidate" + ); + Ok(Some(true)) + } + LiftDecision::KeepAborting => { + self.lift_held.lock().expect("lift held mutex").remove(id); + self.clear_fault(id); + Ok(None) + } + } + } + + /// The lift's predicate and write for one record. An abort proceeds only + /// over positive evidence that its candidate is dead; an observation that + /// merely errs is undecided. + fn consider_lifting_continuity_abort( + &self, + snapshot: &ControlSnapshot, + stored: &Stored, + ) -> LiftDecision { + let transaction = &stored.value; + let Some(abort) = &transaction.post_fencing_abort else { + return LiftDecision::KeepAborting; + }; + if transaction.command_kind != CommandKind::Continuity + || transaction.completion.is_some() + || abort.candidate_cleanup != CleanupEvidence::Pending + || abort.lease_withdrawal == CleanupEvidence::Complete + || snapshot + .on_target(&transaction.target) + .any(|other| other.value.awaits_handover()) + { + return LiftDecision::KeepAborting; + } + let lifted = (|| -> Result { + let plan = required(&transaction.plan, "transaction plan")?; + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(plan)? + .observe(expected, activation, workload)?; + if let Some(lease) = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + { + let path = plan + .parsed_inputs()? + .1 + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + if !CultCacheWriteLeaseDriver::new(&transaction.target, path) + .observe_exact(lease)? + { + return Ok(false); + } + } + let now = now_millis()?; + let mut next = transaction.clone(); + next.post_fencing_abort = None; + next.last_error = Some(abort.error.clone()); + next.updated_at_unix_millis = now; + next.retime(now); + replace_transaction(&self.options.state_store, stored, &next)?; + Ok(true) + })(); + match lifted { + Ok(true) => LiftDecision::Lifted, + Ok(false) => LiftDecision::KeepAborting, + Err(error) => match self.candidate_is_permanently_stopped(transaction) { + Ok(true) => LiftDecision::KeepAborting, + _ => LiftDecision::Undecided(error), + }, + } + } + /// Once at boot: demote the activation a failed continuity left standing, /// where the projection still names exactly the activation that failed /// transaction issued. A continuity abort before the single resolution @@ -6086,6 +6175,9 @@ impl Engine { /// transaction would report progress every tick. fn resume_candidate(&self, current: &Stored) -> Result { let id = current.value.transaction_id.as_str(); + if let Some(moved) = self.retry_held_lift(current)? { + return Ok(moved); + } if let Some(moved) = self.resolve_phase_deadline(current)? { if moved { // A phase-ending write is movement: the next step is not held @@ -9665,7 +9757,8 @@ impl Engine { self.leave_to_operator( transaction, &format!( - "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, stop its unit by hand" + "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, {}", + transaction.manual_exit() ), ); Ok(None) @@ -16173,6 +16266,21 @@ mod tests { } } + /// The operator's exit from a forward-retrying continuity is worded from + /// the driver the plan binds: a host-actuator target has no unit to stop. + #[test] + fn the_manual_exit_is_worded_from_the_targets_driver() -> Result<()> { + let (world, _deploy, host_continuity) = stop_first_world(SwitchWorkload::new())?; + let host = resident_id(&world, &host_continuity)?.value; + assert!(host.manual_exit().contains("managed host"), "{}", host.manual_exit()); + assert!(!host.manual_exit().contains("unit"), "{}", host.manual_exit()); + let systemd = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&systemd)?; + let unit = seeded_transaction(&systemd, DeploymentPhase::Fencing, CommandKind::Continuity, Some(&incumbent))?; + assert!(unit.manual_exit().contains("unit"), "{}", unit.manual_exit()); + Ok(()) + } + /// The step that ends a transaction archives it in its own write, so a /// record gone from the live set is finished only if history holds it. One /// that vanished from both is a fault, never progress. @@ -24080,6 +24188,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(abort.is_some(), "the lift withdrew the yield"); clear_other_live_transactions(&routed.world, owner)?; } + if matches!(case, Case::LeaseWithdrawalRecordedComplete) { + // The record's claim is made true, so the abort can finish. + let (lease, _) = lease_of(&routed)?; + lease_driver(&routed).0.revoke_exact(Some(&lease))?; + } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), From 1beca61ac458081fa171fbc1884c9d5e65ecaa0f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:31:36 +0200 Subject: [PATCH 116/196] c2c fix 3/3: the refusal table's recorded-withdrawal case ends at the lift Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 786486f..3f9ef43 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -24189,9 +24189,9 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide clear_other_live_transactions(&routed.world, owner)?; } if matches!(case, Case::LeaseWithdrawalRecordedComplete) { - // The record's claim is made true, so the abort can finish. - let (lease, _) = lease_of(&routed)?; - lease_driver(&routed).0.revoke_exact(Some(&lease))?; + // An edited record, not a reachable one: the lift's refusal + // is the whole claim, and its abort is not run to the end. + continue; } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( From 04389e6afc2696d895dca271a208e2625484f8ee Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:38:10 +0200 Subject: [PATCH 117/196] c2d fix 1/2: negative tests, no yield to a serving continuity or a braked Deploy Fail at 1beca61 (the c2d logic is 81bfb03's): a Committing continuity past its deadline yields to a waiting Deploy and stops serving; a stop-then-start Deploy at its Sealing tail with the deployment brake unreleased draws the yield. The second test releases the brake and pins that the yield then happens and closes the continuity's operator-required incident Recovered. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 95 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3f9ef43..e34c1c4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17945,7 +17945,7 @@ mod tests { Ok(()) } - /// `awaits_handover` and `retries_forward` row by row. + /// `awaits_handover` and `yields_to_a_deploy` row by row. #[test] fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; @@ -17999,6 +17999,93 @@ mod tests { Ok(()) } + /// NEGATIVE. A continuity past its route proof is serving. Past its deadline + /// it keeps retrying and pages; a waiting Deploy does not stop it. + #[test] + fn a_committing_continuity_past_its_deadline_keeps_serving_and_pages() -> Result<()> { + let (world, workload, _, _, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + rephase(&world, &owner_id, DeploymentPhase::Committing)?; + pass_deadline_of(&world, &owner_id)?; + let stops = stops_of(&workload); + for _ in 0..4 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + let owner = resident_id(&world, &owner_id)?.value; + assert!(owner.post_fencing_abort.is_none(), "a serving continuity yielded"); + assert_eq!(owner.phase, DeploymentPhase::Committing); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + Ok(()) + } + + /// NEGATIVE. A Deploy held at its Sealing tail by the deployment brake + /// cannot take the handover, so a continuity does not stop its candidate + /// for it; once the brake releases, the continuity yields. The yield closes + /// the continuity's operator-required incident Recovered. + #[test] + fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + make_stop_first(&world, &deploy_id)?; + let sealing = resident_id(&world, &deploy_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + assert!(resident_id(&world, &deploy_id)?.value.awaits_handover()); + pass_deadline_of(&world, &owner_id)?; + let stops = stops_of(&workload); + for _ in 0..4 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + let held = resident_id(&world, &owner_id)?.value; + assert!(held.post_fencing_abort.is_none(), "the continuity yielded to a braked Deploy"); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + + // The operator releases the brake for this exact Deploy. + let deploy = resident_id(&world, &deploy_id)?.value; + let authorization = deployment_authorization( + &world, + deploy.expected.as_ref().context("no Expected")?, + &deploy_id, + now_millis()?, + )?; + let store = deploy.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + assert!(SingleFileMessagePackBackingStore::new(&store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + key: record.brake_id.clone(), + current: None, + }], + &[CultCacheEnvelope { + key: record.brake_id.clone(), + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + payload: authorization.canonical_brake_bytes.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), + }], + )?); + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + let yielded = resident_id(&world, &owner_id)?.value; + let abort = yielded.post_fencing_abort.as_ref().context("the continuity did not yield")?; + assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); + // The yield retimed the record: the next pass closes its incident + // Recovered, and it stays closed. + for _ in 0..3 { + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_one_transaction()?; + } + assert!(operator_required(&world)?.is_empty()); + let records = world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].subject, owner_id); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + Ok(()) + } + /// The c2b waiter still ends on an owner that will not yield: the yield's /// own abort wedged past its deadline. #[test] @@ -18311,7 +18398,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide bind_provider( world, &format!("{head}{HOST}[runtime_identity]{identity}[brakes]{brakes}") - .replace("candidate-then-promote", "stop-then-start"), + .replace("candidate-then-promote", "stop-then-start") + .replace( + "/var/lib/gamecult/idunn/service-deployment-brake.cc", + &world.root.join("service-deployment-brake.cc").display().to_string(), + ), ) } From b119e500adb0d318de56a48a1739947c58f34532 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:43:34 +0200 Subject: [PATCH 118/196] c2d fix 2/2: a continuity yields only while unproven, and only to a Deploy that can move serving_proven (Committing, or routing evidence recorded) ends the yield: yields_to_a_deploy replaces retries_forward, and wait_for_owner reads it, so a Deploy no longer waits on an owner that will keep serving. deploy_can_move asks the deployment brake live for a Deploy at its Sealing tail, so a braked Deploy draws no yield and the yield happens once the brake releases. The tests of 04389e6 are repaired to reach their worlds (the failures they showed at 1beca61 were the right ones). Rulings c2d-yield-only-unproven-to-movable. Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 12 ++-- src/control_plane.rs | 109 ++++++++++++++++++++++++-------- 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index c014a2a..7c25d0c 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -57,11 +57,13 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: incumbent. A continuity past its deadline is left to the operator (operator-required incident) and retries its phase forward on a capped backoff, because aborting it would stop the only running copy of the admitted - body (ruling post-fence-forward-retry), until a deployment awaits the - handover of its target: then it yields through its own post-fencing abort, - and the deployment fences what is admitted (ruling - continuity-yields-to-deploy). A stateful deployment is left to the - operator. + body (ruling post-fence-forward-retry), until a deployment that the + deployment brake has released awaits the handover of its target and the + continuity's route proof has not succeeded: then it yields through its own + post-fencing abort, and the deployment fences what is admitted (rulings + continuity-yields-to-deploy, c2d-yield-only-unproven-to-movable). A + continuity that has proven its route is serving and never yields. A + stateful deployment is left to the operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never meets drift between the receipt and the projection. Rollback demotes the exact diff --git a/src/control_plane.rs b/src/control_plane.rs index e34c1c4..5d1389a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1880,12 +1880,23 @@ impl DeploymentTransaction { } } + /// The candidate's route proof has succeeded: it is Committing, or its + /// routing evidence is recorded and only the transition remains. It serves, + /// so no waiting deployment stops it (ruling c2d-yield-only-unproven-to- + /// movable). Derived, never stored. + fn serving_proven(&self) -> bool { + self.phase >= DeploymentPhase::Committing || self.routing.is_some() + } + /// A continuity at or past Fencing that its deadline leaves to the operator - /// while its step runs. It yields to a deployment awaiting the handover - /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. - /// Derived, never stored. - fn retries_forward(&self) -> bool { - self.resolves_by_deadline() && self.phase_end() == PhaseEnd::RetryForward + /// while its step runs, and whose candidate is still unproven: it yields to + /// a deployment that can take the handover (ruling continuity-yields-to- + /// deploy), so a waiter never ends on it. A proven continuity keeps serving + /// and paging and does not yield. Derived, never stored. + fn yields_to_a_deploy(&self) -> bool { + self.resolves_by_deadline() + && self.phase_end() == PhaseEnd::RetryForward + && !self.serving_proven() } /// What an ended post-fencing phase resolves to (Q3). A continuity at or @@ -8546,6 +8557,17 @@ impl Engine { })) } + /// Whether a deployment awaiting the handover is free to take it: the + /// deployment brake has released its exact transaction, so the stop a yield + /// makes is followed by the deployment's own landing. A Deploy at its + /// Sealing tail has not passed the brake yet and is read live; one that is + /// still building, or whose brake cannot be read, is held. An operator's + /// hold is a reason to leave a serving or retrying body alone. + fn deploy_can_move(&self, deploy: &Stored, now: u64) -> bool { + deploy.value.deployment_authorization.is_some() + || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) + } + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { let plan = required(¤t.value.plan, "continuity plan")?; let binding = plan.parsed_inputs()?.1; @@ -8673,7 +8695,7 @@ impl Engine { if snapshot.transactions.iter().any(|stored| { stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) - && !stored.value.retries_forward() + && !stored.value.yields_to_a_deploy() }) { bail!( "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", @@ -9740,10 +9762,12 @@ impl Engine { } PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - if let Some(deploy) = snapshot - .on_target(&transaction.target) - .find(|other| other.value.awaits_handover()) - { + let now = now_millis()?; + if let Some(deploy) = snapshot.on_target(&transaction.target).find(|other| { + transaction.yields_to_a_deploy() + && other.value.awaits_handover() + && self.deploy_can_move(other, now) + }) { self.begin_post_fencing_abort( current, anyhow!( @@ -17947,7 +17971,7 @@ mod tests { /// `awaits_handover` and `yields_to_a_deploy` row by row. #[test] - fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { + fn awaits_handover_and_yields_to_a_deploy_are_exact_functions_of_the_record() -> Result<()> { let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; let waiting = resident_id(&world, &deploy_id)?.value; assert_eq!(waiting.phase, DeploymentPhase::Warming); @@ -17980,22 +18004,29 @@ mod tests { sealing.phase = DeploymentPhase::Warming; assert!(!sealing.awaits_handover()); - // A continuity retries forward from Fencing to Committing, until aborting. + // A continuity yields from Fencing until its route proof succeeds, and + // until aborting; a proven one keeps serving. for phase in [ DeploymentPhase::Fencing, DeploymentPhase::Leasing, DeploymentPhase::AwaitingReady, DeploymentPhase::Routing, - DeploymentPhase::Committing, ] { continuity.phase = phase; - assert!(continuity.retries_forward(), "{phase:?}"); + assert!(continuity.yields_to_a_deploy(), "{phase:?}"); + assert!(!continuity.serving_proven(), "{phase:?}"); } + continuity.phase = DeploymentPhase::Committing; + assert!(continuity.serving_proven() && !continuity.yields_to_a_deploy()); + continuity.phase = DeploymentPhase::Routing; + continuity.routing = Some(RoutingEvidence::SkippedUnrouted); + assert!(continuity.serving_proven() && !continuity.yields_to_a_deploy()); + continuity.routing = None; let mut deploy_record = waiting.clone(); deploy_record.phase = DeploymentPhase::Fencing; - assert!(!deploy_record.retries_forward()); + assert!(!deploy_record.yields_to_a_deploy()); continuity.post_fencing_abort = Some(post_fencing_abort_intent(&continuity, "x")); - assert!(!continuity.retries_forward()); + assert!(!continuity.yields_to_a_deploy()); Ok(()) } @@ -18003,15 +18034,43 @@ mod tests { /// it keeps retrying and pages; a waiting Deploy does not stop it. #[test] fn a_committing_continuity_past_its_deadline_keeps_serving_and_pages() -> Result<()> { - let (world, workload, _, _, owner_id, _) = waiter_beside_gate_waiting_continuity()?; - rephase(&world, &owner_id, DeploymentPhase::Committing)?; + let (world, workload, _, _, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + let owner = resident_id(&world, &owner_id)?.value; + // The continuity's incarnation already has a correlation (the + // incumbent's, same release): Odin re-stamps it Ready for this activation. + let expected = owner.expected.as_ref().context("no Expected")?; + let backing = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); + let key = crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?); + let current = backing + .pull_all_read_only_snapshot()? + .into_iter() + .find(|entry| entry.key == key); + assert!(backing.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + key: key.clone(), + current, + }], + &[CultCacheEnvelope { + key, + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(&world, &owner, seeded_sequence(Some(&incumbent)) + 10, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }], + )?); + let phases = drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::Committing)?; + let at = latest_id(&world, &owner_id)?; + assert_eq!(at.phase, DeploymentPhase::Committing, "{phases:?} {:?} {:?}", at.completion, at.last_error); + // Its Committing step now fails on every attempt: the body is gone. + workload.kill(); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); for _ in 0..4 { world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; } - let owner = resident_id(&world, &owner_id)?.value; + let owner = latest_id(&world, &owner_id)?; assert!(owner.post_fencing_abort.is_none(), "a serving continuity yielded"); assert_eq!(owner.phase, DeploymentPhase::Committing); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); @@ -18026,25 +18085,25 @@ mod tests { #[test] fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; - make_stop_first(&world, &deploy_id)?; - let sealing = resident_id(&world, &deploy_id)?; + make_stop_first(&world, &deploy_id).context("A make_stop_first")?; + let sealing = resident_id(&world, &deploy_id).context("B sealing")?; let mut tail = sealing.value.clone(); tail.deployment_authorization = None; replace_transaction(&world.state_store, &sealing, &tail)?; - assert!(resident_id(&world, &deploy_id)?.value.awaits_handover()); + assert!(resident_id(&world, &deploy_id).context("C awaits")?.value.awaits_handover()); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); for _ in 0..4 { world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; } - let held = resident_id(&world, &owner_id)?.value; + let held = latest_id(&world, &owner_id)?; assert!(held.post_fencing_abort.is_none(), "the continuity yielded to a braked Deploy"); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); // The operator releases the brake for this exact Deploy. - let deploy = resident_id(&world, &deploy_id)?.value; + let deploy = resident_id(&world, &deploy_id).context("E deploy")?.value; let authorization = deployment_authorization( &world, deploy.expected.as_ref().context("no Expected")?, @@ -18069,7 +18128,7 @@ mod tests { )?); world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; - let yielded = resident_id(&world, &owner_id)?.value; + let yielded = resident_id(&world, &owner_id).context("F yielded")?.value; let abort = yielded.post_fencing_abort.as_ref().context("the continuity did not yield")?; assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); // The yield retimed the record: the next pass closes its incident From b993615c69eb34dc8d07e1eb47af8260b100f952 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:47:48 +0200 Subject: [PATCH 119/196] Negative tests: the lift must not yield to a braked waiting Deploy Two tests, from Soul's h2 attacks: a running candidate at boot beside a braked stop-then-start Deploy at its Sealing tail, and a held (unobservable) candidate when such a Deploy appears. Both must leave the candidate unstopped. They fail here: the lift treats any awaits_handover Deploy as a reason to keep aborting. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5d1389a..bf08f52 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17914,6 +17914,74 @@ mod tests { workload.stops.load(std::sync::atomic::Ordering::SeqCst) } + /// A Deploy at its Sealing tail, braked, beside a continuity: it awaits the + /// handover but cannot take it (the brake has not released it). + fn braked_sealing_deploy(world: &EngineFixture, deploy_id: &str) -> Result<()> { + make_stop_first(world, deploy_id)?; + let sealing = resident_id(world, deploy_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + let tail = resident_id(world, deploy_id)?.value; + assert!(tail.awaits_handover()); + assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?, now_millis()?)); + Ok(()) + } + + /// NEGATIVE. The boot lift yields to a Deploy only by the continuity's own + /// yield test: a braked Deploy awaiting the handover is no reason to keep a + /// pre-c2c abort running over the only copy of the body. + #[test] + fn the_boot_lift_returns_a_running_candidate_forward_beside_a_braked_waiting_deploy() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + braked_sealing_deploy(&world, &deploy_id)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + let returned = world.engine.return_wedged_continuity_aborts_forward()?; + assert_eq!(returned, vec![owner_id.clone()]); + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the running candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + Ok(()) + } + + /// NEGATIVE. A held record (its candidate unobservable, not provably dead) + /// is not released to its abort because a braked Deploy starts awaiting the + /// handover. + #[test] + fn a_held_continuity_abort_is_not_released_by_a_braked_waiting_deploy() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + workload.silence(true); + let returned = world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty()); + assert!(world.engine.lift_held.lock().unwrap().contains(&owner_id)); + for _ in 0..3 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + braked_sealing_deploy(&world, &deploy_id)?; + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the unobservable candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + assert!(world.engine.lift_held.lock().unwrap().contains(&owner_id)); + Ok(()) + } + /// The yield is a function of its inputs: the deadline, the continuity's /// own record, and a deployment that awaits the handover. #[test] From c4edc66c48ae80ea8a72ac37a450c16de4bd5ba2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:50:46 +0200 Subject: [PATCH 120/196] The boot lift yields to a Deploy by the continuity's own yield test The lift kept an abort running for any awaits_handover Deploy, so a braked Deploy released a held record to an abort that stopped the only copy of the body. One predicate, deploy_the_continuity_yields_to (candidate unproven, a waiting Deploy that deploy_can_move), is now what both the deadline's yield and the lift ask (rulings c2d-yield-only-unproven-to-movable, c2c-lift-defers-on-observe-error). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 46 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index bf08f52..c681051 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -2104,7 +2104,8 @@ impl DeploymentTransaction { /// A live deployment whose next move is the handover: not ended, not /// aborting, one phase short of `takes_target_at`. Derived, never stored. - /// Read by the continuity's yield and the boot lift. + /// Read by `deploy_the_continuity_yields_to`: the continuity's yield and the + /// boot lift. fn awaits_handover(&self) -> bool { self.command_kind == CommandKind::Deploy && self.completion.is_none() @@ -5810,12 +5811,20 @@ impl Engine { || transaction.completion.is_some() || abort.candidate_cleanup != CleanupEvidence::Pending || abort.lease_withdrawal == CleanupEvidence::Complete - || snapshot - .on_target(&transaction.target) - .any(|other| other.value.awaits_handover()) { return LiftDecision::KeepAborting; } + match now_millis() { + Ok(now) => { + if self + .deploy_the_continuity_yields_to(snapshot, transaction, now) + .is_some() + { + return LiftDecision::KeepAborting; + } + } + Err(error) => return LiftDecision::Undecided(error), + } let lifted = (|| -> Result { let plan = required(&transaction.plan, "transaction plan")?; let expected = required(&transaction.expected, "Expected projection")?; @@ -8568,6 +8577,25 @@ impl Engine { || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) } + /// The deployment a continuity gives way to: one on its target that awaits + /// the handover and can take it, while the continuity's own candidate is + /// unproven (ruling c2d-yield-only-unproven-to-movable). The one test the + /// deadline's yield and the boot lift's abort both ask, so neither stops a + /// serving candidate, or one a braked Deploy cannot replace. + fn deploy_the_continuity_yields_to<'a>( + &self, + snapshot: &'a ControlSnapshot, + continuity: &DeploymentTransaction, + now: u64, + ) -> Option<&'a Stored> { + if continuity.serving_proven() { + return None; + } + snapshot.on_target(&continuity.target).find(|other| { + other.value.awaits_handover() && self.deploy_can_move(other, now) + }) + } + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { let plan = required(¤t.value.plan, "continuity plan")?; let binding = plan.parsed_inputs()?.1; @@ -9763,11 +9791,11 @@ impl Engine { PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let now = now_millis()?; - if let Some(deploy) = snapshot.on_target(&transaction.target).find(|other| { - transaction.yields_to_a_deploy() - && other.value.awaits_handover() - && self.deploy_can_move(other, now) - }) { + if let Some(deploy) = transaction + .yields_to_a_deploy() + .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction, now)) + .flatten() + { self.begin_post_fencing_abort( current, anyhow!( From 8760a4fab377d3153a5621a64336e82a4e509ee7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:51:57 +0200 Subject: [PATCH 121/196] A boot lift that errs fails closed If the boot lift returned Err, boot logged and served with nothing held, so every pending continuity abort ran and stopped a running candidate. Boot now sets lift_unsettled; retry_held_lift runs the lift for each continuity abort until one succeeds, and no abort step runs before. Boot's call moves into Engine::boot_lift so the failure is testable. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 97 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c681051..d2a7635 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5133,6 +5133,10 @@ struct Engine { /// not be observed): no step of them runs until `retry_held_lift` decides. /// Process-local: a restart asks again at boot. lift_held: Mutex>, + /// Set when the boot lift itself failed, so no record's decision exists: + /// every continuity with a post-fencing abort is held until a later lift + /// succeeds. Process-local, like `lift_held`. + lift_unsettled: std::sync::atomic::AtomicBool, /// When each routed target's next challenge is due, held only when the /// stored `RouteSupervisionState` could not record the last one. The stored /// value owns the schedule and survives restarts; this spaces attempts @@ -5246,6 +5250,7 @@ impl Engine { fault_reports: Mutex::default(), resume_backoff: Mutex::default(), lift_held: Mutex::default(), + lift_unsettled: std::sync::atomic::AtomicBool::new(false), challenge_wait: Mutex::default(), }) } @@ -5731,6 +5736,28 @@ enum LiftDecision { } impl Engine { + /// Boot's call of the lift. A lift that errs decided nothing, so boot fails + /// closed: every continuity abort is held, none of its steps run, until + /// `retry_held_lift` completes a lift. + fn boot_lift(&self) { + match self.return_wedged_continuity_aborts_forward() { + Ok(returned) => { + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + } + Err(error) => { + self.lift_unsettled + .store(true, std::sync::atomic::Ordering::SeqCst); + eprintln!( + "Idunn boot could not lift wedged continuity aborts; every continuity abort is held until a lift succeeds: {error:#}" + ); + } + } + } + /// Once at boot: withdraw the abort of a continuity that has not stopped /// its running candidate, and retime the record so its phase retries /// forward. A pre-c2c Idunn aborted a continuity at its deadline; that @@ -5770,6 +5797,29 @@ impl Engine { /// pass is spent on the record; `None` lets the pass go on. fn retry_held_lift(&self, current: &Stored) -> Result> { let id = current.value.transaction_id.as_str(); + if current.value.command_kind == CommandKind::Continuity + && current.value.post_fencing_abort.is_some() + && self.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst) + { + match self.return_wedged_continuity_aborts_forward() { + Err(error) => { + self.note_fault("holds continuity abort", id, &error); + return Ok(Some(false)); + } + Ok(returned) => { + self.lift_unsettled + .store(false, std::sync::atomic::Ordering::SeqCst); + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + if returned.iter().any(|lifted| lifted == id) { + return Ok(Some(true)); + } + } + } + } if !self.lift_held.lock().expect("lift held mutex").contains(id) { return Ok(None); } @@ -5983,16 +6033,7 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; - match engine.return_wedged_continuity_aborts_forward() { - Ok(returned) => { - for transaction_id in &returned { - println!( - "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" - ); - } - } - Err(error) => eprintln!("Idunn boot could not lift wedged continuity aborts: {error:#}"), - } + engine.boot_lift(); match engine.reconcile_failed_continuity_projections() { Ok(outcome) => { for transaction_id in &outcome.demoted { @@ -17979,6 +18020,42 @@ mod tests { Ok(()) } + /// NEGATIVE. A boot lift that errs decided nothing: no continuity abort + /// runs a step until a later lift succeeds, and then the running candidate + /// is returned forward instead of stopped. + #[test] + fn a_boot_lift_that_errs_holds_every_continuity_abort_until_a_lift_succeeds() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + braked_sealing_deploy(&world, &deploy_id)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + // The lift's only fallible read fails: the store is unreadable at boot. + let intact = std::fs::read(&world.state_store)?; + std::fs::write(&world.state_store, b"not a control store")?; + world.engine.boot_lift(); + assert!(world.engine.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)); + std::fs::write(&world.state_store, &intact)?; + let stored = resident_id(&world, &owner_id)?; + assert!(stored.value.post_fencing_abort.is_some()); + // Still failing: the abort is held, no step runs. + std::fs::write(&world.state_store, b"not a control store")?; + assert!(!world.engine.resume_candidate(&stored)?, "a held abort moved"); + std::fs::write(&world.state_store, &intact)?; + assert_eq!(stops_of(&workload), 0); + // A later lift succeeds: the running candidate is returned forward. + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the running candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(!world.engine.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)); + Ok(()) + } + /// NEGATIVE. A held record (its candidate unobservable, not provably dead) /// is not released to its abort because a braked Deploy starts awaiting the /// handover. From 2f8671ac355da1d3e0d404468158f1b57a3b1ca4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 02:36:25 +0200 Subject: [PATCH 122/196] Reincarnation map: cut 2 installed Co-Authored-By: Claude Opus 5.5 --- docs/reincarnation-map.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md index d63a6b3..4f99aed 100644 --- a/docs/reincarnation-map.md +++ b/docs/reincarnation-map.md @@ -575,3 +575,26 @@ its stores and `idunn.running` together, because the cb60965 binary cannot read "migrated 5 control record(s) … idunn.target_supervision.v2". streampixels-service kept running through the restart (unit 73ca40ba, PID 968752, NRestarts 0). The catalog returned 200, and 4443/udp was listening. + +## Installed: cut 2 + +Cuts c2-phase-split, c2b-owner-deadlines, c2c-post-fence-forward and +c2d-continuity-yields (branch tip `8760a4f`) were installed on Yggdrasil on +2026-10-20 at 00:34:06 UTC, with the operator's OK. + +- **The hotfix before it.** `ec45906` (`bounded_detail`) was installed on its + own on 2026-10-19 at 17:32 UTC. It unwedged tx-f978ab86. +- **Build.** `/srv/build/idunn-8760a4f`; `cargo test --lib` passed 471. +- **Backup.** `/root/idunn-store-backup-*-pre-reincarnation-c2` holds the + stores, `incidents.cc` and the binaries. A rollback must restore them + together: older binaries refuse two live transactions on one target, and + they refuse the `operator-required` incident condition. +- **First boot.** The boot lift returned exactly tx-039b91e9 (ghostlight) and + tx-4eaed669 (odin) to forward retry, as the Soul install gate predicted. + Odin's continuity, wedged in Routing since the 2026-10-18 reboot, committed + 4 s later. Generation tx-4eaed669 is admitted, its route is healthy, and the + same PID 2571 still runs. Ghostlight's continuity is retrying forward in + AwaitingReady. Its daemon's readiness is a ghostlight-side fault + (`Permission denied` reading its session-wrapping key). +- **Live processes.** Every live MainPID was the same before and after the + install. The streampixels catalog returned 200, and 4443/udp was listening. From 6e2a31cf31dcc3394bc6cee01f1ca46829f3caf7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 02:40:55 +0200 Subject: [PATCH 123/196] Body map after reincarnation cut 2: two slots, take_target, forward retry, operator-required Co-Authored-By: Claude Opus 5.5 --- docs/authority-map.md | 52 ++++++++++++++++++++----------- docs/deployment-authority.md | 34 +++++++++++++++++--- docs/host-actuator.md | 3 +- docs/watchdog-reporting-map.md | 11 ++++--- docs/watchdog-reporting-target.md | 13 +++++--- 5 files changed, 80 insertions(+), 33 deletions(-) diff --git a/docs/authority-map.md b/docs/authority-map.md index 1cbf3ca..7cf027b 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -173,24 +173,40 @@ health plus Ready correlation. It changes and observes route membership only afterward. Only after the proxy has adopted the candidate may Idunn clear the promotion fence and drain the incumbent. -Ownership of a target is exclusive at each instant. A live -`DeploymentTransaction` owns it from Fencing on (a stop-first rollout, from -Starting); before Fencing a deployment owns only its private candidate and -continuity owns the incumbent slot. Continuity owns the admitted generation's -survival and never stops. There is no stopped state: six charged attempts in -an hour (`restarts_page`) open the ContinuityExhausted incident and gate -nothing, and no deployment path yields to continuity. When continuity next acts -is derived only from its own record in `TargetSupervision` (schema v2, a v1 -record lifts at store open): the attempt streak, the backoff anchor (written -by the dead-and-free pass, the only writer), and the once-per-admission -re-materialization bound. The workload driver alone answers whether a lost unit -was a host reboot (`WorkloadPort::lost_to_host_reboot`). Crash restart, restart -after a sustained run and reboot re-materialization all mint through one CAS; -continuity supervision never aborts a transaction. Operator-facing incidents -and `idunn status` are projections that decide nothing. Cut line: cuts 2 to 4 -of `reincarnation-map.md` still owe continuity restarting alongside a -pre-fencing build, dependency failures not growing the streak, and the -dead-generation route refresh. +A target has two slots: the incumbent (the admitted generation) and a +transaction's candidate. Continuity owns the incumbent slot and keeps the +admitted generation alive; it never stops. A live `DeploymentTransaction` owns +its private candidate before Fencing and takes the target itself at Fencing (a +stop-then-start rollout, at Starting) through one `take_target` CAS, so of two +transactions entering from one view exactly one succeeds. A continuity +restart is itself a transaction and takes the target the same way. + +Phases from Warming through Committing carry deadlines, as do aborts and +drains; an expired deadline never waits silently. A stateless deployment +aborts to its incumbent. A deployment whose candidate was issued a write lease +is left to the operator (`operator-required` incident, the second incident +condition beside `continuity-exhausted`; its subject is the transaction id and +it closes when the record ends or its step goes on). A continuity after +Fencing retries its phase forward on a capped backoff and never stops its live +candidate, since that is the only running copy of the admitted body; the +deadline opens the same incident. The one exception is yielding: a continuity +whose route proof has not succeeded gives way, through its own post-fencing +abort, to a deployment the deployment brake has released and that awaits the +handover; a continuity that has proven its route never yields. The +`ContinuityExhausted` incident (six charged attempts in an hour, +`restarts_page`) pages and gates nothing. When continuity next acts is derived +only from its own record in `TargetSupervision` (schema v2): the attempt +streak, the backoff anchor (written by the dead-and-free pass, the only +writer), and the once-per-admission re-materialization bound. The workload +driver alone answers whether a lost unit was a host reboot +(`WorkloadPort::lost_to_host_reboot`). Operator-facing incidents and +`idunn status` are projections that decide nothing. A boot lift returns a +continuity abort left by an older Idunn, one that had not stopped its running +candidate, to forward retry; if it cannot decide, every continuity abort is +held until a lift succeeds. It is a migration with a deletion line (follow-up +`c2c-delete-boot-lift`). Cut line: cuts 3 and 4 of `reincarnation-map.md` +still owe dependency failures not growing the streak and the dead-generation +route refresh. The deployment brake gates changes to source, release, configuration, schema, unit, or authority binding. The lifecycle brake separately gates restart of an diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index 6d464e5..bdc7efc 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -313,10 +313,36 @@ written when the pass first finds the admitted generation dead and the target free, and a capped doubling wait from that anchor (5 s base, 300 s cap). A run of 600 s or more by the admitted body resets the streak. A generation lost to a host reboot (the workload driver's evidence, `lost_to_host_reboot`) is -re-materialized once per admission without charge or wait. The target is free -for a deployment's candidate between attempts, and a deployment never yields to -continuity before Fencing; from Fencing on (or from Starting for a stop-first -rollout) the live transaction alone owns the target. +re-materialized once per admission without charge or wait. + +Each target has two slots, the incumbent and a transaction's candidate. The +incumbent is continuity's until a transaction takes the target, at Fencing (at +Starting for a stop-then-start rollout), through one `take_target` CAS; of two +transactions entering from one view exactly one succeeds. Before that, a +deployment owns only its private candidate and the target is free for it +between continuity attempts. + +Phases Warming through Committing, post-fencing aborts and drains all carry +deadlines (the plan's frozen `PhaseDeadlines`; `WARMING_DEADLINE_SECONDS`, and +`PRE_FENCING_ABORT_DEADLINE_SECONDS` for a pre-fencing abort). What an expired +post-fencing deadline does depends on the owner. A deployment whose candidate +was never issued a write lease aborts to the incumbent. One whose candidate was +issued a lease may have written under a new state contract and is left to the +operator. A continuity retries its phase forward on a capped backoff and never +stops its live candidate, because that candidate is the only running copy of +the admitted body. Both operator cases open an `operator-required` incident +(subject: the transaction id) that closes when the record ends or its step goes +on. + +The one yield: a continuity whose route proof has not succeeded gives way to a +deployment the deployment brake has released that awaits the handover of the +target. It begins its own post-fencing abort and the deployment fences what is +admitted. A continuity that has proven its route is serving never yields. + +At boot, a lift returns to forward retry any continuity abort an older Idunn +left over a running candidate. If the lift errs or cannot observe the +candidate, the abort is held, not run. The lift is a migration with a deletion +line (follow-up `c2c-delete-boot-lift`). An admitted generation keeps its Ready and latest Odin receipts as durable proof of the exact activation at their recorded admission times. Dependency diff --git a/docs/host-actuator.md b/docs/host-actuator.md index cdb8cde..c595c5d 100644 --- a/docs/host-actuator.md +++ b/docs/host-actuator.md @@ -97,7 +97,8 @@ Each of these was found by running the pipe against Raven, not by reading. bind time. `rollout.strategy = "stop-then-start"` exists for exactly a route-less host-actuator workload: the admitted incumbent is stopped at Starting, before the candidate starts, and supervision leaves the target - alone from Starting on. A systemd-transient workload may not declare it. + alone from Starting on: the transaction takes the target there (one + `take_target` CAS) rather than at Fencing. A systemd-transient workload may not declare it. - **An unobservable host is silence.** The hub returns a typed `HostUnobservable` for not-attached, detached, reconnected and unanswered. Supervision skips that tick and logs it. Before this, an Idunn restart diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index 4c9ada7..4bc8814 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -21,8 +21,8 @@ probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC | I2 | Continuity exhaustion (since `idunn-reincarnation` cut 1: `restarts_page`, a paging threshold only; nothing stops, and the line numbers below are pre-cut) | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | | I3 | Route degradation | `RouteSupervisionState` `:890-942`; `record_failed_challenge` sets `degraded_since_unix_millis` on the **first** failed challenge, `record_proved_challenge` clears it; reported at `:5975` under `route:` | Durable per incarnation (inside `admitted_generation.v4`); a new generation starts from `default()`. Backoff doubles from the topology max age to a cap. | | I4 | Other operator-facing faults | `:5645-5651` (`generation-down:`, "holds a dead admitted generation down"), `:6016` (`route-record:`), `note_fault` per transaction | Also stderr plus `last_error`. Not named by the target; listed so the paging-set question sees them. | -| I5 | `OperatorRequired` on main | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | -| I6 | `OperatorRequired` on B5 | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | +| I5 | `OperatorRequired` on main (pre-cut-2 reading; since `idunn-reincarnation` cut 2 `resolve` reaches it in production through `PhaseEnd::OperatorRequired` and `PhaseEnd::RetryForward`, both calling `leave_to_operator`, which opens the `operator-required` incident `IncidentCondition::OperatorRequired`, subject the transaction id; `close_spent_operator_incidents` closes it when the record no longer `leaves_to_operator`) | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | +| I6 | `OperatorRequired` on B5 (superseded by cut 2: B5's deadlines merged as `PhaseDeadline`, and a continuity past its deadline now also lands here) | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | | I7 | Store kinds | `:55-61`, `:936-966`; B5 adds `idunn.expiry_request.v1` | `control.cc`: `idunn.deployment_command`, `idunn.deployment_transaction`, `idunn.admitted_generation.v4`, `idunn.target_supervision.v1`. `history.cc` beside it holds terminal transactions (`:3653`), retired one per tick (`:5068-5091`). | | I8 | Single-writer precedent inside `control.cc` | B5 `ExpiryRequest` doc comment | "The CLI only writes this record. The daemon's deadline resolver reads it ... No CLI writes a transaction field." One writer per kind. | | I9 | Serve loop | `:5041-5058`, `run_scheduler_tick` `:5098-5110` | 500 ms poll. A tick error is logged, never propagated. | @@ -166,9 +166,10 @@ cut spec fixes them. Empty cells, stated plainly: -- `operator-required` closure: no path on `hands/b5` ends a transaction left to - the operator (I6). The incident can open and never close until B5 names that - action. +- `operator-required` closure: the incident closes when the transaction ends or + its step goes on (`close_spent_operator_incidents`). A stateful deployment + left to the operator still has no Idunn-owned terminal action; the operator's + `idunn expire` re-enters the same arm. - Delivery-failure visibility: no owner. - Discord `enforce_nonce` window: not verified against Discord's current docs in this pass. diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index 5d21c75..509cb6e 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -30,9 +30,12 @@ journald (`report_once`, `control_plane.rs:4599`): - An admitted route is degraded. It fails its stable-route challenge (`supervise_admitted_route`, ~5973). The degraded mark is durable, but no one is told. -- A post-fencing phase deadline expires into `OperatorRequired`. This lives on - B5, which is unmerged on `hands/b5`. The failure record is durable and the - notice is a log line. +- A post-fencing phase deadline leaves a transaction to the operator: a + stateful deployment whose candidate was issued a lease, or a continuity + retrying forward past its deadline. Since `idunn-reincarnation` cut 2 it is + the second incident condition, `operator-required` (subject: the transaction + id), beside `continuity-exhausted`; it closes when the record ends or its + step goes on. Odin's own store condition (`odin.store_condition.v1`, `contested`) is mapped but not built. Idunn reads nothing about Odin's store health today. @@ -85,8 +88,8 @@ the machine has a detector with no record, and a deliverer that nothing feeds. - Odin's store-recovery cuts, O2 included. They produce `odin.store_condition` and are Odin's campaign, still in prose. This campaign consumes the record once it exists. -- Idunn's B5 phase deadlines. B5 is in prose on `hands/b5`. This campaign - reports its `OperatorRequired` outcome once B5 merges. +- Idunn's phase deadlines. They landed in `idunn-reincarnation` cut 2, which + also owns the `operator-required` incident condition. - VoidBot's `notify_owner` MCP tool and its separate DM code. - Bifrost's Persona delivery crossing. - Eve/TUI lowering of incidents. From 491459e6a04fd93036207370da8646647de03123 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 18:47:20 +0200 Subject: [PATCH 124/196] State kind external-directory: process-owned directory, lease-gated like external-file Not built or tested yet; verified at the end of the cut. Co-Authored-By: Claude Sonnet 5.5 --- docs/authority-map.md | 5 ++++- src/deployment.rs | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/authority-map.md b/docs/authority-map.md index 7cf027b..270d592 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -55,7 +55,10 @@ A deployable repository publishes one strict declaration, normally - immutable artifact outputs, optionally pinned to an expected SHA-256; - a constrained launch contract; - one health-contract identity; -- an optional typed state contract with state schema generation and slots; +- an optional typed state contract with state schema generation and slots + (kinds `cultcache-file`, `external-file` and `external-directory`; the + external kinds are process-owned state an external runtime imposes, which + Idunn preserves and fences but never interprets); - provided capabilities, dependency kinds, startup requirements, capacity, and conflicts. diff --git a/src/deployment.rs b/src/deployment.rs index a67b4bc..a81dff4 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -184,6 +184,11 @@ pub enum StateKind { /// Process-owned state imposed by an external runtime contract. Idunn /// preserves and fences the file but never interprets its payload. ExternalFile, + /// A process-owned directory imposed by an external runtime contract (for + /// example Synapse's media store). Idunn preserves and fences it and never + /// interprets its contents. Same writer, recovery and startup rules as + /// `ExternalFile`. + ExternalDirectory, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -2261,6 +2266,17 @@ nodes = ["yggdrasil"] assert!(declaration.write_lease_required()); } + #[test] + fn external_directory_state_is_lease_gated_like_external_file() { + let input = RECIPE.replace("kind = \"cultcache-file\"", "kind = \"external-directory\""); + let declaration = TargetDeclaration::parse(&input).unwrap(); + let slot = &declaration.state.as_ref().unwrap().slots[0]; + assert_eq!(slot.kind, StateKind::ExternalDirectory); + assert_eq!(slot.writer, StateWriter::ProcessBoundSingleWriter); + assert_eq!(slot.startup, StateStartup::CreateOrOpenAfterWriteLease); + assert!(declaration.write_lease_required()); + } + #[test] fn state_slots_cannot_claim_overlapping_paths() { let input = RECIPE.replace( From b9f27ce1e70d6cf01bc02ccddaaee27b78946204 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 18:53:29 +0200 Subject: [PATCH 125/196] idunn-launch: presence launcher, lease hold, closed-set config renderer, route acceptor (unbuilt) Does not build-verified yet; tests follow. drivers.rs: holder-side lease read, one lease-binding rule shared with grant. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.toml | 4 + src/bin/idunn-launch.rs | 86 +++ src/drivers.rs | 105 +++- src/launcher.rs | 1277 +++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + 5 files changed, 1446 insertions(+), 28 deletions(-) create mode 100644 src/bin/idunn-launch.rs create mode 100644 src/launcher.rs diff --git a/Cargo.toml b/Cargo.toml index 713ad33..dcaca2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,10 @@ path = "src/bin/idunn-provision.rs" name = "idunn-host" path = "src/bin/idunn-host.rs" +[[bin]] +name = "idunn-launch" +path = "src/bin/idunn-launch.rs" + [lib] name = "idunn_daemon" path = "src/lib.rs" diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs new file mode 100644 index 0000000..32c6291 --- /dev/null +++ b/src/bin/idunn-launch.rs @@ -0,0 +1,86 @@ +//! The unit's process for an upstream daemon: parse the recipe's argv, read +//! what Idunn set, run the launcher, and map the outcome to an exit code. +#[cfg(unix)] +fn main() -> std::process::ExitCode { + use std::sync::atomic::{AtomicBool, Ordering}; + + use anyhow::Context; + use idunn_daemon::deployment::{ + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + }; + use idunn_daemon::launcher::{self, *}; + + static STOP: AtomicBool = AtomicBool::new(false); + extern "C" fn on_signal(_: libc::c_int) { + STOP.store(true, Ordering::SeqCst); + } + + fn launch() -> anyhow::Result { + let contract = LaunchContract::parse(std::env::args().skip(1))?; + let bundle = std::env::var_os(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT) + .context("the Idunn runtime bundle is not set")?; + let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + .context("the Idunn candidate bind is not set")?; + let (activation, provider) = launcher::inherited_authority_files()?; + let authority = RuntimeAuthority::open(bundle.as_ref(), activation, provider)?; + authority.require_candidate_bind(&bind)?; + let lease_path = std::env::var_os(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT); + anyhow::ensure!( + lease_path.is_some() == authority.expected().write_lease_required, + "the process write lease binding differs from Expected" + ); + let route = std::net::TcpListener::bind(&bind).context("binding the candidate endpoint")?; + let sink = OdinRudpSink::from_expected(authority.expected())?; + let lease = DriverLeaseReader::new( + &authority.expected().target, + lease_path.as_deref().unwrap_or_default().as_ref(), + ); + let health_path = contract.health_path.clone(); + let ports = Ports { + credentials: std::sync::Arc::new(EnvCredentials), + clock: std::sync::Arc::new(SystemClock), + sink: std::sync::Arc::new(sink), + lease: std::sync::Arc::new(lease), + spawner: Box::new(ProcessSpawner), + health: Box::new(move |endpoint| { + std::sync::Arc::new(HttpHealthProbe { + endpoint: endpoint.clone(), + path: health_path.clone(), + }) + }), + }; + // SAFETY: the handler only stores to an atomic. + unsafe { + libc::signal(libc::SIGTERM, on_signal as libc::sighandler_t); + libc::signal(libc::SIGINT, on_signal as libc::sighandler_t); + } + Ok(launcher::run( + &contract, + authority, + route, + &std::env::temp_dir(), + ports, + &STOP, + )) + } + + match launch() { + Ok(outcome) => { + if let Outcome::Failed(code) = &outcome { + eprintln!("idunn-launch ended: {code}"); + } + std::process::ExitCode::from(outcome.exit_code() as u8) + } + Err(error) => { + eprintln!("idunn-launch refused: {error}"); + std::process::ExitCode::from(2) + } + } +} + +#[cfg(not(unix))] +fn main() { + eprintln!("idunn-launch runs only on Unix"); + std::process::exit(2); +} diff --git a/src/drivers.rs b/src/drivers.rs index fdf925c..4bf500a 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -4110,6 +4110,20 @@ impl CultCacheWriteLeaseDriver { fn current(&self) -> Result> { validate_root_authority_path(&self.record_path)?; + self.read_record() + } + + /// The lease as its holder reads it: the same custody, envelope, key and + /// target validation Idunn applies, without requiring the reader to be + /// root. The holder is a service uid by construction; root is who writes. + pub(crate) fn current_for_holder( + &self, + ) -> Result> { + validate_authority_path_custody(&self.record_path)?; + self.read_record() + } + + fn read_record(&self) -> Result> { if !self.record_path.exists() { return Ok(None); } @@ -4156,36 +4170,61 @@ impl CultCacheWriteLeaseDriver { expected.validate()?; activation.validate()?; lease.validate()?; - let expected_sha256 = expected.canonical_sha256()?; - let activation_sha256 = activation.canonical_sha256()?; ensure!( - self.target == expected.target - && lease.target == expected.target - && lease.expected_projection_sha256 == expected_sha256 - && lease.plan_id == expected.plan_id - && lease.incarnation_id == expected.incarnation_id - && lease.sealed_release_id == expected.sealed_release_id - && lease.activation_witness_sha256 == activation_sha256 - && lease.runtime_id == expected.runtime_id - && lease.runtime_instance_id == activation.runtime_instance_id - && lease.state_schema_generation - == expected - .state_schema_generation - .as_deref() - .context("write-lease Expected has no state generation")? - && lease.state_contract_sha256 - == expected - .state_contract_sha256 - .as_deref() - .context("write-lease Expected has no state contract")? - && lease.warming_presence_sha256 == warming.signed_presence_sha256() - && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + self.target == expected.target, "process write lease does not bind the exact warming candidate" ); - Ok(()) + ensure!( + warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + "process write lease does not bind the exact warming candidate" + ); + lease_binds_candidate( + expected, + activation, + lease, + std::slice::from_ref(&warming.signed_presence_sha256().to_owned()), + ) } } +/// The one rule for what makes a process write lease this incarnation's: it +/// binds this Expected, this activation and one of the given provider-signed +/// Warming proofs. Idunn grants with it and the lease holder accepts with it. +pub(crate) fn lease_binds_candidate( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, + warming_presence_sha256es: &[String], +) -> Result<()> { + let expected_sha256 = expected.canonical_sha256()?; + let activation_sha256 = activation.canonical_sha256()?; + ensure!( + lease.target == expected.target + && lease.expected_projection_sha256 == expected_sha256 + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation_sha256 + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id + && lease.state_schema_generation + == expected + .state_schema_generation + .as_deref() + .context("write-lease Expected has no state generation")? + && lease.state_contract_sha256 + == expected + .state_contract_sha256 + .as_deref() + .context("write-lease Expected has no state contract")? + && warming_presence_sha256es + .iter() + .any(|sha256| sha256 == &lease.warming_presence_sha256), + "process write lease does not bind the exact warming candidate" + ); + Ok(()) +} + impl WriteLeasePort for CultCacheWriteLeaseDriver { fn revoke_exact(&self, incumbent: Option<&IdunnProcessWriteLeaseRecord>) -> Result<()> { if let Some(incumbent) = incumbent { @@ -5861,7 +5900,7 @@ impl Answer { } } -fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { +pub(crate) fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) .with_context(|| format!("connecting stable CultNet TCP route {target}")) .map_err(silent)?; @@ -6070,7 +6109,7 @@ fn connect_route_socket(target: SocketAddr) -> std::io::Result { } } -fn exact_route_snapshot_response( +pub(crate) fn exact_route_snapshot_response( response: CultNetMessage, message_id: &str, target: &str, @@ -8335,12 +8374,17 @@ fn authority_lock_path(path: &Path) -> PathBuf { #[cfg(unix)] fn validate_root_authority_path(path: &Path) -> Result<()> { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - ensure!( unsafe { libc::geteuid() } == 0, "process write-lease actuation requires root Idunn" ); + validate_authority_path_custody(path) +} + +#[cfg(unix)] +fn validate_authority_path_custody(path: &Path) -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let parent = path .parent() .context("process write-lease path has no parent")?; @@ -8383,6 +8427,11 @@ fn validate_root_authority_path(_path: &Path) -> Result<()> { bail!("process write-lease actuation requires a Unix authority path") } +#[cfg(not(unix))] +fn validate_authority_path_custody(_path: &Path) -> Result<()> { + bail!("process write-lease actuation requires a Unix authority path") +} + #[cfg(unix)] fn harden_root_authority_file(path: &Path) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; diff --git a/src/launcher.rs b/src/launcher.rs new file mode 100644 index 0000000..f3bee58 --- /dev/null +++ b/src/launcher.rs @@ -0,0 +1,1277 @@ +//! idunn-launch: the unit's process for an upstream daemon that cannot speak +//! Idunn's runtime contract itself (Synapse, MAS). +//! +//! Idunn observes the unit; the launcher, inside it and supervising the child, +//! signs the presence statement with the provider identity and the activation +//! key. The launcher binds the candidate endpoint Idunn challenges, because the +//! child cannot answer a CultNet frame. It starts a stateful child only after +//! it holds the process write lease, renders the child's config from a closed +//! set of placeholders, and ends with the child. +//! +//! Errors, logs and refusals name a field and a code. They never echo an +//! argument, a template line, a path or a credential. + +use std::collections::VecDeque; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::net::UnixStream; +use std::path::{Component, Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultnet_rs::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, + CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, CultNetWireContract, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, + GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, + IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationRecord, + IdunnRuntimeActivationSigner, ServiceIdentitySigner, decode_cultnet_message_from_slice, + encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, +}; + +use crate::deployment::RUNTIME_PRESENCE_IDENTITY_FD_NAME; +use crate::drivers::{CultCacheWriteLeaseDriver, lease_binds_candidate}; + +/// The route challenge is the only thing that begins with this byte: a +/// CultNet frame's 4-byte big-endian length is bounded at 1 MiB, and an HTTP +/// request starts with a method letter. +const CHALLENGE_FIRST_BYTE: u8 = 0x00; +const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; +const CHALLENGE_IO_TIMEOUT: Duration = Duration::from_secs(3); +const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); +const POLL: Duration = Duration::from_millis(100); +const MAX_RECENT_WARMING_PROOFS: usize = 64; +const TERMINATE_GRACE: Duration = Duration::from_secs(20); +const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; +const ODIN_CAPABILITY: &str = "odin.verse-rendezvous"; + +// ---------------------------------------------------------------- contract + +/// Where the child listens. The launcher chooses the address; the child is +/// told it through `{{child.listen}}`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChildListener { + UnixSocket, + LoopbackPort, +} + +/// Everything the recipe declares about the child, parsed from the launcher's +/// argv (`--flag value` pairs; `--child-arg` repeats). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LaunchContract { + pub release_root: PathBuf, + pub child: PathBuf, + pub child_args: Vec, + /// The flag that carries the rendered config path to the child, when the + /// child takes one. + pub config_flag: Option, + pub template: PathBuf, + pub listener: ChildListener, + pub health_path: String, + pub health_contract: String, + pub state_root: Option, + pub lease_wait: Duration, + pub startup: Duration, + pub heartbeat: Duration, +} + +impl LaunchContract { + pub fn parse(args: impl IntoIterator) -> Result { + let mut release_root = None; + let mut child = None; + let mut child_args = Vec::new(); + let mut config_flag = None; + let mut template = None; + let mut listener = None; + let mut health_path = None; + let mut health_contract = None; + let mut state_root = None; + let (mut lease_wait, mut startup, mut heartbeat) = (300_u64, 120_u64, 5_u64); + let mut args = args.into_iter(); + while let Some(flag) = args.next() { + let value = args.next().context("launcher argument has no value")?; + let seconds = |field: &str| -> Result { + value + .parse::() + .ok() + .filter(|seconds| *seconds > 0) + .with_context(|| format!("launcher argument {field} is not a positive integer")) + }; + match flag.as_str() { + "--release-root" => release_root = Some(PathBuf::from(value)), + "--child" => child = Some(release_relative(&value, "--child")?), + "--child-arg" => child_args.push(value), + "--config-flag" => config_flag = Some(value), + "--template" => template = Some(release_relative(&value, "--template")?), + "--listener" => { + listener = Some(match value.as_str() { + "unix" => ChildListener::UnixSocket, + "loopback" => ChildListener::LoopbackPort, + _ => bail!("launcher argument --listener is not unix or loopback"), + }) + } + "--health-path" => { + ensure!( + value.starts_with('/'), + "launcher argument --health-path is not absolute" + ); + health_path = Some(value) + } + "--health-contract" => health_contract = Some(value), + "--state-root" => state_root = Some(PathBuf::from(value)), + "--lease-wait-secs" => lease_wait = seconds("--lease-wait-secs")?, + "--startup-secs" => startup = seconds("--startup-secs")?, + "--heartbeat-secs" => heartbeat = seconds("--heartbeat-secs")?, + _ => bail!("unknown launcher argument"), + } + } + let release_root = release_root.context("launcher argument --release-root is required")?; + ensure!( + release_root.is_absolute(), + "launcher argument --release-root is not absolute" + ); + Ok(Self { + release_root, + child: child.context("launcher argument --child is required")?, + child_args, + config_flag, + template: template.context("launcher argument --template is required")?, + listener: listener.context("launcher argument --listener is required")?, + health_path: health_path.context("launcher argument --health-path is required")?, + health_contract: health_contract + .context("launcher argument --health-contract is required")?, + state_root, + lease_wait: Duration::from_secs(lease_wait), + startup: Duration::from_secs(startup), + heartbeat: Duration::from_secs(heartbeat), + }) + } +} + +fn release_relative(value: &str, field: &str) -> Result { + let path = PathBuf::from(value); + ensure!( + !value.is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))), + "launcher argument {field} is not release-relative" + ); + Ok(path) +} + +// ---------------------------------------------------------------- authority + +/// The bundle, activation credential and provider identity Idunn hands the +/// unit, and the two signatures every presence statement needs. +pub struct RuntimeAuthority { + expected: IdunnExpectedIncarnationRecord, + expected_sha256: String, + activation: IdunnRuntimeActivationRecord, + activation_sha256: String, + activation_signer: IdunnRuntimeActivationSigner, + provider_signer: ServiceIdentitySigner, + sequence: u64, +} + +impl RuntimeAuthority { + pub fn open( + bundle: &Path, + activation_credential: impl Read, + provider_credential: impl Read, + ) -> Result { + ensure!(bundle.is_absolute(), "Idunn runtime bundle is not absolute"); + let (key, payload) = read_single_record( + &bundle.join("expected.cc"), + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + )?; + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&payload)?; + expected.validate()?; + ensure!(key == expected.target, "Expected key is substituted"); + let (key, payload) = read_single_record( + &bundle.join("activation.cc"), + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + )?; + let activation = IdunnRuntimeActivationRecord::decode_canonical(&payload)?; + activation.validate()?; + ensure!(key == expected.target, "activation key is substituted"); + let expected_sha256 = expected.canonical_sha256()?; + ensure!( + activation.expected_projection_sha256 == expected_sha256 + && activation.runtime_id == expected.runtime_id, + "activation does not bind this Expected projection" + ); + let activation_signer = IdunnRuntimeActivationSigner::from_credential_reader( + activation_credential, + )?; + ensure!( + activation_signer.identity_id() == activation.activation_signer_identity_id + && activation_signer.public_key() == activation.activation_signer_public_key, + "activation credential does not belong to this activation" + ); + let provider_signer = + open_service_identity_credential_reader::( + provider_credential, + )?; + ensure!( + provider_signer.entry().identity_id == expected.expected_signer_identity_id, + "provider identity is not the signer selected by Expected" + ); + Ok(Self { + activation_sha256: activation.canonical_sha256()?, + expected, + expected_sha256, + activation, + activation_signer, + provider_signer, + sequence: 0, + }) + } + + pub fn expected(&self) -> &IdunnExpectedIncarnationRecord { + &self.expected + } + + pub fn activation(&self) -> &IdunnRuntimeActivationRecord { + &self.activation + } + + /// The candidate endpoint Idunn will challenge, which the launcher binds. + pub fn require_candidate_bind(&self, bind: &str) -> Result<()> { + let route = self.expected.route.as_ref().context("Expected has no route")?; + ensure!( + route.transport == "tcp" && route.candidate_endpoint == format!("tcp://{bind}"), + "candidate bind differs from the Expected route" + ); + Ok(()) + } + + /// The next presence record, signed by the provider identity and the + /// activation. The sequence is the clock: strictly increasing here and + /// above anything an earlier incarnation of this signer published. + pub fn sign( + &mut self, + state: &str, + detail: &str, + write_lease_sha256: Option, + now_millis: u64, + ) -> Result { + self.sequence = now_millis.max( + self.sequence + .checked_add(1) + .ok_or_else(|| anyhow!("runtime presence sequence exhausted"))?, + ); + let expected = &self.expected; + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.expected_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation_sha256.clone(), + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: state.into(), + detail: detail.into(), + write_lease_sha256, + signer_identity_id: self.provider_signer.entry().identity_id.clone(), + publisher_sequence: self.sequence, + observed_at_unix_millis: now_millis, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + let payload = record.canonical_proof_payload()?; + record.signature = self + .provider_signer + .sign::(&payload) + .signature; + record.activation_signature = self.activation_signer.sign_presence_proof(&record)?; + record.validate()?; + Ok(record) + } +} + +fn read_single_record(path: &Path, record_type: &str, schema: &str) -> Result<(String, Vec)> { + let entries = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading a runtime bundle record")?; + let [envelope] = entries.as_slice() else { + bail!("runtime bundle record does not hold exactly one entry"); + }; + ensure!( + envelope.r#type == record_type && envelope.schema_id.as_deref() == Some(schema), + "runtime bundle record has the wrong type or schema" + ); + Ok((envelope.key.clone(), envelope.payload.clone())) +} + +/// The two authority descriptors systemd opens for the unit, in the order and +/// under the names Idunn's launch contract fixes. +pub fn inherited_authority_files() -> Result<(File, File)> { + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let fds = std::env::var("LISTEN_FDS").context("systemd did not bind authority FDs")?; + let names = std::env::var("LISTEN_FDNAMES").context("systemd did not name authority FDs")?; + ensure!( + fds == "2" + && names + == format!( + "{IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME}:{RUNTIME_PRESENCE_IDENTITY_FD_NAME}" + ), + "systemd authority FDs differ from the launch contract" + ); + for fd in [3, 4] { + // SAFETY: F_GETFD observes one integer descriptor and transfers nothing. + ensure!( + unsafe { libc::fcntl(fd, libc::F_GETFD) } != -1, + "an authority descriptor is not open" + ); + } + // SAFETY: the LISTEN_* contract above assigns these two distinct, open + // descriptors to this process; this is their first and only consumer. + let (activation, provider) = unsafe { (File::from_raw_fd(3), File::from_raw_fd(4)) }; + for (file, exact) in [(&activation, Some(32_u64)), (&provider, None)] { + // OpenFile descriptors arrive without close-on-exec; the child must + // never inherit a signer. + // SAFETY: F_SETFD on a descriptor `file` owns. + ensure!( + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) } != -1, + "an authority descriptor could not be sealed" + ); + let metadata = file.metadata()?; + ensure!( + metadata.is_file() + && metadata.uid() == 0 + && metadata.permissions().mode() & 0o777 == 0o400 + && metadata.nlink() == 1 + && exact.map_or(metadata.len() > 0, |size| metadata.len() == size), + "an authority descriptor is not one root-owned 0400 file" + ); + } + Ok((activation, provider)) +} + +// ---------------------------------------------------------------- ports + +pub trait Clock: Send + Sync { + fn now_millis(&self) -> u64; + fn sleep(&self, duration: Duration); +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_millis(&self) -> u64 { + u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0) + } + + fn sleep(&self, duration: Duration) { + thread::sleep(duration); + } +} + +/// Systemd credentials as Idunn binds them: the environment variable `NAME` +/// holds the path of the credential file. +pub trait CredentialSource: Send + Sync { + fn contents(&self, name: &str) -> Result>; + fn path(&self, name: &str) -> Result; +} + +pub struct EnvCredentials; + +impl CredentialSource for EnvCredentials { + fn contents(&self, name: &str) -> Result> { + fs::read(self.path(name)?).map_err(|_| anyhow!("a credential could not be read")) + } + + fn path(&self, name: &str) -> Result { + std::env::var_os(name) + .map(PathBuf::from) + .ok_or_else(|| anyhow!("a credential is not bound")) + } +} + +/// Where signed presence goes (Odin). +pub trait PresenceSink: Send + Sync { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()>; +} + +/// The lease as the launcher sees it: the current record, and a shared lock on +/// its store that Idunn's revocation cannot take while the child lives. +pub trait LeaseReader: Send + Sync { + fn read(&self) -> Result>; + fn hold_shared(&self) -> Result>; +} + +pub trait HealthProbe: Send + Sync { + fn healthy(&self) -> bool; +} + +pub trait ChildHandle: Send { + /// The exit status once the child has exited. + fn exited(&mut self) -> Option; + /// SIGTERM, then SIGKILL after the grace period; returns once it is gone. + fn terminate(&mut self, grace: Duration); +} + +pub trait ChildSpawner: Send { + fn spawn(&self, program: &Path, args: &[String]) -> Result>; +} + +// ---------------------------------------------------------------- production ports + +pub struct DriverLeaseReader { + driver: CultCacheWriteLeaseDriver, +} + +impl DriverLeaseReader { + pub fn new(target: &str, record_path: &Path) -> Self { + Self { + driver: CultCacheWriteLeaseDriver::new(target, record_path), + } + } +} + +struct SharedLockGuard { + release: Option>, + worker: Option>, +} + +impl Drop for SharedLockGuard { + fn drop(&mut self) { + drop(self.release.take()); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl LeaseReader for DriverLeaseReader { + fn read(&self) -> Result> { + self.driver + .current_for_holder() + .map(|current| current.map(|(_, lease)| lease)) + .map_err(|_| anyhow!("write-lease store refused custody, envelope or target")) + } + + fn hold_shared(&self) -> Result> { + let path = self.driver.record_path.clone(); + let (ready_tx, ready_rx) = mpsc::sync_channel::(1); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let worker = thread::spawn(move || { + let held = SingleFileMessagePackBackingStore::new(&path).with_read_only_shared_snapshot( + |_| { + let _ = ready_tx.send(true); + // Returns when the guard drops its sender. + let _ = release_rx.recv(); + Ok(()) + }, + ); + if held.is_err() { + let _ = ready_tx.send(false); + } + }); + if ready_rx.recv() != Ok(true) { + let _ = worker.join(); + bail!("write-lease lock could not be held"); + } + Ok(Box::new(SharedLockGuard { + release: Some(release_tx), + worker: Some(worker), + })) + } +} + +/// Odin takes presence over RUDP at the endpoint Expected's Odin dependency +/// names. +pub struct OdinRudpSink { + endpoint: SocketAddr, +} + +impl OdinRudpSink { + pub fn from_expected(expected: &IdunnExpectedIncarnationRecord) -> Result { + let dependency = expected + .dependencies + .iter() + .find(|dependency| dependency.capability == ODIN_CAPABILITY) + .context("Expected has no Odin dependency")?; + let endpoint = dependency + .provider_endpoint + .as_deref() + .context("Expected Odin dependency has no endpoint")?; + let endpoint = endpoint.strip_prefix("rudp://").unwrap_or(endpoint); + Ok(Self { + endpoint: endpoint + .parse() + .map_err(|_| anyhow!("Expected Odin endpoint is not a socket address"))?, + }) + } +} + +impl PresenceSink for OdinRudpSink { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { + let message = CultNetMessage::DocumentPutRaw { + message_id: format!( + "runtime-presence:{}:{}:{}", + record.target, record.runtime_instance_id, record.publisher_sequence + ), + document: presence_document( + record, + Some(vec![ + "cultnet.transport.rudp.v0".into(), + "runtime-presence".into(), + ]), + )?, + }; + let bind = if self.endpoint.is_ipv4() { + "0.0.0.0:0" + } else { + "[::]:0" + }; + let socket = UdpSocket::bind(bind)?; + socket.set_read_timeout(Some(Duration::from_millis(100)))?; + let mut transport = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client( + &record.runtime_id, + socket, + self.endpoint, + ODIN_CULTMESH_CATALOG_CONNECTION_ID, + ), + )?; + transport.connect(Vec::new())?; + let deadline = Instant::now() + Duration::from_millis(500); + while !transport.connected() { + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "Odin did not accept the connection"); + } + let receipt = transport.send_reliable( + "schema", + encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?, + )?; + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match transport.reliable_send_status(&receipt) { + CultNetRudpReliableSendStatus::Acknowledged => return Ok(()), + CultNetRudpReliableSendStatus::Invalidated => { + bail!("Odin invalidated the presence acknowledgement") + } + CultNetRudpReliableSendStatus::Pending => {} + } + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "Odin did not acknowledge presence"); + } + } +} + +fn presence_document( + record: &GameCultRuntimePresenceHealthRecord, + tags: Option>, +) -> Result { + let payload = rmp_serde::to_vec(record).context("encoding runtime presence")?; + Ok(CultNetRawDocumentRecord { + schema_id: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: record.target.clone(), + stored_at: chrono::DateTime::from_timestamp_millis( + record.observed_at_unix_millis.try_into()?, + ) + .context("runtime presence timestamp is invalid")? + .to_rfc3339(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: Some(record.runtime_id.clone()), + source_agent_id: Some(record.signer_identity_id.clone()), + source_role: Some("runtime-presence-publisher".into()), + tags, + }) +} + +/// The child's private listener. +#[derive(Clone, Debug)] +pub enum ChildEndpoint { + Unix(PathBuf), + Loopback(SocketAddr), +} + +pub enum ChildStream { + Unix(UnixStream), + Tcp(TcpStream), +} + +impl ChildEndpoint { + pub fn connect(&self) -> io::Result { + match self { + Self::Unix(path) => UnixStream::connect(path).map(ChildStream::Unix), + Self::Loopback(address) => { + TcpStream::connect_timeout(address, HEALTH_PROBE_TIMEOUT).map(ChildStream::Tcp) + } + } + } + + /// What `{{child.listen}}` renders to. + fn listen_value(&self) -> String { + match self { + Self::Unix(path) => path.display().to_string(), + Self::Loopback(address) => address.to_string(), + } + } +} + +impl ChildStream { + fn try_clone(&self) -> io::Result { + match self { + Self::Unix(stream) => stream.try_clone().map(Self::Unix), + Self::Tcp(stream) => stream.try_clone().map(Self::Tcp), + } + } + + fn shutdown_write(&self) { + let _ = match self { + Self::Unix(stream) => stream.shutdown(Shutdown::Write), + Self::Tcp(stream) => stream.shutdown(Shutdown::Write), + }; + } + + fn set_timeout(&self, timeout: Duration) { + let _ = match self { + Self::Unix(stream) => stream + .set_read_timeout(Some(timeout)) + .and_then(|()| stream.set_write_timeout(Some(timeout))), + Self::Tcp(stream) => stream + .set_read_timeout(Some(timeout)) + .and_then(|()| stream.set_write_timeout(Some(timeout))), + }; + } +} + +impl Read for ChildStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self { + Self::Unix(stream) => stream.read(buffer), + Self::Tcp(stream) => stream.read(buffer), + } + } +} + +impl Write for ChildStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + match self { + Self::Unix(stream) => stream.write(buffer), + Self::Tcp(stream) => stream.write(buffer), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Unix(stream) => stream.flush(), + Self::Tcp(stream) => stream.flush(), + } + } +} + +/// `GET ` over the child's private listener; healthy on a 200. +pub struct HttpHealthProbe { + pub endpoint: ChildEndpoint, + pub path: String, +} + +impl HealthProbe for HttpHealthProbe { + fn healthy(&self) -> bool { + let Ok(mut stream) = self.endpoint.connect() else { + return false; + }; + stream.set_timeout(HEALTH_PROBE_TIMEOUT); + let request = format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + self.path + ); + let mut head = [0_u8; 12]; + stream.write_all(request.as_bytes()).is_ok() + && stream.read_exact(&mut head).is_ok() + && head.starts_with(b"HTTP/1.") + && &head[8..12] == b" 200" + } +} + +pub struct ProcessSpawner; + +struct ProcessChild(Child); + +impl ChildSpawner for ProcessSpawner { + fn spawn(&self, program: &Path, args: &[String]) -> Result> { + let mut command = Command::new(program); + // The child gets a bare environment: no Idunn variable, no candidate + // bind, no credential path leaks into it. + command.args(args).env_clear().stdin(Stdio::null()); + for name in ["PATH", "TMPDIR", "LANG", "LC_ALL"] { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } + let child = command.spawn().map_err(|_| anyhow!("the child could not be spawned"))?; + Ok(Box::new(ProcessChild(child))) + } +} + +impl ChildHandle for ProcessChild { + fn exited(&mut self) -> Option { + match self.0.try_wait() { + Ok(Some(status)) => Some(status.code().unwrap_or(1)), + Ok(None) => None, + Err(_) => Some(1), + } + } + + fn terminate(&mut self, grace: Duration) { + if self.exited().is_some() { + return; + } + // SAFETY: the pid is our own live child. + unsafe { libc::kill(self.0.id() as libc::pid_t, libc::SIGTERM) }; + let deadline = Instant::now() + grace; + while Instant::now() < deadline { + if self.exited().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +// ---------------------------------------------------------------- config + +/// The closed placeholder set (decision D6). Any other `{{...}}` refuses. +pub struct ConfigRenderer<'a> { + pub credentials: &'a dyn CredentialSource, + pub listen: &'a str, + pub state_root: Option<&'a Path>, +} + +impl ConfigRenderer<'_> { + pub fn render(&self, template: &str) -> Result { + let mut rendered = String::with_capacity(template.len()); + let mut rest = template; + while let Some(open) = rest.find("{{") { + rendered.push_str(&rest[..open]); + let after = &rest[open + 2..]; + let close = after + .find("}}") + .context("config template has an unterminated placeholder")?; + rendered.push_str(&self.expand(&after[..close])?); + rest = &after[close + 2..]; + } + rendered.push_str(rest); + Ok(rendered) + } + + fn expand(&self, key: &str) -> Result { + if key == "child.listen" { + return Ok(self.listen.to_owned()); + } + if key == "state_root" { + return Ok(self + .state_root + .context("config template needs a state root the launch did not give")? + .display() + .to_string()); + } + if let Some(name) = key.strip_prefix("credential:") { + ensure!(valid_credential_name(name), "credential placeholder name is invalid"); + let bytes = self.credentials.contents(name)?; + let text = String::from_utf8(bytes) + .map_err(|_| anyhow!("a credential is not UTF-8 text"))?; + return Ok(text.strip_suffix('\n').unwrap_or(&text).to_owned()); + } + if let Some(name) = key.strip_prefix("credential_path:") { + ensure!(valid_credential_name(name), "credential placeholder name is invalid"); + return Ok(self.credentials.path(name)?.display().to_string()); + } + bail!("config template uses a placeholder outside the closed set") + } +} + +fn valid_credential_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') +} + +fn write_private_file(directory: &Path, name: &str, contents: &str) -> Result { + let path = directory.join(name); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .map_err(|_| anyhow!("the rendered config could not be created"))?; + file.write_all(contents.as_bytes())?; + Ok(path) +} + +// ---------------------------------------------------------------- lease + +/// A write lease this launcher holds: the record binds this Expected, +/// activation and one of the launcher's own Warming proofs, and its store is +/// share-locked until the hold drops. +pub struct LeaseHold { + reader: Arc, + lease: IdunnProcessWriteLeaseRecord, + sha256: String, + _lock: Box, +} + +impl LeaseHold { + pub fn acquire( + reader: Arc, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + warming_proofs: &[String], + ) -> Result { + let lease = reader.read()?.context("no write lease is granted")?; + lease.validate()?; + lease_binds_candidate(expected, activation, &lease, warming_proofs)?; + let lock = reader.hold_shared()?; + ensure!( + reader.read()?.as_ref() == Some(&lease), + "write lease changed while it was being held" + ); + Ok(Self { + sha256: lease.canonical_sha256()?, + reader, + lease, + _lock: lock, + }) + } + + pub fn sha256(&self) -> &str { + &self.sha256 + } + + pub fn still_current(&self) -> bool { + matches!(self.reader.read(), Ok(Some(current)) if current == self.lease) + } +} + +// ---------------------------------------------------------------- route + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Phase { + Warming, + Active(Option), + Failed, +} + +/// What the launcher currently is, shared with the route acceptor. +struct Shared { + authority: Mutex, + phase: Mutex, + lease: Mutex>, + lease_required: bool, + health: Arc, + clock: Arc, +} + +impl Shared { + fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { + self.authority + .lock() + .map_err(|_| anyhow!("authority lock poisoned"))? + .sign(state, detail, lease, self.clock.now_millis()) + } + + fn lease_current(&self) -> bool { + !self.lease_required + || self + .lease + .lock() + .is_ok_and(|hold| hold.as_ref().is_some_and(LeaseHold::still_current)) + } + + /// The answer to a route observation: active only if the child is healthy + /// and the lease is still ours. + fn observe(&self, request: CultNetMessage) -> Result { + let CultNetMessage::SnapshotRequest { message_id, .. } = request else { + bail!("route challenge is not a snapshot request"); + }; + let detail = format!("route-observation:{message_id}"); + let phase = self.phase.lock().map_err(|_| anyhow!("phase lock poisoned"))?.clone(); + let record = match phase { + Phase::Warming => self.sign("warming", &detail, None)?, + Phase::Failed => self.sign("failed", &detail, None)?, + Phase::Active(lease) => { + let state = if self.health.healthy() && self.lease_current() { + "active" + } else { + "degraded" + }; + self.sign(state, &detail, lease)? + } + }; + Ok(CultNetMessage::SnapshotResponseRaw { + message_id, + documents: vec![presence_document(&record, None)?], + }) + } +} + +type Responder = Arc Result + Send + Sync>; + +/// Owns the candidate endpoint. A leading 0x00 is an Idunn challenge frame and +/// is answered here; every other byte is copied untouched to the child. +pub struct RouteAcceptor { + listener: TcpListener, + child: ChildEndpoint, + responder: Responder, +} + +impl RouteAcceptor { + pub fn new(listener: TcpListener, child: ChildEndpoint, responder: Responder) -> Self { + Self { + listener, + child, + responder, + } + } + + pub fn serve(self, halt: Arc) -> Result<()> { + self.listener.set_nonblocking(true)?; + let acceptor = Arc::new(self); + while !halt.load(Ordering::SeqCst) { + match acceptor.listener.accept() { + Ok((connection, _)) => { + let acceptor = Arc::clone(&acceptor); + thread::spawn(move || { + let _ = connection.set_nonblocking(false); + let _ = acceptor.handle(connection); + }); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => thread::sleep(Duration::from_millis(10)), + } + } + Ok(()) + } + + fn handle(&self, mut connection: TcpStream) -> Result<()> { + let mut first = [0_u8; 1]; + if connection.read(&mut first)? == 0 { + return Ok(()); + } + if first[0] == CHALLENGE_FIRST_BYTE { + return self.answer_challenge(&mut connection); + } + let mut upstream = self.child.connect()?; + upstream.write_all(&first)?; + let mut upstream_reader = upstream.try_clone()?; + let mut client_reader = connection.try_clone()?; + let forward = thread::spawn(move || { + let _ = io::copy(&mut client_reader, &mut upstream); + upstream.shutdown_write(); + }); + let _ = io::copy(&mut upstream_reader, &mut connection); + let _ = connection.shutdown(Shutdown::Write); + let _ = forward.join(); + Ok(()) + } + + fn answer_challenge(&self, connection: &mut TcpStream) -> Result<()> { + connection.set_read_timeout(Some(CHALLENGE_IO_TIMEOUT))?; + connection.set_write_timeout(Some(CHALLENGE_IO_TIMEOUT))?; + let mut rest = [0_u8; 3]; + connection.read_exact(&mut rest)?; + let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; + ensure!( + (1..=CHALLENGE_MAX_BYTES).contains(&length), + "challenge frame is outside its bound" + ); + let mut payload = vec![0_u8; length]; + connection.read_exact(&mut payload)?; + let request = + decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; + let response = (self.responder)(request)?; + let frame = encode_frame(&encode_cultnet_message_to_vec( + &response, + CultNetWireContract::CultNetSchemaV0, + )?)?; + connection.write_all(&frame)?; + connection.flush()?; + Ok(()) + } +} + +// ---------------------------------------------------------------- lifecycle + +pub struct Ports { + pub credentials: Arc, + pub clock: Arc, + pub sink: Arc, + pub lease: Arc, + pub spawner: Box, + /// Built from the endpoint `run` chooses. + pub health: Box Arc + Send>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + /// SIGTERM: the child was stopped and waited for. + Stopped, + /// The unit must die so continuity sees it; the code names why. + Failed(&'static str), +} + +impl Outcome { + pub fn exit_code(&self) -> i32 { + match self { + Self::Stopped => 0, + Self::Failed(_) => 1, + } + } +} + +fn child_endpoint(listener: ChildListener, tmp: &Path) -> Result { + Ok(match listener { + ChildListener::UnixSocket => ChildEndpoint::Unix(tmp.join("child.sock")), + ChildListener::LoopbackPort => { + let probe = TcpListener::bind("127.0.0.1:0")?; + ChildEndpoint::Loopback(probe.local_addr()?) + } + }) +} + +/// The unit's whole life: Warming, lease, child, Active, until the child or the +/// lease ends or SIGTERM arrives. `route` is the already-bound candidate +/// listener; `tmp` is the unit's private temporary directory. +pub fn run( + contract: &LaunchContract, + authority: RuntimeAuthority, + route: TcpListener, + tmp: &Path, + ports: Ports, + stop: &AtomicBool, +) -> Outcome { + let expected = authority.expected().clone(); + let activation = authority.activation().clone(); + if expected.health_contract != contract.health_contract { + return Outcome::Failed("health-contract-mismatch"); + } + let Ok(endpoint) = child_endpoint(contract.listener, tmp) else { + return Outcome::Failed("child-endpoint"); + }; + let health = (ports.health)(&endpoint); + let shared = Arc::new(Shared { + authority: Mutex::new(authority), + phase: Mutex::new(Phase::Warming), + lease: Mutex::new(None), + lease_required: expected.write_lease_required, + health: Arc::clone(&health), + clock: Arc::clone(&ports.clock), + }); + let halt = Arc::new(AtomicBool::new(false)); + let responder: Responder = { + let shared = Arc::clone(&shared); + Arc::new(move |request| shared.observe(request)) + }; + let serving = { + let acceptor = RouteAcceptor::new(route, endpoint.clone(), responder); + let halt = Arc::clone(&halt); + thread::spawn(move || acceptor.serve(halt)) + }; + let outcome = lifecycle(contract, &expected, &activation, &endpoint, tmp, &ports, &shared, stop); + halt.store(true, Ordering::SeqCst); + let _ = serving.join(); + outcome +} + +#[allow(clippy::too_many_arguments)] +fn lifecycle( + contract: &LaunchContract, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + endpoint: &ChildEndpoint, + tmp: &Path, + ports: &Ports, + shared: &Shared, + stop: &AtomicBool, +) -> Outcome { + let clock = &*ports.clock; + let publish = |state: &str, detail: &str, lease: Option| -> Option { + let record = shared.sign(state, detail, lease).ok()?; + let sha = record.canonical_sha256().ok()?; + // A refused or lost publish is retried by the heartbeat; Odin may + // refuse Warming for a while after Idunn starts the unit. + let _ = ports.sink.put(&record); + Some(sha) + }; + let set_phase = |phase: Phase| { + if let Ok(mut current) = shared.phase.lock() { + *current = phase; + } + }; + let fail = |child: Option<&mut Box>, code: &'static str| { + if let Some(child) = child { + child.terminate(TERMINATE_GRACE); + } + set_phase(Phase::Failed); + publish("failed", code, None); + Outcome::Failed(code) + }; + + // 1. Warming reaches Odin before any child exists. + let mut proofs: VecDeque = VecDeque::new(); + let mut remember = |sha: Option, proofs: &mut VecDeque| { + if let Some(sha) = sha { + proofs.push_back(sha); + while proofs.len() > MAX_RECENT_WARMING_PROOFS { + proofs.pop_front(); + } + } + }; + remember(publish("warming", "launcher-started", None), &mut proofs); + let heartbeat = u64::try_from(contract.heartbeat.as_millis()).unwrap_or(u64::MAX); + let mut next_heartbeat = clock.now_millis() + heartbeat; + + // 2. A stateful child waits for the lease; a stateless one does not. + let mut lease_sha = None; + if expected.write_lease_required { + let deadline = clock.now_millis() + contract.lease_wait.as_millis() as u64; + loop { + if stop.load(Ordering::SeqCst) { + return Outcome::Stopped; + } + let held = LeaseHold::acquire( + Arc::clone(&ports.lease), + expected, + activation, + proofs.make_contiguous(), + ); + if let Ok(hold) = held { + lease_sha = Some(hold.sha256().to_owned()); + if let Ok(mut slot) = shared.lease.lock() { + *slot = Some(hold); + } + break; + } + let now = clock.now_millis(); + if now >= deadline { + return fail(None, "lease-timeout"); + } + if now >= next_heartbeat { + remember(publish("warming", "write-lease-pending", None), &mut proofs); + next_heartbeat = now + heartbeat; + } + clock.sleep(POLL); + } + } + + // 3. Render, spawn, and wait for health. + let Some(mut child) = start_child(contract, endpoint, tmp, ports) else { + return fail(None, "child-start-refused"); + }; + let startup_deadline = clock.now_millis() + contract.startup.as_millis() as u64; + loop { + if stop.load(Ordering::SeqCst) { + child.terminate(TERMINATE_GRACE); + return Outcome::Stopped; + } + if child.exited().is_some() { + return fail(None, "child-exited"); + } + if shared.health.healthy() { + break; + } + if clock.now_millis() >= startup_deadline { + return fail(Some(&mut child), "startup-timeout"); + } + clock.sleep(POLL); + } + + // 4. Active, and heartbeats while it holds. + set_phase(Phase::Active(lease_sha.clone())); + publish("active", "child-healthy", lease_sha.clone()); + next_heartbeat = clock.now_millis() + heartbeat; + loop { + if stop.load(Ordering::SeqCst) { + child.terminate(TERMINATE_GRACE); + return Outcome::Stopped; + } + if child.exited().is_some() { + return fail(None, "child-exited"); + } + if !shared.lease_current() { + return fail(Some(&mut child), "lease-lost"); + } + let now = clock.now_millis(); + if now >= next_heartbeat { + let state = if shared.health.healthy() { "active" } else { "degraded" }; + publish(state, "heartbeat", lease_sha.clone()); + next_heartbeat = now + heartbeat; + } + clock.sleep(POLL); + } +} + +fn start_child( + contract: &LaunchContract, + endpoint: &ChildEndpoint, + tmp: &Path, + ports: &Ports, +) -> Option> { + let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; + let listen = endpoint.listen_value(); + let rendered = ConfigRenderer { + credentials: &*ports.credentials, + listen: &listen, + state_root: contract.state_root.as_deref(), + } + .render(&template) + .ok()?; + let config = write_private_file(tmp, "child-config", &rendered).ok()?; + let mut args = contract.child_args.clone(); + if let Some(flag) = &contract.config_flag { + args.push(flag.clone()); + args.push(config.display().to_string()); + } + ports + .spawner + .spawn(&contract.release_root.join(&contract.child), &args) + .ok() +} diff --git a/src/lib.rs b/src/lib.rs index b96d835..e71a17f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,5 +4,7 @@ pub mod deployment_plan; pub mod drivers; pub mod host; pub mod host_actuator; +#[cfg(unix)] +pub mod launcher; mod incident; pub mod provisioning; From 6866d755364b7af08016196df1bcbfe02608bda3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 19:10:58 +0200 Subject: [PATCH 126/196] idunn-launch tests: lease gate, foreign lease, lease loss, stateless, route challenge, verbatim splice, closed placeholders, child exit (unbuilt) Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 713 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 713 insertions(+) diff --git a/src/launcher.rs b/src/launcher.rs index f3bee58..25e38db 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -1275,3 +1275,716 @@ fn start_child( .spawn(&contract.release_root.join(&contract.child), &args) .ok() } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::{AtomicU64, AtomicUsize}; + + use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; + use cultnet_rs::{ + IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, + IdunnServiceIdentity, RuntimePresenceAuthenticationContext, VerifiedRuntimeAuthority, + authenticate_runtime_presence_claim, enroll_service_identity_at, verify_runtime_authority, + }; + + use super::*; + use crate::drivers::{WriteLeasePort, exact_route_snapshot_response, request_tcp_snapshot}; + + fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) + } + + struct Fixture { + dir: tempfile::TempDir, + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + activation_credential: Vec, + provider_credential: Vec, + verified: VerifiedRuntimeAuthority, + } + + fn write_record(path: &Path, key: &str, kind: &str, schema: &str, payload: Vec) { + let envelope = CultCacheEnvelope { + key: key.into(), + r#type: kind.into(), + payload, + stored_at: chrono::Utc::now().to_rfc3339(), + schema_id: Some(schema.into()), + }; + let outcome = SingleFileMessagePackBackingStore::new(path) + .try_compare_exchange_snapshot(&[], &[envelope]) + .unwrap(); + assert!(matches!( + outcome, + TryCompareExchangeSnapshotOutcome::Exchanged + )); + } + + fn fixture(lease_required: bool) -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let provider_path = root.join("provider.cc"); + let provider = + enroll_service_identity_at::(&provider_path).unwrap(); + let idunn = + enroll_service_identity_at::(&root.join("idunn.cc")).unwrap(); + let expected = IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: "synapse".into(), + plan_id: digest('1'), + incarnation_id: "incarnation-1".into(), + sealed_release_id: digest('2'), + source_repository: "github.com/GameCult/gamecult-chat".into(), + source_revision: "3".repeat(40), + recipe_sha256: digest('4'), + runtime_id: "synapse-yggdrasil".into(), + expected_signer_identity_id: provider.entry().identity_id.clone(), + health_contract: "gamecult.chat.synapse.presence-launcher.v1".into(), + artifact_sha256: digest('5'), + state_schema_generation: lease_required.then(|| "state-v1".into()), + state_contract_sha256: lease_required.then(|| digest('6')), + write_lease_required: lease_required, + route: Some(IdunnExpectedRoute { + route_id: "synapse-tcp".into(), + transport: "tcp".into(), + stable_endpoint: "tcp://127.0.0.1:18834".into(), + candidate_endpoint: "tcp://127.0.0.1:18835".into(), + }), + capabilities: vec![IdunnExpectedCapability { + capability: "gamecult.chat.homeserver".into(), + schema: "matrix.client.v3".into(), + compatibility: "v3".into(), + minimum_capacity: 1, + }], + dependencies: Vec::new(), + }; + let launch = + IdunnRuntimeActivationLaunch::issue(&expected, digest('7'), 100, &idunn).unwrap(); + let activation = launch.activation().clone(); + let mut activation_credential = Vec::new(); + launch.write_credential(&mut activation_credential).unwrap(); + let verified = verify_runtime_authority( + &expected, + &activation, + &idunn.trust_anchor().unwrap(), + &provider.entry().public_key, + ) + .unwrap(); + let bundle = root.join("bundle"); + fs::create_dir(&bundle).unwrap(); + write_record( + &bundle.join("expected.cc"), + &expected.target, + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + expected.canonical_bytes().unwrap(), + ); + write_record( + &bundle.join("activation.cc"), + &expected.target, + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + activation.canonical_bytes().unwrap(), + ); + let provider_credential = fs::read(&provider_path).unwrap(); + for directory in ["release/bin", "tmp"] { + fs::create_dir_all(root.join(directory)).unwrap(); + } + Fixture { + dir, + expected, + activation, + activation_credential, + provider_credential, + verified, + } + } + + impl Fixture { + fn authority(&self) -> RuntimeAuthority { + RuntimeAuthority::open( + &self.dir.path().join("bundle"), + &self.activation_credential[..], + &self.provider_credential[..], + ) + .unwrap() + } + + fn lease(&self, warming_sha256: &str) -> IdunnProcessWriteLeaseRecord { + IdunnProcessWriteLeaseRecord { + schema_version: cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: self.expected.target.clone(), + expected_projection_sha256: self.expected.canonical_sha256().unwrap(), + plan_id: self.expected.plan_id.clone(), + incarnation_id: self.expected.incarnation_id.clone(), + sealed_release_id: self.expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256().unwrap(), + state_schema_generation: self.expected.state_schema_generation.clone().unwrap(), + state_contract_sha256: self.expected.state_contract_sha256.clone().unwrap(), + runtime_id: self.expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + warming_presence_sha256: warming_sha256.into(), + lease_epoch: 1, + issued_at_unix_millis: 200, + } + } + + fn contract(&self, template: &str) -> LaunchContract { + fs::write(self.dir.path().join("release/config.tmpl"), template).unwrap(); + LaunchContract { + release_root: self.dir.path().join("release"), + child: PathBuf::from("bin/child"), + child_args: vec!["--serve".into()], + config_flag: Some("--config".into()), + template: PathBuf::from("config.tmpl"), + listener: ChildListener::UnixSocket, + health_path: "/health".into(), + health_contract: self.expected.health_contract.clone(), + state_root: Some(PathBuf::from("/var/lib/synapse")), + lease_wait: Duration::from_secs(3600), + startup: Duration::from_secs(3600), + heartbeat: Duration::from_secs(5), + } + } + } + + /// Virtual time runs a hundred times faster than the wall clock. + struct FastClock(AtomicU64); + + impl Clock for FastClock { + fn now_millis(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } + + fn sleep(&self, duration: Duration) { + self.0 + .fetch_add(duration.as_millis() as u64, Ordering::SeqCst); + thread::sleep(Duration::from_millis(1)); + } + } + + #[derive(Default)] + struct Sink(Mutex>); + + impl PresenceSink for Sink { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { + self.0.lock().unwrap().push(record.clone()); + Ok(()) + } + } + + impl Sink { + fn states(&self) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .map(|record| record.state.clone()) + .collect() + } + + fn first(&self, state: &str) -> Option { + self.0 + .lock() + .unwrap() + .iter() + .find(|record| record.state == state) + .cloned() + } + } + + #[derive(Default)] + struct FakeLease(Mutex>); + + impl LeaseReader for FakeLease { + fn read(&self) -> Result> { + Ok(self.0.lock().unwrap().clone()) + } + + fn hold_shared(&self) -> Result> { + Ok(Box::new(())) + } + } + + struct Health(AtomicBool); + + impl HealthProbe for Health { + fn healthy(&self) -> bool { + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct Child { + spawns: AtomicUsize, + exit: Mutex>, + terminated: AtomicBool, + args: Mutex>>, + } + + struct FakeChild(Arc); + + impl ChildHandle for FakeChild { + fn exited(&mut self) -> Option { + *self.0.exit.lock().unwrap() + } + + fn terminate(&mut self, _grace: Duration) { + self.0.terminated.store(true, Ordering::SeqCst); + *self.0.exit.lock().unwrap() = Some(143); + } + } + + struct FakeSpawner(Arc); + + impl ChildSpawner for FakeSpawner { + fn spawn(&self, _program: &Path, args: &[String]) -> Result> { + self.0.spawns.fetch_add(1, Ordering::SeqCst); + self.0.args.lock().unwrap().push(args.to_vec()); + Ok(Box::new(FakeChild(Arc::clone(&self.0)))) + } + } + + struct Credentials(BTreeMap>); + + impl CredentialSource for Credentials { + fn contents(&self, name: &str) -> Result> { + self.0.get(name).cloned().ok_or_else(|| anyhow!("unbound")) + } + + fn path(&self, name: &str) -> Result { + self.0 + .contains_key(name) + .then(|| PathBuf::from(format!("/run/credentials/unit/{name}"))) + .ok_or_else(|| anyhow!("unbound")) + } + } + + struct Run { + sink: Arc, + lease: Arc, + child: Arc, + stop: Arc, + outcome: Option>, + } + + fn start(fx: &Fixture, contract: LaunchContract, healthy: bool) -> Run { + let sink = Arc::new(Sink::default()); + let lease = Arc::new(FakeLease::default()); + let health = Arc::new(Health(AtomicBool::new(healthy))); + let child = Arc::new(Child::default()); + let stop = Arc::new(AtomicBool::new(false)); + let mut secrets = BTreeMap::new(); + secrets.insert("SYNAPSE_KEY".to_owned(), b"s3cret-value\n".to_vec()); + let ports = Ports { + credentials: Arc::new(Credentials(secrets)), + clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + sink: sink.clone(), + lease: lease.clone(), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(move |_| health.clone()), + }; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let flag = stop.clone(); + let outcome = thread::spawn(move || run(&contract, authority, route, &tmp, ports, &flag)); + Run { + sink, + lease, + child, + stop, + outcome: Some(outcome), + } + } + + impl Run { + fn finish(&mut self) -> Outcome { + self.outcome.take().unwrap().join().unwrap() + } + + fn stop(&mut self) -> Outcome { + self.stop.store(true, Ordering::SeqCst); + self.finish() + } + } + + fn wait_until(what: &str, condition: impl Fn() -> bool) { + for _ in 0..4000 { + if condition() { + return; + } + thread::sleep(Duration::from_millis(5)); + } + panic!("timed out waiting for {what}"); + } + + #[test] + fn stateful_child_never_starts_before_its_lease() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("listen {{child.listen}}"), true); + wait_until("two warming heartbeats", || { + launch.sink.states().iter().filter(|s| *s == "warming").count() >= 2 + }); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); + let warming = launch.sink.first("warming").unwrap(); + assert!(warming.write_lease_sha256.is_none()); + + // The lease binds the launcher's own Warming proof; only then it starts. + let lease = fx.lease(&warming.canonical_sha256().unwrap()); + *launch.lease.0.lock().unwrap() = Some(lease.clone()); + wait_until("child spawn", || { + launch.child.spawns.load(Ordering::SeqCst) == 1 + }); + wait_until("active", || launch.sink.first("active").is_some()); + let active = launch.sink.first("active").unwrap(); + assert_eq!( + active.write_lease_sha256.as_deref(), + Some(lease.canonical_sha256().unwrap().as_str()) + ); + assert_eq!(launch.stop(), Outcome::Stopped); + assert!(launch.child.terminated.load(Ordering::SeqCst)); + } + + #[test] + fn a_lease_for_another_activation_or_expected_is_not_accepted() { + let fx = fixture(true); + let foreign: [(&str, fn(&mut IdunnProcessWriteLeaseRecord)); 3] = [ + ("another runtime instance", |lease| { + lease.runtime_instance_id = "other".into() + }), + ("another Expected", |lease| { + lease.expected_projection_sha256 = digest('9') + }), + ("another warming proof", |lease| { + lease.warming_presence_sha256 = digest('9') + }), + ]; + for (name, edit) in foreign { + let mut contract = fx.contract("x"); + contract.lease_wait = Duration::from_secs(5); + let mut launch = start(&fx, contract, true); + // Installed at once; the launcher's own proofs are never digest('8'). + let mut lease = fx.lease(&digest('8')); + edit(&mut lease); + *launch.lease.0.lock().unwrap() = Some(lease); + assert_eq!(launch.finish(), Outcome::Failed("lease-timeout"), "{name}"); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0, "{name}"); + assert_eq!(launch.sink.states().last().unwrap(), "failed", "{name}"); + } + } + + #[test] + fn lease_loss_stops_the_child() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("warming", || launch.sink.first("warming").is_some()); + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + *launch.lease.0.lock().unwrap() = None; + let outcome = launch.finish(); + assert_eq!(outcome, Outcome::Failed("lease-lost")); + assert_ne!(outcome.exit_code(), 0); + assert!(launch.child.terminated.load(Ordering::SeqCst)); + assert_eq!(launch.sink.states().last().unwrap(), "failed"); + } + + #[test] + fn stateless_child_starts_without_a_lease() { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 1); + assert_eq!(launch.sink.states().first().unwrap(), "warming"); + assert!( + launch + .sink + .first("active") + .unwrap() + .write_lease_sha256 + .is_none() + ); + launch.stop(); + } + + #[test] + fn child_exit_ends_the_launcher() { + for code in [0, 3] { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + *launch.child.exit.lock().unwrap() = Some(code); + let outcome = launch.finish(); + assert_eq!(outcome, Outcome::Failed("child-exited"), "exit {code}"); + assert_eq!(outcome.exit_code(), 1); + assert_eq!(launch.sink.states().last().unwrap(), "failed"); + } + } + + #[test] + fn closed_placeholder_set_renders_and_refuses() { + let fx = fixture(false); + let template = "listen={{child.listen}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; + let mut launch = start(&fx, fx.contract(template), true); + wait_until("spawn", || launch.child.spawns.load(Ordering::SeqCst) == 1); + let args = launch.child.args.lock().unwrap()[0].clone(); + assert_eq!(args[..2], ["--serve", "--config"]); + let config = PathBuf::from(&args[2]); + assert_eq!(config.parent().unwrap(), fx.dir.path().join("tmp")); + assert_eq!( + fs::metadata(&config).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let rendered = fs::read_to_string(&config).unwrap(); + let socket = fx.dir.path().join("tmp/child.sock"); + assert_eq!( + rendered, + format!( + "listen={}\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n", + socket.display() + ) + ); + assert!( + !rendered.contains("18835"), + "the candidate bind never reaches the child" + ); + launch.stop(); + + // Any other placeholder refuses to start, and the refusal names no input. + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("a={{canary_bind}}"), true); + assert_eq!(launch.finish(), Outcome::Failed("child-start-refused")); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); + let credentials = Credentials(BTreeMap::new()); + let renderer = ConfigRenderer { + credentials: &credentials, + listen: "L", + state_root: None, + }; + for template in [ + "{{canary_bind}}", + "{{credential:CANARY-MISSING}}", + "{{credential:canary b}}", + "{{child.listen}} {{canary", + "{{state_root}}", + ] { + let error = format!("{:#}", renderer.render(template).unwrap_err()); + assert!(!error.to_lowercase().contains("canary"), "{error}"); + } + } + + fn shared(fx: &Fixture, health: Arc) -> Arc { + Arc::new(Shared { + authority: Mutex::new(fx.authority()), + phase: Mutex::new(Phase::Active(None)), + lease: Mutex::new(None), + lease_required: false, + health, + clock: Arc::new(SystemClock), + }) + } + + #[test] + fn route_answer_degrades_with_the_child() { + let fx = fixture(false); + let health = Arc::new(Health(AtomicBool::new(true))); + let shared = shared(&fx, health.clone()); + let state_of = |shared: &Shared| { + let request = CultNetMessage::SnapshotRequest { + message_id: "challenge-1".into(), + schema_ids: None, + record_keys: None, + }; + let CultNetMessage::SnapshotResponseRaw { documents, .. } = + shared.observe(request).unwrap() + else { + panic!("not a raw snapshot response") + }; + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&documents[0].payload).unwrap(); + assert_eq!(record.detail, "route-observation:challenge-1"); + record.state + }; + assert_eq!(state_of(&shared), "active"); + health.0.store(false, Ordering::SeqCst); + assert_eq!(state_of(&shared), "degraded"); + } + + #[test] + fn route_challenge_is_answered_with_both_signatures_through_idunns_client() { + let fx = fixture(false); + let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let responder: Responder = Arc::new(move |request| shared.observe(request)); + let halt = Arc::new(AtomicBool::new(false)); + let acceptor = RouteAcceptor::new( + listener, + ChildEndpoint::Unix("/nonexistent".into()), + responder, + ); + let serving = { + let halt = halt.clone(); + thread::spawn(move || acceptor.serve(halt)) + }; + let request = CultNetMessage::SnapshotRequest { + message_id: "challenge-7".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + let response = request_tcp_snapshot(address, &payload).ok().unwrap(); + let message = + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .unwrap(); + let answer = + exact_route_snapshot_response(message, "challenge-7", &fx.expected.target).unwrap(); + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&answer.canonical_presence).unwrap(); + let claim = authenticate_runtime_presence_claim( + &answer.canonical_presence, + &fx.verified, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: record.observed_at_unix_millis, + maximum_age_millis: 10_000, + maximum_future_skew_millis: 10, + }, + ) + .unwrap(); + assert_eq!(claim.record().state, "active"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn member_bytes_reach_the_child_verbatim_and_only_challenge_frames_are_answered() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("child.sock"); + let child = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + let request: Vec = + b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 6\r\n\r\n\x00\x01\x00\x02\x00\x03".to_vec(); + let response: Vec = + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n\x00\xff\x00\x10".to_vec(); + let received = Arc::new(Mutex::new(Vec::new())); + let connections = Arc::new(AtomicUsize::new(0)); + let fake = { + let (received, connections, response, want) = ( + received.clone(), + connections.clone(), + response.clone(), + request.len(), + ); + thread::spawn(move || { + for stream in child.incoming() { + let mut stream = stream.unwrap(); + connections.fetch_add(1, Ordering::SeqCst); + let mut buffer = vec![0_u8; want]; + if stream.read_exact(&mut buffer).is_err() { + continue; + } + *received.lock().unwrap() = buffer; + stream.write_all(&response).unwrap(); + return; + } + }) + }; + let challenges = Arc::new(AtomicUsize::new(0)); + let responder: Responder = { + let challenges = challenges.clone(); + Arc::new(move |_| { + challenges.fetch_add(1, Ordering::SeqCst); + bail!("not under test") + }) + }; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let halt = Arc::new(AtomicBool::new(false)); + let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder); + let serving = { + let halt = halt.clone(); + thread::spawn(move || acceptor.serve(halt)) + }; + + // A leading 0x00 that is not a challenge frame never reaches the child. + let mut probe = TcpStream::connect(address).unwrap(); + probe.write_all(&[0x00, 0xff, 0xff, 0xff, 1, 2, 3]).unwrap(); + let mut answered = Vec::new(); + let _ = probe.read_to_end(&mut answered); + assert!(answered.is_empty()); + assert_eq!(connections.load(Ordering::SeqCst), 0); + + // Everything else is copied untouched, both ways, 0x00 bytes included. + let mut member = TcpStream::connect(address).unwrap(); + member.write_all(&request).unwrap(); + let mut answer = Vec::new(); + member.read_to_end(&mut answer).unwrap(); + assert_eq!(answer, response); + fake.join().unwrap(); + assert_eq!(*received.lock().unwrap(), request); + assert_eq!( + challenges.load(Ordering::SeqCst), + 0, + "a malformed frame fails before the responder; member traffic never reaches it" + ); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn contract_parsing_refuses_escapes_and_echoes_nothing() { + let base = |extra: &[&str]| -> Result { + let mut args: Vec = [ + "--release-root", + "/r", + "--child", + "bin/c", + "--template", + "t", + "--listener", + "unix", + "--health-path", + "/health", + "--health-contract", + "h", + ] + .map(String::from) + .to_vec(); + args.extend(extra.iter().map(|value| value.to_string())); + LaunchContract::parse(args) + }; + assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); + assert!(base(&["--child", "../escape"]).is_err()); + assert!(base(&["--template", "/etc/passwd"]).is_err()); + let error = base(&["--canary-flag", "canary-value"]) + .unwrap_err() + .to_string(); + assert!(!error.contains("canary")); + } + + #[test] + fn real_lease_reader_holds_the_shared_lock_idunn_must_respect() { + let fx = fixture(true); + let path = fx.dir.path().join("lease.cc"); + let warming = digest('a'); + let lease = fx.lease(&warming); + write_record( + &path, + &lease.target, + IdunnProcessWriteLeaseRecord::TYPE, + cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes().unwrap(), + ); + let reader: Arc = Arc::new(DriverLeaseReader::new(&lease.target, &path)); + let hold = LeaseHold::acquire(reader, &fx.expected, &fx.activation, &[warming]).unwrap(); + assert!(hold.still_current()); + let idunn = CultCacheWriteLeaseDriver::new(&lease.target, &path); + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer")); + drop(hold); + idunn.revoke_exact(Some(&lease)).unwrap(); + } +} From d94150d396135759ec1cab13242609e87ee1d892 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 19:58:21 +0200 Subject: [PATCH 127/196] idunn-launch: release root is the launcher's own release; docs for foreign workloads; probe, spawner and authority tests Not built at this commit; verified at the head by the final job. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 25 ++++++++++++++ docs/authority-map.md | 36 ++++++++++++++++++++ src/bin/idunn-launch.rs | 10 ++++-- src/launcher.rs | 74 +++++++++++++++++++++++++++++++++-------- 4 files changed, 129 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 503409e..225c5d8 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,31 @@ omits one. Unknown fields are rejected on both sides. A repository that gets compromised can therefore change what is built, but not what it is allowed to touch on the host. +An upstream daemon that does not speak Idunn's runtime contract is deployed +with `idunn-launch` as its executable artifact and the upstream tree as a +required adjacent artifact; the launcher finds the child inside the release that +holds it. The launcher is the unit's process: it binds the +candidate endpoint, answers Idunn's challenge, renders the child's config, +starts a stateful child only once it holds the write lease, and signs presence. + +```toml +# recipe fragment — the upstream tree is built or fetched as an artifact +[service] +executable_artifact = "idunn-launch" +arguments = [ + { kind = "literal", value = "--child" }, + { kind = "literal", value = "upstream/bin/daemon" }, + { kind = "literal", value = "--template" }, + { kind = "literal", value = "config.tmpl" }, + { kind = "literal", value = "--listener" }, + { kind = "literal", value = "unix" }, + { kind = "literal", value = "--health-path" }, + { kind = "literal", value = "/health" }, + { kind = "literal", value = "--health-contract" }, + { kind = "literal", value = "example.presence-launcher.v1" }, +] +``` + ## Contracts Idunn's typed contracts are published by diff --git a/docs/authority-map.md b/docs/authority-map.md index 270d592..b61a8f2 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -70,6 +70,42 @@ recipe fields. Stateless services declare no state block. Their operator binding therefore has no state root, state-transition policy, or process write-lease record. +## Foreign workloads: idunn-launch + +An upstream daemon that cannot speak Idunn's runtime contract (Synapse, MAS) +runs under `idunn-launch`, the unit's own process. Idunn's two-witness rule is +unchanged: Idunn observes the unit, and the launcher, inside the unit and +supervising the child, signs presence with the provider identity and the +activation key. + +- **Owner:** the launcher owns presence, config rendering and the child's + lifetime inside the unit, and the route answers it signs for that child. + Idunn owns admission, lease, route and continuity, as before. +- **Inputs:** the runtime bundle, the activation credential and provider + identity descriptors, the process write lease, the systemd credentials the + binding names, and the child's health over its private listener. +- **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active + and failed to Odin; route observations on the candidate endpoint), one child + process, and a rendered config in the unit's private tmp that is never read + back. +- **Route:** the launcher, not the child, binds the candidate endpoint. A first + byte of 0x00 is an Idunn challenge frame and is answered by the launcher; + every other byte is copied untouched to the child's private listener. +- **Lease:** with `write_lease_required` the launcher starts the child only + after it holds the lease (bound to this Expected, this activation and one of + its own Warming proofs) and keeps the lease store share-locked while the + child lives. Losing the lease stops the child. +- **Config:** one template from the release is rendered with the closed + placeholder set `{{child.listen}}`, `{{credential:NAME}}`, + `{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder + refuses to start. +- **Forbidden:** the child binding the candidate endpoint; a child of a + lease-requiring target running without a held lease; any presence not signed + by both keys. +- **State kind `external-directory`:** a process-owned directory imposed by an + external runtime contract. Same writer, recovery and startup rules as + `external-file`: it is lease-gated, preserved and fenced, never interpreted. + ## Operator binding The root-admitted operator binding supplies the physical policy a repository diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs index 32c6291..b743f06 100644 --- a/src/bin/idunn-launch.rs +++ b/src/bin/idunn-launch.rs @@ -17,7 +17,11 @@ fn main() -> std::process::ExitCode { } fn launch() -> anyhow::Result { - let contract = LaunchContract::parse(std::env::args().skip(1))?; + let release_root = std::env::current_exe()? + .parent() + .context("the launcher is not inside a release")? + .to_owned(); + let contract = LaunchContract::parse(release_root, std::env::args().skip(1))?; let bundle = std::env::var_os(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT) .context("the Idunn runtime bundle is not set")?; let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) @@ -52,8 +56,8 @@ fn main() -> std::process::ExitCode { }; // SAFETY: the handler only stores to an atomic. unsafe { - libc::signal(libc::SIGTERM, on_signal as libc::sighandler_t); - libc::signal(libc::SIGINT, on_signal as libc::sighandler_t); + libc::signal(libc::SIGTERM, on_signal as *const () as libc::sighandler_t); + libc::signal(libc::SIGINT, on_signal as *const () as libc::sighandler_t); } Ok(launcher::run( &contract, diff --git a/src/launcher.rs b/src/launcher.rs index 25e38db..7748177 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -66,7 +66,8 @@ pub enum ChildListener { } /// Everything the recipe declares about the child, parsed from the launcher's -/// argv (`--flag value` pairs; `--child-arg` repeats). +/// argv (`--flag value` pairs; `--child-arg` repeats). The release root is the +/// sealed release directory that holds the launcher itself. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LaunchContract { pub release_root: PathBuf, @@ -86,8 +87,8 @@ pub struct LaunchContract { } impl LaunchContract { - pub fn parse(args: impl IntoIterator) -> Result { - let mut release_root = None; + pub fn parse(release_root: PathBuf, args: impl IntoIterator) -> Result { + ensure!(release_root.is_absolute(), "the release root is not absolute"); let mut child = None; let mut child_args = Vec::new(); let mut config_flag = None; @@ -108,7 +109,6 @@ impl LaunchContract { .with_context(|| format!("launcher argument {field} is not a positive integer")) }; match flag.as_str() { - "--release-root" => release_root = Some(PathBuf::from(value)), "--child" => child = Some(release_relative(&value, "--child")?), "--child-arg" => child_args.push(value), "--config-flag" => config_flag = Some(value), @@ -135,11 +135,6 @@ impl LaunchContract { _ => bail!("unknown launcher argument"), } } - let release_root = release_root.context("launcher argument --release-root is required")?; - ensure!( - release_root.is_absolute(), - "launcher argument --release-root is not absolute" - ); Ok(Self { release_root, child: child.context("launcher argument --child is required")?, @@ -1157,7 +1152,7 @@ fn lifecycle( // 1. Warming reaches Odin before any child exists. let mut proofs: VecDeque = VecDeque::new(); - let mut remember = |sha: Option, proofs: &mut VecDeque| { + let remember = |sha: Option, proofs: &mut VecDeque| { if let Some(sha) = sha { proofs.push_back(sha); while proofs.len() > MAX_RECENT_WARMING_PROOFS { @@ -1938,8 +1933,6 @@ mod tests { fn contract_parsing_refuses_escapes_and_echoes_nothing() { let base = |extra: &[&str]| -> Result { let mut args: Vec = [ - "--release-root", - "/r", "--child", "bin/c", "--template", @@ -1954,7 +1947,7 @@ mod tests { .map(String::from) .to_vec(); args.extend(extra.iter().map(|value| value.to_string())); - LaunchContract::parse(args) + LaunchContract::parse(PathBuf::from("/r"), args) }; assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); assert!(base(&["--child", "../escape"]).is_err()); @@ -1987,4 +1980,59 @@ mod tests { drop(hold); idunn.revoke_exact(Some(&lease)).unwrap(); } + + #[test] + fn authority_binds_the_candidate_and_signs_strictly_increasing_sequences() { + let fx = fixture(false); + let mut authority = fx.authority(); + assert!(authority.require_candidate_bind("127.0.0.1:18835").is_ok()); + assert!(authority.require_candidate_bind("127.0.0.1:18836").is_err()); + let first = authority.sign("warming", "a", None, 5_000).unwrap(); + let second = authority.sign("warming", "b", None, 5_000).unwrap(); + assert_eq!(first.publisher_sequence, 5_000); + assert_eq!(second.publisher_sequence, 5_001); + assert_eq!( + first.bound_endpoint.as_deref(), + Some("tcp://127.0.0.1:18835") + ); + } + + #[test] + fn health_probe_is_healthy_only_on_a_200_from_the_child() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("child.sock"); + let probe = HttpHealthProbe { + endpoint: ChildEndpoint::Unix(socket.clone()), + path: "/health".into(), + }; + assert!(!probe.healthy(), "no listener is not healthy"); + let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + let server = thread::spawn(move || { + for status in ["200 OK", "503 Service Unavailable"] { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 64]; + let read = stream.read(&mut request).unwrap(); + assert!(request[..read].starts_with(b"GET /health HTTP/1.1\r\n")); + write!(stream, "HTTP/1.1 {status}\r\nContent-Length: 2\r\n\r\nOK").unwrap(); + } + }); + assert!(probe.healthy()); + assert!(!probe.healthy()); + server.join().unwrap(); + } + + #[test] + fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { + let mut gone = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()]) + .unwrap(); + wait_until("child exit", || gone.exited().is_some()); + assert_eq!(gone.exited(), Some(3)); + let mut running = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()]) + .unwrap(); + assert_eq!(running.exited(), None); + running.terminate(Duration::from_secs(5)); + assert!(running.exited().is_some()); + } } From 0e09488c19a9fd564cfd82596fd9f664268e5236 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 20:09:26 +0200 Subject: [PATCH 128/196] idunn-launch tests: wait_until takes FnMut Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/launcher.rs b/src/launcher.rs index 7748177..b6946fa 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -1606,7 +1606,7 @@ mod tests { } } - fn wait_until(what: &str, condition: impl Fn() -> bool) { + fn wait_until(what: &str, mut condition: impl FnMut() -> bool) { for _ in 0..4000 { if condition() { return; From 9c69d006410e4b92bd42c1c606221ed2d4e9a3ad Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 20:48:29 +0200 Subject: [PATCH 129/196] Tests kill survivors: grant binding and holder custody, parse fields, credential names, env credentials, Odin endpoint, half-close splice, heartbeat pacing, SIGTERM escalation Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 34 ++++++++++ src/launcher.rs | 166 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 4bf500a..df361c3 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -13473,6 +13473,40 @@ processes 9 Ok(()) } + #[cfg(unix)] + #[test] + fn write_lease_grant_binds_the_exact_warming_candidate_and_the_holder_read_keeps_custody() + -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let path = temp.path().join("lease.cc"); + let driver = CultCacheWriteLeaseDriver::new("service", &path); + let (expected, activation, warming, _) = authenticated_warming(temp.path())?; + let good = lease(&expected, &activation, &warming); + let mut foreign = good.clone(); + foreign.warming_presence_sha256 = digest('8'); + assert!( + driver + .grant(&expected, &activation, &warming, &foreign) + .is_err() + ); + let other = CultCacheWriteLeaseDriver::new("other", temp.path().join("other.cc")); + assert!(other.grant(&expected, &activation, &warming, &good).is_err()); + assert!(driver.current_for_holder()?.is_none()); + driver.grant(&expected, &activation, &warming, &good)?; + assert_eq!( + driver.current_for_holder()?.map(|(_, held)| held), + Some(good) + ); + fs::set_permissions(&path, fs::Permissions::from_mode(0o666))?; + assert!( + driver.current_for_holder().is_err(), + "a service-writable lease store is not authority" + ); + Ok(()) + } + #[cfg(unix)] #[test] fn installed_release_is_root_owned_and_nonwritable() -> Result<()> { diff --git a/src/launcher.rs b/src/launcher.rs index b6946fa..b1eeb69 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2021,6 +2021,172 @@ mod tests { server.join().unwrap(); } + #[test] + fn contract_parsing_reads_every_declared_field() { + let args: Vec = [ + "--child", "bin/c", "--child-arg", "-x", "--child-arg", "y", "--config-flag", "-c", + "--template", "t", "--listener", "loopback", "--health-path", "/h", + "--health-contract", "hc", "--state-root", "/s", "--lease-wait-secs", "11", + "--startup-secs", "22", "--heartbeat-secs", "33", + ] + .map(String::from) + .to_vec(); + let contract = LaunchContract::parse(PathBuf::from("/r"), args.clone()).unwrap(); + assert_eq!(contract.child_args, ["-x", "y"]); + assert_eq!(contract.config_flag.as_deref(), Some("-c")); + assert_eq!(contract.listener, ChildListener::LoopbackPort); + assert_eq!(contract.state_root, Some(PathBuf::from("/s"))); + assert_eq!( + (contract.lease_wait, contract.startup, contract.heartbeat), + (Duration::from_secs(11), Duration::from_secs(22), Duration::from_secs(33)) + ); + for zero in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { + let mut bad = args.clone(); + bad.extend([zero.to_owned(), "0".to_owned()]); + assert!(LaunchContract::parse(PathBuf::from("/r"), bad).is_err(), "{zero}"); + } + } + + #[test] + fn credential_names_cannot_escape_or_be_empty() { + struct Any; + impl CredentialSource for Any { + fn contents(&self, _: &str) -> Result> { + Ok(b"v".to_vec()) + } + fn path(&self, name: &str) -> Result { + Ok(PathBuf::from(name)) + } + } + let renderer = ConfigRenderer { credentials: &Any, listen: "L", state_root: None }; + for ok in ["A", "a-b_9", "Z0"] { + assert_eq!(renderer.render(&format!("{{{{credential:{ok}}}}}")).unwrap(), "v"); + } + for bad in ["", "../x", "a/b", "a b", "a.b", "a:b", "é"] { + assert!(renderer.render(&format!("{{{{credential:{bad}}}}}")).is_err(), "{bad}"); + assert!(renderer.render(&format!("{{{{credential_path:{bad}}}}}")).is_err(), "{bad}"); + } + assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "aLbLc"); + } + + #[test] + fn env_credentials_resolve_through_the_variable_idunn_sets() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("cred"); + fs::write(&file, b"abc\n").unwrap(); + // SAFETY: a variable no other test reads. + unsafe { std::env::set_var("IDUNN_LAUNCH_TEST_CREDENTIAL", &file) }; + assert_eq!(EnvCredentials.path("IDUNN_LAUNCH_TEST_CREDENTIAL").unwrap(), file); + assert_eq!(EnvCredentials.contents("IDUNN_LAUNCH_TEST_CREDENTIAL").unwrap(), b"abc\n"); + assert!(EnvCredentials.path("IDUNN_LAUNCH_TEST_UNBOUND").is_err()); + } + + #[test] + fn odin_sink_takes_its_endpoint_only_from_the_odin_dependency() { + let fx = fixture(false); + let dependency = |capability: &str, endpoint: &str| cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: capability.into(), + schema: "s".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: None, + provider_authority: None, + provider_expected_projection_sha256: None, + provider_endpoint: Some(endpoint.into()), + }; + let mut expected = fx.expected.clone(); + assert!(OdinRudpSink::from_expected(&expected).is_err()); + expected.dependencies = vec![dependency("other.thing", "rudp://127.0.0.1:1")]; + assert!(OdinRudpSink::from_expected(&expected).is_err()); + expected.dependencies = vec![ + dependency("other.thing", "rudp://127.0.0.1:1"), + dependency(ODIN_CAPABILITY, "rudp://127.0.0.1:4100"), + ]; + assert_eq!( + OdinRudpSink::from_expected(&expected).unwrap().endpoint, + "127.0.0.1:4100".parse().unwrap() + ); + expected.dependencies = vec![dependency(ODIN_CAPABILITY, "rudp://not-an-address")]; + assert!(OdinRudpSink::from_expected(&expected).is_err()); + } + + #[test] + fn half_closed_member_requests_still_reach_a_child_that_waits_for_eof() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("child.sock"); + let child = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + let fake = thread::spawn(move || { + let (mut stream, _) = child.accept().unwrap(); + let mut received = Vec::new(); + stream.read_to_end(&mut received).unwrap(); + stream.write_all(&received.iter().rev().copied().collect::>()).unwrap(); + }); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let halt = Arc::new(AtomicBool::new(false)); + let responder: Responder = Arc::new(|_| bail!("never")); + let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder); + let serving = { + let halt = halt.clone(); + thread::spawn(move || acceptor.serve(halt)) + }; + let mut member = TcpStream::connect(address).unwrap(); + member.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + member.write_all(b"GET /abc").unwrap(); + member.shutdown(Shutdown::Write).unwrap(); + let mut answer = Vec::new(); + member.read_to_end(&mut answer).unwrap(); + assert_eq!(answer, b"cba/ TEG"); + fake.join().unwrap(); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn heartbeats_are_paced_and_a_stop_exits_zero() { + let fx = fixture(true); + let launch = start(&fx, fx.contract("x"), true); + thread::sleep(Duration::from_millis(250)); + let waiting = launch + .sink + .0 + .lock() + .unwrap() + .iter() + .filter(|record| record.detail == "write-lease-pending") + .count(); + assert!((1..=12).contains(&waiting), "{waiting} lease-wait heartbeats"); + let mut launch = launch; + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + thread::sleep(Duration::from_millis(250)); + let beats = launch + .sink + .0 + .lock() + .unwrap() + .iter() + .filter(|record| record.detail == "heartbeat") + .count(); + assert!((1..=12).contains(&beats), "{beats} active heartbeats"); + assert_eq!(launch.stop().exit_code(), 0); + } + + #[test] + fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { + let mut stubborn = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()]) + .unwrap(); + thread::sleep(Duration::from_millis(200)); + let started = Instant::now(); + stubborn.terminate(Duration::from_millis(400)); + assert!(started.elapsed() >= Duration::from_millis(400), "SIGTERM gets its grace period"); + assert!(stubborn.exited().is_some(), "then SIGKILL ends it"); + } + #[test] fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { let mut gone = ProcessSpawner From d36786f76c61cae332f80cef5a171463d6f613ca Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 23:16:42 +0200 Subject: [PATCH 130/196] idunn-launch: credentials only under CREDENTIALS_DIRECTORY, bounded route acceptor, lease lock inherited by the child, startup probe rules pinned Soul s1 fixes. Credential placeholders resolve files under $CREDENTIALS_DIRECTORY only, never an environment variable. The route acceptor has a first-byte deadline, a total challenge deadline and a connection cap with unspawned refusal; a dead acceptor degrades presence. The shared lease lock is a plain descriptor the child inherits, so the lock lives as long as any writer. Tests pin a late-healthy child and a hung probe. One commit: the spawner and lease trait signatures change together with their fakes. Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/bin/idunn-launch.rs | 3 +- src/launcher.rs | 715 ++++++++++++++++++++++++++++++++++------ 2 files changed, 624 insertions(+), 94 deletions(-) diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs index b743f06..41b6c76 100644 --- a/src/bin/idunn-launch.rs +++ b/src/bin/idunn-launch.rs @@ -42,11 +42,12 @@ fn main() -> std::process::ExitCode { ); let health_path = contract.health_path.clone(); let ports = Ports { - credentials: std::sync::Arc::new(EnvCredentials), + credentials: std::sync::Arc::new(CredentialsDirectory::from_environment()), clock: std::sync::Arc::new(SystemClock), sink: std::sync::Arc::new(sink), lease: std::sync::Arc::new(lease), spawner: Box::new(ProcessSpawner), + route_bounds: RouteBounds::default(), health: Box::new(move |endpoint| { std::sync::Arc::new(HttpHealthProbe { endpoint: endpoint.clone(), diff --git a/src/launcher.rs b/src/launcher.rs index b1eeb69..cf34c9d 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -15,12 +15,14 @@ use std::collections::VecDeque; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::fd::{AsRawFd, RawFd}; use std::os::unix::fs::OpenOptionsExt; use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt; use std::path::{Component, Path, PathBuf}; use std::process::{Child, Command, Stdio}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, mpsc}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -48,6 +50,12 @@ use crate::drivers::{CultCacheWriteLeaseDriver, lease_binds_candidate}; const CHALLENGE_FIRST_BYTE: u8 = 0x00; const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; const CHALLENGE_IO_TIMEOUT: Duration = Duration::from_secs(3); +/// How long a public connection may stay silent before its first byte. +const FIRST_BYTE_DEADLINE: Duration = Duration::from_secs(5); +/// A whole challenge, from its first byte to its answer, however it trickles. +const CHALLENGE_DEADLINE: Duration = Duration::from_secs(5); +/// Concurrent connections the acceptor serves; the rest are refused unspawned. +const MAX_CONNECTIONS: usize = 256; const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); const POLL: Duration = Duration::from_millis(100); const MAX_RECENT_WARMING_PROOFS: usize = 64; @@ -398,24 +406,44 @@ impl Clock for SystemClock { } } -/// Systemd credentials as Idunn binds them: the environment variable `NAME` -/// holds the path of the credential file. +/// Systemd credentials as Idunn binds them: the files `NAME` under the unit's +/// credentials directory. pub trait CredentialSource: Send + Sync { fn contents(&self, name: &str) -> Result>; fn path(&self, name: &str) -> Result; } -pub struct EnvCredentials; +/// Resolves a credential only as a file directly under the credentials +/// directory systemd gives the unit (`$CREDENTIALS_DIRECTORY`, which Idunn +/// pins to `/run/credentials/`). The process environment is never a +/// credential namespace: a name is a plain file name, never a variable. +pub struct CredentialsDirectory(Option); -impl CredentialSource for EnvCredentials { +impl CredentialsDirectory { + pub fn new(directory: PathBuf) -> Self { + Self(Some(directory)) + } + + /// The directory systemd exported; no directory means no credential is bound. + pub fn from_environment() -> Self { + Self(std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from)) + } +} + +impl CredentialSource for CredentialsDirectory { fn contents(&self, name: &str) -> Result> { fs::read(self.path(name)?).map_err(|_| anyhow!("a credential could not be read")) } fn path(&self, name: &str) -> Result { - std::env::var_os(name) - .map(PathBuf::from) - .ok_or_else(|| anyhow!("a credential is not bound")) + ensure!(valid_credential_name(name), "credential name is invalid"); + let path = self + .0 + .as_ref() + .ok_or_else(|| anyhow!("a credential is not bound"))? + .join(name); + ensure!(fs::symlink_metadata(&path).is_ok(), "a credential is not bound"); + Ok(path) } } @@ -428,7 +456,14 @@ pub trait PresenceSink: Send + Sync { /// its store that Idunn's revocation cannot take while the child lives. pub trait LeaseReader: Send + Sync { fn read(&self) -> Result>; - fn hold_shared(&self) -> Result>; + fn hold_shared(&self) -> Result>; +} + +/// The shared lock on the lease store, held until dropped. Its descriptor is +/// handed to the child (`ChildSpawner::spawn`), so the lock lives as long as +/// the launcher or any process that inherited it. +pub trait LeaseLock: Send { + fn descriptor(&self) -> Option; } pub trait HealthProbe: Send + Sync { @@ -443,7 +478,14 @@ pub trait ChildHandle: Send { } pub trait ChildSpawner: Send { - fn spawn(&self, program: &Path, args: &[String]) -> Result>; + /// `inherit` is a descriptor the child keeps open across exec (the lease + /// lock); every other descriptor of the launcher is close-on-exec. + fn spawn( + &self, + program: &Path, + args: &[String], + inherit: Option, + ) -> Result>; } // ---------------------------------------------------------------- production ports @@ -460,17 +502,11 @@ impl DriverLeaseReader { } } -struct SharedLockGuard { - release: Option>, - worker: Option>, -} +struct SharedLockFile(File); -impl Drop for SharedLockGuard { - fn drop(&mut self) { - drop(self.release.take()); - if let Some(worker) = self.worker.take() { - let _ = worker.join(); - } +impl LeaseLock for SharedLockFile { + fn descriptor(&self) -> Option { + Some(self.0.as_raw_fd()) } } @@ -482,31 +518,20 @@ impl LeaseReader for DriverLeaseReader { .map_err(|_| anyhow!("write-lease store refused custody, envelope or target")) } - fn hold_shared(&self) -> Result> { - let path = self.driver.record_path.clone(); - let (ready_tx, ready_rx) = mpsc::sync_channel::(1); - let (release_tx, release_rx) = mpsc::channel::<()>(); - let worker = thread::spawn(move || { - let held = SingleFileMessagePackBackingStore::new(&path).with_read_only_shared_snapshot( - |_| { - let _ = ready_tx.send(true); - // Returns when the guard drops its sender. - let _ = release_rx.recv(); - Ok(()) - }, - ); - if held.is_err() { - let _ = ready_tx.send(false); - } - }); - if ready_rx.recv() != Ok(true) { - let _ = worker.join(); - bail!("write-lease lock could not be held"); - } - Ok(Box::new(SharedLockGuard { - release: Some(release_tx), - worker: Some(worker), - })) + /// LOCK_SH on `.lock`, the lock Idunn's revocation respects. The + /// descriptor is a plain file, so flock's lock belongs to its open file + /// description: a child that inherits it keeps the lock after the launcher + /// dies. + fn hold_shared(&self) -> Result> { + let mut lock = self.driver.record_path.clone().into_os_string(); + lock.push(".lock"); + let file = File::open(&lock).map_err(|_| anyhow!("write-lease lock could not be held"))?; + // SAFETY: flock on a descriptor this function owns. + ensure!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0, + "write-lease lock could not be held" + ); + Ok(Box::new(SharedLockFile(file))) } } @@ -727,7 +752,12 @@ pub struct ProcessSpawner; struct ProcessChild(Child); impl ChildSpawner for ProcessSpawner { - fn spawn(&self, program: &Path, args: &[String]) -> Result> { + fn spawn( + &self, + program: &Path, + args: &[String], + inherit: Option, + ) -> Result> { let mut command = Command::new(program); // The child gets a bare environment: no Idunn variable, no candidate // bind, no credential path leaks into it. @@ -737,6 +767,18 @@ impl ChildSpawner for ProcessSpawner { command.env(name, value); } } + if let Some(descriptor) = inherit { + // SAFETY: runs in the forked child before exec; fcntl is + // async-signal-safe and the parent's descriptor stays close-on-exec. + unsafe { + command.pre_exec(move || { + if libc::fcntl(descriptor, libc::F_SETFD, 0) == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + } let child = command.spawn().map_err(|_| anyhow!("the child could not be spawned"))?; Ok(Box::new(ProcessChild(child))) } @@ -849,7 +891,7 @@ pub struct LeaseHold { reader: Arc, lease: IdunnProcessWriteLeaseRecord, sha256: String, - _lock: Box, + lock: Box, } impl LeaseHold { @@ -871,7 +913,7 @@ impl LeaseHold { sha256: lease.canonical_sha256()?, reader, lease, - _lock: lock, + lock, }) } @@ -879,6 +921,11 @@ impl LeaseHold { &self.sha256 } + /// The locked descriptor, for the child to inherit. + pub fn lock_descriptor(&self) -> Option { + self.lock.descriptor() + } + pub fn still_current(&self) -> bool { matches!(self.reader.read(), Ok(Some(current)) if current == self.lease) } @@ -901,9 +948,21 @@ struct Shared { lease_required: bool, health: Arc, clock: Arc, + /// False once the route acceptor has stopped for any reason but a halt: + /// the launcher cannot answer Idunn's challenge, so it must not say active. + acceptor_alive: AtomicBool, } impl Shared { + /// Active only while the child is healthy and the route is being served. + fn serving_state(&self) -> &'static str { + if self.health.healthy() && self.acceptor_alive.load(Ordering::SeqCst) { + "active" + } else { + "degraded" + } + } + fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { self.authority .lock() @@ -931,8 +990,8 @@ impl Shared { Phase::Warming => self.sign("warming", &detail, None)?, Phase::Failed => self.sign("failed", &detail, None)?, Phase::Active(lease) => { - let state = if self.health.healthy() && self.lease_current() { - "active" + let state = if self.lease_current() { + self.serving_state() } else { "degraded" }; @@ -948,39 +1007,90 @@ impl Shared { type Responder = Arc Result + Send + Sync>; +/// Bounds on what the public route endpoint spends on one peer. +#[derive(Clone, Copy, Debug)] +pub struct RouteBounds { + pub first_byte: Duration, + pub challenge: Duration, + pub max_connections: usize, +} + +impl Default for RouteBounds { + fn default() -> Self { + Self { + first_byte: FIRST_BYTE_DEADLINE, + challenge: CHALLENGE_DEADLINE, + max_connections: MAX_CONNECTIONS, + } + } +} + +/// Counts a live connection; dropping it frees the slot. +struct Slot(Arc); + +impl Drop for Slot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + /// Owns the candidate endpoint. A leading 0x00 is an Idunn challenge frame and -/// is answered here; every other byte is copied untouched to the child. +/// is answered here; every other byte is copied untouched to the child. The +/// endpoint is public, so it spends bounded resources on a peer: a first-byte +/// deadline, a total challenge deadline, and a cap on live connections whose +/// overflow is closed without a thread. pub struct RouteAcceptor { listener: TcpListener, child: ChildEndpoint, responder: Responder, + bounds: RouteBounds, + live: Arc, } impl RouteAcceptor { - pub fn new(listener: TcpListener, child: ChildEndpoint, responder: Responder) -> Self { + pub fn new(listener: TcpListener, child: ChildEndpoint, responder: Responder, bounds: RouteBounds) -> Self { Self { listener, child, responder, + bounds, + live: Arc::new(AtomicUsize::new(0)), } } + /// Serves until `halt`. Any other end is an error: a failed accept or a + /// thread that cannot be created means the route is no longer served. pub fn serve(self, halt: Arc) -> Result<()> { self.listener.set_nonblocking(true)?; let acceptor = Arc::new(self); while !halt.load(Ordering::SeqCst) { match acceptor.listener.accept() { Ok((connection, _)) => { - let acceptor = Arc::clone(&acceptor); - thread::spawn(move || { - let _ = connection.set_nonblocking(false); - let _ = acceptor.handle(connection); - }); + if acceptor.live.fetch_add(1, Ordering::SeqCst) >= acceptor.bounds.max_connections { + acceptor.live.fetch_sub(1, Ordering::SeqCst); + continue; + } + let slot = Slot(Arc::clone(&acceptor.live)); + let worker = Arc::clone(&acceptor); + thread::Builder::new() + .spawn(move || { + let _slot = slot; + let _ = connection.set_nonblocking(false); + let _ = worker.handle(connection); + }) + .map_err(|_| anyhow!("a route connection thread could not be created"))?; } - Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock + | io::ErrorKind::Interrupted + | io::ErrorKind::ConnectionAborted + ) => + { thread::sleep(Duration::from_millis(10)); } - Err(_) => thread::sleep(Duration::from_millis(10)), + Err(_) => bail!("the route listener could not accept"), } } Ok(()) @@ -988,9 +1098,7 @@ impl RouteAcceptor { fn handle(&self, mut connection: TcpStream) -> Result<()> { let mut first = [0_u8; 1]; - if connection.read(&mut first)? == 0 { - return Ok(()); - } + read_by(&mut connection, &mut first, Instant::now() + self.bounds.first_byte)?; if first[0] == CHALLENGE_FIRST_BYTE { return self.answer_challenge(&mut connection); } @@ -998,10 +1106,15 @@ impl RouteAcceptor { upstream.write_all(&first)?; let mut upstream_reader = upstream.try_clone()?; let mut client_reader = connection.try_clone()?; - let forward = thread::spawn(move || { - let _ = io::copy(&mut client_reader, &mut upstream); - upstream.shutdown_write(); - }); + let forward = thread::Builder::new() + .spawn(move || { + let _ = io::copy(&mut client_reader, &mut upstream); + upstream.shutdown_write(); + }) + .map_err(|_| anyhow!("a route splice thread could not be created"))?; + // The first-byte deadline bounded the silent peer; a spliced session + // runs as long as the member and the child keep it. + connection.set_read_timeout(None)?; let _ = io::copy(&mut upstream_reader, &mut connection); let _ = connection.shutdown(Shutdown::Write); let _ = forward.join(); @@ -1009,17 +1122,19 @@ impl RouteAcceptor { } fn answer_challenge(&self, connection: &mut TcpStream) -> Result<()> { - connection.set_read_timeout(Some(CHALLENGE_IO_TIMEOUT))?; + // The challenge began with its first byte; the rest must arrive within + // the total deadline, however slowly the peer sends it. + let deadline = Instant::now() + self.bounds.challenge; connection.set_write_timeout(Some(CHALLENGE_IO_TIMEOUT))?; let mut rest = [0_u8; 3]; - connection.read_exact(&mut rest)?; + read_by(connection, &mut rest, deadline)?; let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; ensure!( (1..=CHALLENGE_MAX_BYTES).contains(&length), "challenge frame is outside its bound" ); let mut payload = vec![0_u8; length]; - connection.read_exact(&mut payload)?; + read_by(connection, &mut payload, deadline)?; let request = decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; let response = (self.responder)(request)?; @@ -1033,6 +1148,25 @@ impl RouteAcceptor { } } +/// Fills `buffer` or fails when `deadline` passes, however the peer paces it. +fn read_by(connection: &mut TcpStream, buffer: &mut [u8], deadline: Instant) -> Result<()> { + let mut filled = 0; + while filled < buffer.len() { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .context("the peer did not send in time")?; + connection.set_read_timeout(Some(remaining))?; + match connection.read(&mut buffer[filled..]) { + Ok(0) => bail!("the peer closed early"), + Ok(count) => filled += count, + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(_) => bail!("the peer did not send in time"), + } + } + Ok(()) +} + // ---------------------------------------------------------------- lifecycle pub struct Ports { @@ -1041,6 +1175,7 @@ pub struct Ports { pub sink: Arc, pub lease: Arc, pub spawner: Box, + pub route_bounds: RouteBounds, /// Built from the endpoint `run` chooses. pub health: Box Arc + Send>, } @@ -1099,6 +1234,7 @@ pub fn run( lease_required: expected.write_lease_required, health: Arc::clone(&health), clock: Arc::clone(&ports.clock), + acceptor_alive: AtomicBool::new(true), }); let halt = Arc::new(AtomicBool::new(false)); let responder: Responder = { @@ -1106,9 +1242,16 @@ pub fn run( Arc::new(move |request| shared.observe(request)) }; let serving = { - let acceptor = RouteAcceptor::new(route, endpoint.clone(), responder); + let acceptor = RouteAcceptor::new(route, endpoint.clone(), responder, ports.route_bounds); let halt = Arc::clone(&halt); - thread::spawn(move || acceptor.serve(halt)) + let shared = Arc::clone(&shared); + thread::spawn(move || { + let served = acceptor.serve(halt); + if served.is_err() { + shared.acceptor_alive.store(false, Ordering::SeqCst); + } + served + }) }; let outcome = lifecycle(contract, &expected, &activation, &endpoint, tmp, &ports, &shared, stop); halt.store(true, Ordering::SeqCst); @@ -1166,6 +1309,7 @@ fn lifecycle( // 2. A stateful child waits for the lease; a stateless one does not. let mut lease_sha = None; + let mut lease_fd = None; if expected.write_lease_required { let deadline = clock.now_millis() + contract.lease_wait.as_millis() as u64; loop { @@ -1180,6 +1324,7 @@ fn lifecycle( ); if let Ok(hold) = held { lease_sha = Some(hold.sha256().to_owned()); + lease_fd = hold.lock_descriptor(); if let Ok(mut slot) = shared.lease.lock() { *slot = Some(hold); } @@ -1198,7 +1343,7 @@ fn lifecycle( } // 3. Render, spawn, and wait for health. - let Some(mut child) = start_child(contract, endpoint, tmp, ports) else { + let Some(mut child) = start_child(contract, endpoint, tmp, ports, lease_fd) else { return fail(None, "child-start-refused"); }; let startup_deadline = clock.now_millis() + contract.startup.as_millis() as u64; @@ -1221,7 +1366,7 @@ fn lifecycle( // 4. Active, and heartbeats while it holds. set_phase(Phase::Active(lease_sha.clone())); - publish("active", "child-healthy", lease_sha.clone()); + publish(shared.serving_state(), "child-healthy", lease_sha.clone()); next_heartbeat = clock.now_millis() + heartbeat; loop { if stop.load(Ordering::SeqCst) { @@ -1236,8 +1381,7 @@ fn lifecycle( } let now = clock.now_millis(); if now >= next_heartbeat { - let state = if shared.health.healthy() { "active" } else { "degraded" }; - publish(state, "heartbeat", lease_sha.clone()); + publish(shared.serving_state(), "heartbeat", lease_sha.clone()); next_heartbeat = now + heartbeat; } clock.sleep(POLL); @@ -1249,6 +1393,7 @@ fn start_child( endpoint: &ChildEndpoint, tmp: &Path, ports: &Ports, + lease_fd: Option, ) -> Option> { let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; let listen = endpoint.listen_value(); @@ -1267,7 +1412,7 @@ fn start_child( } ports .spawner - .spawn(&contract.release_root.join(&contract.child), &args) + .spawn(&contract.release_root.join(&contract.child), &args, lease_fd) .ok() } @@ -1276,6 +1421,7 @@ mod tests { use std::collections::BTreeMap; use std::os::unix::fs::PermissionsExt; use std::sync::atomic::{AtomicU64, AtomicUsize}; + use std::sync::mpsc; use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; use cultnet_rs::{ @@ -1498,8 +1644,19 @@ mod tests { Ok(self.0.lock().unwrap().clone()) } - fn hold_shared(&self) -> Result> { - Ok(Box::new(())) + fn hold_shared(&self) -> Result> { + Ok(Box::new(FakeLock)) + } + } + + /// The descriptor number a fake lock claims; the launcher must hand it on. + const FAKE_LOCK_FD: RawFd = 1234; + + struct FakeLock; + + impl LeaseLock for FakeLock { + fn descriptor(&self) -> Option { + Some(FAKE_LOCK_FD) } } @@ -1517,6 +1674,7 @@ mod tests { exit: Mutex>, terminated: AtomicBool, args: Mutex>>, + inherited: Mutex>>, } struct FakeChild(Arc); @@ -1535,7 +1693,13 @@ mod tests { struct FakeSpawner(Arc); impl ChildSpawner for FakeSpawner { - fn spawn(&self, _program: &Path, args: &[String]) -> Result> { + fn spawn( + &self, + _program: &Path, + args: &[String], + inherit: Option, + ) -> Result> { + self.0.inherited.lock().unwrap().push(inherit); self.0.spawns.fetch_add(1, Ordering::SeqCst); self.0.args.lock().unwrap().push(args.to_vec()); Ok(Box::new(FakeChild(Arc::clone(&self.0)))) @@ -1579,6 +1743,7 @@ mod tests { sink: sink.clone(), lease: lease.clone(), spawner: Box::new(FakeSpawner(child.clone())), + route_bounds: RouteBounds::default(), health: Box::new(move |_| health.clone()), }; let authority = fx.authority(); @@ -1779,6 +1944,7 @@ mod tests { lease_required: false, health, clock: Arc::new(SystemClock), + acceptor_alive: AtomicBool::new(true), }) } @@ -1820,6 +1986,7 @@ mod tests { listener, ChildEndpoint::Unix("/nonexistent".into()), responder, + RouteBounds::default(), ); let serving = { let halt = halt.clone(); @@ -1898,7 +2065,7 @@ mod tests { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let halt = Arc::new(AtomicBool::new(false)); - let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder); + let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder, RouteBounds::default()); let serving = { let halt = halt.clone(); thread::spawn(move || acceptor.serve(halt)) @@ -2070,15 +2237,51 @@ mod tests { } #[test] - fn env_credentials_resolve_through_the_variable_idunn_sets() { + fn credentials_resolve_only_as_files_under_the_credentials_directory() { let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("cred"); - fs::write(&file, b"abc\n").unwrap(); - // SAFETY: a variable no other test reads. - unsafe { std::env::set_var("IDUNN_LAUNCH_TEST_CREDENTIAL", &file) }; - assert_eq!(EnvCredentials.path("IDUNN_LAUNCH_TEST_CREDENTIAL").unwrap(), file); - assert_eq!(EnvCredentials.contents("IDUNN_LAUNCH_TEST_CREDENTIAL").unwrap(), b"abc\n"); - assert!(EnvCredentials.path("IDUNN_LAUNCH_TEST_UNBOUND").is_err()); + fs::write(dir.path().join("cred"), b"abc\n").unwrap(); + let credentials = CredentialsDirectory::new(dir.path().to_owned()); + assert_eq!(credentials.path("cred").unwrap(), dir.path().join("cred")); + assert_eq!(credentials.contents("cred").unwrap(), b"abc\n"); + assert!(credentials.path("unbound").is_err()); + for bad in ["", "..", "../cred", "a/b", ".", "a.b"] { + assert!(credentials.path(bad).is_err(), "{bad}"); + assert!(credentials.contents(bad).is_err(), "{bad}"); + } + assert!(CredentialsDirectory(None).path("cred").is_err()); + } + + #[test] + fn credential_placeholders_never_read_the_process_environment() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("elsewhere"); + fs::write(&file, b"canary-file\n").unwrap(); + // SAFETY: variables no other test reads. + unsafe { + std::env::set_var("GAMECULT_IDUNN_CANDIDATE_BIND", "127.0.0.1:18835"); + std::env::set_var("IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", &file); + } + let credentials = CredentialsDirectory::new(dir.path().to_owned()); + let renderer = ConfigRenderer { credentials: &credentials, listen: "L", state_root: None }; + for name in [ + "GAMECULT_IDUNN_CANDIDATE_BIND", + "IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", + "../elsewhere", + "a/b", + "..", + ] { + for kind in ["credential", "credential_path"] { + let error = renderer.render(&format!("x={{{{{kind}:{name}}}}}")); + assert!(error.is_err(), "{kind}:{name}"); + assert!(!format!("{:#}", error.unwrap_err()).contains("18835")); + } + } + // The same name resolves when it is a credential file, never otherwise. + fs::write(dir.path().join("GAMECULT_IDUNN_CANDIDATE_BIND"), b"file-value\n").unwrap(); + assert_eq!( + renderer.render("{{credential:GAMECULT_IDUNN_CANDIDATE_BIND}}").unwrap(), + "file-value" + ); } #[test] @@ -2127,7 +2330,7 @@ mod tests { let address = listener.local_addr().unwrap(); let halt = Arc::new(AtomicBool::new(false)); let responder: Responder = Arc::new(|_| bail!("never")); - let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder); + let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder, RouteBounds::default()); let serving = { let halt = halt.clone(); thread::spawn(move || acceptor.serve(halt)) @@ -2178,7 +2381,7 @@ mod tests { #[test] fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { let mut stubborn = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()]) + .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()], None) .unwrap(); thread::sleep(Duration::from_millis(200)); let started = Instant::now(); @@ -2190,15 +2393,341 @@ mod tests { #[test] fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { let mut gone = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()]) + .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()], None) .unwrap(); wait_until("child exit", || gone.exited().is_some()); assert_eq!(gone.exited(), Some(3)); let mut running = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()]) + .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()], None) .unwrap(); assert_eq!(running.exited(), None); running.terminate(Duration::from_secs(5)); assert!(running.exited().is_some()); } + + // ------------------------------------------------------------ Soul s1 fixes + + fn serve_with( + fx: &Fixture, + bounds: RouteBounds, + ) -> (SocketAddr, Arc, thread::JoinHandle>) { + let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let responder: Responder = Arc::new(move |request| shared.observe(request)); + let acceptor = + RouteAcceptor::new(listener, ChildEndpoint::Unix("/nonexistent".into()), responder, bounds); + let halt = Arc::new(AtomicBool::new(false)); + let flag = halt.clone(); + (address, halt, thread::spawn(move || acceptor.serve(flag))) + } + + fn quick(first_byte_ms: u64, challenge_ms: u64, max_connections: usize) -> RouteBounds { + RouteBounds { + first_byte: Duration::from_millis(first_byte_ms), + challenge: Duration::from_millis(challenge_ms), + max_connections, + } + } + + /// True when the peer closed (EOF or reset) within `within`. + fn closed_within(stream: &mut TcpStream, within: Duration) -> bool { + stream.set_read_timeout(Some(within)).unwrap(); + let mut byte = [0_u8; 64]; + match stream.read(&mut byte) { + Ok(0) => true, + Ok(_) => false, + Err(error) => !matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut), + } + } + + fn challenge_answered(address: SocketAddr, fx: &Fixture) -> bool { + let request = CultNetMessage::SnapshotRequest { + message_id: "bounded".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + request_tcp_snapshot(address, &payload).is_ok() + } + + #[test] + fn an_idle_connection_is_closed_at_the_first_byte_deadline() { + let fx = fixture(false); + let (address, halt, serving) = serve_with(&fx, quick(300, 5000, 64)); + let mut idle = TcpStream::connect(address).unwrap(); + let started = Instant::now(); + assert!(closed_within(&mut idle, Duration::from_secs(4)), "never closed"); + assert!(started.elapsed() >= Duration::from_millis(250), "closed before its deadline"); + assert!(challenge_answered(address, &fx)); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_its_total_deadline() { + let fx = fixture(false); + // Every pause is far inside the first-byte deadline: only the total + // challenge deadline can end this. + let (address, halt, serving) = serve_with(&fx, quick(5000, 600, 64)); + let mut slow = TcpStream::connect(address).unwrap(); + slow.write_all(&[0x00, 0x00, 0x01, 0x00]).unwrap(); + let started = Instant::now(); + let mut closed = false; + while started.elapsed() < Duration::from_secs(4) { + if closed_within(&mut slow, Duration::from_millis(150)) || slow.write_all(&[0x01]).is_err() { + closed = true; + break; + } + } + assert!(closed, "the trickle was never cut off"); + assert!(started.elapsed() >= Duration::from_millis(500), "cut before its deadline"); + assert!(challenge_answered(address, &fx)); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn live_connections_are_capped_and_the_overflow_is_refused_unspawned() { + let fx = fixture(false); + let (address, halt, serving) = serve_with(&fx, quick(30_000, 5000, 4)); + let mut held: Vec = (0..4).map(|_| TcpStream::connect(address).unwrap()).collect(); + thread::sleep(Duration::from_millis(300)); + let threads_before = thread_count(); + let mut overflow: Vec = (0..40).map(|_| TcpStream::connect(address).unwrap()).collect(); + for stream in &mut overflow { + assert!(closed_within(stream, Duration::from_secs(3)), "an overflow connection was served"); + } + assert!(thread_count() < threads_before + 8, "overflow connections cost threads"); + for stream in &mut held { + assert!(!closed_within(stream, Duration::from_millis(100)), "a held connection was cut"); + } + // A freed slot serves the next peer. + held.pop(); + let mut answered = false; + for _ in 0..40 { + if challenge_answered(address, &fx) { + answered = true; + break; + } + thread::sleep(Duration::from_millis(50)); + } + assert!(answered, "a freed slot was never reused"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + fn thread_count() -> usize { + fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("Threads:")) + .unwrap() + .trim() + .parse() + .unwrap() + } + + /// A listener whose accept always fails: a datagram socket's descriptor. + fn failing_listener() -> TcpListener { + use std::os::fd::{FromRawFd, IntoRawFd}; + let datagram = UdpSocket::bind("127.0.0.1:0").unwrap(); + // SAFETY: the descriptor is owned by the new listener alone. + unsafe { TcpListener::from_raw_fd(datagram.into_raw_fd()) } + } + + #[test] + fn an_acceptor_that_cannot_accept_ends_with_an_error() { + let fx = fixture(false); + let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); + let responder: Responder = Arc::new(move |request| shared.observe(request)); + let acceptor = RouteAcceptor::new( + failing_listener(), + ChildEndpoint::Unix("/nonexistent".into()), + responder, + RouteBounds::default(), + ); + assert!(acceptor.serve(Arc::new(AtomicBool::new(false))).is_err()); + } + + #[test] + fn presence_stops_claiming_active_when_the_route_acceptor_dies() { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let health = Arc::new(Health(AtomicBool::new(true))); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(child)), + route_bounds: RouteBounds::default(), + health: Box::new(move |_| health.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = thread::spawn(move || run(&contract, authority, failing_listener(), &tmp, ports, &flag)); + wait_until("two degraded heartbeats", || { + sink.states().iter().filter(|state| *state == "degraded").count() >= 2 + }); + // The child is healthy throughout; only the dead acceptor degrades it, + // and nothing says active after that. + let states = sink.states(); + let first_degraded = states.iter().position(|state| state == "degraded").unwrap(); + assert!(states[first_degraded..].iter().all(|state| state != "active"), "{states:?}"); + stop.store(true, Ordering::SeqCst); + assert_eq!(handle.join().unwrap(), Outcome::Stopped); + } + + #[test] + fn a_live_acceptor_keeps_presence_active() { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("two active heartbeats", || { + launch.sink.states().iter().filter(|state| *state == "active").count() >= 2 + }); + assert!(launch.sink.first("degraded").is_none()); + launch.stop(); + } + + /// A store with a lease for `fx`, and Idunn's own driver over it. + fn lease_store( + fx: &Fixture, + name: &str, + ) -> (DriverLeaseReader, CultCacheWriteLeaseDriver, IdunnProcessWriteLeaseRecord) { + let dir = fx.dir.path().join(name); + fs::create_dir(&dir).unwrap(); + let path = dir.join("lease.cc"); + let lease = fx.lease(&digest('a')); + write_record( + &path, + &lease.target, + IdunnProcessWriteLeaseRecord::TYPE, + cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes().unwrap(), + ); + fs::write(dir.join("lease.cc.lock"), b"").unwrap(); + ( + DriverLeaseReader::new(&lease.target, &path), + CultCacheWriteLeaseDriver::new(&lease.target, &path), + lease, + ) + } + + #[test] + fn a_child_that_inherits_the_lease_lock_holds_it_after_the_launcher_is_gone() { + let fx = fixture(true); + let (reader, idunn, lease) = lease_store(&fx, "inherited"); + let guard = reader.hold_shared().unwrap(); + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], guard.descriptor()) + .unwrap(); + // The launcher dies: its own descriptor goes with it. + drop(guard); + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); + child.terminate(Duration::from_secs(5)); + wait_until("child gone", || child.exited().is_some()); + idunn.revoke_exact(Some(&lease)).unwrap(); + } + + #[test] + fn a_child_spawned_without_the_descriptor_leaves_the_lock_with_the_launcher() { + let fx = fixture(true); + let (reader, idunn, lease) = lease_store(&fx, "uninherited"); + let guard = reader.hold_shared().unwrap(); + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], None) + .unwrap(); + drop(guard); + idunn.revoke_exact(Some(&lease)).unwrap(); + child.terminate(Duration::from_secs(5)); + } + + #[test] + fn the_launcher_hands_its_lease_descriptor_to_the_child_it_spawns() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("warming", || launch.sink.first("warming").is_some()); + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(*launch.child.inherited.lock().unwrap(), [Some(FAKE_LOCK_FD)]); + launch.stop(); + + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(*launch.child.inherited.lock().unwrap(), [None]); + launch.stop(); + } + + /// Unhealthy for its first `probes` calls, healthy from then on. + struct LateHealth { + calls: AtomicUsize, + probes: usize, + } + + impl HealthProbe for LateHealth { + fn healthy(&self) -> bool { + self.calls.fetch_add(1, Ordering::SeqCst) >= self.probes + } + } + + #[test] + fn a_child_that_becomes_healthy_late_is_not_killed_at_its_first_probe() { + let fx = fixture(false); + let mut contract = fx.contract("x"); + contract.startup = Duration::from_secs(60); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let health = Arc::new(LateHealth { calls: AtomicUsize::new(0), probes: 5 }); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(child.clone())), + route_bounds: RouteBounds::default(), + health: Box::new(move |_| health.clone()), + }; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = thread::spawn(move || run(&contract, authority, route, &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert!(!child.terminated.load(Ordering::SeqCst), "killed while it was still starting"); + assert_eq!(child.spawns.load(Ordering::SeqCst), 1); + assert!(sink.first("failed").is_none()); + stop.store(true, Ordering::SeqCst); + assert_eq!(handle.join().unwrap(), Outcome::Stopped); + } + + #[test] + fn a_hung_health_probe_times_out_instead_of_blocking() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = ChildEndpoint::Loopback(listener.local_addr().unwrap()); + // Accepts and never answers. + let held = thread::spawn(move || { + let _connection = listener.accept().unwrap(); + thread::sleep(Duration::from_secs(8)); + }); + let probe = HttpHealthProbe { endpoint, path: "/health".into() }; + let (done, verdict) = mpsc::channel(); + thread::spawn(move || { + let _ = done.send(probe.healthy()); + }); + let healthy = verdict + .recv_timeout(Duration::from_secs(5)) + .expect("the probe outlived its timeout against a hung child"); + assert!(!healthy); + drop(held); + } } From 8246de7c3737aa7ff95d0378863a0977dcfc4400 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 23:26:35 +0200 Subject: [PATCH 131/196] idunn-launch tests: lock-release wait tolerates a concurrent fork; health probe test reads the request line, not a 64-byte read Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index cf34c9d..6c72c0b 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2177,9 +2177,10 @@ mod tests { let server = thread::spawn(move || { for status in ["200 OK", "503 Service Unavailable"] { let (mut stream, _) = listener.accept().unwrap(); - let mut request = [0_u8; 64]; - let read = stream.read(&mut request).unwrap(); - assert!(request[..read].starts_with(b"GET /health HTTP/1.1\r\n")); + let mut line = String::new(); + io::BufRead::read_line(&mut io::BufReader::new(stream.try_clone().unwrap()), &mut line) + .unwrap(); + assert_eq!(line, "GET /health HTTP/1.1\r\n"); write!(stream, "HTTP/1.1 {status}\r\nContent-Length: 2\r\n\r\nOK").unwrap(); } }); @@ -2645,7 +2646,9 @@ mod tests { .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], None) .unwrap(); drop(guard); - idunn.revoke_exact(Some(&lease)).unwrap(); + // Another test's fork may hold a copy for the instant before its exec; + // the child under test would hold it for thirty seconds. + wait_until("lock released", || idunn.revoke_exact(Some(&lease)).is_ok()); child.terminate(Duration::from_secs(5)); } From e2dee7c16a96e179116e6429cb43f081eed3b401 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 00:15:33 +0200 Subject: [PATCH 132/196] idunn-launch tests: the inherited lease lock must stay held for a second, so a concurrent fork cannot mask a missing descriptor Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 6c72c0b..689c35a 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2630,8 +2630,13 @@ mod tests { .unwrap(); // The launcher dies: its own descriptor goes with it. drop(guard); - let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); - assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); + // Held for a full second: a fork of some other test could pin the lock + // for an instant, only the child under test keeps it that long. + for _ in 0..10 { + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); + thread::sleep(Duration::from_millis(100)); + } child.terminate(Duration::from_secs(5)); wait_until("child gone", || child.exited().is_some()); idunn.revoke_exact(Some(&lease)).unwrap(); From 7805c0aa9ac40dcc14821141fb765b2b6fd020ac Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 01:18:26 +0200 Subject: [PATCH 133/196] Idunn routes: an optional [route.member] renders a second nginx server block and sets GAMECULT_IDUNN_MEMBER_BIND; Idunn-owned launch names live in one list The member candidate port is derived from the candidate port; both stable sockets get firewall rules; native targets render the same bytes as before. Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 1 + src/deployment.rs | 337 +++++++++++++++++++++++++++++++++++++------ src/drivers.rs | 256 ++++++++++++++++++++++++++------ 3 files changed, 512 insertions(+), 82 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d2a7635..9597d43 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -10723,6 +10723,7 @@ mod tests { private_port_end, config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), reload_unit: "nginx.service".into(), + member: None, } } diff --git a/src/deployment.rs b/src/deployment.rs index a81dff4..b01deed 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -9,12 +9,27 @@ pub const TARGET_DECLARATION_SCHEMA: &str = "gamecult.idunn.target_declaration.v pub const OPERATOR_BINDING_SCHEMA: &str = "gamecult.idunn.operator_binding.v2"; pub const IDUNN_RUNTIME_BUNDLE_ENVIRONMENT: &str = "GAMECULT_IDUNN_RUNTIME_BUNDLE"; pub const IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_CANDIDATE_BIND"; +pub const IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_MEMBER_BIND"; pub const IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT: &str = "GAMECULT_IDUNN_PROCESS_WRITE_LEASE"; pub const RUNTIME_PRESENCE_IDENTITY_BINDING: &str = "GAMECULT_RUNTIME_PRESENCE_IDENTITY"; /// A host actuator hands the workload its activation credential as a file /// named by this variable; on the Idunn host it arrives as a parent-only /// descriptor instead and this name is never set. pub const IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT: &str = "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL"; +/// systemd hands a unit its credential directory under this name; Idunn owns +/// the directory, so no binding may set it. +pub const SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT: &str = "CREDENTIALS_DIRECTORY"; +/// Every launch-environment name Idunn sets or reserves. A binding may name +/// none of them; this is the one list both the binding check and the +/// launch-input check read. +pub(crate) const IDUNN_OWNED_LAUNCH_ENVIRONMENT: [&str; 6] = [ + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT, +]; pub const RUNTIME_PRESENCE_IDENTITY_FD_NAME: &str = "gamecult-runtime-presence-identity"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -509,8 +524,7 @@ impl WorkloadBinding { { require_environment_name(name)?; ensure!( - name != IDUNN_RUNTIME_BUNDLE_ENVIRONMENT - && name != IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + !IDUNN_OWNED_LAUNCH_ENVIRONMENT.contains(&name.as_str()), "operator binding attempts to replace Idunn-owned workload environment" ); } @@ -667,40 +681,87 @@ pub struct RouteBinding { pub private_port_end: u16, pub config_path: PathBuf, pub reload_unit: String, + /// A second listener for the route's members. When set, members reach the + /// workload straight through a second nginx server block, and the + /// workload's launcher answers presence only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub member: Option, +} + +/// The member listener of a route: where members connect, and the first of the +/// private ports the workload binds for them. The private range is as wide as +/// the route's own. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemberListenerBinding { + pub stable_endpoint: String, + pub private_port_start: u16, } impl RouteBinding { pub(crate) fn stable_socket(&self) -> Result<(IpAddr, u16)> { - let authority = match self.driver { - RouteDriver::NginxStreamTcp => self - .stable_endpoint - .strip_prefix("http://") - .or_else(|| self.stable_endpoint.strip_prefix("tcp://")), - RouteDriver::NginxStreamUdp => self.stable_endpoint.strip_prefix("rudp://"), - } - .context("stable endpoint transport does not match its route driver")?; - ensure!( - !authority - .chars() - .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), - "stable endpoint is not a plain IP address and port" - ); - let (host, port) = authority - .rsplit_once(':') - .context("stable endpoint has no port")?; - let host = host - .parse::() - .context("stable endpoint host is not an IP address")?; - ensure!( - matches!(host, IpAddr::V4(_)), - "route binding v2 stable endpoints use explicit IPv4 addresses" - ); - let port = port - .parse::() - .context("stable endpoint port is not a u16")?; - ensure!(port > 0, "stable endpoint port is zero"); - Ok((host, port)) + parse_stable_socket(self.driver, &self.stable_endpoint, "stable endpoint") } + + /// The member listener's stable socket, when the route has one. + pub(crate) fn member_stable_socket(&self) -> Result> { + self.member + .as_ref() + .map(|member| { + parse_stable_socket(self.driver, &member.stable_endpoint, "member endpoint") + }) + .transpose() + } + + /// The private port the workload binds for members when its presence + /// candidate is `candidate_port`. Derived, never stored: the member range + /// is the private range shifted to the member start. + pub(crate) fn member_candidate_port(&self, candidate_port: u16) -> Result> { + let Some(member) = self.member.as_ref() else { + return Ok(None); + }; + ensure!( + (self.private_port_start..=self.private_port_end).contains(&candidate_port), + "candidate port is outside the route binding" + ); + let offset = candidate_port - self.private_port_start; + let port = member + .private_port_start + .checked_add(offset) + .context("member private port overflows")?; + Ok(Some(port)) + } +} + +fn parse_stable_socket(driver: RouteDriver, endpoint: &str, what: &str) -> Result<(IpAddr, u16)> { + let authority = match driver { + RouteDriver::NginxStreamTcp => endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("tcp://")), + RouteDriver::NginxStreamUdp => endpoint.strip_prefix("rudp://"), + } + .with_context(|| format!("{what} transport does not match its route driver"))?; + ensure!( + !authority + .chars() + .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), + "{what} is not a plain IP address and port" + ); + let (host, port) = authority + .rsplit_once(':') + .with_context(|| format!("{what} has no port"))?; + let host = host + .parse::() + .with_context(|| format!("{what} host is not an IP address"))?; + ensure!( + matches!(host, IpAddr::V4(_)), + "route binding v2 endpoints use explicit IPv4 addresses" + ); + let port = port + .parse::() + .with_context(|| format!("{what} port is not a u16"))?; + ensure!(port > 0, "{what} port is zero"); + Ok((host, port)) } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -924,6 +985,14 @@ impl TargetDeclaration { == self.service.route_required, "routed services must require exactly one Idunn-owned candidate bind" ); + ensure!( + !self + .service + .required_environment + .contains(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + || self.service.route_required, + "only routed services may require the Idunn-owned member bind" + ); ensure!( self.service .required_environment @@ -1381,7 +1450,35 @@ impl OperatorBinding { route.private_port_start < route.private_port_end, "candidate rollout requires at least two private ports" ); - route.stable_socket()?; + let stable = route.stable_socket()?; + if let Some(member) = route.member.as_ref() { + ensure!( + route.driver == RouteDriver::NginxStreamTcp, + "only a TCP route may carry a member listener" + ); + require_value(&member.stable_endpoint, "member endpoint")?; + let member_stable = route + .member_stable_socket()? + .context("route has no member listener")?; + ensure!( + member_stable != stable, + "member endpoint equals the route's stable endpoint" + ); + ensure!( + member.private_port_start > 0, + "member private port range starts at zero" + ); + let width = route.private_port_end - route.private_port_start; + let member_end = member + .private_port_start + .checked_add(width) + .context("member private port range overflows")?; + ensure!( + member_end < route.private_port_start + || member.private_port_start > route.private_port_end, + "member private port range overlaps the route's private range" + ); + } require_absolute_path(&route.config_path, "route config")?; require_unit(&route.reload_unit, "route reload unit")?; } @@ -1559,12 +1656,7 @@ impl OperatorBinding { .keys() .chain(self.workload.secret_environment_names()) .collect(); - for owned in [ - IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, - IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, - IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, - ] { + for owned in IDUNN_OWNED_LAUNCH_ENVIRONMENT { ensure!( !workload_names.contains(&owned.to_string()), "operator binding cannot replace Idunn-owned launch environment {owned}" @@ -1586,6 +1678,9 @@ impl OperatorBinding { } if declaration.service.route_required { available_environment.insert(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT.into()); + if self.route.as_ref().is_some_and(|route| route.member.is_some()) { + available_environment.insert(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into()); + } } if declaration.write_lease_required() { available_environment.insert(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT.into()); @@ -1933,10 +2028,10 @@ fn require_host(value: &str) -> Result<()> { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - const RECIPE: &str = r#" + pub(crate) const RECIPE: &str = r#" schema = "gamecult.idunn.target_declaration.v1" target = "test-service" source_stamp_environment = "TEST_SERVICE_BUILD_COMMIT" @@ -2008,7 +2103,7 @@ compatibility = "v1" startup = "before-promotion" "#; - const BINDING: &str = r#" + pub(crate) const BINDING: &str = r#" schema = "gamecult.idunn.operator_binding.v2" target = "test-service" profiles = ["aetheria", "full-gamecult"] @@ -2235,6 +2330,166 @@ nodes = ["yggdrasil"] } } + /// The recipe and binding of a routed service whose members connect to a + /// listener of their own. + pub(crate) fn member_listener_pair() -> (String, String) { + let recipe = RECIPE.replace( + "required_environment = [", + "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", + ); + let binding = BINDING.replace( + "reload_unit = \"nginx.service\"\n", + "reload_unit = \"nginx.service\"\n\n[route.member]\nstable_endpoint = \"http://127.0.0.1:18840\"\nprivate_port_start = 18841\n", + ); + (recipe, binding) + } + + fn member_binding_with(replacement: (&str, &str)) -> Result { + let (recipe, binding) = member_listener_pair(); + let recipe = TargetDeclaration::parse(&recipe)?; + let binding = OperatorBinding::parse(&binding.replace(replacement.0, replacement.1))?; + binding.admit(&recipe)?; + Ok(binding) + } + + #[test] + fn a_member_listener_admits_with_a_recipe_that_requires_the_member_bind() { + let binding = member_binding_with(("never-matches", "")).unwrap(); + let route = binding.route.unwrap(); + assert_eq!(route.member_stable_socket().unwrap().unwrap().1, 18840); + assert_eq!(route.member_candidate_port(18831).unwrap(), Some(18841)); + assert_eq!(route.member_candidate_port(18839).unwrap(), Some(18849)); + assert!(route.member_candidate_port(18840).is_err()); + } + + #[test] + fn a_binding_without_a_member_listener_has_no_member_ports() { + let binding = OperatorBinding::parse(BINDING).unwrap(); + let route = binding.route.unwrap(); + assert_eq!(route.member_stable_socket().unwrap(), None); + assert_eq!(route.member_candidate_port(18831).unwrap(), None); + } + + #[test] + fn a_member_listener_needs_a_recipe_that_requires_the_member_bind() { + let (recipe, binding) = member_listener_pair(); + // Bound but not required: the recipe never asked for the variable. + let plain_recipe = TargetDeclaration::parse(RECIPE).unwrap(); + assert!( + OperatorBinding::parse(&binding) + .unwrap() + .admit(&plain_recipe) + .is_err() + ); + // Required but not bound: the variable would never be set. + let recipe = TargetDeclaration::parse(&recipe).unwrap(); + assert!( + OperatorBinding::parse(BINDING) + .unwrap() + .admit(&recipe) + .is_err() + ); + } + + #[test] + fn only_a_routed_recipe_may_require_the_member_bind() { + let recipe = RECIPE + .replace("route_required = true", "route_required = false") + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", ", + "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", + ); + assert!(recipe.contains("GAMECULT_IDUNN_MEMBER_BIND")); + assert!(TargetDeclaration::parse(&recipe).is_err()); + } + + #[test] + fn member_listener_bindings_are_validated() { + let start = |value: &str| { + member_binding_with(( + "private_port_start = 18841", + &format!("private_port_start = {value}"), + )) + }; + // Overlapping the route's own private range, in either direction. + assert!(start("18835").is_err()); + assert!(start("18825").is_err()); + // Adjacent but disjoint is fine. + assert!(start("18840").is_ok()); + // Zero, and a width that overflows u16. + assert!(start("0").is_err()); + assert!(start("65530").is_err()); + // The member endpoint may not be the route's stable endpoint. + assert!(member_binding_with(("http://127.0.0.1:18840", "http://127.0.0.1:18830")).is_err()); + // The same transport and address rules as the stable endpoint. + assert!(member_binding_with(("http://127.0.0.1:18840", "rudp://127.0.0.1:18840")).is_err()); + assert!(member_binding_with(("http://127.0.0.1:18840", "http://example.test:18840")).is_err()); + } + + #[test] + fn a_udp_route_cannot_carry_a_member_listener() { + let (recipe, binding) = member_listener_pair(); + let binding = binding + .replace("nginx-stream-tcp", "nginx-stream-udp") + .replace("http://127.0.0.1:18830", "rudp://127.0.0.1:18830") + .replace("http://127.0.0.1:18840", "rudp://127.0.0.1:18840"); + let recipe = TargetDeclaration::parse(&recipe).unwrap(); + assert!( + OperatorBinding::parse(&binding) + .and_then(|binding| binding.admit(&recipe)) + .is_err() + ); + } + + #[test] + fn idunn_owned_names_are_reserved_in_one_list() { + let recipe = TargetDeclaration::parse(RECIPE).unwrap(); + let owned = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", + "GAMECULT_IDUNN_PROCESS_WRITE_LEASE", + "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", + "CREDENTIALS_DIRECTORY", + ]; + let mut listed = IDUNN_OWNED_LAUNCH_ENVIRONMENT.to_vec(); + listed.sort_unstable(); + let mut expected = owned.to_vec(); + expected.sort_unstable(); + assert_eq!(listed, expected); + + let as_environment = |name: &str| { + BINDING.replace( + "[workload.argument_bindings]", + &format!("[workload.environment]\n{name} = \"value\"\n\n[workload.argument_bindings]"), + ) + }; + let as_secret = |name: &str| { + BINDING.replace( + "[workload.secret_files]", + &format!("[workload.secret_files]\n{name} = \"/etc/gamecult/test-service/other.cc\""), + ) + }; + // The fixtures are well formed: an unowned name parses in both + // shapes, so the refusals below are about the name. + assert!(OperatorBinding::parse(&as_environment("UNOWNED_NAME")).is_ok()); + assert!(OperatorBinding::parse(&as_secret("UNOWNED_NAME")).is_ok()); + for name in owned { + assert!( + OperatorBinding::parse(&as_environment(name)) + .and_then(|binding| binding.admit(&recipe)) + .is_err(), + "{name} accepted as environment" + ); + assert!( + OperatorBinding::parse(&as_secret(name)) + .and_then(|binding| binding.admit(&recipe)) + .is_err(), + "{name} accepted as a secret file" + ); + } + } + #[test] fn state_slot_path_cannot_escape_the_operator_owned_root() { let input = RECIPE.replace( diff --git a/src/drivers.rs b/src/drivers.rs index df361c3..fb4ff1b 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -22,7 +22,8 @@ use crate::control_plane::SequenceAdmittedWarming; use crate::deployment::{ ArtifactOutput, ArtifactSource, DockerRunnerBinding, GitlinkBinding, IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, LaunchArgument, OperatorBinding, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + LaunchArgument, OperatorBinding, RUNTIME_PRESENCE_IDENTITY_BINDING, RUNTIME_PRESENCE_IDENTITY_FD_NAME, RouteBinding, RouteDriver, SourceSelectionPolicy, TargetDeclaration, WorkloadNetwork, }; @@ -3377,10 +3378,32 @@ impl SystemdTransientWorkloadDriver { .is_none(), "operator binding attempts to replace the Idunn candidate bind" ); + let member_port = binding + .route + .as_ref() + .map(|binding_route| binding_route.member_candidate_port(port)) + .transpose()? + .flatten(); + match member_port { + Some(member_port) => ensure!( + environment + .insert( + IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into(), + format!("{host}:{member_port}"), + ) + .is_none(), + "operator binding attempts to replace the Idunn member bind" + ), + None => ensure!( + !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), + "workload without a member listener carries an Idunn member bind" + ), + } } None => ensure!( - !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT), - "unrouted workload carries an Idunn candidate bind" + !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + && !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), + "unrouted workload carries an Idunn route bind" ), } match &binding.process_write_lease { @@ -5241,50 +5264,67 @@ impl NginxRouteDriver { } } - /// The one firewall rule this route owns: inbound to the stable endpoint's - /// address and port, its transport's protocol, tagged with the route id. - fn endpoint_rule(&self) -> Result> { - let (host, port) = self.binding.stable_socket()?; + /// The firewall rules this route owns, one per stable socket (the + /// presence endpoint, then the member endpoint when the binding has one): + /// inbound to the socket's address and port, its transport's protocol. + fn endpoint_rules(&self) -> Result>> { let protocol = match self.binding.driver { RouteDriver::NginxStreamTcp => "tcp", RouteDriver::NginxStreamUdp => "udp", }; - Ok(vec![ - OsString::from("allow"), - OsString::from("in"), - OsString::from("to"), - OsString::from(host.to_string()), - OsString::from("port"), - OsString::from(port.to_string()), - OsString::from("proto"), - OsString::from(protocol), - ]) + let mut sockets = vec![self.binding.stable_socket()?]; + sockets.extend(self.binding.member_stable_socket()?); + Ok(sockets + .into_iter() + .map(|(host, port)| { + vec![ + OsString::from("allow"), + OsString::from("in"), + OsString::from("to"), + OsString::from(host.to_string()), + OsString::from("port"), + OsString::from(port.to_string()), + OsString::from("proto"), + OsString::from(protocol), + ] + }) + .collect()) } fn admit_endpoint(&self) -> Result<()> { - let mut args = self.endpoint_rule()?; - args.push(OsString::from("comment")); - args.push(OsString::from(format!( - "Idunn route {}", - self.binding.route_id - ))); - self.command(&self.ufw_program, args) - .context("admitting the stable endpoint on the host firewall")?; + for mut args in self.endpoint_rules()? { + args.push(OsString::from("comment")); + args.push(OsString::from(format!( + "Idunn route {}", + self.binding.route_id + ))); + self.command(&self.ufw_program, args) + .context("admitting a route endpoint on the host firewall")?; + } Ok(()) } fn withdraw_endpoint(&self) -> Result<()> { - let mut args = vec![OsString::from("delete")]; - args.extend(self.endpoint_rule()?); - match self.command(&self.ufw_program, args) { - Ok(_) => Ok(()), - // ufw reports a rule that is already gone as a failure; for a - // withdrawal that is the state being asked for. - Err(error) if format!("{error:#}").contains("non-existent") => Ok(()), - Err(error) => { - Err(error).context("withdrawing the stable endpoint from the host firewall") + let mut first_error = None; + for rule in self.endpoint_rules()? { + let mut args = vec![OsString::from("delete")]; + args.extend(rule); + match self.command(&self.ufw_program, args) { + Ok(_) => {} + // ufw reports a rule that is already gone as a failure; for a + // withdrawal that is the state being asked for. + Err(error) if format!("{error:#}").contains("non-existent") => {} + Err(error) => { + first_error.get_or_insert( + error.context("withdrawing a route endpoint from the host firewall"), + ); + } } } + match first_error { + Some(error) => Err(error), + None => Ok(()), + } } fn command(&self, program: &Path, args: I) -> Result @@ -5345,8 +5385,20 @@ impl NginxRouteDriver { RouteDriver::NginxStreamTcp => { let (stable_host, stable_port) = endpoint_host_port(&route.stable_endpoint, endpoint_prefix)?; + let member_blocks = match ( + self.binding.member_stable_socket()?, + self.binding.member_candidate_port(candidate_port)?, + ) { + (Some((member_host, member_stable_port)), Some(member_port)) => { + let member_upstream = format!("{upstream}_member"); + format!( + "upstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n proxy_pass {member_upstream};\n}}\n" + ) + } + _ => String::new(), + }; format!( - "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n{member_blocks}" ) } RouteDriver::NginxStreamUdp => { @@ -8366,7 +8418,7 @@ fn ensure_source_directory(path: &Path, identity: Option) -> Re } } -fn authority_lock_path(path: &Path) -> PathBuf { +pub(crate) fn authority_lock_path(path: &Path) -> PathBuf { let mut lock = path.as_os_str().to_os_string(); lock.push(".lock"); PathBuf::from(lock) @@ -8698,6 +8750,7 @@ mod tests { private_port_end: port, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }) .request_candidate_runtime_presence(&candidate, "candidate-probe") } @@ -9521,6 +9574,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 17998, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -12253,6 +12307,7 @@ processes 9 private_port_end: 17979, config_path: temp.path().join("odin.conf"), reload_unit: "nginx.service".into(), + member: None, }, nginx_program: program("nginx", "#!/bin/sh\nexit 0\n")?, systemd_run_program: program("systemd-run", "#!/bin/sh\nexit 64\n")?, @@ -12262,12 +12317,15 @@ processes 9 }; assert_eq!( driver - .endpoint_rule()? + .endpoint_rules()? .iter() - .map(|value| value.to_string_lossy().into_owned()) - .collect::>() - .join(" "), - "allow in to 10.77.0.1 port 17971 proto udp" + .map(|rule| rule + .iter() + .map(|value| value.to_string_lossy().into_owned()) + .collect::>() + .join(" ")) + .collect::>(), + vec!["allow in to 10.77.0.1 port 17971 proto udp"] ); driver.restore(None, &Unmetered)?; @@ -12327,6 +12385,7 @@ processes 9 private_port_end: 4109, config_path: temp.join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }, &RouteActuators { nginx: program("nginx")?, @@ -12361,6 +12420,117 @@ processes 9 }) } + fn with_member_listener(driver: &mut NginxRouteDriver) { + driver.binding.member = Some(crate::deployment::MemberListenerBinding { + stable_endpoint: "http://127.0.0.1:4120".into(), + private_port_start: 4121, + }); + } + + /// The fragment as the base commit rendered it, computed from its format + /// string and not from the code under test. + #[cfg(unix)] + fn base_fragment(sha: &str, candidate_port: u16) -> String { + format!( + "# Idunn Expected {sha}\nupstream idunn_service_route {{\n server 127.0.0.1:{candidate_port};\n}}\nserver {{\n listen 127.0.0.1:4103;\n proxy_pass idunn_service_route;\n}}\n" + ) + } + + /// A route without a member listener renders the bytes it always did; a + /// member listener adds a second upstream and server block to the one + /// fragment, under the one Expected comment. + #[cfg(unix)] + #[test] + fn a_member_listener_renders_a_second_block_only_when_bound() -> Result<()> { + let temp = tempfile::tempdir()?; + let (mut driver, mut candidate, _calls) = stubbed_route_driver(temp.path())?; + let sha = candidate.canonical_sha256()?; + assert_eq!( + String::from_utf8(driver.render(&candidate)?)?, + base_fragment(&sha, 4104) + ); + + with_member_listener(&mut driver); + let member_blocks = |member_port: u16| { + format!( + "upstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n proxy_pass idunn_service_route_member;\n}}\n" + ) + }; + let rendered = String::from_utf8(driver.render(&candidate)?)?; + assert_eq!(rendered, base_fragment(&sha, 4104) + &member_blocks(4121)); + assert_eq!(rendered.matches("# Idunn Expected").count(), 1); + assert_eq!(rendered.matches("server {").count(), 2); + + candidate.route.as_mut().unwrap().candidate_endpoint = "http://127.0.0.1:4106".into(); + let sha = candidate.canonical_sha256()?; + assert_eq!( + String::from_utf8(driver.render(&candidate)?)?, + base_fragment(&sha, 4106) + &member_blocks(4123) + ); + Ok(()) + } + + /// Both stable sockets are admitted on the firewall, and both withdrawn. + #[cfg(unix)] + #[test] + fn both_route_endpoints_are_admitted_and_withdrawn() -> Result<()> { + let temp = tempfile::tempdir()?; + let (mut driver, _candidate, calls) = stubbed_route_driver(temp.path())?; + with_member_listener(&mut driver); + driver.admit_endpoint()?; + driver.withdraw_endpoint()?; + let log = fs::read_to_string(&calls)?; + let ufw: Vec<&str> = log.lines().filter(|line| line.starts_with("ufw ")).collect(); + assert_eq!( + ufw, + vec![ + "ufw allow in to 127.0.0.1 port 4103 proto tcp comment Idunn route service-route", + "ufw allow in to 127.0.0.1 port 4120 proto tcp comment Idunn route service-route", + "ufw delete allow in to 127.0.0.1 port 4103 proto tcp", + "ufw delete allow in to 127.0.0.1 port 4120 proto tcp", + ] + ); + Ok(()) + } + + /// Idunn sets the member bind for a bound member listener, derived from the + /// candidate, and for no other workload. + #[test] + fn the_member_bind_is_set_only_for_a_bound_member_listener() -> Result<()> { + use crate::deployment::tests::{BINDING, RECIPE, member_listener_pair}; + + let driver = SystemdTransientWorkloadDriver::default(); + let mut routed = expected(); + routed.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "test-service-public".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:18830".into(), + candidate_endpoint: "http://127.0.0.1:18833".into(), + }); + let bundle = Path::new("/etc/gamecult/test-service/runtime/bundle"); + let unit = "idunn-test-service-abc"; + + let (recipe, binding) = member_listener_pair(); + let binding = OperatorBinding::parse(&binding)?; + binding.admit(&TargetDeclaration::parse(&recipe)?)?; + let environment = driver.launch_environment(&binding, bundle, &routed, unit)?; + assert_eq!( + environment.get(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT).map(String::as_str), + Some("127.0.0.1:18843") + ); + assert_eq!( + environment.get(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT).map(String::as_str), + Some("127.0.0.1:18833") + ); + + let plain = OperatorBinding::parse(BINDING)?; + plain.admit(&TargetDeclaration::parse(RECIPE)?)?; + let environment = driver.launch_environment(&plain, bundle, &routed, unit)?; + assert!(!environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT)); + assert!(environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT)); + Ok(()) + } + /// A refused forward change stops before any host mutation: no program /// runs, no fragment is written, and no private-mount unit starts. #[cfg(unix)] @@ -12680,6 +12850,7 @@ processes 9 private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), + member: None, }, nginx_program: program("nginx")?, systemd_run_program: program("systemd-run")?, @@ -12740,6 +12911,7 @@ processes 9 private_port_end: 27879, config_path: config_path.clone(), reload_unit: "nginx.service".into(), + member: None, }); assert_eq!(driver.current_configuration()?, None); // What systemd leaves behind when it materializes the preflight's bind @@ -12766,6 +12938,7 @@ processes 9 private_port_end: 27879, config_path: "/etc/nginx/idunn-stream-routes/odin-rudp.conf".into(), reload_unit: "nginx.service".into(), + member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -12827,6 +13000,7 @@ processes 9 private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), + member: None, }; let driver = NginxRouteDriver { binding, From 9173a0e8e65a204c8111b1310a658b4e2292d590 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 01:28:30 +0200 Subject: [PATCH 134/196] idunn-launch answers presence only: the splice, the unix child endpoint and --listener are deleted; the child binds the member bind and the probe reaches it over TCP PresenceAcceptor serves one connection at a time within a 1 s deadline and never ends on an accept error; a dead presence thread fails the unit. Builds. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 29 +- docs/authority-map.md | 20 +- src/bin/idunn-launch.rs | 13 +- src/deployment.rs | 4 +- src/launcher.rs | 865 ++++++++++++++++------------------------ 5 files changed, 396 insertions(+), 535 deletions(-) diff --git a/README.md b/README.md index 225c5d8..3714ea9 100644 --- a/README.md +++ b/README.md @@ -154,8 +154,11 @@ An upstream daemon that does not speak Idunn's runtime contract is deployed with `idunn-launch` as its executable artifact and the upstream tree as a required adjacent artifact; the launcher finds the child inside the release that holds it. The launcher is the unit's process: it binds the -candidate endpoint, answers Idunn's challenge, renders the child's config, -starts a stateful child only once it holds the write lease, and signs presence. +candidate endpoint and answers Idunn's challenge there, renders the child's +config, starts a stateful child only once it holds the write lease, and signs +presence. Members never pass through it: the binding's `[route.member]` makes +Idunn route a second listener straight to the child, which binds the address +Idunn gives it in `GAMECULT_IDUNN_MEMBER_BIND`. ```toml # recipe fragment — the upstream tree is built or fetched as an artifact @@ -166,13 +169,31 @@ arguments = [ { kind = "literal", value = "upstream/bin/daemon" }, { kind = "literal", value = "--template" }, { kind = "literal", value = "config.tmpl" }, - { kind = "literal", value = "--listener" }, - { kind = "literal", value = "unix" }, { kind = "literal", value = "--health-path" }, { kind = "literal", value = "/health" }, { kind = "literal", value = "--health-contract" }, { kind = "literal", value = "example.presence-launcher.v1" }, ] +required_environment = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", +] +``` + +```toml +# binding fragment — members connect to the member endpoint, which Idunn +# forwards to the private port derived from the candidate: member +# private_port_start + (candidate port - route private_port_start) +[route] +stable_endpoint = "tcp://10.77.0.1:8008" +private_port_start = 18831 +private_port_end = 18839 +# ... + +[route.member] +stable_endpoint = "tcp://10.77.0.1:8448" +private_port_start = 18841 ``` ## Contracts diff --git a/docs/authority-map.md b/docs/authority-map.md index b61a8f2..2db896f 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -83,24 +83,30 @@ activation key. Idunn owns admission, lease, route and continuity, as before. - **Inputs:** the runtime bundle, the activation credential and provider identity descriptors, the process write lease, the systemd credentials the - binding names, and the child's health over its private listener. + binding names, and the child's health over TCP at the member bind. - **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active and failed to Odin; route observations on the candidate endpoint), one child process, and a rendered config in the unit's private tmp that is never read back. -- **Route:** the launcher, not the child, binds the candidate endpoint. A first - byte of 0x00 is an Idunn challenge frame and is answered by the launcher; - every other byte is copied untouched to the child's private listener. +- **Route:** Idunn owns both listeners of the route: the Expected route's + stable endpoint (presence) and, when the binding has `[route.member]`, the + member endpoint, rendered as a second server block in the one nginx fragment. + The launcher binds the candidate endpoint and answers Idunn's challenge there, + one connection at a time within a one-second deadline; any other first byte + closes the connection. The child binds the member bind Idunn sets in + `GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes + never touch the launcher. - **Lease:** with `write_lease_required` the launcher starts the child only after it holds the lease (bound to this Expected, this activation and one of its own Warming proofs) and keeps the lease store share-locked while the child lives. Losing the lease stops the child. - **Config:** one template from the release is rendered with the closed - placeholder set `{{child.listen}}`, `{{credential:NAME}}`, + placeholder set `{{child.listen}}`, `{{child.listen_host}}`, + `{{child.listen_port}}` (all three the member bind), `{{credential:NAME}}`, `{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder refuses to start. -- **Forbidden:** the child binding the candidate endpoint; a child of a - lease-requiring target running without a held lease; any presence not signed +- **Forbidden:** member traffic through the launcher; the child binding the + candidate endpoint; a child of a lease-requiring target running without a held lease; any presence not signed by both keys. - **State kind `external-directory`:** a process-owned directory imposed by an external runtime contract. Same writer, recovery and startup rules as diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs index 41b6c76..3cf71e6 100644 --- a/src/bin/idunn-launch.rs +++ b/src/bin/idunn-launch.rs @@ -7,7 +7,7 @@ fn main() -> std::process::ExitCode { use anyhow::Context; use idunn_daemon::deployment::{ IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, }; use idunn_daemon::launcher::{self, *}; @@ -26,6 +26,11 @@ fn main() -> std::process::ExitCode { .context("the Idunn runtime bundle is not set")?; let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) .context("the Idunn candidate bind is not set")?; + let member = launcher::parse_member_bind( + &std::env::var(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + .context("the Idunn member bind is not set")?, + &bind, + )?; let (activation, provider) = launcher::inherited_authority_files()?; let authority = RuntimeAuthority::open(bundle.as_ref(), activation, provider)?; authority.require_candidate_bind(&bind)?; @@ -47,10 +52,9 @@ fn main() -> std::process::ExitCode { sink: std::sync::Arc::new(sink), lease: std::sync::Arc::new(lease), spawner: Box::new(ProcessSpawner), - route_bounds: RouteBounds::default(), - health: Box::new(move |endpoint| { + health: Box::new(move |member| { std::sync::Arc::new(HttpHealthProbe { - endpoint: endpoint.clone(), + member, path: health_path.clone(), }) }), @@ -64,6 +68,7 @@ fn main() -> std::process::ExitCode { &contract, authority, route, + member, &std::env::temp_dir(), ports, &STOP, diff --git a/src/deployment.rs b/src/deployment.rs index b01deed..067f3b8 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -657,9 +657,7 @@ pub struct ProcessWriteLeaseBinding { impl ProcessWriteLeaseBinding { pub fn lock_path(&self) -> PathBuf { - let mut lock_path = self.record_path.as_os_str().to_os_string(); - lock_path.push(".lock"); - PathBuf::from(lock_path) + crate::drivers::authority_lock_path(&self.record_path) } } diff --git a/src/launcher.rs b/src/launcher.rs index 689c35a..391ee9e 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -4,9 +4,11 @@ //! Idunn observes the unit; the launcher, inside it and supervising the child, //! signs the presence statement with the provider identity and the activation //! key. The launcher binds the candidate endpoint Idunn challenges, because the -//! child cannot answer a CultNet frame. It starts a stateful child only after -//! it holds the process write lease, renders the child's config from a closed -//! set of placeholders, and ends with the child. +//! child cannot answer a CultNet frame, and answers presence there and nowhere +//! else: members reach the child on the member bind Idunn routes them to. It +//! starts a stateful child only after it holds the process write lease, renders +//! the child's config from a closed set of placeholders, and ends with the +//! child. //! //! Errors, logs and refusals name a field and a code. They never echo an //! argument, a template line, a path or a credential. @@ -14,14 +16,13 @@ use std::collections::VecDeque; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; -use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::net::{SocketAddr, TcpListener, TcpStream, UdpSocket}; use std::os::fd::{AsRawFd, RawFd}; use std::os::unix::fs::OpenOptionsExt; -use std::os::unix::net::UnixStream; use std::os::unix::process::CommandExt; use std::path::{Component, Path, PathBuf}; use std::process::{Child, Command, Stdio}; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -41,21 +42,22 @@ use cultnet_rs::{ encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, }; -use crate::deployment::RUNTIME_PRESENCE_IDENTITY_FD_NAME; -use crate::drivers::{CultCacheWriteLeaseDriver, lease_binds_candidate}; +use crate::deployment::{RUNTIME_PRESENCE_IDENTITY_FD_NAME, SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT}; +use crate::drivers::{CultCacheWriteLeaseDriver, authority_lock_path, lease_binds_candidate}; /// The route challenge is the only thing that begins with this byte: a /// CultNet frame's 4-byte big-endian length is bounded at 1 MiB, and an HTTP /// request starts with a method letter. const CHALLENGE_FIRST_BYTE: u8 = 0x00; const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; -const CHALLENGE_IO_TIMEOUT: Duration = Duration::from_secs(3); -/// How long a public connection may stay silent before its first byte. -const FIRST_BYTE_DEADLINE: Duration = Duration::from_secs(5); -/// A whole challenge, from its first byte to its answer, however it trickles. -const CHALLENGE_DEADLINE: Duration = Duration::from_secs(5); -/// Concurrent connections the acceptor serves; the rest are refused unspawned. -const MAX_CONNECTIONS: usize = 256; +/// One whole presence exchange on the candidate endpoint, from accept to +/// answer, however the peer paces it. It is shorter than the 3 s Idunn waits +/// for a challenge answer, so one stalled peer cannot cost the next challenge +/// its reply. +const PRESENCE_DEADLINE: Duration = Duration::from_secs(1); +/// How long the presence acceptor waits after an accept error before trying +/// again. No accept error ends the acceptor. +const ACCEPT_BACKOFF: Duration = Duration::from_millis(50); const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); const POLL: Duration = Duration::from_millis(100); const MAX_RECENT_WARMING_PROOFS: usize = 64; @@ -65,14 +67,6 @@ const ODIN_CAPABILITY: &str = "odin.verse-rendezvous"; // ---------------------------------------------------------------- contract -/// Where the child listens. The launcher chooses the address; the child is -/// told it through `{{child.listen}}`. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ChildListener { - UnixSocket, - LoopbackPort, -} - /// Everything the recipe declares about the child, parsed from the launcher's /// argv (`--flag value` pairs; `--child-arg` repeats). The release root is the /// sealed release directory that holds the launcher itself. @@ -85,7 +79,6 @@ pub struct LaunchContract { /// child takes one. pub config_flag: Option, pub template: PathBuf, - pub listener: ChildListener, pub health_path: String, pub health_contract: String, pub state_root: Option, @@ -101,7 +94,6 @@ impl LaunchContract { let mut child_args = Vec::new(); let mut config_flag = None; let mut template = None; - let mut listener = None; let mut health_path = None; let mut health_contract = None; let mut state_root = None; @@ -121,13 +113,6 @@ impl LaunchContract { "--child-arg" => child_args.push(value), "--config-flag" => config_flag = Some(value), "--template" => template = Some(release_relative(&value, "--template")?), - "--listener" => { - listener = Some(match value.as_str() { - "unix" => ChildListener::UnixSocket, - "loopback" => ChildListener::LoopbackPort, - _ => bail!("launcher argument --listener is not unix or loopback"), - }) - } "--health-path" => { ensure!( value.starts_with('/'), @@ -149,7 +134,6 @@ impl LaunchContract { child_args, config_flag, template: template.context("launcher argument --template is required")?, - listener: listener.context("launcher argument --listener is required")?, health_path: health_path.context("launcher argument --health-path is required")?, health_contract: health_contract .context("launcher argument --health-contract is required")?, @@ -426,7 +410,7 @@ impl CredentialsDirectory { /// The directory systemd exported; no directory means no credential is bound. pub fn from_environment() -> Self { - Self(std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from)) + Self(std::env::var_os(SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT).map(PathBuf::from)) } } @@ -523,9 +507,7 @@ impl LeaseReader for DriverLeaseReader { /// description: a child that inherits it keeps the lock after the launcher /// dies. fn hold_shared(&self) -> Result> { - let mut lock = self.driver.record_path.clone().into_os_string(); - lock.push(".lock"); - let file = File::open(&lock).map_err(|_| anyhow!("write-lease lock could not be held"))?; + let file = File::open(authority_lock_path(&self.driver.record_path)).map_err(|_| anyhow!("write-lease lock could not be held"))?; // SAFETY: flock on a descriptor this function owns. ensure!( unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0, @@ -640,108 +622,34 @@ fn presence_document( }) } -/// The child's private listener. -#[derive(Clone, Debug)] -pub enum ChildEndpoint { - Unix(PathBuf), - Loopback(SocketAddr), -} - -pub enum ChildStream { - Unix(UnixStream), - Tcp(TcpStream), -} - -impl ChildEndpoint { - pub fn connect(&self) -> io::Result { - match self { - Self::Unix(path) => UnixStream::connect(path).map(ChildStream::Unix), - Self::Loopback(address) => { - TcpStream::connect_timeout(address, HEALTH_PROBE_TIMEOUT).map(ChildStream::Tcp) - } - } - } - - /// What `{{child.listen}}` renders to. - fn listen_value(&self) -> String { - match self { - Self::Unix(path) => path.display().to_string(), - Self::Loopback(address) => address.to_string(), - } - } -} - -impl ChildStream { - fn try_clone(&self) -> io::Result { - match self { - Self::Unix(stream) => stream.try_clone().map(Self::Unix), - Self::Tcp(stream) => stream.try_clone().map(Self::Tcp), - } - } - - fn shutdown_write(&self) { - let _ = match self { - Self::Unix(stream) => stream.shutdown(Shutdown::Write), - Self::Tcp(stream) => stream.shutdown(Shutdown::Write), - }; - } - - fn set_timeout(&self, timeout: Duration) { - let _ = match self { - Self::Unix(stream) => stream - .set_read_timeout(Some(timeout)) - .and_then(|()| stream.set_write_timeout(Some(timeout))), - Self::Tcp(stream) => stream - .set_read_timeout(Some(timeout)) - .and_then(|()| stream.set_write_timeout(Some(timeout))), - }; - } -} - -impl Read for ChildStream { - fn read(&mut self, buffer: &mut [u8]) -> io::Result { - match self { - Self::Unix(stream) => stream.read(buffer), - Self::Tcp(stream) => stream.read(buffer), - } - } -} - -impl Write for ChildStream { - fn write(&mut self, buffer: &[u8]) -> io::Result { - match self { - Self::Unix(stream) => stream.write(buffer), - Self::Tcp(stream) => stream.write(buffer), - } - } - - fn flush(&mut self) -> io::Result<()> { - match self { - Self::Unix(stream) => stream.flush(), - Self::Tcp(stream) => stream.flush(), - } - } -} - -/// `GET ` over the child's private listener; healthy on a 200. +/// `GET ` over TCP at the child's member bind; healthy on a 200. One +/// deadline covers connect, write and read, so a full backlog or a silent child +/// answers false within `HEALTH_PROBE_TIMEOUT`. pub struct HttpHealthProbe { - pub endpoint: ChildEndpoint, + pub member: SocketAddr, pub path: String, } impl HealthProbe for HttpHealthProbe { fn healthy(&self) -> bool { - let Ok(mut stream) = self.endpoint.connect() else { + let deadline = Instant::now() + HEALTH_PROBE_TIMEOUT; + let Ok(mut stream) = TcpStream::connect_timeout(&self.member, HEALTH_PROBE_TIMEOUT) else { + return false; + }; + let Some(remaining) = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + else { return false; }; - stream.set_timeout(HEALTH_PROBE_TIMEOUT); let request = format!( "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", self.path ); let mut head = [0_u8; 12]; - stream.write_all(request.as_bytes()).is_ok() - && stream.read_exact(&mut head).is_ok() + stream.set_write_timeout(Some(remaining)).is_ok() + && stream.write_all(request.as_bytes()).is_ok() + && read_by(&mut stream, &mut head, deadline).is_ok() && head.starts_with(b"HTTP/1.") && &head[8..12] == b" 200" } @@ -816,7 +724,8 @@ impl ChildHandle for ProcessChild { /// The closed placeholder set (decision D6). Any other `{{...}}` refuses. pub struct ConfigRenderer<'a> { pub credentials: &'a dyn CredentialSource, - pub listen: &'a str, + /// The member bind the child serves on. + pub listen: SocketAddr, pub state_root: Option<&'a Path>, } @@ -838,8 +747,11 @@ impl ConfigRenderer<'_> { } fn expand(&self, key: &str) -> Result { - if key == "child.listen" { - return Ok(self.listen.to_owned()); + match key { + "child.listen" => return Ok(self.listen.to_string()), + "child.listen_host" => return Ok(self.listen.ip().to_string()), + "child.listen_port" => return Ok(self.listen.port().to_string()), + _ => {} } if key == "state_root" { return Ok(self @@ -940,7 +852,7 @@ enum Phase { Failed, } -/// What the launcher currently is, shared with the route acceptor. +/// What the launcher currently is, shared with the presence acceptor. struct Shared { authority: Mutex, phase: Mutex, @@ -948,15 +860,13 @@ struct Shared { lease_required: bool, health: Arc, clock: Arc, - /// False once the route acceptor has stopped for any reason but a halt: - /// the launcher cannot answer Idunn's challenge, so it must not say active. - acceptor_alive: AtomicBool, } impl Shared { - /// Active only while the child is healthy and the route is being served. + /// Active only while the child is healthy. A dead presence acceptor does + /// not degrade presence: it fails the unit. fn serving_state(&self) -> &'static str { - if self.health.healthy() && self.acceptor_alive.load(Ordering::SeqCst) { + if self.health.healthy() { "active" } else { "degraded" @@ -1007,134 +917,66 @@ impl Shared { type Responder = Arc Result + Send + Sync>; -/// Bounds on what the public route endpoint spends on one peer. -#[derive(Clone, Copy, Debug)] -pub struct RouteBounds { - pub first_byte: Duration, - pub challenge: Duration, - pub max_connections: usize, -} - -impl Default for RouteBounds { - fn default() -> Self { - Self { - first_byte: FIRST_BYTE_DEADLINE, - challenge: CHALLENGE_DEADLINE, - max_connections: MAX_CONNECTIONS, - } - } -} - -/// Counts a live connection; dropping it frees the slot. -struct Slot(Arc); - -impl Drop for Slot { - fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::SeqCst); - } -} - -/// Owns the candidate endpoint. A leading 0x00 is an Idunn challenge frame and -/// is answered here; every other byte is copied untouched to the child. The -/// endpoint is public, so it spends bounded resources on a peer: a first-byte -/// deadline, a total challenge deadline, and a cap on live connections whose -/// overflow is closed without a thread. -pub struct RouteAcceptor { +/// Owns the candidate endpoint and answers presence there and nowhere else. +/// Members never reach it: Idunn routes them to the child's member bind. +/// +/// The endpoint is public, so it spends bounded resources on a peer: one +/// connection at a time, so a connection holds at most one descriptor; a total +/// deadline for the whole exchange; and any first byte but a challenge's closes +/// the connection unanswered. No accept error ends it: it backs off and accepts +/// again, and only `halt` stops it. +pub struct PresenceAcceptor { listener: TcpListener, - child: ChildEndpoint, responder: Responder, - bounds: RouteBounds, - live: Arc, } -impl RouteAcceptor { - pub fn new(listener: TcpListener, child: ChildEndpoint, responder: Responder, bounds: RouteBounds) -> Self { - Self { - listener, - child, - responder, - bounds, - live: Arc::new(AtomicUsize::new(0)), - } +impl PresenceAcceptor { + pub fn new(listener: TcpListener, responder: Responder) -> Self { + Self { listener, responder } } - /// Serves until `halt`. Any other end is an error: a failed accept or a - /// thread that cannot be created means the route is no longer served. - pub fn serve(self, halt: Arc) -> Result<()> { - self.listener.set_nonblocking(true)?; - let acceptor = Arc::new(self); + pub fn serve(self, halt: Arc) { + self.serve_accepting(halt, |listener| listener.accept()); + } + + fn serve_accepting( + self, + halt: Arc, + mut accept: impl FnMut(&TcpListener) -> io::Result<(TcpStream, SocketAddr)>, + ) { + // A listener that cannot be made non-blocking could never observe + // `halt`; ending here is a dead acceptor, which fails the unit. + if self.listener.set_nonblocking(true).is_err() { + return; + } while !halt.load(Ordering::SeqCst) { - match acceptor.listener.accept() { + match accept(&self.listener) { Ok((connection, _)) => { - if acceptor.live.fetch_add(1, Ordering::SeqCst) >= acceptor.bounds.max_connections { - acceptor.live.fetch_sub(1, Ordering::SeqCst); - continue; - } - let slot = Slot(Arc::clone(&acceptor.live)); - let worker = Arc::clone(&acceptor); - thread::Builder::new() - .spawn(move || { - let _slot = slot; - let _ = connection.set_nonblocking(false); - let _ = worker.handle(connection); - }) - .map_err(|_| anyhow!("a route connection thread could not be created"))?; + let _ = connection.set_nonblocking(false); + let _ = self.answer(connection); } - Err(error) - if matches!( - error.kind(), - io::ErrorKind::WouldBlock - | io::ErrorKind::Interrupted - | io::ErrorKind::ConnectionAborted - ) => - { + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { thread::sleep(Duration::from_millis(10)); } - Err(_) => bail!("the route listener could not accept"), + Err(_) => thread::sleep(ACCEPT_BACKOFF), } } - Ok(()) } - fn handle(&self, mut connection: TcpStream) -> Result<()> { + fn answer(&self, mut connection: TcpStream) -> Result<()> { + let deadline = Instant::now() + PRESENCE_DEADLINE; let mut first = [0_u8; 1]; - read_by(&mut connection, &mut first, Instant::now() + self.bounds.first_byte)?; - if first[0] == CHALLENGE_FIRST_BYTE { - return self.answer_challenge(&mut connection); - } - let mut upstream = self.child.connect()?; - upstream.write_all(&first)?; - let mut upstream_reader = upstream.try_clone()?; - let mut client_reader = connection.try_clone()?; - let forward = thread::Builder::new() - .spawn(move || { - let _ = io::copy(&mut client_reader, &mut upstream); - upstream.shutdown_write(); - }) - .map_err(|_| anyhow!("a route splice thread could not be created"))?; - // The first-byte deadline bounded the silent peer; a spliced session - // runs as long as the member and the child keep it. - connection.set_read_timeout(None)?; - let _ = io::copy(&mut upstream_reader, &mut connection); - let _ = connection.shutdown(Shutdown::Write); - let _ = forward.join(); - Ok(()) - } - - fn answer_challenge(&self, connection: &mut TcpStream) -> Result<()> { - // The challenge began with its first byte; the rest must arrive within - // the total deadline, however slowly the peer sends it. - let deadline = Instant::now() + self.bounds.challenge; - connection.set_write_timeout(Some(CHALLENGE_IO_TIMEOUT))?; + read_by(&mut connection, &mut first, deadline)?; + ensure!(first[0] == CHALLENGE_FIRST_BYTE, "not a challenge frame"); let mut rest = [0_u8; 3]; - read_by(connection, &mut rest, deadline)?; + read_by(&mut connection, &mut rest, deadline)?; let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; ensure!( (1..=CHALLENGE_MAX_BYTES).contains(&length), "challenge frame is outside its bound" ); let mut payload = vec![0_u8; length]; - read_by(connection, &mut payload, deadline)?; + read_by(&mut connection, &mut payload, deadline)?; let request = decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; let response = (self.responder)(request)?; @@ -1142,6 +984,11 @@ impl RouteAcceptor { &response, CultNetWireContract::CultNetSchemaV0, )?)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .context("the answer was not ready in time")?; + connection.set_write_timeout(Some(remaining))?; connection.write_all(&frame)?; connection.flush()?; Ok(()) @@ -1175,9 +1022,8 @@ pub struct Ports { pub sink: Arc, pub lease: Arc, pub spawner: Box, - pub route_bounds: RouteBounds, - /// Built from the endpoint `run` chooses. - pub health: Box Arc + Send>, + /// Built from the member bind the child serves on. + pub health: Box Arc + Send>, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -1197,23 +1043,32 @@ impl Outcome { } } -fn child_endpoint(listener: ChildListener, tmp: &Path) -> Result { - Ok(match listener { - ChildListener::UnixSocket => ChildEndpoint::Unix(tmp.join("child.sock")), - ChildListener::LoopbackPort => { - let probe = TcpListener::bind("127.0.0.1:0")?; - ChildEndpoint::Loopback(probe.local_addr()?) - } - }) +/// The member bind Idunn hands the unit: a socket address on a loopback IP, +/// other than the candidate bind. Errors name the variable, never its value. +pub fn parse_member_bind(raw: &str, candidate_bind: &str) -> Result { + let member: SocketAddr = raw + .parse() + .map_err(|_| anyhow!("the Idunn member bind is not a socket address"))?; + ensure!( + member.ip().is_loopback(), + "the Idunn member bind is not on a loopback address" + ); + let candidate: SocketAddr = candidate_bind + .parse() + .map_err(|_| anyhow!("the Idunn candidate bind is not a socket address"))?; + ensure!(member != candidate, "the Idunn member bind equals the candidate bind"); + Ok(member) } /// The unit's whole life: Warming, lease, child, Active, until the child or the /// lease ends or SIGTERM arrives. `route` is the already-bound candidate -/// listener; `tmp` is the unit's private temporary directory. +/// listener, answered for presence only; `member` is the address the child +/// serves members on; `tmp` is the unit's private temporary directory. pub fn run( contract: &LaunchContract, authority: RuntimeAuthority, route: TcpListener, + member: SocketAddr, tmp: &Path, ports: Ports, stop: &AtomicBool, @@ -1223,10 +1078,7 @@ pub fn run( if expected.health_contract != contract.health_contract { return Outcome::Failed("health-contract-mismatch"); } - let Ok(endpoint) = child_endpoint(contract.listener, tmp) else { - return Outcome::Failed("child-endpoint"); - }; - let health = (ports.health)(&endpoint); + let health = (ports.health)(member); let shared = Arc::new(Shared { authority: Mutex::new(authority), phase: Mutex::new(Phase::Warming), @@ -1234,26 +1086,23 @@ pub fn run( lease_required: expected.write_lease_required, health: Arc::clone(&health), clock: Arc::clone(&ports.clock), - acceptor_alive: AtomicBool::new(true), }); let halt = Arc::new(AtomicBool::new(false)); let responder: Responder = { let shared = Arc::clone(&shared); Arc::new(move |request| shared.observe(request)) }; + let acceptor = PresenceAcceptor::new(route, responder); let serving = { - let acceptor = RouteAcceptor::new(route, endpoint.clone(), responder, ports.route_bounds); let halt = Arc::clone(&halt); - let shared = Arc::clone(&shared); - thread::spawn(move || { - let served = acceptor.serve(halt); - if served.is_err() { - shared.acceptor_alive.store(false, Ordering::SeqCst); - } - served - }) + thread::Builder::new() + .name("presence-acceptor".into()) + .spawn(move || acceptor.serve(halt)) }; - let outcome = lifecycle(contract, &expected, &activation, &endpoint, tmp, &ports, &shared, stop); + let Ok(serving) = serving else { + return Outcome::Failed("presence-acceptor"); + }; + let outcome = lifecycle(contract, &expected, &activation, member, tmp, &ports, &shared, &serving, stop); halt.store(true, Ordering::SeqCst); let _ = serving.join(); outcome @@ -1264,10 +1113,11 @@ fn lifecycle( contract: &LaunchContract, expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, - endpoint: &ChildEndpoint, + member: SocketAddr, tmp: &Path, ports: &Ports, shared: &Shared, + presence: &thread::JoinHandle<()>, stop: &AtomicBool, ) -> Outcome { let clock = &*ports.clock; @@ -1316,6 +1166,9 @@ fn lifecycle( if stop.load(Ordering::SeqCst) { return Outcome::Stopped; } + if presence.is_finished() { + return fail(None, "presence-acceptor"); + } let held = LeaseHold::acquire( Arc::clone(&ports.lease), expected, @@ -1343,7 +1196,7 @@ fn lifecycle( } // 3. Render, spawn, and wait for health. - let Some(mut child) = start_child(contract, endpoint, tmp, ports, lease_fd) else { + let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { return fail(None, "child-start-refused"); }; let startup_deadline = clock.now_millis() + contract.startup.as_millis() as u64; @@ -1355,6 +1208,9 @@ fn lifecycle( if child.exited().is_some() { return fail(None, "child-exited"); } + if presence.is_finished() { + return fail(Some(&mut child), "presence-acceptor"); + } if shared.health.healthy() { break; } @@ -1376,6 +1232,9 @@ fn lifecycle( if child.exited().is_some() { return fail(None, "child-exited"); } + if presence.is_finished() { + return fail(Some(&mut child), "presence-acceptor"); + } if !shared.lease_current() { return fail(Some(&mut child), "lease-lost"); } @@ -1390,16 +1249,15 @@ fn lifecycle( fn start_child( contract: &LaunchContract, - endpoint: &ChildEndpoint, + member: SocketAddr, tmp: &Path, ports: &Ports, lease_fd: Option, ) -> Option> { let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; - let listen = endpoint.listen_value(); let rendered = ConfigRenderer { credentials: &*ports.credentials, - listen: &listen, + listen: member, state_root: contract.state_root.as_deref(), } .render(&template) @@ -1421,7 +1279,6 @@ mod tests { use std::collections::BTreeMap; use std::os::unix::fs::PermissionsExt; use std::sync::atomic::{AtomicU64, AtomicUsize}; - use std::sync::mpsc; use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; use cultnet_rs::{ @@ -1433,6 +1290,11 @@ mod tests { use super::*; use crate::drivers::{WriteLeasePort, exact_route_snapshot_response, request_tcp_snapshot}; + /// The member bind Idunn would hand these fixtures' unit. + fn member() -> SocketAddr { + "127.0.0.1:18845".parse().unwrap() + } + fn digest(byte: char) -> String { format!("sha256-{}", byte.to_string().repeat(64)) } @@ -1580,7 +1442,6 @@ mod tests { child_args: vec!["--serve".into()], config_flag: Some("--config".into()), template: PathBuf::from("config.tmpl"), - listener: ChildListener::UnixSocket, health_path: "/health".into(), health_contract: self.expected.health_contract.clone(), state_root: Some(PathBuf::from("/var/lib/synapse")), @@ -1743,14 +1604,14 @@ mod tests { sink: sink.clone(), lease: lease.clone(), spawner: Box::new(FakeSpawner(child.clone())), - route_bounds: RouteBounds::default(), health: Box::new(move |_| health.clone()), }; let authority = fx.authority(); let route = TcpListener::bind("127.0.0.1:0").unwrap(); let tmp = fx.dir.path().join("tmp"); let flag = stop.clone(); - let outcome = thread::spawn(move || run(&contract, authority, route, &tmp, ports, &flag)); + let outcome = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); Run { sink, lease, @@ -1887,7 +1748,7 @@ mod tests { #[test] fn closed_placeholder_set_renders_and_refuses() { let fx = fixture(false); - let template = "listen={{child.listen}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; + let template = "listen={{child.listen}}\nhost={{child.listen_host}}\nport={{child.listen_port}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; let mut launch = start(&fx, fx.contract(template), true); wait_until("spawn", || launch.child.spawns.load(Ordering::SeqCst) == 1); let args = launch.child.args.lock().unwrap()[0].clone(); @@ -1899,13 +1760,9 @@ mod tests { 0o600 ); let rendered = fs::read_to_string(&config).unwrap(); - let socket = fx.dir.path().join("tmp/child.sock"); assert_eq!( rendered, - format!( - "listen={}\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n", - socket.display() - ) + "listen=127.0.0.1:18845\nhost=127.0.0.1\nport=18845\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n" ); assert!( !rendered.contains("18835"), @@ -1921,7 +1778,7 @@ mod tests { let credentials = Credentials(BTreeMap::new()); let renderer = ConfigRenderer { credentials: &credentials, - listen: "L", + listen: member(), state_root: None, }; for template in [ @@ -1944,7 +1801,6 @@ mod tests { lease_required: false, health, clock: Arc::new(SystemClock), - acceptor_alive: AtomicBool::new(true), }) } @@ -1982,12 +1838,7 @@ mod tests { let address = listener.local_addr().unwrap(); let responder: Responder = Arc::new(move |request| shared.observe(request)); let halt = Arc::new(AtomicBool::new(false)); - let acceptor = RouteAcceptor::new( - listener, - ChildEndpoint::Unix("/nonexistent".into()), - responder, - RouteBounds::default(), - ); + let acceptor = PresenceAcceptor::new(listener, responder); let serving = { let halt = halt.clone(); thread::spawn(move || acceptor.serve(halt)) @@ -2019,81 +1870,7 @@ mod tests { .unwrap(); assert_eq!(claim.record().state, "active"); halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); - } - - #[test] - fn member_bytes_reach_the_child_verbatim_and_only_challenge_frames_are_answered() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("child.sock"); - let child = std::os::unix::net::UnixListener::bind(&socket).unwrap(); - let request: Vec = - b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 6\r\n\r\n\x00\x01\x00\x02\x00\x03".to_vec(); - let response: Vec = - b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n\x00\xff\x00\x10".to_vec(); - let received = Arc::new(Mutex::new(Vec::new())); - let connections = Arc::new(AtomicUsize::new(0)); - let fake = { - let (received, connections, response, want) = ( - received.clone(), - connections.clone(), - response.clone(), - request.len(), - ); - thread::spawn(move || { - for stream in child.incoming() { - let mut stream = stream.unwrap(); - connections.fetch_add(1, Ordering::SeqCst); - let mut buffer = vec![0_u8; want]; - if stream.read_exact(&mut buffer).is_err() { - continue; - } - *received.lock().unwrap() = buffer; - stream.write_all(&response).unwrap(); - return; - } - }) - }; - let challenges = Arc::new(AtomicUsize::new(0)); - let responder: Responder = { - let challenges = challenges.clone(); - Arc::new(move |_| { - challenges.fetch_add(1, Ordering::SeqCst); - bail!("not under test") - }) - }; - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let halt = Arc::new(AtomicBool::new(false)); - let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder, RouteBounds::default()); - let serving = { - let halt = halt.clone(); - thread::spawn(move || acceptor.serve(halt)) - }; - - // A leading 0x00 that is not a challenge frame never reaches the child. - let mut probe = TcpStream::connect(address).unwrap(); - probe.write_all(&[0x00, 0xff, 0xff, 0xff, 1, 2, 3]).unwrap(); - let mut answered = Vec::new(); - let _ = probe.read_to_end(&mut answered); - assert!(answered.is_empty()); - assert_eq!(connections.load(Ordering::SeqCst), 0); - - // Everything else is copied untouched, both ways, 0x00 bytes included. - let mut member = TcpStream::connect(address).unwrap(); - member.write_all(&request).unwrap(); - let mut answer = Vec::new(); - member.read_to_end(&mut answer).unwrap(); - assert_eq!(answer, response); - fake.join().unwrap(); - assert_eq!(*received.lock().unwrap(), request); - assert_eq!( - challenges.load(Ordering::SeqCst), - 0, - "a malformed frame fails before the responder; member traffic never reaches it" - ); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); + serving.join().unwrap(); } #[test] @@ -2104,8 +1881,6 @@ mod tests { "bin/c", "--template", "t", - "--listener", - "unix", "--health-path", "/health", "--health-contract", @@ -2118,6 +1893,9 @@ mod tests { }; assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); assert!(base(&["--child", "../escape"]).is_err()); + // The child serves members on Idunn's member bind: there is no + // listener to choose. + assert!(base(&["--listener", "loopback"]).is_err()); assert!(base(&["--template", "/etc/passwd"]).is_err()); let error = base(&["--canary-flag", "canary-value"]) .unwrap_err() @@ -2166,14 +1944,9 @@ mod tests { #[test] fn health_probe_is_healthy_only_on_a_200_from_the_child() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("child.sock"); - let probe = HttpHealthProbe { - endpoint: ChildEndpoint::Unix(socket.clone()), - path: "/health".into(), - }; - assert!(!probe.healthy(), "no listener is not healthy"); - let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + let probe = HttpHealthProbe { member, path: "/health".into() }; let server = thread::spawn(move || { for status in ["200 OK", "503 Service Unavailable"] { let (mut stream, _) = listener.accept().unwrap(); @@ -2187,13 +1960,14 @@ mod tests { assert!(probe.healthy()); assert!(!probe.healthy()); server.join().unwrap(); + assert!(!probe.healthy(), "no listener is not healthy"); } #[test] fn contract_parsing_reads_every_declared_field() { let args: Vec = [ "--child", "bin/c", "--child-arg", "-x", "--child-arg", "y", "--config-flag", "-c", - "--template", "t", "--listener", "loopback", "--health-path", "/h", + "--template", "t", "--health-path", "/h", "--health-contract", "hc", "--state-root", "/s", "--lease-wait-secs", "11", "--startup-secs", "22", "--heartbeat-secs", "33", ] @@ -2202,7 +1976,6 @@ mod tests { let contract = LaunchContract::parse(PathBuf::from("/r"), args.clone()).unwrap(); assert_eq!(contract.child_args, ["-x", "y"]); assert_eq!(contract.config_flag.as_deref(), Some("-c")); - assert_eq!(contract.listener, ChildListener::LoopbackPort); assert_eq!(contract.state_root, Some(PathBuf::from("/s"))); assert_eq!( (contract.lease_wait, contract.startup, contract.heartbeat), @@ -2226,7 +1999,7 @@ mod tests { Ok(PathBuf::from(name)) } } - let renderer = ConfigRenderer { credentials: &Any, listen: "L", state_root: None }; + let renderer = ConfigRenderer { credentials: &Any, listen: member(), state_root: None }; for ok in ["A", "a-b_9", "Z0"] { assert_eq!(renderer.render(&format!("{{{{credential:{ok}}}}}")).unwrap(), "v"); } @@ -2234,7 +2007,7 @@ mod tests { assert!(renderer.render(&format!("{{{{credential:{bad}}}}}")).is_err(), "{bad}"); assert!(renderer.render(&format!("{{{{credential_path:{bad}}}}}")).is_err(), "{bad}"); } - assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "aLbLc"); + assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "a127.0.0.1:18845b127.0.0.1:18845c"); } #[test] @@ -2263,7 +2036,7 @@ mod tests { std::env::set_var("IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", &file); } let credentials = CredentialsDirectory::new(dir.path().to_owned()); - let renderer = ConfigRenderer { credentials: &credentials, listen: "L", state_root: None }; + let renderer = ConfigRenderer { credentials: &credentials, listen: member(), state_root: None }; for name in [ "GAMECULT_IDUNN_CANDIDATE_BIND", "IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", @@ -2316,38 +2089,6 @@ mod tests { assert!(OdinRudpSink::from_expected(&expected).is_err()); } - #[test] - fn half_closed_member_requests_still_reach_a_child_that_waits_for_eof() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("child.sock"); - let child = std::os::unix::net::UnixListener::bind(&socket).unwrap(); - let fake = thread::spawn(move || { - let (mut stream, _) = child.accept().unwrap(); - let mut received = Vec::new(); - stream.read_to_end(&mut received).unwrap(); - stream.write_all(&received.iter().rev().copied().collect::>()).unwrap(); - }); - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let halt = Arc::new(AtomicBool::new(false)); - let responder: Responder = Arc::new(|_| bail!("never")); - let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder, RouteBounds::default()); - let serving = { - let halt = halt.clone(); - thread::spawn(move || acceptor.serve(halt)) - }; - let mut member = TcpStream::connect(address).unwrap(); - member.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); - member.write_all(b"GET /abc").unwrap(); - member.shutdown(Shutdown::Write).unwrap(); - let mut answer = Vec::new(); - member.read_to_end(&mut answer).unwrap(); - assert_eq!(answer, b"cba/ TEG"); - fake.join().unwrap(); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); - } - #[test] fn heartbeats_are_paced_and_a_stop_exits_zero() { let fx = fixture(true); @@ -2406,32 +2147,35 @@ mod tests { assert!(running.exited().is_some()); } - // ------------------------------------------------------------ Soul s1 fixes + // ------------------------------------------------------------ presence acceptor - fn serve_with( + type Accepted = io::Result<(TcpStream, SocketAddr)>; + + fn serve_presence_accepting( fx: &Fixture, - bounds: RouteBounds, - ) -> (SocketAddr, Arc, thread::JoinHandle>) { - let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + responder: Responder, + accept: impl FnMut(&TcpListener) -> Accepted + Send + 'static, + ) -> (SocketAddr, Arc, thread::JoinHandle<()>) { + let _ = fx; let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); - let responder: Responder = Arc::new(move |request| shared.observe(request)); - let acceptor = - RouteAcceptor::new(listener, ChildEndpoint::Unix("/nonexistent".into()), responder, bounds); + let acceptor = PresenceAcceptor::new(listener, responder); let halt = Arc::new(AtomicBool::new(false)); let flag = halt.clone(); - (address, halt, thread::spawn(move || acceptor.serve(flag))) + (address, halt, thread::spawn(move || acceptor.serve_accepting(flag, accept))) } - fn quick(first_byte_ms: u64, challenge_ms: u64, max_connections: usize) -> RouteBounds { - RouteBounds { - first_byte: Duration::from_millis(first_byte_ms), - challenge: Duration::from_millis(challenge_ms), - max_connections, - } + fn observing(fx: &Fixture) -> Responder { + let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + Arc::new(move |request| shared.observe(request)) } - /// True when the peer closed (EOF or reset) within `within`. + fn serve_presence(fx: &Fixture) -> (SocketAddr, Arc, thread::JoinHandle<()>) { + serve_presence_accepting(fx, observing(fx), |listener| listener.accept()) + } + + /// True when the peer closed (EOF or reset) within `within`, having sent + /// nothing. fn closed_within(stream: &mut TcpStream, within: Duration) -> bool { stream.set_read_timeout(Some(within)).unwrap(); let mut byte = [0_u8; 64]; @@ -2454,24 +2198,58 @@ mod tests { } #[test] - fn an_idle_connection_is_closed_at_the_first_byte_deadline() { + fn only_challenge_frames_are_answered_and_anything_else_is_closed_unanswered() { let fx = fixture(false); - let (address, halt, serving) = serve_with(&fx, quick(300, 5000, 64)); - let mut idle = TcpStream::connect(address).unwrap(); - let started = Instant::now(); - assert!(closed_within(&mut idle, Duration::from_secs(4)), "never closed"); - assert!(started.elapsed() >= Duration::from_millis(250), "closed before its deadline"); + let reached = Arc::new(AtomicUsize::new(0)); + let responder: Responder = { + let (inner, reached) = (observing(&fx), reached.clone()); + Arc::new(move |request| { + reached.fetch_add(1, Ordering::SeqCst); + inner(request) + }) + }; + let (address, halt, serving) = + serve_presence_accepting(&fx, responder, |listener| listener.accept()); + // Member traffic is not presence: an HTTP request gets no byte back. + let mut member = TcpStream::connect(address).unwrap(); + member + .write_all(b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 2\r\n\r\n\x00\x01") + .unwrap(); + assert!(closed_within(&mut member, Duration::from_secs(3)), "member bytes were answered"); + // A leading 0x00 with a frame length outside its bound never reaches + // the responder. + let mut malformed = TcpStream::connect(address).unwrap(); + malformed.write_all(&[0x00, 0xff, 0xff, 0xff]).unwrap(); + assert!(closed_within(&mut malformed, Duration::from_secs(3)), "a malformed frame was answered"); + assert_eq!(reached.load(Ordering::SeqCst), 0); + // A real challenge is answered, and is the only thing that reaches it. assert!(challenge_answered(address, &fx)); + assert_eq!(reached.load(Ordering::SeqCst), 1); halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); + serving.join().unwrap(); } #[test] - fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_its_total_deadline() { + fn an_idle_peer_holds_the_acceptor_for_the_presence_deadline_and_no_longer() { let fx = fixture(false); - // Every pause is far inside the first-byte deadline: only the total - // challenge deadline can end this. - let (address, halt, serving) = serve_with(&fx, quick(5000, 600, 64)); + let (address, halt, serving) = serve_presence(&fx); + let mut idle = TcpStream::connect(address).unwrap(); + let started = Instant::now(); + // Idunn waits 3 s for a challenge answer; one stalled peer ahead of it + // must not cost it that. + assert!(challenge_answered(address, &fx)); + let waited = started.elapsed(); + assert!(waited >= Duration::from_millis(800), "the idle peer was cut early: {waited:?}"); + assert!(waited < Duration::from_millis(2200), "the idle peer held the acceptor: {waited:?}"); + assert!(closed_within(&mut idle, Duration::from_secs(1)), "the idle peer stayed connected"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_the_presence_deadline() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); let mut slow = TcpStream::connect(address).unwrap(); slow.write_all(&[0x00, 0x00, 0x01, 0x00]).unwrap(); let started = Instant::now(); @@ -2483,51 +2261,34 @@ mod tests { } } assert!(closed, "the trickle was never cut off"); - assert!(started.elapsed() >= Duration::from_millis(500), "cut before its deadline"); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(800), "cut before its deadline: {elapsed:?}"); + assert!(elapsed < Duration::from_millis(2200), "cut after its deadline: {elapsed:?}"); assert!(challenge_answered(address, &fx)); halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); + serving.join().unwrap(); } #[test] - fn live_connections_are_capped_and_the_overflow_is_refused_unspawned() { + fn accept_errors_never_end_the_presence_acceptor() { let fx = fixture(false); - let (address, halt, serving) = serve_with(&fx, quick(30_000, 5000, 4)); - let mut held: Vec = (0..4).map(|_| TcpStream::connect(address).unwrap()).collect(); - thread::sleep(Duration::from_millis(300)); - let threads_before = thread_count(); - let mut overflow: Vec = (0..40).map(|_| TcpStream::connect(address).unwrap()).collect(); - for stream in &mut overflow { - assert!(closed_within(stream, Duration::from_secs(3)), "an overflow connection was served"); - } - assert!(thread_count() < threads_before + 8, "overflow connections cost threads"); - for stream in &mut held { - assert!(!closed_within(stream, Duration::from_millis(100)), "a held connection was cut"); - } - // A freed slot serves the next peer. - held.pop(); - let mut answered = false; - for _ in 0..40 { - if challenge_answered(address, &fx) { - answered = true; - break; - } - thread::sleep(Duration::from_millis(50)); - } - assert!(answered, "a freed slot was never reused"); + // Descriptor exhaustion: the first accepts fail, then descriptors free. + let failures = Arc::new(AtomicUsize::new(0)); + let (address, halt, serving) = { + let failures = failures.clone(); + serve_presence_accepting(&fx, observing(&fx), move |listener| { + if failures.fetch_add(1, Ordering::SeqCst) < 3 { + Err(io::Error::from_raw_os_error(libc::EMFILE)) + } else { + listener.accept() + } + }) + }; + assert!(challenge_answered(address, &fx), "the acceptor did not recover"); + assert!(failures.load(Ordering::SeqCst) >= 4); + assert!(!serving.is_finished()); halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); - } - - fn thread_count() -> usize { - fs::read_to_string("/proc/self/status") - .unwrap() - .lines() - .find_map(|line| line.strip_prefix("Threads:")) - .unwrap() - .trim() - .parse() - .unwrap() + serving.join().unwrap(); } /// A listener whose accept always fails: a datagram socket's descriptor. @@ -2539,50 +2300,57 @@ mod tests { } #[test] - fn an_acceptor_that_cannot_accept_ends_with_an_error() { + fn an_acceptor_whose_every_accept_fails_runs_until_halted() { let fx = fixture(false); - let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); - let responder: Responder = Arc::new(move |request| shared.observe(request)); - let acceptor = RouteAcceptor::new( - failing_listener(), - ChildEndpoint::Unix("/nonexistent".into()), - responder, - RouteBounds::default(), - ); - assert!(acceptor.serve(Arc::new(AtomicBool::new(false))).is_err()); + let acceptor = PresenceAcceptor::new(failing_listener(), observing(&fx)); + let halt = Arc::new(AtomicBool::new(false)); + let flag = halt.clone(); + let serving = thread::spawn(move || acceptor.serve(flag)); + thread::sleep(Duration::from_millis(400)); + assert!(!serving.is_finished(), "the acceptor ended on an accept error"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + /// Healthy for everyone but the presence thread, which it kills: the + /// stand-in for a presence acceptor that dies while the child is fine. + struct PresenceTripwire; + + impl HealthProbe for PresenceTripwire { + fn healthy(&self) -> bool { + assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); + true + } } #[test] - fn presence_stops_claiming_active_when_the_route_acceptor_dies() { + fn a_dead_presence_acceptor_fails_the_unit() { let fx = fixture(false); let sink = Arc::new(Sink::default()); let child = Arc::new(Child::default()); - let health = Arc::new(Health(AtomicBool::new(true))); let ports = Ports { credentials: Arc::new(Credentials(BTreeMap::new())), clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), sink: sink.clone(), lease: Arc::new(FakeLease::default()), - spawner: Box::new(FakeSpawner(child)), - route_bounds: RouteBounds::default(), - health: Box::new(move |_| health.clone()), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(|_| Arc::new(PresenceTripwire)), }; let contract = fx.contract("x"); let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); let tmp = fx.dir.path().join("tmp"); let stop = Arc::new(AtomicBool::new(false)); let flag = stop.clone(); - let handle = thread::spawn(move || run(&contract, authority, failing_listener(), &tmp, ports, &flag)); - wait_until("two degraded heartbeats", || { - sink.states().iter().filter(|state| *state == "degraded").count() >= 2 - }); - // The child is healthy throughout; only the dead acceptor degrades it, - // and nothing says active after that. - let states = sink.states(); - let first_degraded = states.iter().position(|state| state == "degraded").unwrap(); - assert!(states[first_degraded..].iter().all(|state| state != "active"), "{states:?}"); - stop.store(true, Ordering::SeqCst); - assert_eq!(handle.join().unwrap(), Outcome::Stopped); + let handle = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + // The first challenge reaches the tripwire on the presence thread. + assert!(!challenge_answered(address, &fx)); + assert_eq!(handle.join().unwrap(), Outcome::Failed("presence-acceptor")); + assert_eq!(sink.first("failed").unwrap().detail, "presence-acceptor"); + assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); } #[test] @@ -2701,7 +2469,6 @@ mod tests { sink: sink.clone(), lease: Arc::new(FakeLease::default()), spawner: Box::new(FakeSpawner(child.clone())), - route_bounds: RouteBounds::default(), health: Box::new(move |_| health.clone()), }; let authority = fx.authority(); @@ -2709,7 +2476,8 @@ mod tests { let tmp = fx.dir.path().join("tmp"); let stop = Arc::new(AtomicBool::new(false)); let flag = stop.clone(); - let handle = thread::spawn(move || run(&contract, authority, route, &tmp, ports, &flag)); + let handle = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); wait_until("active", || sink.first("active").is_some()); assert!(!child.terminated.load(Ordering::SeqCst), "killed while it was still starting"); assert_eq!(child.spawns.load(Ordering::SeqCst), 1); @@ -2719,23 +2487,86 @@ mod tests { } #[test] - fn a_hung_health_probe_times_out_instead_of_blocking() { + fn a_silent_child_fails_the_probe_within_its_timeout() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let endpoint = ChildEndpoint::Loopback(listener.local_addr().unwrap()); + let member = listener.local_addr().unwrap(); // Accepts and never answers. let held = thread::spawn(move || { let _connection = listener.accept().unwrap(); - thread::sleep(Duration::from_secs(8)); + thread::sleep(Duration::from_secs(4)); }); - let probe = HttpHealthProbe { endpoint, path: "/health".into() }; - let (done, verdict) = mpsc::channel(); - thread::spawn(move || { - let _ = done.send(probe.healthy()); - }); - let healthy = verdict - .recv_timeout(Duration::from_secs(5)) - .expect("the probe outlived its timeout against a hung child"); - assert!(!healthy); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let started = Instant::now(); + assert!(!probe.healthy()); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(900), "gave up before its timeout: {elapsed:?}"); + assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its timeout: {elapsed:?}"); drop(held); } + + #[test] + fn the_probe_deadline_covers_the_whole_response() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + // Every byte arrives inside any per-read timeout, but the head takes + // three seconds in all. + let dribbler = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + for byte in b"HTTP/1.1 200 OK" { + if stream.write_all(&[*byte]).is_err() { + return; + } + thread::sleep(Duration::from_millis(250)); + } + }); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let started = Instant::now(); + assert!(!probe.healthy()); + let elapsed = started.elapsed(); + assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its deadline: {elapsed:?}"); + let _ = dribbler.join(); + } + + #[test] + fn a_process_spawner_refuses_an_invalid_lease_descriptor_and_runs_no_child() { + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("ran"); + let script = format!("touch '{}'", marker.display()); + for invalid in [-1, 9999] { + let refused = ProcessSpawner.spawn( + Path::new("/bin/sh"), + &["-c".to_owned(), script.clone()], + Some(invalid), + ); + assert!(refused.is_err(), "descriptor {invalid} was accepted"); + } + thread::sleep(Duration::from_millis(300)); + assert!(!marker.exists(), "a child ran past an invalid descriptor"); + // Control: the same spawn without a descriptor runs the child. + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".to_owned(), script], None) + .unwrap(); + wait_until("the control child to exit", || child.exited().is_some()); + assert!(marker.exists()); + } + + #[test] + fn the_member_bind_is_a_loopback_address_other_than_the_candidate() { + let candidate = "127.0.0.1:18835"; + assert_eq!( + parse_member_bind("127.0.0.1:18845", candidate).unwrap(), + "127.0.0.1:18845".parse().unwrap() + ); + for bad in [ + "127.0.0.1:18835", + "10.0.0.5:18845", + "0.0.0.0:18845", + "canary-not-an-address", + "", + ] { + let error = parse_member_bind(bad, candidate).unwrap_err().to_string(); + assert!(!error.contains("canary") && !error.contains("10.0.0.5"), "{error}"); + } + assert!(parse_member_bind("127.0.0.1:18845", "not-an-address").is_err()); + } } From b379fbd70035d55b745442a9419a64da19268f62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 01:45:19 +0200 Subject: [PATCH 135/196] Tests kill mutation survivors: each owned-name site is pinned alone, the member range below the route's is accepted, a challenge near the frame bound is answered; accept errors share one backoff Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/deployment.rs | 39 ++++++++++++++++++++++++++++++--------- src/launcher.rs | 29 ++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/deployment.rs b/src/deployment.rs index 067f3b8..6fdf339 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -2412,8 +2412,10 @@ nodes = ["yggdrasil"] // Overlapping the route's own private range, in either direction. assert!(start("18835").is_err()); assert!(start("18825").is_err()); - // Adjacent but disjoint is fine. + // Adjacent but disjoint is fine, above the route's range or below it. assert!(start("18840").is_ok()); + assert!(start("18822").is_ok()); + assert!(start("18823").is_err()); // Zero, and a width that overflows u16. assert!(start("0").is_err()); assert!(start("65530").is_err()); @@ -2473,18 +2475,37 @@ nodes = ["yggdrasil"] assert!(OperatorBinding::parse(&as_environment("UNOWNED_NAME")).is_ok()); assert!(OperatorBinding::parse(&as_secret("UNOWNED_NAME")).is_ok()); for name in owned { + // Refused when the binding is parsed... assert!( - OperatorBinding::parse(&as_environment(name)) - .and_then(|binding| binding.admit(&recipe)) - .is_err(), - "{name} accepted as environment" + OperatorBinding::parse(&as_environment(name)).is_err(), + "{name} accepted as environment at parse" ); assert!( - OperatorBinding::parse(&as_secret(name)) - .and_then(|binding| binding.admit(&recipe)) - .is_err(), - "{name} accepted as a secret file" + OperatorBinding::parse(&as_secret(name)).is_err(), + "{name} accepted as a secret file at parse" ); + // ...and again at admission, for a binding that did not come + // through parse. Each site reads the one list. + for secret in [false, true] { + let mut binding = OperatorBinding::parse(BINDING).unwrap(); + let WorkloadBinding::SystemdTransient(workload) = &mut binding.workload else { + panic!("the fixture is a systemd workload"); + }; + if secret { + workload.secret_files.insert(name.into(), "/etc/gamecult/test-service/other.cc".into()); + } else { + workload.environment.insert(name.into(), "forged".into()); + } + assert!( + binding.workload.validate_launch_environment().is_err(), + "{name} (secret: {secret}) accepted by the binding check" + ); + let error = binding.admit(&recipe).unwrap_err().to_string(); + assert!( + error.contains("Idunn-owned launch environment"), + "{name} (secret: {secret}) accepted by the launch-input check: {error}" + ); + } } } diff --git a/src/launcher.rs b/src/launcher.rs index 391ee9e..539c396 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -955,9 +955,8 @@ impl PresenceAcceptor { let _ = connection.set_nonblocking(false); let _ = self.answer(connection); } - Err(error) if error.kind() == io::ErrorKind::WouldBlock => { - thread::sleep(Duration::from_millis(10)); - } + // An idle non-blocking accept is an error too; every one waits + // and tries again. Err(_) => thread::sleep(ACCEPT_BACKOFF), } } @@ -2229,6 +2228,30 @@ mod tests { serving.join().unwrap(); } + #[test] + fn a_challenge_up_to_the_frame_bound_is_answered_and_one_past_it_is_not() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); + // A request far larger than a few KiB, well inside the 1 MiB bound. + let request = CultNetMessage::SnapshotRequest { + message_id: "large".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some((0..300).map(|index| format!("{index:0>40}")).collect()), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert!(payload.len() > 10_000 && payload.len() < CHALLENGE_MAX_BYTES); + assert!(request_tcp_snapshot(address, &payload).is_ok()); + // One byte past the bound is closed unanswered. + let past = (CHALLENGE_MAX_BYTES as u32 + 1).to_be_bytes(); + assert_eq!(past[0], CHALLENGE_FIRST_BYTE); + let mut oversized = TcpStream::connect(address).unwrap(); + oversized.write_all(&past).unwrap(); + assert!(closed_within(&mut oversized, Duration::from_secs(3)), "an oversized frame was read"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + #[test] fn an_idle_peer_holds_the_acceptor_for_the_presence_deadline_and_no_longer() { let fx = fixture(false); From 2218ffb8a82cb037a0d78b3184ea1aa4f4335e19 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:10:12 +0200 Subject: [PATCH 136/196] Idunn routes: the member server block renders a limit_conn on a per-route zone; [route.member].max_connections defaults to 128 Co-Authored-By: Claude Sonnet 5.5 --- src/deployment.rs | 30 +++++++++ src/drivers.rs | 151 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/src/deployment.rs b/src/deployment.rs index 6fdf339..75a30ab 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -694,6 +694,18 @@ pub struct RouteBinding { pub struct MemberListenerBinding { pub stable_endpoint: String, pub private_port_start: u16, + /// Most member connections nginx holds open at once for this route. Each + /// costs nginx two connections (client and upstream) from the pool the + /// route's presence block shares, so this limit is what keeps members from + /// exhausting it. + #[serde(default = "default_member_max_connections")] + pub max_connections: u32, +} + +const DEFAULT_MEMBER_MAX_CONNECTIONS: u32 = 128; + +fn default_member_max_connections() -> u32 { + DEFAULT_MEMBER_MAX_CONNECTIONS } impl RouteBinding { @@ -1466,6 +1478,10 @@ impl OperatorBinding { member.private_port_start > 0, "member private port range starts at zero" ); + ensure!( + member.max_connections > 0, + "member connection limit is zero" + ); let width = route.private_port_end - route.private_port_start; let member_end = member .private_port_start @@ -2416,6 +2432,20 @@ nodes = ["yggdrasil"] assert!(start("18840").is_ok()); assert!(start("18822").is_ok()); assert!(start("18823").is_err()); + // The connection limit defaults, and zero is refused. + let limit = |value: &str| { + member_binding_with(( + "private_port_start = 18841", + &format!("private_port_start = 18841 +max_connections = {value}"), + )) + }; + assert_eq!( + member_binding_with(("never-matches", "")).unwrap().route.unwrap().member.unwrap().max_connections, + 128 + ); + assert_eq!(limit("7").unwrap().route.unwrap().member.unwrap().max_connections, 7); + assert!(limit("0").is_err()); // Zero, and a width that overflows u16. assert!(start("0").is_err()); assert!(start("65530").is_err()); diff --git a/src/drivers.rs b/src/drivers.rs index fb4ff1b..d3f6abf 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5391,8 +5391,16 @@ impl NginxRouteDriver { ) { (Some((member_host, member_stable_port)), Some(member_port)) => { let member_upstream = format!("{upstream}_member"); + let max_connections = self + .binding + .member + .as_ref() + .map_or(0, |member| member.max_connections); + // One constant key: the limit is on the route's members + // together, so a flood of them can never take the + // connections the presence block needs. format!( - "upstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n proxy_pass {member_upstream};\n}}\n" + "limit_conn_zone \"{member_upstream}\" zone={member_upstream}:64k;\nupstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n limit_conn {member_upstream} {max_connections};\n proxy_pass {member_upstream};\n}}\n" ) } _ => String::new(), @@ -12424,6 +12432,7 @@ processes 9 driver.binding.member = Some(crate::deployment::MemberListenerBinding { stable_endpoint: "http://127.0.0.1:4120".into(), private_port_start: 4121, + max_connections: 200, }); } @@ -12493,6 +12502,146 @@ processes 9 Ok(()) } + /// Real nginx over the rendered two-block fragment, a real presence + /// acceptor behind the presence block and a flood of idle members on the + /// member block. Needs an `nginx` on PATH and the fixed loopback ports + /// 4103, 4104, 4120 and 4121, so it runs on demand in a container: + /// `cargo test --lib real_nginx -- --ignored`. + #[cfg(unix)] + fn challenges_answered_during_a_member_flood(max_connections: u32) -> Result { + use std::net::{TcpListener, TcpStream}; + use std::process::{Child, Command}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + struct Nginx(Child); + impl Drop for Nginx { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + let temp = tempfile::tempdir()?; + let (mut driver, candidate, _calls) = stubbed_route_driver(temp.path())?; + with_member_listener(&mut driver); + driver.binding.member.as_mut().unwrap().max_connections = max_connections; + let fragment = temp.path().join("route.conf"); + fs::write(&fragment, driver.render(&candidate)?)?; + // Debian builds the stream module dynamic. + let module = Path::new("/usr/lib/nginx/modules/ngx_stream_module.so"); + let load = if module.exists() { + format!("load_module {}; +", module.display()) + } else { + String::new() + }; + let conf = temp.path().join("nginx.conf"); + fs::write( + &conf, + format!( + "{load}worker_processes 1;\npid {dir}/nginx.pid;\nerror_log {dir}/error.log;\nevents {{ worker_connections 512; }}\nstream {{ include {fragment}; }}\n", + dir = temp.path().display(), + fragment = fragment.display() + ), + )?; + let halt = Arc::new(AtomicBool::new(false)); + let presence = crate::launcher::PresenceAcceptor::new( + TcpListener::bind("127.0.0.1:4104")?, + Arc::new(|request| match request { + CultNetMessage::SnapshotRequest { message_id, .. } => { + Ok(CultNetMessage::SnapshotResponseRaw { message_id, documents: vec![] }) + } + _ => bail!("not a snapshot request"), + }), + ); + let presence_halt = halt.clone(); + let presence = thread::spawn(move || presence.serve(presence_halt)); + // The member backend accepts and holds every connection. + let members = TcpListener::bind("127.0.0.1:4121")?; + members.set_nonblocking(true)?; + let member_halt = halt.clone(); + let members = thread::spawn(move || { + let mut held = Vec::new(); + while !member_halt.load(Ordering::SeqCst) { + match members.accept() { + Ok((stream, _)) => held.push(stream), + Err(_) => thread::sleep(Duration::from_millis(5)), + } + } + }); + let _nginx = Nginx( + Command::new("nginx") + .args(["-p"]) + .arg(temp.path()) + .arg("-c") + .arg(&conf) + .args(["-g", "daemon off;"]) + .spawn()?, + ); + let stable: SocketAddr = "127.0.0.1:4103".parse()?; + let ready = Instant::now(); + while TcpStream::connect(stable).is_err() { + ensure!(ready.elapsed() < Duration::from_secs(10), "nginx never listened"); + thread::sleep(Duration::from_millis(50)); + } + // The probe connection above reached the presence acceptor as an idle + // peer; let its deadline lapse so only the flood is measured. + thread::sleep(Duration::from_millis(1200)); + + let mut flood = Vec::new(); + for _ in 0..300 { + if let Ok(mut stream) = TcpStream::connect("127.0.0.1:4120") { + let _ = stream.write_all(b"x"); + flood.push(stream); + } + } + thread::sleep(Duration::from_millis(500)); + + let payload = encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: "flood".into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + )?; + let mut answered = 0; + for _ in 0..20 { + let started = Instant::now(); + if request_tcp_snapshot(stable, &payload).is_ok() + && started.elapsed() < Duration::from_secs(3) + { + answered += 1; + } + } + drop(flood); + halt.store(true, Ordering::SeqCst); + presence.join().unwrap(); + members.join().unwrap(); + Ok(answered) + } + + #[cfg(unix)] + #[test] + #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] + fn real_nginx_keeps_presence_answerable_under_a_member_flood() -> Result<()> { + // The rendered default limit keeps the route's members out of the + // presence block's share of a 512-connection pool. + assert_eq!(challenges_answered_during_a_member_flood(128)?, 20); + Ok(()) + } + + #[cfg(unix)] + #[test] + #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] + fn real_nginx_flood_starves_presence_without_a_member_limit() -> Result<()> { + // The control: the same flood against a limit too high to bind starves + // presence, so the test above measures the limit and not the load. + assert!(challenges_answered_during_a_member_flood(100_000)? < 20); + Ok(()) + } + /// Idunn sets the member bind for a bound member listener, derived from the /// candidate, and for no other workload. #[test] From 36114b8b7baa6dcfb656428c14b0250ba72aa228 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:12:56 +0200 Subject: [PATCH 137/196] Idunn routes tests: the rendered member block pins its limit zone; real-nginx flood tests prove presence stays answerable Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers.rs b/src/drivers.rs index d3f6abf..060917d 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -12462,7 +12462,7 @@ processes 9 with_member_listener(&mut driver); let member_blocks = |member_port: u16| { format!( - "upstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n proxy_pass idunn_service_route_member;\n}}\n" + "limit_conn_zone \"idunn_service_route_member\" zone=idunn_service_route_member:64k;\nupstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n limit_conn idunn_service_route_member 200;\n proxy_pass idunn_service_route_member;\n}}\n" ) }; let rendered = String::from_utf8(driver.render(&candidate)?)?; From e5294db40cce39f0b9d013503f5b29bf077c5630 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:12:57 +0200 Subject: [PATCH 138/196] idunn-launch: a challenge answers from the lifecycle's last probe, stale or failed reads degraded; tests bound their waits and pin startup timeout and heartbeat cadence Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 331 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 294 insertions(+), 37 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 539c396..f15882e 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -59,6 +59,11 @@ const PRESENCE_DEADLINE: Duration = Duration::from_secs(1); /// again. No accept error ends the acceptor. const ACCEPT_BACKOFF: Duration = Duration::from_millis(50); const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); +/// How often the lifecycle probes the child once it is Active. +const HEALTH_REFRESH: Duration = Duration::from_millis(500); +/// A probe result older than this is not evidence of health: the lifecycle +/// that refreshes it is itself stuck. +const HEALTH_STALE: Duration = Duration::from_secs(5); const POLL: Duration = Duration::from_millis(100); const MAX_RECENT_WARMING_PROOFS: usize = 64; const TERMINATE_GRACE: Duration = Duration::from_secs(20); @@ -859,18 +864,36 @@ struct Shared { lease: Mutex>, lease_required: bool, health: Arc, + /// The last probe result and the clock time it started. Only the lifecycle + /// probes; a challenge reads this and never waits on the child. + probed: Mutex>, clock: Arc, } impl Shared { - /// Active only while the child is healthy. A dead presence acceptor does - /// not degrade presence: it fails the unit. - fn serving_state(&self) -> &'static str { - if self.health.healthy() { - "active" - } else { - "degraded" + /// Probes the child now and keeps the result for presence to answer from. + fn refresh_health(&self) -> bool { + let started = self.clock.now_millis(); + let healthy = self.health.healthy(); + if let Ok(mut probed) = self.probed.lock() { + *probed = Some((healthy, started)); } + healthy + } + + /// Active only while the last probe found the child healthy and is recent. + /// A child never probed, wedged, or probed too long ago reads degraded. A + /// dead presence acceptor does not degrade presence: it fails the unit. + fn serving_state(&self) -> &'static str { + let now = self.clock.now_millis(); + let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); + let healthy = self + .probed + .lock() + .ok() + .and_then(|probed| *probed) + .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); + if healthy { "active" } else { "degraded" } } fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { @@ -1084,6 +1107,7 @@ pub fn run( lease: Mutex::new(None), lease_required: expected.write_lease_required, health: Arc::clone(&health), + probed: Mutex::new(None), clock: Arc::clone(&ports.clock), }); let halt = Arc::new(AtomicBool::new(false)); @@ -1210,7 +1234,7 @@ fn lifecycle( if presence.is_finished() { return fail(Some(&mut child), "presence-acceptor"); } - if shared.health.healthy() { + if shared.refresh_health() { break; } if clock.now_millis() >= startup_deadline { @@ -1222,6 +1246,8 @@ fn lifecycle( // 4. Active, and heartbeats while it holds. set_phase(Phase::Active(lease_sha.clone())); publish(shared.serving_state(), "child-healthy", lease_sha.clone()); + let refresh = u64::try_from(HEALTH_REFRESH.as_millis()).unwrap_or(u64::MAX); + let mut next_probe = clock.now_millis() + refresh; next_heartbeat = clock.now_millis() + heartbeat; loop { if stop.load(Ordering::SeqCst) { @@ -1238,6 +1264,10 @@ fn lifecycle( return fail(Some(&mut child), "lease-lost"); } let now = clock.now_millis(); + if now >= next_probe { + shared.refresh_health(); + next_probe = now + refresh; + } if now >= next_heartbeat { publish(shared.serving_state(), "heartbeat", lease_sha.clone()); next_heartbeat = now + heartbeat; @@ -1793,13 +1823,18 @@ mod tests { } fn shared(fx: &Fixture, health: Arc) -> Arc { + shared_on(fx, health, Arc::new(SystemClock)) + } + + fn shared_on(fx: &Fixture, health: Arc, clock: Arc) -> Arc { Arc::new(Shared { authority: Mutex::new(fx.authority()), phase: Mutex::new(Phase::Active(None)), lease: Mutex::new(None), lease_required: false, health, - clock: Arc::new(SystemClock), + probed: Mutex::new(None), + clock, }) } @@ -1824,8 +1859,14 @@ mod tests { assert_eq!(record.detail, "route-observation:challenge-1"); record.state }; + // Never probed: nothing says the child is healthy. + assert_eq!(state_of(&shared), "degraded"); + shared.refresh_health(); assert_eq!(state_of(&shared), "active"); + // A challenge answers from the last probe and never runs one. health.0.store(false, Ordering::SeqCst); + assert_eq!(state_of(&shared), "active"); + shared.refresh_health(); assert_eq!(state_of(&shared), "degraded"); } @@ -1833,6 +1874,7 @@ mod tests { fn route_challenge_is_answered_with_both_signatures_through_idunns_client() { let fx = fixture(false); let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); + shared.refresh_health(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let responder: Responder = Arc::new(move |request| shared.observe(request)); @@ -2088,34 +2130,248 @@ mod tests { assert!(OdinRudpSink::from_expected(&expected).is_err()); } + #[test] - fn heartbeats_are_paced_and_a_stop_exits_zero() { + fn a_probe_result_is_evidence_only_while_it_is_recent() { + let fx = fixture(false); + let clock = Arc::new(FastClock(AtomicU64::new(1_000_000))); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + let stale = HEALTH_STALE.as_millis() as u64; + clock.0.fetch_add(stale, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "active", "a result exactly at the bound is still recent"); + clock.0.fetch_add(1, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "degraded", "a result past the bound is not evidence"); + shared.refresh_health(); + assert_eq!(shared.serving_state(), "active"); + } + + /// Runs `work` and returns its result, failing the test when it has not + /// finished within `limit`, so a mutant that makes it wait forever fails + /// instead of hanging the suite. + fn within(limit: Duration, work: impl FnOnce() -> T + Send + 'static) -> T { + join_within(limit, thread::spawn(work)) + } + + fn join_within(limit: Duration, handle: thread::JoinHandle) -> T { + let started = Instant::now(); + while !handle.is_finished() { + assert!(started.elapsed() < limit, "still running after {limit:?}"); + thread::sleep(Duration::from_millis(10)); + } + handle.join().unwrap() + } + + /// The state the presence endpoint signs in answer to a challenge, or None + /// when it is not answered promptly. + fn challenge_state(address: SocketAddr, fx: &Fixture) -> Option { + let request = CultNetMessage::SnapshotRequest { + message_id: "wedge".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + let started = Instant::now(); + let answer = request_tcp_snapshot(address, &payload).ok()?; + if started.elapsed() > Duration::from_millis(750) { + return None; + } + let CultNetMessage::SnapshotResponseRaw { documents, .. } = + decode_cultnet_message_from_slice(&answer, CultNetWireContract::CultNetSchemaV0).ok()? + else { + return None; + }; + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&documents.first()?.payload).ok()?; + Some(record.state) + } + + #[derive(Clone, Copy)] + enum Wedge { + /// Accepts and never answers. + Silent, + /// Never accepts; the kernel backlog fills. + FullBacklog, + /// Accepts and answers 200 after 1.1 s, past the probe's deadline. + Slow, + } + + type WedgeSwitch = Arc>>; + + /// A child that answers 200 until wedged. Returns the member address, the + /// switch, and the thread to join after `halt`. + fn serving_child(halt: Arc) -> (SocketAddr, WedgeSwitch, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let wedge: WedgeSwitch = Arc::default(); + let mode = wedge.clone(); + let serving = thread::spawn(move || { + while !halt.load(Ordering::SeqCst) { + let current = *mode.lock().unwrap(); + if matches!(current, Some(Wedge::FullBacklog)) { + thread::sleep(Duration::from_millis(5)); + continue; + } + let Ok((mut stream, _)) = listener.accept() else { + thread::sleep(Duration::from_millis(5)); + continue; + }; + stream.set_nonblocking(false).unwrap(); + match current { + None => { + let mut request = [0_u8; 256]; + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + let _ = stream.read(&mut request); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + } + Some(Wedge::Slow) => { + thread::spawn(move || { + thread::sleep(Duration::from_millis(1100)); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + }); + } + _ => { + thread::spawn(move || { + thread::sleep(Duration::from_secs(8)); + drop(stream); + }); + } + } + } + }); + (address, wedge, serving) + } + + /// Whatever wedges the child, every challenge is still answered promptly, + /// and the answer turns to a signed degraded within a few probes. + fn a_wedged_child_is_answered_degraded(wedge: Wedge) { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let halt = Arc::new(AtomicBool::new(false)); + let (member, mode, child) = serving_child(halt.clone()); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(SystemClock), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(|member| Arc::new(HttpHealthProbe { member, path: "/health".into() })), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member, &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); + + *mode.lock().unwrap() = Some(wedge); + // A backlog fills only once clients have connected without being + // accepted. + let mut queued = Vec::new(); + if matches!(wedge, Wedge::FullBacklog) { + for _ in 0..4096 { + match TcpStream::connect_timeout(&member, Duration::from_millis(20)) { + Ok(stream) => queued.push(stream), + Err(_) => break, + } + } + } + let started = Instant::now(); + let mut degraded = false; + while started.elapsed() < Duration::from_secs(6) { + let state = challenge_state(address, &fx); + assert!(state.is_some(), "a challenge went unanswered after {:?}", started.elapsed()); + if state.as_deref() == Some("degraded") { + degraded = true; + break; + } + thread::sleep(Duration::from_millis(100)); + } + assert!(degraded, "the wedged child was never reported degraded"); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + halt.store(true, Ordering::SeqCst); + join_within(Duration::from_secs(10), child); + drop(queued); + } + + #[test] + fn a_silent_child_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::Silent); + } + + #[test] + fn a_child_with_a_full_backlog_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::FullBacklog); + } + + #[test] + fn a_child_answering_after_the_probe_deadline_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::Slow); + } + + /// The virtual time (milliseconds) of each record that has the given + /// detail. + fn times_of(sink: &Sink, detail: &str) -> Vec { + sink.0 + .lock() + .unwrap() + .iter() + .filter(|record| record.detail == detail) + .map(|record| record.observed_at_unix_millis) + .collect() + } + + #[test] + fn a_child_that_never_becomes_healthy_fails_the_unit_at_the_startup_timeout() { + let fx = fixture(false); + let mut contract = fx.contract("x"); + contract.startup = Duration::from_secs(5); + let mut launch = start(&fx, contract, false); + let sink = launch.sink.clone(); + let child = launch.child.clone(); + let outcome = within(Duration::from_secs(20), move || launch.finish()); + assert_eq!(outcome, Outcome::Failed("startup-timeout")); + assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); + let started = times_of(&sink, "launcher-started")[0]; + let failed = times_of(&sink, "startup-timeout")[0]; + // Virtual time: the deadline, found within one poll of it. + let waited = failed - started; + assert!( + (5_000..=5_000 + POLL.as_millis() as u64).contains(&waited), + "failed after {waited} ms of virtual time" + ); + } + + #[test] + fn heartbeats_follow_the_contracts_cadence_while_waiting_for_the_lease_and_while_active() { let fx = fixture(true); - let launch = start(&fx, fx.contract("x"), true); - thread::sleep(Duration::from_millis(250)); - let waiting = launch - .sink - .0 - .lock() - .unwrap() - .iter() - .filter(|record| record.detail == "write-lease-pending") - .count(); - assert!((1..=12).contains(&waiting), "{waiting} lease-wait heartbeats"); - let mut launch = launch; + let mut contract = fx.contract("x"); + contract.heartbeat = Duration::from_secs(1); + let mut launch = start(&fx, contract, true); + let cadence = 1_000..=1_000 + POLL.as_millis() as u64; + wait_until("lease-wait heartbeats", || { + times_of(&launch.sink, "write-lease-pending").len() >= 4 + }); let warming = launch.sink.first("warming").unwrap(); + let mut beats = vec![warming.observed_at_unix_millis]; + beats.extend(times_of(&launch.sink, "write-lease-pending").into_iter().take(4)); + for pair in beats.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "lease-wait beats {beats:?}"); + } *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); - wait_until("active", || launch.sink.first("active").is_some()); - thread::sleep(Duration::from_millis(250)); - let beats = launch - .sink - .0 - .lock() - .unwrap() - .iter() - .filter(|record| record.detail == "heartbeat") - .count(); - assert!((1..=12).contains(&beats), "{beats} active heartbeats"); + wait_until("active heartbeats", || times_of(&launch.sink, "heartbeat").len() >= 4); + let mut beats = vec![times_of(&launch.sink, "child-healthy")[0]]; + beats.extend(times_of(&launch.sink, "heartbeat").into_iter().take(4)); + for pair in beats.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "active beats {beats:?}"); + } assert_eq!(launch.stop().exit_code(), 0); } @@ -2166,6 +2422,7 @@ mod tests { fn observing(fx: &Fixture) -> Responder { let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + shared.refresh_health(); Arc::new(move |request| shared.observe(request)) } @@ -2371,7 +2628,7 @@ mod tests { wait_until("active", || sink.first("active").is_some()); // The first challenge reaches the tripwire on the presence thread. assert!(!challenge_answered(address, &fx)); - assert_eq!(handle.join().unwrap(), Outcome::Failed("presence-acceptor")); + assert_eq!(join_within(Duration::from_secs(20), handle), Outcome::Failed("presence-acceptor")); assert_eq!(sink.first("failed").unwrap().detail, "presence-acceptor"); assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); } @@ -2520,7 +2777,7 @@ mod tests { }); let probe = HttpHealthProbe { member, path: "/health".into() }; let started = Instant::now(); - assert!(!probe.healthy()); + assert!(!within(Duration::from_secs(5), move || probe.healthy())); let elapsed = started.elapsed(); assert!(elapsed >= Duration::from_millis(900), "gave up before its timeout: {elapsed:?}"); assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its timeout: {elapsed:?}"); @@ -2544,10 +2801,10 @@ mod tests { }); let probe = HttpHealthProbe { member, path: "/health".into() }; let started = Instant::now(); - assert!(!probe.healthy()); + assert!(!within(Duration::from_secs(5), move || probe.healthy())); let elapsed = started.elapsed(); assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its deadline: {elapsed:?}"); - let _ = dribbler.join(); + join_within(Duration::from_secs(10), dribbler); } #[test] From c38bb365601cdf5d9a9a01e86cf007de15f5f3fe Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:15:50 +0200 Subject: [PATCH 139/196] Fix route limit bound (nginx allows below 65536); the dead-acceptor test kills the presence thread through the clock Co-Authored-By: Claude Sonnet 5.5 --- src/deployment.rs | 8 +++++--- src/drivers.rs | 6 +++--- src/launcher.rs | 17 +++++++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/deployment.rs b/src/deployment.rs index 75a30ab..56cd8d8 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -699,12 +699,12 @@ pub struct MemberListenerBinding { /// route's presence block shares, so this limit is what keeps members from /// exhausting it. #[serde(default = "default_member_max_connections")] - pub max_connections: u32, + pub max_connections: u16, } -const DEFAULT_MEMBER_MAX_CONNECTIONS: u32 = 128; +const DEFAULT_MEMBER_MAX_CONNECTIONS: u16 = 128; -fn default_member_max_connections() -> u32 { +fn default_member_max_connections() -> u16 { DEFAULT_MEMBER_MAX_CONNECTIONS } @@ -2446,6 +2446,8 @@ max_connections = {value}"), ); assert_eq!(limit("7").unwrap().route.unwrap().member.unwrap().max_connections, 7); assert!(limit("0").is_err()); + assert!(limit("65535").is_ok()); + assert!(limit("65536").is_err()); // Zero, and a width that overflows u16. assert!(start("0").is_err()); assert!(start("65530").is_err()); diff --git a/src/drivers.rs b/src/drivers.rs index 060917d..73feb5f 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -12508,7 +12508,7 @@ processes 9 /// 4103, 4104, 4120 and 4121, so it runs on demand in a container: /// `cargo test --lib real_nginx -- --ignored`. #[cfg(unix)] - fn challenges_answered_during_a_member_flood(max_connections: u32) -> Result { + fn challenges_answered_during_a_member_flood(max_connections: u16) -> Result { use std::net::{TcpListener, TcpStream}; use std::process::{Child, Command}; use std::sync::Arc; @@ -12636,9 +12636,9 @@ processes 9 #[test] #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] fn real_nginx_flood_starves_presence_without_a_member_limit() -> Result<()> { - // The control: the same flood against a limit too high to bind starves + // The control: the same flood against a limit too high to bind (nginx allows up to 65535) starves // presence, so the test above measures the limit and not the load. - assert!(challenges_answered_during_a_member_flood(100_000)? < 20); + assert!(challenges_answered_during_a_member_flood(60_000)? < 20); Ok(()) } diff --git a/src/launcher.rs b/src/launcher.rs index f15882e..b787386 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -28,7 +28,7 @@ use std::thread; use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultcacheclock: Arc::new(PresenceTripwire(FastClock(AtomicU64::new(1_000_000)))),health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))),rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, @@ -2592,14 +2592,19 @@ mod tests { serving.join().unwrap(); } - /// Healthy for everyone but the presence thread, which it kills: the + /// A clock that kills the presence thread when it reads the time: the /// stand-in for a presence acceptor that dies while the child is fine. - struct PresenceTripwire; + /// Answering a challenge signs, and signing reads the clock. + struct PresenceTripwire(FastClock); - impl HealthProbe for PresenceTripwire { - fn healthy(&self) -> bool { + impl Clock for PresenceTripwire { + fn now_millis(&self) -> u64 { assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); - true + self.0.now_millis() + } + + fn sleep(&self, duration: Duration) { + self.0.sleep(duration); } } From d6ebd328e156b1538e35851e9fc5f94a32c5f2ab Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:16:44 +0200 Subject: [PATCH 140/196] Fix route limit bound (nginx allows below 65536); the dead-acceptor test kills the presence thread through the clock Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index b787386..85ed592 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -28,7 +28,7 @@ use std::thread; use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcacheclock: Arc::new(PresenceTripwire(FastClock(AtomicU64::new(1_000_000)))),health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))),rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, @@ -2615,11 +2615,11 @@ mod tests { let child = Arc::new(Child::default()); let ports = Ports { credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + clock: Arc::new(PresenceTripwire(FastClock(AtomicU64::new(1_000_000)))), sink: sink.clone(), lease: Arc::new(FakeLease::default()), spawner: Box::new(FakeSpawner(child.clone())), - health: Box::new(|_| Arc::new(PresenceTripwire)), + health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), }; let contract = fx.contract("x"); let authority = fx.authority(); From 60b64e283d577785f7803ee2754f0c6bea1bc3f6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:33:23 +0200 Subject: [PATCH 141/196] Real-nginx tests stop nginx through nginx so its workers do not hold the test's pipe Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 73feb5f..4fc1577 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -12514,10 +12514,18 @@ processes 9 use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; - struct Nginx(Child); + /// Stopped through nginx itself: killing the master would orphan its + /// workers, which keep the test's output pipe open. + struct Nginx(Child, PathBuf, PathBuf); impl Drop for Nginx { fn drop(&mut self) { - let _ = self.0.kill(); + let _ = Command::new("nginx") + .arg("-p") + .arg(&self.1) + .arg("-c") + .arg(&self.2) + .args(["-s", "stop"]) + .status(); let _ = self.0.wait(); } } @@ -12577,7 +12585,11 @@ processes 9 .arg("-c") .arg(&conf) .args(["-g", "daemon off;"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) .spawn()?, + temp.path().to_path_buf(), + conf.clone(), ); let stable: SocketAddr = "127.0.0.1:4103".parse()?; let ready = Instant::now(); From 1207f219487383653d5e0b7459c67a880e555608 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:34:14 +0200 Subject: [PATCH 142/196] Real-nginx tests report how many challenges were answered Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/drivers.rs b/src/drivers.rs index 4fc1577..44d9976 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -12627,6 +12627,7 @@ processes 9 answered += 1; } } + eprintln!("answered {answered} of 20 challenges with a member limit of {max_connections}"); drop(flood); halt.store(true, Ordering::SeqCst); presence.join().unwrap(); From 2ea0572a4cdd80f1f9fe43f507ab573a2bc422e9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:34:28 +0200 Subject: [PATCH 143/196] Docs: member connection limit and the cached presence answer Co-Authored-By: Claude Sonnet 5.5 --- README.md | 1 + docs/authority-map.md | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3714ea9..db728a9 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ private_port_end = 18839 [route.member] stable_endpoint = "tcp://10.77.0.1:8448" private_port_start = 18841 +# max_connections = 128 # nginx limit_conn on the member listener (1-65535) ``` ## Contracts diff --git a/docs/authority-map.md b/docs/authority-map.md index 2db896f..c8a8546 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -90,10 +90,14 @@ activation key. back. - **Route:** Idunn owns both listeners of the route: the Expected route's stable endpoint (presence) and, when the binding has `[route.member]`, the - member endpoint, rendered as a second server block in the one nginx fragment. + member endpoint, rendered as a second server block in the one nginx fragment + with a `limit_conn` on one per-route zone (`max_connections`, default 128), so + members cannot exhaust the connections the presence block shares with them. The launcher binds the candidate endpoint and answers Idunn's challenge there, one connection at a time within a one-second deadline; any other first byte - closes the connection. The child binds the member bind Idunn sets in + closes the connection. The answer never waits on the child: it reports the + lifecycle's last probe, and a failed, missing or stale probe reads `degraded`. + The child binds the member bind Idunn sets in `GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes never touch the launcher. - **Lease:** with `write_lease_required` the launcher starts the child only From 6b1d14d1596b3acdb0e83b0bb18a3649b65fb985 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:43:04 +0200 Subject: [PATCH 144/196] idunn-launch tests: an Active child is probed at the refresh cadence Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/launcher.rs b/src/launcher.rs index 85ed592..62a04b1 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2375,6 +2375,53 @@ mod tests { assert_eq!(launch.stop().exit_code(), 0); } + + /// Healthy, and notes the virtual time of every probe. + struct RecordingProbe { + clock: Arc, + at: Mutex>, + } + + impl HealthProbe for RecordingProbe { + fn healthy(&self) -> bool { + self.at.lock().unwrap().push(self.clock.now_millis()); + true + } + } + + #[test] + fn an_active_child_is_probed_at_the_refresh_cadence() { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let clock = Arc::new(FastClock(AtomicU64::new(1_000_000))); + let probe = Arc::new(RecordingProbe { clock: clock.clone(), at: Mutex::new(Vec::new()) }); + let seen = probe.clone(); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock, + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(move |_| probe.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("five probes", || seen.at.lock().unwrap().len() >= 5); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + let at = seen.at.lock().unwrap().clone(); + // The first probe ends startup; the rest follow the refresh cadence. + let cadence = HEALTH_REFRESH.as_millis() as u64..=HEALTH_REFRESH.as_millis() as u64 + POLL.as_millis() as u64; + for pair in at[1..].windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "probes at {at:?}"); + } + } + #[test] fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { let mut stubborn = ProcessSpawner From b5d13652e10cf5eda6a324043a91d51d8481a799 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:50:09 +0200 Subject: [PATCH 145/196] idunn-launch tests: the first Active probe is also held to the refresh cadence Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 62a04b1..b2a17be 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2415,9 +2415,9 @@ mod tests { stop.store(true, Ordering::SeqCst); assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); let at = seen.at.lock().unwrap().clone(); - // The first probe ends startup; the rest follow the refresh cadence. + // The first probe ends startup; every later one follows the cadence. let cadence = HEALTH_REFRESH.as_millis() as u64..=HEALTH_REFRESH.as_millis() as u64 + POLL.as_millis() as u64; - for pair in at[1..].windows(2) { + for pair in at.windows(2) { assert!(cadence.contains(&(pair[1] - pair[0])), "probes at {at:?}"); } } From 5d8f7e18c538cb0b098116c2d6c2680b9b050aae Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 12:26:22 +0200 Subject: [PATCH 146/196] idunn-launch: freshness, deadlines and the probe schedule run on a monotonic clock Clock gains monotonic_millis; the cached health result's age, the freshness decision, the probe schedule, heartbeat and lease/startup deadlines all read it. Wall time stays on the signed record only. A result stamped after now is not evidence. Tests step the wall clock both ways, pin the warming-proof cap, and bound the lease-timeout join. Co-Authored-By: Claude Opus 5.5 --- src/launcher.rs | 288 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 261 insertions(+), 27 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index b2a17be..4b775f0 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -379,7 +379,12 @@ pub fn inherited_authority_files() -> Result<(File, File)> { // ---------------------------------------------------------------- ports pub trait Clock: Send + Sync { + /// Wall-clock time, for values signed or published to another party. No + /// decision about freshness, deadlines or schedules reads it. fn now_millis(&self) -> u64; + /// Monotonic time: never steps back, unaffected by wall-clock changes. Every + /// age, deadline and schedule in the launcher reads this one source. + fn monotonic_millis(&self) -> u64; fn sleep(&self, duration: Duration); } @@ -390,6 +395,12 @@ impl Clock for SystemClock { u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0) } + fn monotonic_millis(&self) -> u64 { + static START: std::sync::OnceLock = std::sync::OnceLock::new(); + u64::try_from(START.get_or_init(std::time::Instant::now).elapsed().as_millis()) + .unwrap_or(u64::MAX) + } + fn sleep(&self, duration: Duration) { thread::sleep(duration); } @@ -864,7 +875,7 @@ struct Shared { lease: Mutex>, lease_required: bool, health: Arc, - /// The last probe result and the clock time it started. Only the lifecycle + /// The last probe result and the monotonic time it started. Only the lifecycle /// probes; a challenge reads this and never waits on the child. probed: Mutex>, clock: Arc, @@ -873,7 +884,7 @@ struct Shared { impl Shared { /// Probes the child now and keeps the result for presence to answer from. fn refresh_health(&self) -> bool { - let started = self.clock.now_millis(); + let started = self.clock.monotonic_millis(); let healthy = self.health.healthy(); if let Ok(mut probed) = self.probed.lock() { *probed = Some((healthy, started)); @@ -885,14 +896,14 @@ impl Shared { /// A child never probed, wedged, or probed too long ago reads degraded. A /// dead presence acceptor does not degrade presence: it fails the unit. fn serving_state(&self) -> &'static str { - let now = self.clock.now_millis(); + let now = self.clock.monotonic_millis(); let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); let healthy = self .probed .lock() .ok() .and_then(|probed| *probed) - .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); + .is_some_and(|(healthy, at)| healthy && now.checked_sub(at).is_some_and(|age| age <= stale)); if healthy { "active" } else { "degraded" } } @@ -1178,13 +1189,13 @@ fn lifecycle( }; remember(publish("warming", "launcher-started", None), &mut proofs); let heartbeat = u64::try_from(contract.heartbeat.as_millis()).unwrap_or(u64::MAX); - let mut next_heartbeat = clock.now_millis() + heartbeat; + let mut next_heartbeat = clock.monotonic_millis() + heartbeat; // 2. A stateful child waits for the lease; a stateless one does not. let mut lease_sha = None; let mut lease_fd = None; if expected.write_lease_required { - let deadline = clock.now_millis() + contract.lease_wait.as_millis() as u64; + let deadline = clock.monotonic_millis() + contract.lease_wait.as_millis() as u64; loop { if stop.load(Ordering::SeqCst) { return Outcome::Stopped; @@ -1206,7 +1217,7 @@ fn lifecycle( } break; } - let now = clock.now_millis(); + let now = clock.monotonic_millis(); if now >= deadline { return fail(None, "lease-timeout"); } @@ -1222,7 +1233,7 @@ fn lifecycle( let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { return fail(None, "child-start-refused"); }; - let startup_deadline = clock.now_millis() + contract.startup.as_millis() as u64; + let startup_deadline = clock.monotonic_millis() + contract.startup.as_millis() as u64; loop { if stop.load(Ordering::SeqCst) { child.terminate(TERMINATE_GRACE); @@ -1237,7 +1248,7 @@ fn lifecycle( if shared.refresh_health() { break; } - if clock.now_millis() >= startup_deadline { + if clock.monotonic_millis() >= startup_deadline { return fail(Some(&mut child), "startup-timeout"); } clock.sleep(POLL); @@ -1247,8 +1258,8 @@ fn lifecycle( set_phase(Phase::Active(lease_sha.clone())); publish(shared.serving_state(), "child-healthy", lease_sha.clone()); let refresh = u64::try_from(HEALTH_REFRESH.as_millis()).unwrap_or(u64::MAX); - let mut next_probe = clock.now_millis() + refresh; - next_heartbeat = clock.now_millis() + heartbeat; + let mut next_probe = clock.monotonic_millis() + refresh; + next_heartbeat = clock.monotonic_millis() + heartbeat; loop { if stop.load(Ordering::SeqCst) { child.terminate(TERMINATE_GRACE); @@ -1263,7 +1274,7 @@ fn lifecycle( if !shared.lease_current() { return fail(Some(&mut child), "lease-lost"); } - let now = clock.now_millis(); + let now = clock.monotonic_millis(); if now >= next_probe { shared.refresh_health(); next_probe = now + refresh; @@ -1307,7 +1318,7 @@ fn start_child( mod tests { use std::collections::BTreeMap; use std::os::unix::fs::PermissionsExt; - use std::sync::atomic::{AtomicU64, AtomicUsize}; + use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize}; use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; use cultnet_rs::{ @@ -1482,15 +1493,34 @@ mod tests { } /// Virtual time runs a hundred times faster than the wall clock. - struct FastClock(AtomicU64); + struct FastClock { + monotonic: AtomicU64, + wall_step: AtomicI64, + } + + impl FastClock { + fn new(start: u64) -> Self { + Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(0) } + } + + /// Steps the wall clock alone, as timesyncd or a VM restore would. + fn step_wall(&self, millis: i64) { + self.wall_step.fetch_add(millis, Ordering::SeqCst); + } + } impl Clock for FastClock { fn now_millis(&self) -> u64 { - self.0.load(Ordering::SeqCst) + let wall = self.monotonic.load(Ordering::SeqCst) as i64 + self.wall_step.load(Ordering::SeqCst); + u64::try_from(wall).unwrap_or(0) + } + + fn monotonic_millis(&self) -> u64 { + self.monotonic.load(Ordering::SeqCst) } fn sleep(&self, duration: Duration) { - self.0 + self.monotonic .fetch_add(duration.as_millis() as u64, Ordering::SeqCst); thread::sleep(Duration::from_millis(1)); } @@ -1629,7 +1659,7 @@ mod tests { secrets.insert("SYNAPSE_KEY".to_owned(), b"s3cret-value\n".to_vec()); let ports = Ports { credentials: Arc::new(Credentials(secrets)), - clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + clock: Arc::new(FastClock::new(1_000_000)), sink: sink.clone(), lease: lease.clone(), spawner: Box::new(FakeSpawner(child.clone())), @@ -1720,9 +1750,13 @@ mod tests { let mut lease = fx.lease(&digest('8')); edit(&mut lease); *launch.lease.0.lock().unwrap() = Some(lease); - assert_eq!(launch.finish(), Outcome::Failed("lease-timeout"), "{name}"); - assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0, "{name}"); - assert_eq!(launch.sink.states().last().unwrap(), "failed", "{name}"); + let (child, sink) = (launch.child.clone(), launch.sink.clone()); + // Bounded: a lease deadline that never arrives fails here, not at the + // suite's outer timeout. + let outcome = within(Duration::from_secs(20), move || launch.finish()); + assert_eq!(outcome, Outcome::Failed("lease-timeout"), "{name}"); + assert_eq!(child.spawns.load(Ordering::SeqCst), 0, "{name}"); + assert_eq!(sink.states().last().unwrap(), "failed", "{name}"); } } @@ -2134,18 +2168,214 @@ mod tests { #[test] fn a_probe_result_is_evidence_only_while_it_is_recent() { let fx = fixture(false); - let clock = Arc::new(FastClock(AtomicU64::new(1_000_000))); + let clock = Arc::new(FastClock::new(1_000_000)); let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); shared.refresh_health(); let stale = HEALTH_STALE.as_millis() as u64; - clock.0.fetch_add(stale, Ordering::SeqCst); + clock.monotonic.fetch_add(stale, Ordering::SeqCst); assert_eq!(shared.serving_state(), "active", "a result exactly at the bound is still recent"); - clock.0.fetch_add(1, Ordering::SeqCst); + clock.monotonic.fetch_add(1, Ordering::SeqCst); assert_eq!(shared.serving_state(), "degraded", "a result past the bound is not evidence"); shared.refresh_health(); assert_eq!(shared.serving_state(), "active"); } + /// A lease reader that grants, at the first read once `after` Warming + /// records exist, a lease bound to the Warming record `back` places before + /// the newest. The launcher reads it from its own thread between publishes, + /// so the record it names is at a known place in the retained window. + struct WindowLease { + sink: Arc, + template: IdunnProcessWriteLeaseRecord, + after: usize, + back: usize, + granted: Mutex>, + } + + impl LeaseReader for WindowLease { + fn read(&self) -> Result> { + let mut granted = self.granted.lock().unwrap(); + if granted.is_none() { + let records = self.sink.0.lock().unwrap(); + let warming: Vec<_> = records.iter().filter(|r| r.state == "warming").collect(); + if warming.len() >= self.after { + let mut lease = self.template.clone(); + lease.warming_presence_sha256 = + warming[warming.len() - self.back].canonical_sha256().unwrap(); + *granted = Some(lease); + } + } + Ok(granted.clone()) + } + + fn hold_shared(&self) -> Result> { + Ok(Box::new(FakeLock)) + } + } + + /// Starts a stateful unit waiting on a `WindowLease` and reports whether it + /// accepted the lease (spawned the child) or retried well past the grant. + fn warming_window_accepts(back: usize) -> bool { + let fx = fixture(true); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let lease = Arc::new(WindowLease { + sink: sink.clone(), + template: fx.lease(""), + after: MAX_RECENT_WARMING_PROOFS + 6, + back, + granted: Mutex::new(None), + }); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock::new(1_000_000)), + sink: sink.clone(), + lease: lease.clone(), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), + }; + let mut contract = fx.contract("x"); + contract.heartbeat = POLL; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("the lease is granted", || lease.granted.lock().unwrap().is_some()); + let warming = || sink.states().iter().filter(|s| *s == "warming").count(); + let granted_at = warming(); + let mut spawned = false; + for _ in 0..4000 { + spawned = child.spawns.load(Ordering::SeqCst) == 1; + if spawned || warming() >= granted_at + 10 { + break; + } + thread::sleep(Duration::from_millis(5)); + } + stop.store(true, Ordering::SeqCst); + // A mutant may wedge the unit; only a unit that took the lease is joined. + if spawned { + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + } + spawned + } + + #[test] + fn the_warming_proof_window_keeps_exactly_the_cap() { + assert!( + warming_window_accepts(MAX_RECENT_WARMING_PROOFS), + "the oldest of the retained Warming proofs is accepted" + ); + assert!( + !warming_window_accepts(MAX_RECENT_WARMING_PROOFS + 1), + "a Warming proof past the cap is no longer accepted" + ); + } + + /// The child is healthy, then the wall clock steps by `step_millis`, then + /// the child wedges. A challenge must keep reading the truth throughout. + fn a_wall_clock_step_never_decides_freshness(step_millis: i64) { + struct Toggle { + healthy: AtomicBool, + probes: AtomicUsize, + } + impl HealthProbe for Toggle { + fn healthy(&self) -> bool { + self.probes.fetch_add(1, Ordering::SeqCst); + self.healthy.load(Ordering::SeqCst) + } + } + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let clock = Arc::new(FastClock::new(100_000_000)); + let probe = Arc::new(Toggle { healthy: AtomicBool::new(true), probes: AtomicUsize::new(0) }); + let given = probe.clone(); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: clock.clone(), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(move |_| given.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); + + clock.step_wall(step_millis); + for _ in 0..10 { + assert_eq!( + challenge_state(address, &fx).as_deref(), + Some("active"), + "a healthy child reads active across a wall-clock step of {step_millis} ms" + ); + thread::sleep(Duration::from_millis(20)); + } + + probe.healthy.store(false, Ordering::SeqCst); + let probed = probe.probes.load(Ordering::SeqCst); + let published = sink.0.lock().unwrap().len(); + let started = Instant::now(); + let mut state = None; + while started.elapsed() < Duration::from_secs(5) { + state = challenge_state(address, &fx); + if state.as_deref() == Some("degraded") { + break; + } + thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + state.as_deref(), + Some("degraded"), + "a wedged child reads degraded within the freshness window after a wall-clock step of {step_millis} ms" + ); + assert!(probe.probes.load(Ordering::SeqCst) > probed, "probing continued"); + wait_until("a heartbeat after the step", || sink.0.lock().unwrap().len() > published); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + } + + #[test] + fn a_backward_wall_clock_step_does_not_keep_a_wedged_child_active() { + a_wall_clock_step_never_decides_freshness(-600_000); + } + + #[test] + fn a_forward_wall_clock_step_does_not_make_a_healthy_child_degraded() { + a_wall_clock_step_never_decides_freshness(600_000); + } + + #[test] + fn the_cached_result_ages_on_the_monotonic_clock_alone() { + let fx = fixture(false); + let clock = Arc::new(FastClock::new(100_000_000)); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + for step in [-600_000, 1_200_000] { + clock.step_wall(step); + assert_eq!(shared.serving_state(), "active", "wall step {step}"); + } + clock.monotonic.fetch_add(HEALTH_STALE.as_millis() as u64 + 1, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "degraded", "monotonic age decides, whatever the wall says"); + } + + #[test] + fn a_result_stamped_after_now_is_not_evidence() { + let fx = fixture(false); + let clock = Arc::new(FastClock::new(100_000_000)); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + *shared.probed.lock().unwrap() = Some((true, 100_000_001)); + assert_eq!(shared.serving_state(), "degraded"); + } + /// Runs `work` and returns its result, failing the test when it has not /// finished within `limit`, so a mutant that makes it wait forever fails /// instead of hanging the suite. @@ -2384,7 +2614,7 @@ mod tests { impl HealthProbe for RecordingProbe { fn healthy(&self) -> bool { - self.at.lock().unwrap().push(self.clock.now_millis()); + self.at.lock().unwrap().push(self.clock.monotonic_millis()); true } } @@ -2393,7 +2623,7 @@ mod tests { fn an_active_child_is_probed_at_the_refresh_cadence() { let fx = fixture(false); let sink = Arc::new(Sink::default()); - let clock = Arc::new(FastClock(AtomicU64::new(1_000_000))); + let clock = Arc::new(FastClock::new(1_000_000)); let probe = Arc::new(RecordingProbe { clock: clock.clone(), at: Mutex::new(Vec::new()) }); let seen = probe.clone(); let ports = Ports { @@ -2650,6 +2880,10 @@ mod tests { self.0.now_millis() } + fn monotonic_millis(&self) -> u64 { + self.0.monotonic_millis() + } + fn sleep(&self, duration: Duration) { self.0.sleep(duration); } @@ -2662,7 +2896,7 @@ mod tests { let child = Arc::new(Child::default()); let ports = Ports { credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(PresenceTripwire(FastClock(AtomicU64::new(1_000_000)))), + clock: Arc::new(PresenceTripwire(FastClock::new(1_000_000))), sink: sink.clone(), lease: Arc::new(FakeLease::default()), spawner: Box::new(FakeSpawner(child.clone())), @@ -2797,7 +3031,7 @@ mod tests { let health = Arc::new(LateHealth { calls: AtomicUsize::new(0), probes: 5 }); let ports = Ports { credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + clock: Arc::new(FastClock::new(1_000_000)), sink: sink.clone(), lease: Arc::new(FakeLease::default()), spawner: Box::new(FakeSpawner(child.clone())), From 06782d38f20387ad47e247ca0f137190377d16b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 12:27:38 +0200 Subject: [PATCH 147/196] idunn-launch tests: the presence tripwire trips on the monotonic read, before any lock is held Co-Authored-By: Claude Opus 5.5 --- src/launcher.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 4b775f0..d1cc49f 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2871,16 +2871,17 @@ mod tests { /// A clock that kills the presence thread when it reads the time: the /// stand-in for a presence acceptor that dies while the child is fine. - /// Answering a challenge signs, and signing reads the clock. + /// Answering a challenge reads the cached result's age, which reads the + /// monotonic clock before any lock is held. struct PresenceTripwire(FastClock); impl Clock for PresenceTripwire { fn now_millis(&self) -> u64 { - assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); self.0.now_millis() } fn monotonic_millis(&self) -> u64 { + assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); self.0.monotonic_millis() } From d111986093ab168ef8360b80981be003819297ce Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 14:23:28 +0200 Subject: [PATCH 148/196] Launcher: pin deadline clock sources, read presence freshness in a race-free order, bound declared durations The test clock's wall now starts a Unix-epoch away from the monotonic clock, so a deadline or schedule that reads the wrong source fails. serving_state reads the cached probe before the clock (a stamp is taken before it is stored, so the later reading is never behind it) and ages with saturating_sub; the test that pinned the old wrong answer is replaced by a deterministic descheduled-reader test. The lease-wait, startup and heartbeat seconds are bounded to one day at parse with a refusal that names the flag and not the value, and every deadline is computed by one saturating helper. Co-Authored-By: Claude Opus 5.5 --- src/launcher.rs | 144 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 119 insertions(+), 25 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index d1cc49f..13937f6 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -65,6 +65,9 @@ const HEALTH_REFRESH: Duration = Duration::from_millis(500); /// that refreshes it is itself stuck. const HEALTH_STALE: Duration = Duration::from_secs(5); const POLL: Duration = Duration::from_millis(100); +/// The largest duration a recipe may declare for the lease wait, the startup +/// wait or the heartbeat: one day. Past it a deadline is not a deadline. +const MAX_DECLARED_SECONDS: u64 = 86_400; const MAX_RECENT_WARMING_PROOFS: usize = 64; const TERMINATE_GRACE: Duration = Duration::from_secs(20); const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; @@ -110,8 +113,10 @@ impl LaunchContract { value .parse::() .ok() - .filter(|seconds| *seconds > 0) - .with_context(|| format!("launcher argument {field} is not a positive integer")) + .filter(|seconds| (1..=MAX_DECLARED_SECONDS).contains(seconds)) + .with_context(|| { + format!("launcher argument {field} is outside 1 to {MAX_DECLARED_SECONDS} seconds") + }) }; match flag.as_str() { "--child" => child = Some(release_relative(&value, "--child")?), @@ -896,14 +901,13 @@ impl Shared { /// A child never probed, wedged, or probed too long ago reads degraded. A /// dead presence acceptor does not degrade presence: it fails the unit. fn serving_state(&self) -> &'static str { - let now = self.clock.monotonic_millis(); let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); - let healthy = self - .probed - .lock() - .ok() - .and_then(|probed| *probed) - .is_some_and(|(healthy, at)| healthy && now.checked_sub(at).is_some_and(|age| age <= stale)); + // The result is read before the clock: a stamp is taken before it is + // stored, so a later reading of the monotonic clock is never behind it, + // whatever the presence thread does between the two. + let probed = self.probed.lock().ok().and_then(|probed| *probed); + let now = self.clock.monotonic_millis(); + let healthy = probed.is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); if healthy { "active" } else { "degraded" } } @@ -1093,6 +1097,12 @@ pub fn parse_member_bind(raw: &str, candidate_bind: &str) -> Result Ok(member) } +/// The monotonic reading `span` after `now`; a span too long to represent is a +/// deadline that never arrives. +fn after(now: u64, span: Duration) -> u64 { + now.saturating_add(u64::try_from(span.as_millis()).unwrap_or(u64::MAX)) +} + /// The unit's whole life: Warming, lease, child, Active, until the child or the /// lease ends or SIGTERM arrives. `route` is the already-bound candidate /// listener, answered for presence only; `member` is the address the child @@ -1188,14 +1198,14 @@ fn lifecycle( } }; remember(publish("warming", "launcher-started", None), &mut proofs); - let heartbeat = u64::try_from(contract.heartbeat.as_millis()).unwrap_or(u64::MAX); - let mut next_heartbeat = clock.monotonic_millis() + heartbeat; + let heartbeat = contract.heartbeat; + let mut next_heartbeat = after(clock.monotonic_millis(), heartbeat); // 2. A stateful child waits for the lease; a stateless one does not. let mut lease_sha = None; let mut lease_fd = None; if expected.write_lease_required { - let deadline = clock.monotonic_millis() + contract.lease_wait.as_millis() as u64; + let deadline = after(clock.monotonic_millis(), contract.lease_wait); loop { if stop.load(Ordering::SeqCst) { return Outcome::Stopped; @@ -1223,7 +1233,7 @@ fn lifecycle( } if now >= next_heartbeat { remember(publish("warming", "write-lease-pending", None), &mut proofs); - next_heartbeat = now + heartbeat; + next_heartbeat = after(now, heartbeat); } clock.sleep(POLL); } @@ -1233,7 +1243,7 @@ fn lifecycle( let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { return fail(None, "child-start-refused"); }; - let startup_deadline = clock.monotonic_millis() + contract.startup.as_millis() as u64; + let startup_deadline = after(clock.monotonic_millis(), contract.startup); loop { if stop.load(Ordering::SeqCst) { child.terminate(TERMINATE_GRACE); @@ -1257,9 +1267,8 @@ fn lifecycle( // 4. Active, and heartbeats while it holds. set_phase(Phase::Active(lease_sha.clone())); publish(shared.serving_state(), "child-healthy", lease_sha.clone()); - let refresh = u64::try_from(HEALTH_REFRESH.as_millis()).unwrap_or(u64::MAX); - let mut next_probe = clock.monotonic_millis() + refresh; - next_heartbeat = clock.monotonic_millis() + heartbeat; + let mut next_probe = after(clock.monotonic_millis(), HEALTH_REFRESH); + next_heartbeat = after(clock.monotonic_millis(), heartbeat); loop { if stop.load(Ordering::SeqCst) { child.terminate(TERMINATE_GRACE); @@ -1277,11 +1286,11 @@ fn lifecycle( let now = clock.monotonic_millis(); if now >= next_probe { shared.refresh_health(); - next_probe = now + refresh; + next_probe = after(now, HEALTH_REFRESH); } if now >= next_heartbeat { publish(shared.serving_state(), "heartbeat", lease_sha.clone()); - next_heartbeat = now + heartbeat; + next_heartbeat = after(now, heartbeat); } clock.sleep(POLL); } @@ -1492,6 +1501,8 @@ mod tests { } } + const WALL_EPOCH_MILLIS: i64 = 1_760_000_000_000; + /// Virtual time runs a hundred times faster than the wall clock. struct FastClock { monotonic: AtomicU64, @@ -1499,8 +1510,11 @@ mod tests { } impl FastClock { + /// The wall clock starts a Unix-epoch-sized distance from the monotonic + /// one, as the real pair does, so a deadline or schedule that reads the + /// wrong source is off by that distance and no test can pass by it. fn new(start: u64) -> Self { - Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(0) } + Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(WALL_EPOCH_MILLIS) } } /// Steps the wall clock alone, as timesyncd or a VM restore would. @@ -2367,13 +2381,93 @@ mod tests { assert_eq!(shared.serving_state(), "degraded", "monotonic age decides, whatever the wall says"); } + /// A monotonic clock whose read, on a thread that marks itself, returns the + /// current value and then waits to be released: the presence thread + /// descheduled right after it reads the time. + struct HeldClock { + monotonic: AtomicU64, + held: Mutex>>, + release: Mutex>>, + } + + thread_local! { + static HOLD_THIS_READ: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + + impl Clock for HeldClock { + fn now_millis(&self) -> u64 { + 0 + } + + fn monotonic_millis(&self) -> u64 { + let value = self.monotonic.load(Ordering::SeqCst); + if HOLD_THIS_READ.with(std::cell::Cell::get) { + self.held.lock().unwrap().take().unwrap().send(()).unwrap(); + self.release.lock().unwrap().take().unwrap().recv().unwrap(); + } + value + } + + fn sleep(&self, _: Duration) {} + } + #[test] - fn a_result_stamped_after_now_is_not_evidence() { + fn a_presence_read_descheduled_across_a_probe_still_sees_a_healthy_child() { let fx = fixture(false); - let clock = Arc::new(FastClock::new(100_000_000)); + let (held_tx, held_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let clock = Arc::new(HeldClock { + monotonic: AtomicU64::new(1_000), + held: Mutex::new(Some(held_tx)), + release: Mutex::new(Some(release_rx)), + }); let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); - *shared.probed.lock().unwrap() = Some((true, 100_000_001)); - assert_eq!(shared.serving_state(), "degraded"); + shared.refresh_health(); + let reader = { + let shared = shared.clone(); + thread::spawn(move || { + HOLD_THIS_READ.with(|hold| hold.set(true)); + shared.serving_state() + }) + }; + // The reader is descheduled in its clock read. The lifecycle probes a + // millisecond later and stores a result stamped after the reader's time. + held_rx.recv().unwrap(); + clock.monotonic.store(1_001, Ordering::SeqCst); + assert!(shared.refresh_health()); + release_tx.send(()).unwrap(); + assert_eq!(reader.join().unwrap(), "active"); + } + + #[test] + fn the_declared_durations_are_bounded_at_parse() { + let base: Vec = [ + "--child", "bin/c", "--template", "t", "--health-path", "/h", "--health-contract", "hc", + ] + .map(String::from) + .to_vec(); + for flag in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { + for (value, accepted) in [ + (MAX_DECLARED_SECONDS.to_string(), true), + ((MAX_DECLARED_SECONDS + 1).to_string(), false), + (u64::MAX.to_string(), false), + ] { + let mut args = base.clone(); + args.extend([flag.to_owned(), value.clone()]); + let parsed = LaunchContract::parse(PathBuf::from("/r"), args); + assert_eq!(parsed.is_ok(), accepted, "{flag} {value}"); + if let Err(error) = parsed { + assert!(!format!("{error:#}").contains(&value), "the refusal echoes its input"); + } + } + } + } + + #[test] + fn a_span_too_long_for_the_clock_is_a_deadline_that_never_arrives() { + assert_eq!(after(10, Duration::from_millis(5)), 15); + assert_eq!(after(u64::MAX - 1, Duration::from_secs(1)), u64::MAX); + assert_eq!(after(10, Duration::MAX), u64::MAX); } /// Runs `work` and returns its result, failing the test when it has not @@ -2872,7 +2966,7 @@ mod tests { /// A clock that kills the presence thread when it reads the time: the /// stand-in for a presence acceptor that dies while the child is fine. /// Answering a challenge reads the cached result's age, which reads the - /// monotonic clock before any lock is held. + /// monotonic clock. struct PresenceTripwire(FastClock); impl Clock for PresenceTripwire { From d74ac8da5c6ba9a2f81ff61599d16a013c3f8303 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 15:01:56 +0200 Subject: [PATCH 149/196] Launcher: presence freshness treats a stamp after the reading as age zero; pin the bound literally Reading the clock after the cached result still left the answer to ordering; the one place freshness is decided is the age subtraction, and monotonic time cannot run backwards, so a result stamped after the reading is fresher than it and ages zero. The bound test names 86400 and 86401 so raising the constant is caught. Co-Authored-By: Claude Opus 5.5 --- src/launcher.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 13937f6..435f8b1 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -902,12 +902,17 @@ impl Shared { /// dead presence acceptor does not degrade presence: it fails the unit. fn serving_state(&self) -> &'static str { let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); - // The result is read before the clock: a stamp is taken before it is - // stored, so a later reading of the monotonic clock is never behind it, - // whatever the presence thread does between the two. - let probed = self.probed.lock().ok().and_then(|probed| *probed); let now = self.clock.monotonic_millis(); - let healthy = probed.is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); + // A probe can finish between this thread reading the clock and locking + // the result, so a stamp later than `now` is a result fresher than the + // reading, age zero: monotonic time does not run backwards, so the stamp + // is not wrong. + let healthy = self + .probed + .lock() + .ok() + .and_then(|probed| *probed) + .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); if healthy { "active" } else { "degraded" } } @@ -2448,8 +2453,8 @@ mod tests { .to_vec(); for flag in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { for (value, accepted) in [ - (MAX_DECLARED_SECONDS.to_string(), true), - ((MAX_DECLARED_SECONDS + 1).to_string(), false), + ("86400".to_owned(), true), + ("86401".to_owned(), false), (u64::MAX.to_string(), false), ] { let mut args = base.clone(); From 332dbd01494909034f8c824a4317c59bf24fe5c6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 19:42:34 +0200 Subject: [PATCH 150/196] deploy-provenance map: tree executable members (body facts, rationale) Imagination pass for cut tree-executable-members (session self-2026-10-23-eureka-b): seal, digest and receipt facts, symlink rule, live binary and install path, disk sizing for the acceptance job. Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-map.md | 113 ++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 docs/deploy-provenance-map.md diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md new file mode 100644 index 0000000..cfbd11b --- /dev/null +++ b/docs/deploy-provenance-map.md @@ -0,0 +1,113 @@ +# Deployment provenance: map + +Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, +session `self-2026-10-23-eureka-b`). The ends are in +[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, +rulings, cut specs and follow-ups live in Eureka's mind; this page holds what +has no kind: body facts, rationale and sequencing. Line numbers are at +`d74ac8d` (origin/main on 2026-10-23). + +## Cut `tree-executable-members` + +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r1`. It +serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's +admission waits on it) and will serve gamecult-chat C3 too: the presence +launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact +(README, launcher fragment), and Synapse needs a Python interpreter there. + +### Body facts + +- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable` + bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and + `validate` (`:740-760`) requires it to equal the declaration. +- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its + executable set from `root.join(receipt.destination)` only. `harden_root_tree` + (`:6858-6892`) seals every directory 0555 and every file 0444 unless that + exact path is in the set. A directory artifact with `executable = true` is + therefore silently meaningless today: its files are all 0444. Nothing refuses + it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin, + StreamPixels service and web surveyed): `executable = true` is on files only. +- T3. A tree output is copied by `copy_artifact`/`copy_tree` + (`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest + (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not + mode bits. So runner or git exec bits never reach the release: harden is the + only mode authority. +- T4. `validate_release_symlink` (`:6895-6914`) keeps a symlink only if its + target is relative with no `..` and lexically inside the release. That rule is + sound without physical resolution (no `..` means a target can only descend). + Absolute links and `..` links refuse the release. The frozen-source path has + a physical resolver (`resolve_frozen_source_symlink`, tested at `:11826`) that + admits in-tree `..`; reuse it if a real tree needs `..` links, not a new rule. +- T5. The sealed release id hashes `(schema, plan_id, artifacts, + external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes + the whole plan including `recipe_blob`. A changed member list is therefore + already a new plan and a new release. The receipt must still carry the list, + because `harden_installed_release` reads receipts only; its two callers + (`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden + every installed release on each observation. +- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with + `#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every + existing receipt's bytes and release id: the plan's `deadlines` field + (`:574-577`) is the precedent. +- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file + whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB) + is held in Idunn's memory three times per install (seal, install, every + validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB + chunks (S9). The cut collapses both onto one streaming helper. +- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the + artifact destination a single filename. Members are paths inside that + destination, so it does not change. `require_relative_path` (`:1936-1949`) + with `allow_current = false` already refuses absolute, `..`, root and prefix + components. +- T9. The host actuator builds receipts separately (`src/host.rs:1043-1081`) + and must carry the same field and the same shape check. +- T10. The transient unit runs with `ProtectSystem=strict` and the release in + `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no + `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. + Shared libraries need no exec bit. +- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`, + installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is + identical between `8760a4f` and `6e2a31c`, so the live code is the + reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in + `src/`: the presence launcher and member route). Installing this cut ships all + of them. Idunn's own body changes only through gamecult-ops + `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, + bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three + copies (runner workspace, staging, installed). Nothing collects old releases or + staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every + Vili deploy leaves about 7 GiB behind. + +### Contract, corrected against the body + +- Any directory artifact may declare `executable_members` (a set of relative + paths). The follow-up limited it to runner-output; that limit protects no + invariant, and the launcher's worktree or fetched trees need the same feature. +- `executable = true` names a file artifact. On a directory output the release + is refused, and `executable_members` on a file output is refused (T2). +- A member is refused at recipe parse if it is empty, absolute, or holds `.`, + `..`, a root or a prefix. At seal it is refused if it is missing, is a + symlink, passes through a symlink or non-directory, or is not a regular file. + Refusal happens before `SealedRelease::new`, so no sealed release can name a + member it cannot seal. +- Harden seals each member 0555 and fails if any declared executable path was + not visited as a regular file. "Every declared executable is sealed 0555, or + the install fails" holds structurally, not by a lookup that can miss. +- Symlinks: relative, no `..`, in-tree links survive (T4). Vili's trees must + not come from `python -m venv`: its `bin/python` is an absolute link and its + `pyvenv.cfg` names the build workspace. Install packages into the relocatable + interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). + +### Authority + +Owner: the seal (`collect_artifact`, and `host.rs` for host targets) decides +membership, from the recipe, and writes it into the receipt. Harden enforces +the receipt and decides nothing. Runner exec bits and git `100755` modes are +dead inputs for release modes. + +### Acceptance sizing + +The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch +on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It +removes it at the end. If the trees hold any absolute or `..` symlink, the job +stops and the T4 resolver reuse becomes a revision of this spec. From 022dbece5becdb2f30af0aed19c28362c42ecc00 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 19:53:16 +0200 Subject: [PATCH 151/196] Directory artifacts declare executable_members; the seal checks them, the receipt carries them, harden seals exactly those 0555 ArtifactOutput and ArtifactReceipt gain executable_members (the receipt field is skipped when empty, so existing sealed release ids are unchanged). check_artifact_modes is the one seal-time owner of mode declarations and runs from both collect_artifact and the host materializer. harden_installed_release drains the declared set and fails when a declared path was never sealed. Digests stream through one hash_file_streamed helper; the encoding is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- src/deployment.rs | 82 ++++++ src/deployment_plan.rs | 117 ++++++++- src/drivers.rs | 581 ++++++++++++++++++++++++++++++++++++++--- src/host.rs | 7 +- 4 files changed, 753 insertions(+), 34 deletions(-) diff --git a/src/deployment.rs b/src/deployment.rs index 56cd8d8..b7cbdc8 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -113,6 +113,10 @@ pub struct ArtifactOutput { pub expected_sha256: Option, #[serde(default)] pub executable: bool, + /// Regular files inside a directory artifact that are sealed executable + /// (0555); every other file in the tree is sealed 0444. + #[serde(default)] + pub executable_members: BTreeSet, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -912,6 +916,14 @@ impl TargetDeclaration { if let Some(expected_sha256) = &artifact.expected_sha256 { require_sha256(expected_sha256, "expected artifact sha256")?; } + ensure!( + !(artifact.executable && !artifact.executable_members.is_empty()), + "artifact {} declares both executable and executable_members", + artifact.id + ); + for member in &artifact.executable_members { + require_relative_path(member, "executable member", false)?; + } ensure!( destinations.insert(artifact.destination.as_path()), "artifact destination {} is declared twice", @@ -2211,6 +2223,76 @@ nodes = ["yggdrasil"] assert!(TargetDeclaration::parse(&input).is_err()); } + const MEMBER_CANARY: &str = "zzcanary"; + + fn recipe_with_members(members: &str) -> String { + let input = RECIPE.replace("executable = true", &format!("executable_members = [{members}]")); + assert_ne!(input, RECIPE, "the fixture lost its executable line"); + input + } + + /// A refused member list is refused by name of the field, never by echoing + /// the member. + fn assert_members_refused(members: &[&str]) { + let control = TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).unwrap(); + assert!(control.artifacts[0].executable_members.contains(Path::new("bin/daemon"))); + for member in members { + let error = TargetDeclaration::parse(&recipe_with_members(&format!("\"{member}\""))) + .expect_err(member); + assert!( + !format!("{error:#}").contains(MEMBER_CANARY), + "the refusal echoed a member path" + ); + } + } + + #[test] + fn recipe_executable_member_refuses_parent_traversal() { + assert_members_refused(&[ + "zzcanary/../b", + "../zzcanary", + "bin/zzcanary/..", + "bin/../../zzcanary", + ]); + } + + #[test] + fn recipe_executable_member_refuses_absolute_path() { + assert_members_refused(&["/zzcanary/sh", "/zzcanary"]); + } + + #[test] + fn recipe_executable_member_refuses_empty_and_current_dir() { + assert_members_refused(&["", ".", "./", "zzcanary/.."]); + } + + #[test] + fn recipe_refuses_executable_flag_with_members() { + let input = RECIPE.replace( + "executable = true", + "executable = true\nexecutable_members = [\"bin/daemon\"]", + ); + let error = TargetDeclaration::parse(&input).unwrap_err(); + assert!(format!("{error:#}").contains("executable_members")); + assert!(!format!("{error:#}").contains("bin/daemon")); + // executable = true alone, and members alone, both stay admitted. + assert!(TargetDeclaration::parse(RECIPE).is_ok()); + assert!(TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).is_ok()); + } + + #[test] + fn recipe_admits_members_on_both_source_kinds() { + let runner_output = TargetDeclaration::parse(&recipe_with_members("\"a/b\", \"c\"")).unwrap(); + assert_eq!(runner_output.artifacts[0].source_kind, ArtifactSource::RunnerOutput); + assert_eq!(runner_output.artifacts[0].executable_members.len(), 2); + let worktree = recipe_with_members("\"a/b\", \"c\"") + .replace("source_kind = \"runner-output\"\nrunner = \"rust\"\n", "source_kind = \"worktree-tree\"\n") + .replace("expected_sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"\n", ""); + let worktree = TargetDeclaration::parse(&worktree).unwrap(); + assert_eq!(worktree.artifacts[0].source_kind, ArtifactSource::WorktreeTree); + assert_eq!(worktree.artifacts[0].executable_members.len(), 2); + } + #[test] fn operator_cannot_omit_an_affordance_derived_from_the_recipe() { let recipe = TargetDeclaration::parse(RECIPE).unwrap(); diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 1363c48..eceb513 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -734,6 +734,11 @@ pub struct ArtifactReceipt { pub sha256: String, pub size_bytes: u64, pub executable: bool, + /// Files sealed executable inside a directory artifact. Skipped when empty + /// so a receipt with no members keeps its original encoding, and so its + /// sealed release id. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub executable_members: BTreeSet, } impl ArtifactReceipt { @@ -747,7 +752,9 @@ impl ArtifactReceipt { .find(|artifact| artifact.id == self.artifact_id) .with_context(|| format!("plan declares no artifact {}", self.artifact_id))?; ensure!( - self.destination == declared.destination && self.executable == declared.executable, + self.destination == declared.destination + && self.executable == declared.executable + && self.executable_members == declared.executable_members, "artifact receipt differs from declared output" ); if let Some(expected_sha256) = &declared.expected_sha256 { @@ -1425,6 +1432,7 @@ nodes = ["yggdrasil"] sha256: digest(b'c'), size_bytes: 42, executable: true, + executable_members: BTreeSet::new(), } } @@ -1791,6 +1799,113 @@ nodes = ["yggdrasil"] assert!(changed.validate_against(&plan).is_err()); } + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + } + + #[test] + fn receipt_without_members_encodes_as_before() { + // Both values were computed outside this code, from the five-field + // msgpack array the receipt encoded to at base d74ac8d. + const BASE_RECEIPT: &str = "95a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac3"; + const BASE_RELEASE_ID: &str = + "sha256-b9143d8f546ad71c493c70037813547aaec10fd87994a6f3bdc77ac3a8b3495e"; + let receipt = artifact_receipt(); + let encoded = rmp_serde::to_vec(&receipt).unwrap(); + assert_eq!(hex(&encoded), BASE_RECEIPT); + let bytes = (0..BASE_RECEIPT.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&BASE_RECEIPT[at..at + 2], 16).unwrap()) + .collect::>(); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), receipt); + let release = SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![receipt], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + assert_eq!(release.recomputed_release_id().unwrap(), BASE_RELEASE_ID); + } + + #[test] + fn sealed_release_address_covers_executable_members() { + let release_with = |members: &[&str]| SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![ArtifactReceipt { + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + ..artifact_receipt() + }], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + let none = release_with(&[]).recomputed_release_id().unwrap(); + let one = release_with(&["bin/daemon"]).recomputed_release_id().unwrap(); + let other = release_with(&["bin/other"]).recomputed_release_id().unwrap(); + let two = release_with(&["bin/daemon", "bin/other"]) + .recomputed_release_id() + .unwrap(); + let all = [&none, &one, &other, &two]; + for (index, left) in all.iter().enumerate() { + for right in &all[index + 1..] { + assert_ne!(left, right); + } + } + + // The declaration is the other half: a receipt must carry exactly the + // declared list, in a plan that declares one and in one that does not. + let plain = plan(); + let mut with_members = artifact_receipt(); + with_members.executable_members = BTreeSet::from([PathBuf::from("bin/daemon")]); + assert!( + SealedRelease::new(&plain, vec![with_members], vec![external_input_receipt()], 120) + .is_err() + ); + let recipe = RECIPE.replace("executable = true", "executable_members = [\"bin/daemon\", \"lib/tool\"]"); + let declaring = compile_deployment_plan( + recipe.as_bytes(), + BINDING.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(18001), + 110, + &[ready_odin_provider("odin", "odin-yggdrasil", 1)], + ) + .unwrap(); + let honest = ArtifactReceipt { + executable: false, + executable_members: BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]), + ..artifact_receipt() + }; + let release = SealedRelease::new( + &declaring, + vec![honest.clone()], + vec![external_input_receipt()], + 120, + ) + .unwrap(); + release.validate_against(&declaring).unwrap(); + for drifted in [ + BTreeSet::from([PathBuf::from("bin/daemon")]), + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/other")]), + BTreeSet::new(), + ] { + let mut receipt = honest.clone(); + receipt.executable_members = drifted; + assert!( + SealedRelease::new(&declaring, vec![receipt], vec![external_input_receipt()], 120) + .is_err() + ); + } + let mut changed = release.clone(); + changed.artifacts[0].executable_members.insert(PathBuf::from("lib/extra")); + assert!(changed.validate_against(&declaring).is_err()); + } + fn binding_with_deadlines(table: &str) -> String { BINDING.replace("[placement]", &format!("{table}\n[placement]")) } diff --git a/src/drivers.rs b/src/drivers.rs index 44d9976..165dfb0 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; @@ -2309,6 +2309,7 @@ impl DockerRunnerDriver { "artifact destination escaped its release staging root" ); copy_artifact(&containment_root, &source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; let (sha256, size_bytes) = digest_artifact(&destination)?; if let Some(expected) = &artifact.expected_sha256 { ensure!( @@ -2322,6 +2323,7 @@ impl DockerRunnerDriver { sha256: format!("sha256-{sha256}"), size_bytes, executable: artifact.executable, + executable_members: artifact.executable_members.clone(), }) } } @@ -6810,21 +6812,10 @@ fn hash_frozen_source_tree(root: &Path, current: &Path, hasher: &mut Sha256) -> ); hasher.update(b"\0"); hasher.update(metadata.len().to_le_bytes()); - // S9: streamed in fixed-size chunks rather than `fs::read`ing the - // whole file, so peak RSS is bounded by the buffer, not by the - // largest file in the tree. - let mut file = fs::File::open(&path) - .with_context(|| format!("opening {} to hash it", path.display()))?; - let mut buffer = [0u8; 1 << 16]; - loop { - let read = file - .read(&mut buffer) - .with_context(|| format!("reading {} to hash it", path.display()))?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - } + ensure!( + hash_file_streamed(&path, hasher)? == metadata.len(), + "frozen source file changed while hashing" + ); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -6846,19 +6837,31 @@ fn frozen_source_sha256(_root: &Path) -> Result { #[cfg(unix)] fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Result<()> { - let executable_paths = artifacts - .iter() - .filter(|artifact| artifact.executable) - .map(|artifact| root.join(&artifact.destination)) - .collect::>(); - harden_root_tree(root, root, &executable_paths) + let mut executable_paths = BTreeSet::new(); + for artifact in artifacts { + let destination = root.join(&artifact.destination); + for member in &artifact.executable_members { + executable_paths.insert(destination.join(member)); + } + if artifact.executable { + executable_paths.insert(destination); + } + } + harden_root_tree(root, root, &mut executable_paths)?; + ensure!( + executable_paths.is_empty(), + "declared executable is missing or not a regular file" + ); + Ok(()) } +/// Each regular file the walk seals 0555 is removed from `executable_paths`; +/// whatever remains was declared executable and never sealed. #[cfg(unix)] fn harden_root_tree( root: &Path, current: &Path, - executable_paths: &std::collections::BTreeSet, + executable_paths: &mut BTreeSet, ) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -6871,13 +6874,13 @@ fn harden_root_tree( let mut entries = fs::read_dir(current)?.collect::>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { - harden_root_tree(root, &entry.path(), executable_paths)?; + harden_root_tree(root, &entry.path(), &mut *executable_paths)?; } fs::set_permissions(current, fs::Permissions::from_mode(0o555))?; } else if metadata.is_file() { fs::set_permissions( current, - fs::Permissions::from_mode(if executable_paths.contains(current) { + fs::Permissions::from_mode(if executable_paths.remove(current) { 0o555 } else { 0o444 @@ -8125,11 +8128,81 @@ pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> R } } +/// The one seal-time owner of mode declarations for a copied artifact: a +/// directory output is never `executable` (it declares `executable_members`), +/// a file output never declares members, and every member is a regular file +/// reached through real directories, never through a symlink. Errors name the +/// rule, never the member path. +pub(crate) fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { + let is_dir = fs::symlink_metadata(destination) + .context("inspecting a copied artifact")? + .is_dir(); + ensure!( + !(is_dir && artifact.executable), + "directory artifact {} cannot be executable; declare executable_members", + artifact.id + ); + if !is_dir { + ensure!( + artifact.executable_members.is_empty(), + "file artifact {} cannot declare executable_members", + artifact.id + ); + return Ok(()); + } + for member in &artifact.executable_members { + ensure!(!member.as_os_str().is_empty(), "executable member is empty"); + let mut current = destination.to_path_buf(); + let mut components = member.components().peekable(); + while let Some(component) = components.next() { + let std::path::Component::Normal(name) = component else { + bail!("executable member has a forbidden path component"); + }; + current.push(name); + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if error.kind() == ErrorKind::NotFound => { + bail!("executable member is missing") + } + Err(error) => return Err(error).context("inspecting an executable member"), + }; + if components.peek().is_some() { + ensure!( + metadata.is_dir(), + "executable member passes through a symlink or non-directory" + ); + } else { + ensure!(metadata.is_file(), "executable member is not a regular file"); + } + } + } + Ok(()) +} + +/// Streams `path` into `hasher` in 64 KiB chunks, so peak memory is the buffer +/// and not the largest file in the tree. Returns the bytes read; a caller that +/// hashed a length first must check the two agree. +pub(crate) fn hash_file_streamed(path: &Path, hasher: &mut Sha256) -> Result { + let mut file = fs::File::open(path).context("opening a file to hash it")?; + let mut buffer = [0u8; 1 << 16]; + let mut total = 0_u64; + loop { + let read = file.read(&mut buffer).context("reading a file to hash it")?; + if read == 0 { + return Ok(total); + } + hasher.update(&buffer[..read]); + total += read as u64; + } +} + pub(crate) fn digest_artifact(path: &Path) -> Result<(String, u64)> { let metadata = fs::symlink_metadata(path)?; if metadata.is_file() { - let bytes = fs::read(path)?; - return Ok((raw_sha256(&bytes), bytes.len().try_into()?)); + let mut hasher = Sha256::new(); + let streamed = hash_file_streamed(path, &mut hasher)?; + ensure!(streamed == metadata.len(), "artifact file changed while hashing"); + return Ok((format!("{:x}", hasher.finalize()), streamed)); } ensure!(metadata.is_dir(), "artifact is not a file or directory"); let mut hasher = Sha256::new(); @@ -8153,13 +8226,16 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) hasher.update(b"\0"); digest_tree(root, &path, hasher, size)?; } else if metadata.is_file() { - let bytes = fs::read(&path)?; + let length = metadata.len(); hasher.update(b"file\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); - hasher.update((bytes.len() as u64).to_le_bytes()); - hasher.update(&bytes); - *size = size.saturating_add(bytes.len().try_into()?); + hasher.update(length.to_le_bytes()); + ensure!( + hash_file_streamed(&path, hasher)? == length, + "artifact file changed while hashing" + ); + *size = size.saturating_add(length); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -13862,6 +13938,7 @@ processes 9 sha256: sha256_id(b"sealed"), size_bytes: 6, executable: true, + executable_members: BTreeSet::new(), }], )?; let root_metadata = fs::symlink_metadata(&root)?; @@ -13873,6 +13950,448 @@ processes 9 Ok(()) } + #[cfg(unix)] + const MEMBER_CANARY: &str = "zzcanary"; + + #[cfg(unix)] + fn tree_output(executable: bool, members: &[&str]) -> ArtifactOutput { + ArtifactOutput { + id: "upstream".into(), + source_kind: ArtifactSource::WorktreeTree, + runner: None, + source: PathBuf::from("upstream"), + destination: PathBuf::from("upstream"), + expected_sha256: None, + executable, + executable_members: members.iter().map(PathBuf::from).collect(), + } + } + + #[cfg(unix)] + fn write_file(path: &Path, bytes: &[u8]) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::create_dir_all(path.parent().context("file has no parent")?)?; + fs::write(path, bytes)?; + // Builds and git leave arbitrary exec bits; the seal must not inherit them. + fs::set_permissions(path, fs::Permissions::from_mode(0o777))?; + Ok(()) + } + + #[cfg(unix)] + fn mode_of(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + + fs::symlink_metadata(path).unwrap().permissions().mode() & 0o777 + } + + /// `/src/upstream` is the worktree tree; `/staging` the release. + #[cfg(unix)] + fn sample_tree(root: &Path) -> Result<(PathBuf, PathBuf)> { + let source = root.join("src"); + let staging = root.join("staging"); + fs::create_dir_all(&staging)?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/daemon"), b"daemon")?; + write_file(&tree.join("bin/helper"), b"helper")?; + write_file(&tree.join("lib/tool"), b"tool")?; + write_file(&tree.join("lib/data.txt"), b"data")?; + Ok((source, staging)) + } + + #[cfg(unix)] + fn collect_tree( + source: &Path, + staging: &Path, + output: ArtifactOutput, + ) -> Result { + let mut declaration = + TargetDeclaration::parse(crate::deployment::tests::RECIPE)?; + let id = output.id.clone(); + declaration.artifacts = vec![output]; + let frozen = FrozenSource { + receipt: FrozenSourceReceipt { + transaction_id: "tx".into(), + plan_id: "plan".into(), + snapshot_sha256: "snapshot".into(), + }, + facts: crate::deployment_plan::tests::source(crate::deployment::tests::RECIPE), + recipe_bytes: Vec::new(), + root: source.to_path_buf(), + }; + DockerRunnerDriver::default().collect_artifact( + &declaration, + &frozen, + &BTreeMap::new(), + staging, + &id, + ) + } + + #[cfg(unix)] + fn refusal_text(result: Result) -> String { + let Err(error) = result else { + panic!("the release was not refused"); + }; + let text = format!("{error:#}"); + assert!( + !text.contains(MEMBER_CANARY), + "the refusal echoed a member path: {text}" + ); + text + } + + #[cfg(unix)] + #[test] + fn tree_member_sealed_0555_and_siblings_0444() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let receipt = collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib/tool"]), + )?; + assert_eq!( + receipt.executable_members, + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]) + ); + harden_installed_release(&staging, &[receipt])?; + let tree = staging.join("upstream"); + assert_eq!(mode_of(&tree.join("bin/daemon")), 0o555); + assert_eq!(mode_of(&tree.join("lib/tool")), 0o555); + assert_eq!(mode_of(&tree.join("bin/helper")), 0o444); + assert_eq!(mode_of(&tree.join("lib/data.txt")), 0o444); + for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { + assert_eq!(mode_of(directory), 0o555); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn missing_member_refuses_the_release() -> Result<()> { + // The bad member sorts after the good one in one case and before it in + // the other, so a check that stops at the first (or last) member fails. + for members in [ + &["bin/daemon", "lib/zzcanary-missing"][..], + &["aaa-zzcanary-missing", "bin/daemon"][..], + &["zzcanary-dir/tool"][..], + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, members))); + assert!(text.contains("executable member is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn symlink_member_refuses_the_release() -> Result<()> { + for link_target in ["daemon", "../zzcanary-nowhere"] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink( + link_target, + source.join("upstream/bin/zzcanary-link"), + )?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "bin/zzcanary-link"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn member_through_symlinked_directory_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("real-bin/zzcanary-tool"), b"tool")?; + std::os::unix::fs::symlink("real-bin", tree.join("linked-bin"))?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["linked-bin/zzcanary-tool"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + // A member below a regular file is refused the same way. + let second_staging = temp.path().join("second-staging"); + fs::create_dir_all(&second_staging)?; + let text = refusal_text(collect_tree( + &source, + &second_staging, + tree_output(false, &["bin/daemon/zzcanary-below-file"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn directory_member_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn executable_flag_on_tree_output_refuses() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(true, &[]))); + assert!(text.contains("cannot be executable"), "{text}"); + // The same flag on a file output stays admitted. + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + let mut output = tree_output(true, &[]); + output.id = "file".into(); + check_artifact_modes(&output, &file)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn members_on_file_output_refuse() -> Result<()> { + let temp = tempfile::tempdir()?; + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + check_artifact_modes(&tree_output(false, &[]), &file)?; + let text = refusal_text(check_artifact_modes( + &tree_output(false, &["zzcanary"]), + &file, + )); + assert!(text.contains("cannot declare executable_members"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn check_artifact_modes_refuses_a_member_that_is_not_a_clean_relative_path() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&temp.path().join("zzcanary-outside"), b"x")?; + for member in ["../../zzcanary-outside", "bin/../bin/daemon", "/zzcanary", ""] { + refusal_text(check_artifact_modes(&tree_output(false, &[member]), &tree)); + } + check_artifact_modes(&tree_output(false, &["bin/daemon"]), &tree)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn harden_fails_when_a_declared_member_is_absent() -> Result<()> { + let receipt = |members: &[&str]| ArtifactReceipt { + artifact_id: "upstream".into(), + destination: PathBuf::from("upstream"), + sha256: sha256_id(b"x"), + size_bytes: 1, + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + }; + let installed = |root: &Path| -> Result<()> { + write_file(&root.join("upstream/bin/daemon"), b"daemon")?; + write_file(&root.join("upstream/lib/tool"), b"tool")?; + fs::create_dir_all(root.join("upstream/zzcanary-dir"))?; + std::os::unix::fs::symlink("daemon", root.join("upstream/bin/zzcanary-link"))?; + Ok(()) + }; + let temp = tempfile::tempdir()?; + installed(temp.path())?; + harden_installed_release(temp.path(), &[receipt(&["bin/daemon", "lib/tool"])])?; + for members in [ + &["bin/daemon", "lib/zzcanary-gone"][..], + &["zzcanary-gone", "bin/daemon"][..], + &["zzcanary-dir"][..], + &["bin/zzcanary-link"][..], + &["bin/daemon/zzcanary-below-file"][..], + ] { + let temp = tempfile::tempdir()?; + installed(temp.path())?; + let text = refusal_text(harden_installed_release(temp.path(), &[receipt(members)])); + assert!(text.contains("declared executable is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn relative_in_tree_symlink_survives_the_seal() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/python3.12"), b"python")?; + std::os::unix::fs::symlink("python3.12", tree.join("bin/python3"))?; + let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/python3.12"]))?; + let (before, _) = digest_artifact(&staging.join("upstream"))?; + harden_installed_release(&staging, &[receipt])?; + let link = staging.join("upstream/bin/python3"); + assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); + assert_eq!(mode_of(&staging.join("upstream/bin/python3.12")), 0o555); + assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); + + // A link out of the release, by `..` or by absolute target, still refuses. + for (target, rule) in [ + ("../../zzcanary-up", "parent traversal"), + ("/zzcanary/absolute", "absolute"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink(target, source.join("upstream/lib/zzcanary-link"))?; + let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))?; + let text = refusal_text(harden_installed_release(&staging, &[receipt])); + assert!(text.contains(rule), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn streamed_digest_is_byte_compatible() -> Result<()> { + // Not a multiple of the 64 KiB read buffer, so a short last chunk + // matters; the aligned and empty files are the other boundaries. + let big = (0..3 * 65536 + 17).map(|i| (i % 251) as u8).collect::>(); + let aligned = vec![7_u8; 65536]; + let temp = tempfile::tempdir()?; + let tree = temp.path().join("tree"); + write_file(&tree.join("a.txt"), b"alpha")?; + write_file(&tree.join("empty"), b"")?; + write_file(&tree.join("sub/aligned.bin"), &aligned)?; + write_file(&tree.join("sub/big.bin"), &big)?; + std::os::unix::fs::symlink("a.txt", tree.join("zlink"))?; + + // The encoding digest_tree had before it streamed, written out by hand. + fn add_file(hasher: &mut Sha256, name: &str, bytes: &[u8]) { + hasher.update(b"file\0"); + hasher.update(name.as_bytes()); + hasher.update(b"\0"); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + let mut expected = Sha256::new(); + add_file(&mut expected, "a.txt", b"alpha"); + add_file(&mut expected, "empty", b""); + let mut expected_size = 5_u64; + expected.update(b"dir\0sub\0"); + add_file(&mut expected, "sub/aligned.bin", &aligned); + add_file(&mut expected, "sub/big.bin", &big); + expected_size += (aligned.len() + big.len()) as u64; + expected.update(b"link\0zlink\0a.txt"); + expected_size += "a.txt".len() as u64; + assert_eq!( + digest_artifact(&tree)?, + (format!("{:x}", expected.finalize()), expected_size) + ); + + let single = tree.join("sub/big.bin"); + assert_eq!( + digest_artifact(&single)?, + (raw_sha256(&big), big.len() as u64) + ); + let mut hasher = Sha256::new(); + assert_eq!(hash_file_streamed(&single, &mut hasher)?, big.len() as u64); + assert_eq!(format!("{:x}", hasher.finalize()), raw_sha256(&big)); + Ok(()) + } + + /// Acceptance on the real trees: `IDUNN_BIG_TREE` holds `python/` and + /// `blender/`. Run on Yggdrasil as its own job (see the cut spec). + #[cfg(unix)] + #[test] + #[ignore = "needs IDUNN_BIG_TREE: a CPython+torch tree and a Blender tree"] + fn big_tree_seals_from_env() -> Result<()> { + use std::time::Instant; + + fn symlinks(path: &Path) -> Result { + let mut count = 0; + for entry in fs::read_dir(path)? { + let entry = entry?; + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.is_dir() { + count += symlinks(&entry.path())?; + } else if metadata.file_type().is_symlink() { + count += 1; + } + } + Ok(count) + } + fn peak_rss() -> String { + fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| { + status + .lines() + .find(|line| line.starts_with("VmHWM")) + .map(str::to_owned) + }) + .unwrap_or_default() + } + + let tree = PathBuf::from( + std::env::var_os("IDUNN_BIG_TREE").context("IDUNN_BIG_TREE is not set")?, + ); + let members = ["python/bin/python3.12", "blender/blender"]; + let temp = tempfile::tempdir()?; + let staging = temp.path().join("staging"); + fs::create_dir_all(&staging)?; + let destination = staging.join("tree"); + let output = ArtifactOutput { + destination: PathBuf::from("tree"), + ..tree_output(false, &members) + }; + + let started = Instant::now(); + copy_artifact(&tree, &tree, &destination)?; + println!("copy {:?} {}", started.elapsed(), peak_rss()); + let started = Instant::now(); + let first = digest_artifact(&destination)?; + let second = digest_artifact(&destination)?; + assert_eq!(first, second); + println!("digest x2 {:?} {}", started.elapsed(), peak_rss()); + check_artifact_modes(&output, &destination)?; + let links = symlinks(&destination)?; + let started = Instant::now(); + harden_installed_release( + &staging, + &[ArtifactReceipt { + artifact_id: "tree".into(), + destination: PathBuf::from("tree"), + sha256: format!("sha256-{}", first.0), + size_bytes: first.1, + executable: false, + executable_members: output.executable_members.clone(), + }], + )?; + println!("harden {:?} {}", started.elapsed(), peak_rss()); + for member in members { + assert_eq!(mode_of(&destination.join(member)), 0o555); + } + let sample = fs::read_dir(&destination)? + .map(|entry| entry.map(|entry| entry.path())) + .collect::>>()?; + for path in sample { + if fs::symlink_metadata(&path)?.is_file() { + assert_eq!(mode_of(&path), 0o444); + } + } + assert_eq!(symlinks(&destination)?, links); + println!("bytes {} symlinks {links}", first.1); + Ok(()) + } + #[cfg(unix)] #[test] fn runner_cache_is_created_once_for_one_exact_identity() -> Result<()> { diff --git a/src/host.rs b/src/host.rs index 5221f12..6d05478 100644 --- a/src/host.rs +++ b/src/host.rs @@ -35,8 +35,9 @@ use crate::deployment::{ }; use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease}; use crate::drivers::{ - HostWorkloadObservation, InstalledReleaseObservation, copy_artifact, copy_tree, - digest_artifact, release_artifact, remove_tree_inside, sha256_id, write_runtime_bundle_records, + HostWorkloadObservation, InstalledReleaseObservation, check_artifact_modes, copy_artifact, + copy_tree, digest_artifact, release_artifact, remove_tree_inside, sha256_id, + write_runtime_bundle_records, }; use crate::host_actuator::{ HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS, @@ -1064,6 +1065,7 @@ fn materialize( "artifact destination escaped its staging root" ); copy_artifact(&source, &source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; let (sha256, size_bytes) = digest_artifact(&destination)?; if let Some(expected) = &artifact.expected_sha256 { ensure!( @@ -1078,6 +1080,7 @@ fn materialize( sha256: format!("sha256-{sha256}"), size_bytes, executable: artifact.executable, + executable_members: artifact.executable_members.clone(), }); } let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?; From 04ff2c8ce3e90d0e8a2a722389d50ef0f5fe4fff Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 19:53:16 +0200 Subject: [PATCH 152/196] README: a directory artifact is sealed 0444 except its executable_members Co-Authored-By: Claude Sonnet 5.5 --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index db728a9..4936cbe 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,26 @@ required_environment = [ ] ``` +A directory artifact is sealed read-only: its files are 0444 and its +directories 0555, whatever mode the build or git gave them. A file the tree +must run, such as the launcher's `upstream/bin/daemon` child, is declared in +`executable_members` and sealed 0555. The members are part of the sealed release +address, so changing the list makes a new release. Idunn refuses the release if a +member is absolute, climbs with `..`, is missing, is a symlink or sits behind +one, or is not a regular file; `executable = true` on a directory output and +`executable_members` on a file output are refused too. + +```toml +# recipe fragment — the upstream artifact declares the child it launches +[[artifacts]] +id = "upstream" +source_kind = "runner-output" +runner = "build" +source = "dist/upstream" +destination = "upstream" +executable_members = ["bin/daemon"] +``` + ```toml # binding fragment — members connect to the member endpoint, which Idunn # forwards to the private port derived from the candidate: member From 76cec0b5403c0bd54d0aa560e68d29e667eec8f4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 19:54:35 +0200 Subject: [PATCH 153/196] Tests: the sealed-mode helper no longer redefines the module's existing mode_of Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 165dfb0..347dd27 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -13979,7 +13979,7 @@ processes 9 } #[cfg(unix)] - fn mode_of(path: &Path) -> u32 { + fn sealed_mode(path: &Path) -> u32 { use std::os::unix::fs::PermissionsExt; fs::symlink_metadata(path).unwrap().permissions().mode() & 0o777 @@ -14057,12 +14057,12 @@ processes 9 ); harden_installed_release(&staging, &[receipt])?; let tree = staging.join("upstream"); - assert_eq!(mode_of(&tree.join("bin/daemon")), 0o555); - assert_eq!(mode_of(&tree.join("lib/tool")), 0o555); - assert_eq!(mode_of(&tree.join("bin/helper")), 0o444); - assert_eq!(mode_of(&tree.join("lib/data.txt")), 0o444); + assert_eq!(sealed_mode(&tree.join("bin/daemon")), 0o555); + assert_eq!(sealed_mode(&tree.join("lib/tool")), 0o555); + assert_eq!(sealed_mode(&tree.join("bin/helper")), 0o444); + assert_eq!(sealed_mode(&tree.join("lib/data.txt")), 0o444); for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { - assert_eq!(mode_of(directory), 0o555); + assert_eq!(sealed_mode(directory), 0o555); } Ok(()) } @@ -14240,7 +14240,7 @@ processes 9 let link = staging.join("upstream/bin/python3"); assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); - assert_eq!(mode_of(&staging.join("upstream/bin/python3.12")), 0o555); + assert_eq!(sealed_mode(&staging.join("upstream/bin/python3.12")), 0o555); assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); // A link out of the release, by `..` or by absolute target, still refuses. @@ -14377,14 +14377,14 @@ processes 9 )?; println!("harden {:?} {}", started.elapsed(), peak_rss()); for member in members { - assert_eq!(mode_of(&destination.join(member)), 0o555); + assert_eq!(sealed_mode(&destination.join(member)), 0o555); } let sample = fs::read_dir(&destination)? .map(|entry| entry.map(|entry| entry.path())) .collect::>>()?; for path in sample { if fs::symlink_metadata(&path)?.is_file() { - assert_eq!(mode_of(&path), 0o444); + assert_eq!(sealed_mode(&path), 0o444); } } assert_eq!(symlinks(&destination)?, links); From fd040f09c162033bd126e54bbdb286b9b574a424 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 20:28:15 +0200 Subject: [PATCH 154/196] Tests: an io error other than NotFound is not reported as a missing member Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 347dd27..2dfeda6 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -14190,6 +14190,22 @@ processes 9 Ok(()) } + #[cfg(unix)] + #[test] + fn an_unreadable_member_is_not_reported_as_missing() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + // A component past NAME_MAX fails with an io error that is not NotFound. + let long = "z".repeat(300); + let text = refusal_text(check_artifact_modes( + &tree_output(false, &[long.as_str()]), + &source.join("upstream"), + )); + assert!(!text.contains("missing"), "{text}"); + assert!(text.contains("inspecting an executable member"), "{text}"); + Ok(()) + } + #[cfg(unix)] #[test] fn harden_fails_when_a_declared_member_is_absent() -> Result<()> { From 07616697aff1098cfda7f23d7162611630024922 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 21:54:00 +0200 Subject: [PATCH 155/196] deploy-provenance map: link containment rule, host targets, shared seal path (r2) Imagination r2 for cut tree-executable-members after Soul s1: link census of the real PBS and Blender trees, why lexical normalisation does not prove containment, the leading-dotdot-within-depth rule and its prior art, members on host targets carried not applied, one seal step for collect and host. Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-map.md | 124 +++++++++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 18 deletions(-) diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md index cfbd11b..195867d 100644 --- a/docs/deploy-provenance-map.md +++ b/docs/deploy-provenance-map.md @@ -4,12 +4,14 @@ Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, session `self-2026-10-23-eureka-b`). The ends are in [`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, rulings, cut specs and follow-ups live in Eureka's mind; this page holds what -has no kind: body facts, rationale and sequencing. Line numbers are at -`d74ac8d` (origin/main on 2026-10-23). +has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are +at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`, +the head of Hands attempt 1, which revision 2 of the spec builds on. ## Cut `tree-executable-members` -Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r1`. It +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2` +(revision 2, after Soul pass 1 found the real interpreter tree refused). It serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's admission waits on it) and will serve gamecult-chat C3 too: the presence launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact @@ -32,12 +34,16 @@ launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not mode bits. So runner or git exec bits never reach the release: harden is the only mode authority. -- T4. `validate_release_symlink` (`:6895-6914`) keeps a symlink only if its - target is relative with no `..` and lexically inside the release. That rule is - sound without physical resolution (no `..` means a target can only descend). - Absolute links and `..` links refuse the release. The frozen-source path has - a physical resolver (`resolve_frozen_source_symlink`, tested at `:11826`) that - admits in-tree `..`; reuse it if a real tree needs `..` links, not a new rule. +- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink + only if its target is relative with no `..`. It runs only inside + `harden_root_tree`, at install, so a tree with a `..` or absolute link copies, + digests and seals, then fails at `install_release` (Soul s1: copying and + digesting the real tree twice took 160 s before harden refused it). The rule + is sound but too narrow: it refuses the real interpreter tree (T13). The + frozen-source path has a physical resolver (`resolve_frozen_source_symlink`, + `:6617`) that walks the filesystem with a 40-link budget and a memo. Its + first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs + the tree on disk to answer. Revision 2 does not reuse it (T14). - T5. The sealed release id hashes `(schema, plan_id, artifacts, external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes the whole plan including `recipe_blob`. A changed member list is therefore @@ -59,8 +65,11 @@ launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact destination, so it does not change. `require_relative_path` (`:1936-1949`) with `allow_current = false` already refuses absolute, `..`, root and prefix components. -- T9. The host actuator builds receipts separately (`src/host.rs:1043-1081`) - and must carry the same field and the same shape check. +- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy, + mode check, digest, pinned-digest check and receipt, the same steps as + `collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode + check survives every test. Revision 2 collapses both onto one `seal_artifact` + in `drivers.rs`. Each caller keeps only how it finds the source path. - T10. The transient unit runs with `ProtectSystem=strict` and the release in `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. @@ -73,6 +82,68 @@ launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact of them. Idunn's own body changes only through gamecult-ops `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T13. Link census of the real interpreter tree (Yggdrasil probe of + `cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`, + sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049 + after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all + their `..` components before the first name (every one is + `share/terminfo// -> ..//`, one `..` each). 0 have a `..` + after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it. + Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1). +- T14. Lexically normalising `/` does not check + containment. The kernel resolves a target one component at a time, so a `..` + after a symlinked name climbs from where that link points, not from where its + name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x` + normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x` + beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same + way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`, + `L -> D/D/../../../etc/passwd` passed a lexical check). + The sound lexical rule constrains the target's shape: all its `..` components + come before its first name, and they number no more than the link's depth + below the artifact's top. The climb then starts from the link's own + directory, which the walk reached through real directories only (copy and + digest never descend through a link), so it lands on a real ancestor inside + the artifact. After that the target only descends, and any link it passes + through obeys the same rule. By induction on the kernel's hops (capped at 40, + then ELOOP), every hop stays inside. The check is one pass over the target's + components with no filesystem reads, so its cost is linear in the tree and no + resolver is needed. The shapes it refuses (`s/../x`) occur in neither real + tree (T13). + Prior art, from documentation as known to Imagination (high confidence for Go + and securejoin, medium for the tar details): + - Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account + for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug. + - Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc, + containerd and umoci) resolve each component inside the root when a path is + opened, refusing or clamping an escape. They guard one later access, need + the tree on disk, and check again on every open. Idunn instead certifies a + sealed tree once, so that the kernel's own resolution stays inside on every + later access. + - Zip-slip defences and archive extractors check entry names lexically and + protect against link targets by refusing to write through links + (libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for + names; GNU tar creates such links last). They protect the extraction, not + what a kept link points at afterwards. + - OCI layer extraction keeps link targets verbatim, absolute ones included, + because the runtime resolves them inside the container's root. Idunn runs a + release in the host's mount namespace, so absolute targets stay refused. + - Nix keeps link targets verbatim and tracks dependencies by scanning for + references. The whole store is its root, so it has no per-artifact + containment to compare with. + None of these is a static per-link rule for a tree that runs unconfined. The + T14 shape rule fits that case. +- T15. Host targets: `idunn-host` observes, starts and stops processes on + Windows only (the non-Windows `platform` module in `src/host.rs` bails), and + `host.rs` sets no file modes. On Windows a file runs by its type, not by a + mode bit, so `executable` and `executable_members` have nothing to set there. + The file `executable` flag has always been carried and never applied on host + targets. A recipe does not know which workload kind its binding will choose. +- T16. The release root holds only artifact destinations. Runner workspaces, + where external inputs land, are removed before `SealedRelease::new` + (`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and + `validate_installed_release` (`:2815-2822`) digest every receipt's + destination before calling harden. So a rule in the digest walk runs at seal, + at install and at every validate, over every link harden would see. - T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three copies (runner workspace, staging, installed). Nothing collects old releases or staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every @@ -93,21 +164,38 @@ launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact - Harden seals each member 0555 and fails if any declared executable path was not visited as a regular file. "Every declared executable is sealed 0555, or the install fails" holds structurally, not by a lookup that can miss. -- Symlinks: relative, no `..`, in-tree links survive (T4). Vili's trees must +- Symlinks: a link in a tree artifact is kept when its target is relative, its + `..` components all come before its first name, and they number no more than + the link's depth below the artifact's top (T14). One function, + `check_release_link`, decides this. Only the digest walk calls it, so a bad + link refuses the release at seal, before `SealedRelease::new`, and again at + install and validate (T16). Harden no longer judges links. The climb is + bounded by the artifact's top, not the release root: no live release has a + `..` link (the old rule refused them all), so nothing installed is newly + refused, and each artifact's digest describes a closed tree. +- Host targets carry `executable` and `executable_members` in the receipt, + shape-checked at seal by the shared `seal_artifact`, and do not apply them + (T15). Refusing members there would make a recipe's validity depend on its + binding. +- Vili's trees must not come from `python -m venv`: its `bin/python` is an absolute link and its `pyvenv.cfg` names the build workspace. Install packages into the relocatable interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). ### Authority -Owner: the seal (`collect_artifact`, and `host.rs` for host targets) decides -membership, from the recipe, and writes it into the receipt. Harden enforces -the receipt and decides nothing. Runner exec bits and git `100755` modes are -dead inputs for release modes. +Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd +targets and by `materialize` in `host.rs` for host targets) decides +membership from the recipe and writes it into the receipt. Harden enforces the +receipt and decides nothing. Runner exec bits and git `100755` modes are dead +inputs for release modes. Link containment belongs to `check_release_link`, +which only `digest_tree` calls. `validate_release_symlink` is deleted, and +harden's symlink arm leaves links as they are. ### Acceptance sizing The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It -removes it at the end. If the trees hold any absolute or `..` symlink, the job -stops and the T4 resolver reuse becomes a revision of this spec. +removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 +terminfo links (T13). If a tree holds a link the T14 rule refuses, the job +prints it and stops, and the link is reported to Self, not deleted. From 68d6f96808c40c04637624c9077d42bcfc02c6dd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 21:57:07 +0200 Subject: [PATCH 156/196] Seal: one seal_artifact for both targets; link containment decided at seal by a shape rule collect_artifact and host.rs materialize both call seal_artifact. digest_tree judges every link with check_release_link (relative, `..` only first, climbs no higher than the link's depth); validate_release_symlink and harden's link judgement are gone. hash_file_exact owns the streamed-length check. A file directly under a tree's top is pinned 0444. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 6 + src/drivers.rs | 297 ++++++++++++++++++++++++++++++------------------- src/host.rs | 48 ++------ 3 files changed, 201 insertions(+), 150 deletions(-) diff --git a/README.md b/README.md index 4936cbe..a5c48bc 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,12 @@ member is absolute, climbs with `..`, is missing, is a symlink or sits behind one, or is not a regular file; `executable = true` on a directory output and `executable_members` on a file output are refused too. +A tree may hold relative symlinks whose `..` steps all come first and climb no +higher than the artifact's top (`share/terminfo/4/4025ex -> ../t/tek4025ex` is +kept). An absolute link, a `..` after a name, or a climb out of the artifact +refuses the release when it is sealed. Host targets (Windows) record +`executable` and `executable_members` in the receipt and do not apply them. + ```toml # recipe fragment — the upstream artifact declares the child it launches [[artifacts]] diff --git a/src/drivers.rs b/src/drivers.rs index 2dfeda6..d82e5ba 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -2302,29 +2302,7 @@ impl DockerRunnerDriver { } ArtifactSource::WorktreeTree => (source.root.clone(), source.root.join(&artifact.source)), }; - ensure!(source_path.exists(), "declared artifact output is absent"); - let destination = staging_root.join(&artifact.destination); - ensure!( - destination.starts_with(staging_root), - "artifact destination escaped its release staging root" - ); - copy_artifact(&containment_root, &source_path, &destination)?; - check_artifact_modes(artifact, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact output differs from its recipe-pinned digest" - ); - } - Ok(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - executable_members: artifact.executable_members.clone(), - }) + seal_artifact(artifact, &containment_root, &source_path, staging_root) } } @@ -6812,10 +6790,7 @@ fn hash_frozen_source_tree(root: &Path, current: &Path, hasher: &mut Sha256) -> ); hasher.update(b"\0"); hasher.update(metadata.len().to_le_bytes()); - ensure!( - hash_file_streamed(&path, hasher)? == metadata.len(), - "frozen source file changed while hashing" - ); + hash_file_exact(&path, metadata.len(), hasher)?; } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -6847,7 +6822,7 @@ fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Resul executable_paths.insert(destination); } } - harden_root_tree(root, root, &mut executable_paths)?; + harden_root_tree(root, &mut executable_paths)?; ensure!( executable_paths.is_empty(), "declared executable is missing or not a regular file" @@ -6858,11 +6833,7 @@ fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Resul /// Each regular file the walk seals 0555 is removed from `executable_paths`; /// whatever remains was declared executable and never sealed. #[cfg(unix)] -fn harden_root_tree( - root: &Path, - current: &Path, - executable_paths: &mut BTreeSet, -) -> Result<()> { +fn harden_root_tree(current: &Path, executable_paths: &mut BTreeSet) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let metadata = fs::symlink_metadata(current)?; @@ -6874,7 +6845,7 @@ fn harden_root_tree( let mut entries = fs::read_dir(current)?.collect::>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { - harden_root_tree(root, &entry.path(), &mut *executable_paths)?; + harden_root_tree(&entry.path(), &mut *executable_paths)?; } fs::set_permissions(current, fs::Permissions::from_mode(0o555))?; } else if metadata.is_file() { @@ -6887,35 +6858,15 @@ fn harden_root_tree( }), )?; } else if metadata.file_type().is_symlink() { - validate_release_symlink(root, current)?; + // Left untouched: containment was decided by check_release_link in + // digest_tree, which install_release and validate_installed_release + // run over every receipt before this walk (map T16). } else { bail!("sealed release contains a special filesystem entry") } Ok(()) } -#[cfg(unix)] -fn validate_release_symlink(root: &Path, path: &Path) -> Result<()> { - let target = fs::read_link(path)?; - ensure!(target.is_relative(), "sealed release symlink is absolute"); - ensure!( - target.components().all(|component| matches!( - component, - std::path::Component::CurDir | std::path::Component::Normal(_) - )), - "sealed release symlink has a parent traversal" - ); - let destination = path - .parent() - .context("sealed release symlink has no parent")? - .join(target); - ensure!( - destination.starts_with(root), - "sealed release symlink escaped its release" - ); - Ok(()) -} - #[cfg(not(unix))] fn harden_installed_release(_root: &Path, _artifacts: &[ArtifactReceipt]) -> Result<()> { bail!("systemd workload installation requires Unix permissions") @@ -8105,7 +8056,7 @@ fn copy_symlink(source: &Path, destination: &Path) -> Result<()> { /// checked to contain the fully resolved `source` with the same /// whole-chain-aware resolver `validate_frozen_source_symlink` uses, before /// anything is read from it. -pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { +fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { let canonical_root = root.canonicalize().context("resolving artifact root")?; let canonical_source = source .canonicalize() @@ -8133,7 +8084,7 @@ pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> R /// a file output never declares members, and every member is a regular file /// reached through real directories, never through a symlink. Errors name the /// rule, never the member path. -pub(crate) fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { +fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { let is_dir = fs::symlink_metadata(destination) .context("inspecting a copied artifact")? .is_dir(); @@ -8179,30 +8130,73 @@ pub(crate) fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path Ok(()) } +/// The one seal step for a declared artifact, for systemd and host targets +/// alike: copy it into the staging root, check its mode declarations, digest it +/// and check the recipe pin. Modes are applied by harden on systemd targets; +/// host targets (Windows, map T15) carry `executable` and `executable_members` +/// in the receipt and do not apply them. +pub(crate) fn seal_artifact( + artifact: &ArtifactOutput, + containment_root: &Path, + source_path: &Path, + staging_root: &Path, +) -> Result { + ensure!( + source_path.exists(), + "declared artifact output {} is absent", + artifact.id + ); + let destination = staging_root.join(&artifact.destination); + ensure!( + destination.starts_with(staging_root), + "artifact {} destination escaped its staging root", + artifact.id + ); + copy_artifact(containment_root, source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; + let (sha256, size_bytes) = digest_artifact(&destination)?; + if let Some(expected) = &artifact.expected_sha256 { + ensure!( + &sha256 == expected, + "artifact {} differs from its recipe-pinned digest", + artifact.id + ); + } + Ok(ArtifactReceipt { + artifact_id: artifact.id.clone(), + destination: artifact.destination.clone(), + sha256: format!("sha256-{sha256}"), + size_bytes, + executable: artifact.executable, + executable_members: artifact.executable_members.clone(), + }) +} + /// Streams `path` into `hasher` in 64 KiB chunks, so peak memory is the buffer -/// and not the largest file in the tree. Returns the bytes read; a caller that -/// hashed a length first must check the two agree. -pub(crate) fn hash_file_streamed(path: &Path, hasher: &mut Sha256) -> Result { +/// and not the largest file in the tree, and refuses if the bytes read differ +/// from `expected_len`. The only owner of the length check. +pub(crate) fn hash_file_exact(path: &Path, expected_len: u64, hasher: &mut Sha256) -> Result<()> { let mut file = fs::File::open(path).context("opening a file to hash it")?; let mut buffer = [0u8; 1 << 16]; let mut total = 0_u64; loop { let read = file.read(&mut buffer).context("reading a file to hash it")?; if read == 0 { - return Ok(total); + break; } hasher.update(&buffer[..read]); total += read as u64; } + ensure!(total == expected_len, "file changed while hashing"); + Ok(()) } pub(crate) fn digest_artifact(path: &Path) -> Result<(String, u64)> { let metadata = fs::symlink_metadata(path)?; if metadata.is_file() { let mut hasher = Sha256::new(); - let streamed = hash_file_streamed(path, &mut hasher)?; - ensure!(streamed == metadata.len(), "artifact file changed while hashing"); - return Ok((format!("{:x}", hasher.finalize()), streamed)); + hash_file_exact(path, metadata.len(), &mut hasher)?; + return Ok((format!("{:x}", hasher.finalize()), metadata.len())); } ensure!(metadata.is_dir(), "artifact is not a file or directory"); let mut hasher = Sha256::new(); @@ -8231,13 +8225,11 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) hasher.update(relative.as_bytes()); hasher.update(b"\0"); hasher.update(length.to_le_bytes()); - ensure!( - hash_file_streamed(&path, hasher)? == length, - "artifact file changed while hashing" - ); + hash_file_exact(&path, length, hasher)?; *size = size.saturating_add(length); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; + check_release_link(path.strip_prefix(root)?, &target)?; hasher.update(b"link\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); @@ -8250,6 +8242,35 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) Ok(()) } +/// The one judge of link containment in a release tree, pure and static: +/// `link` is the link's path relative to its artifact's top and `target` what +/// it reads. A target is relative, its `..` steps all come first, and they +/// climb no higher than the link's own directory depth, so the link can only +/// land inside the artifact. Lexical normalisation of parent and target is not +/// enough: with `s -> ..` it admits `s/../x` and escapes (map T14). +fn check_release_link(link: &Path, target: &Path) -> Result<()> { + use std::path::Component; + + let depth = link.components().count().saturating_sub(1); + let mut climbs = 0_usize; + let mut named = false; + for component in target.components() { + match component { + Component::CurDir => {} + Component::RootDir | Component::Prefix(_) => { + bail!("sealed release symlink is absolute") + } + Component::ParentDir => { + ensure!(!named, "sealed release symlink climbs after a name"); + climbs += 1; + } + Component::Normal(_) => named = true, + } + } + ensure!(climbs <= depth, "sealed release symlink escapes its artifact"); + Ok(()) +} + pub(crate) fn raw_sha256(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } @@ -13996,6 +14017,7 @@ processes 9 write_file(&tree.join("bin/helper"), b"helper")?; write_file(&tree.join("lib/tool"), b"tool")?; write_file(&tree.join("lib/data.txt"), b"data")?; + write_file(&tree.join("NOTICE"), b"notice")?; Ok((source, staging)) } @@ -14061,12 +14083,44 @@ processes 9 assert_eq!(sealed_mode(&tree.join("lib/tool")), 0o555); assert_eq!(sealed_mode(&tree.join("bin/helper")), 0o444); assert_eq!(sealed_mode(&tree.join("lib/data.txt")), 0o444); + assert_eq!(sealed_mode(&tree.join("NOTICE")), 0o444); for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { assert_eq!(sealed_mode(directory), 0o555); } Ok(()) } + #[test] + fn release_link_rule_is_containment() { + for (link, target) in [ + ("a/b/l", "../t/x"), + ("a/l", ".."), + ("a/l", "./../b"), + ("a/l", "b/./c"), + ("l", "python3.12"), + ("a/b/c/l", "../../x"), + ] { + check_release_link(Path::new(link), Path::new(target)) + .unwrap_or_else(|error| panic!("({link}, {target}) refused: {error:#}")); + } + for (link, target, rule) in [ + ("l", "/x", "absolute"), + ("a/l", "s/../x", "climbs after a name"), + ("a/l", "D/D/../../../etc/passwd", "climbs after a name"), + ("a/l", "../../x", "escapes its artifact"), + ("l", "..", "escapes its artifact"), + ("a/b/l", "../../../x", "escapes its artifact"), + ("a/l", "./s/../x", "climbs after a name"), + ] { + let text = format!( + "{:#}", + check_release_link(Path::new(link), Path::new(target)) + .expect_err(&format!("({link}, {target}) was admitted")) + ); + assert!(text.contains(rule), "({link}, {target}): {text}"); + } + } + #[cfg(unix)] #[test] fn missing_member_refuses_the_release() -> Result<()> { @@ -14250,6 +14304,9 @@ processes 9 let tree = source.join("upstream"); write_file(&tree.join("bin/python3.12"), b"python")?; std::os::unix::fs::symlink("python3.12", tree.join("bin/python3"))?; + write_file(&tree.join("lib/terminfo/b/real"), b"terminfo")?; + fs::create_dir_all(tree.join("lib/terminfo/a"))?; + std::os::unix::fs::symlink("../b/real", tree.join("lib/terminfo/a/alias"))?; let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/python3.12"]))?; let (before, _) = digest_artifact(&staging.join("upstream"))?; harden_installed_release(&staging, &[receipt])?; @@ -14257,18 +14314,22 @@ processes 9 assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); assert_eq!(sealed_mode(&staging.join("upstream/bin/python3.12")), 0o555); + let alias = staging.join("upstream/lib/terminfo/a/alias"); + assert!(fs::symlink_metadata(&alias)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&alias)?, PathBuf::from("../b/real")); assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); - // A link out of the release, by `..` or by absolute target, still refuses. + // A link that escapes, is absolute or climbs after a name is refused at + // the seal, before any receipt exists. for (target, rule) in [ - ("../../zzcanary-up", "parent traversal"), + ("../../zzcanary-up", "escapes its artifact"), ("/zzcanary/absolute", "absolute"), + ("zzcanary-s/../x", "climbs after a name"), ] { let temp = tempfile::tempdir()?; let (source, staging) = sample_tree(temp.path())?; std::os::unix::fs::symlink(target, source.join("upstream/lib/zzcanary-link"))?; - let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))?; - let text = refusal_text(harden_installed_release(&staging, &[receipt])); + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); assert!(text.contains(rule), "{text}"); } Ok(()) @@ -14318,8 +14379,12 @@ processes 9 (raw_sha256(&big), big.len() as u64) ); let mut hasher = Sha256::new(); - assert_eq!(hash_file_streamed(&single, &mut hasher)?, big.len() as u64); + hash_file_exact(&single, big.len() as u64, &mut hasher)?; assert_eq!(format!("{:x}", hasher.finalize()), raw_sha256(&big)); + for wrong in [big.len() as u64 + 1, big.len() as u64 - 1] { + let text = refusal_text(hash_file_exact(&single, wrong, &mut Sha256::new())); + assert!(text.contains("changed while hashing"), "{text}"); + } Ok(()) } @@ -14331,18 +14396,40 @@ processes 9 fn big_tree_seals_from_env() -> Result<()> { use std::time::Instant; - fn symlinks(path: &Path) -> Result { - let mut count = 0; + /// (links, links with a `..` step, regular files, files at 0555), never + /// following a link; every regular file must be 0444 or a member's 0555. + fn census( + root: &Path, + path: &Path, + members: &BTreeSet, + modes: bool, + totals: &mut [usize; 4], + ) -> Result<()> { for entry in fs::read_dir(path)? { let entry = entry?; let metadata = fs::symlink_metadata(entry.path())?; if metadata.is_dir() { - count += symlinks(&entry.path())?; + census(root, &entry.path(), members, modes, totals)?; } else if metadata.file_type().is_symlink() { - count += 1; + totals[0] += 1; + let target = fs::read_link(entry.path())?; + if target + .components() + .any(|part| matches!(part, std::path::Component::ParentDir)) + { + totals[1] += 1; + } + } else { + totals[2] += 1; + let member = members.contains(entry.path().strip_prefix(root)?); + let mode = sealed_mode(&entry.path()); + if modes { + assert_eq!(mode, if member { 0o555 } else { 0o444 }); + totals[3] += usize::from(mode == 0o555); + } } } - Ok(count) + Ok(()) } fn peak_rss() -> String { fs::read_to_string("/proc/self/status") @@ -14370,41 +14457,27 @@ processes 9 }; let started = Instant::now(); - copy_artifact(&tree, &tree, &destination)?; - println!("copy {:?} {}", started.elapsed(), peak_rss()); + let receipt = seal_artifact(&output, &tree, &tree, &staging)?; + println!("seal {:?} {}", started.elapsed(), peak_rss()); let started = Instant::now(); - let first = digest_artifact(&destination)?; - let second = digest_artifact(&destination)?; - assert_eq!(first, second); - println!("digest x2 {:?} {}", started.elapsed(), peak_rss()); - check_artifact_modes(&output, &destination)?; - let links = symlinks(&destination)?; + let again = digest_artifact(&destination)?; + assert_eq!(format!("sha256-{}", again.0), receipt.sha256); + assert_eq!(again.1, receipt.size_bytes); + println!("digest {:?} {}", started.elapsed(), peak_rss()); + let mut before = [0; 4]; + census(&destination, &destination, &output.executable_members, false, &mut before)?; let started = Instant::now(); - harden_installed_release( - &staging, - &[ArtifactReceipt { - artifact_id: "tree".into(), - destination: PathBuf::from("tree"), - sha256: format!("sha256-{}", first.0), - size_bytes: first.1, - executable: false, - executable_members: output.executable_members.clone(), - }], - )?; + harden_installed_release(&staging, &[receipt.clone()])?; println!("harden {:?} {}", started.elapsed(), peak_rss()); - for member in members { - assert_eq!(sealed_mode(&destination.join(member)), 0o555); - } - let sample = fs::read_dir(&destination)? - .map(|entry| entry.map(|entry| entry.path())) - .collect::>>()?; - for path in sample { - if fs::symlink_metadata(&path)?.is_file() { - assert_eq!(sealed_mode(&path), 0o444); - } - } - assert_eq!(symlinks(&destination)?, links); - println!("bytes {} symlinks {links}", first.1); + let mut totals = [0; 4]; + census(&destination, &destination, &output.executable_members, true, &mut totals)?; + assert_eq!(totals[0], before[0], "links changed across harden"); + assert_eq!(totals[3], members.len()); + println!( + "dotdot links {} links before {} after {} files {} exec {}", + totals[1], before[0], totals[0], totals[2], totals[3] + ); + println!("bytes {}", receipt.size_bytes); Ok(()) } diff --git a/src/host.rs b/src/host.rs index 6d05478..4c6cbec 100644 --- a/src/host.rs +++ b/src/host.rs @@ -30,14 +30,13 @@ use serde::Deserialize; use crate::control_plane::read_trust_anchor; use crate::deployment::{ - ArtifactSource, HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, LaunchArgument, OperatorBinding, TargetDeclaration, }; -use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease}; +use crate::deployment_plan::{CompiledDeploymentPlan, SealedRelease}; use crate::drivers::{ - HostWorkloadObservation, InstalledReleaseObservation, check_artifact_modes, copy_artifact, - copy_tree, digest_artifact, release_artifact, remove_tree_inside, sha256_id, - write_runtime_bundle_records, + HostWorkloadObservation, InstalledReleaseObservation, copy_tree, digest_artifact, + release_artifact, remove_tree_inside, seal_artifact, sha256_id, write_runtime_bundle_records, }; use crate::host_actuator::{ HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS, @@ -1049,39 +1048,12 @@ fn materialize( fs::create_dir_all(&staging)?; let mut artifacts = Vec::new(); for artifact in &declaration.artifacts { - let source_path = match artifact.source_kind { - ArtifactSource::RunnerOutput | ArtifactSource::WorktreeTree => { - source.join(&artifact.source) - } - }; - ensure!( - source_path.exists(), - "declared artifact output {} is absent", - artifact.id - ); - let destination = staging.join(&artifact.destination); - ensure!( - destination.starts_with(&staging), - "artifact destination escaped its staging root" - ); - copy_artifact(&source, &source_path, &destination)?; - check_artifact_modes(artifact, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact {} differs from its recipe-pinned digest", - artifact.id - ); - } - artifacts.push(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - executable_members: artifact.executable_members.clone(), - }); + artifacts.push(seal_artifact( + artifact, + &source, + &source.join(&artifact.source), + &staging, + )?); } let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?; Ok((release, staging)) From e8c4c6972881b465b30e9e1e896272a8ab7d3ceb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 23:26:50 +0200 Subject: [PATCH 157/196] Seal tests: refused links whose targets resolve (absolute, escaping, after-a-name; file and directory) Closes the live-target bypass: a judge guarded by exists() or is_dir() passed every dangling fixture. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index d82e5ba..0eb0928 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -14335,6 +14335,44 @@ processes 9 Ok(()) } + #[cfg(unix)] + #[test] + fn links_to_existing_targets_are_refused_at_the_seal() -> Result<()> { + // Every refused shape, aimed at a target that EXISTS (file and + // directory): a judge guarded by `exists()`, `is_dir()` or a file-type + // test passes the dangling fixtures and fails these. + for kind in ["file", "dir"] { + for (shape, rule) in [ + ("absolute", "absolute"), + ("escape", "escapes its artifact"), + ("after-name", "climbs after a name"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let target = match (shape, kind) { + ("absolute", "file") => source.join("upstream/bin/daemon"), + ("absolute", _) => source.join("upstream/lib"), + ("escape", "file") => { + write_file(&source.join("outside-file"), b"outside")?; + PathBuf::from("../../outside-file") + } + ("escape", _) => { + fs::create_dir_all(source.join("outside-dir"))?; + PathBuf::from("../../outside-dir") + } + (_, "file") => PathBuf::from("../bin/../NOTICE"), + _ => PathBuf::from("../bin/../lib"), + }; + let link = source.join("upstream/lib/zzcanary-link"); + std::os::unix::fs::symlink(&target, &link)?; + assert!(fs::metadata(&link).is_ok(), "{shape} {kind} link must resolve"); + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); + assert!(text.contains(rule), "{shape} {kind}: {text}"); + } + } + Ok(()) + } + #[cfg(unix)] #[test] fn streamed_digest_is_byte_compatible() -> Result<()> { From a3c6386aa4af33c7fc8a5d2fb5178bd85ef36f36 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 00:14:39 +0200 Subject: [PATCH 158/196] Grant Idunn Vili's runtime, state and release roots motion-pipeline:follow_up:idunn-unit-vili-roots: Vili's admission (motion-pipeline:cut_spec:cut-vili-admission.r1) needs its three roots writable under ProtectSystem=full, beside StreamPixels'. Lands before the d3222e5 install, which restarts idunn-yggdrasil anyway. Co-Authored-By: Claude Opus 5.5 --- deploy/idunn-yggdrasil.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index da82bd2..c63c1ed 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -104,6 +104,8 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web +# Vili (motion worker): runtime bundle, state, and releases. +ReadWritePaths=/etc/gamecult/vili/runtime /var/lib/gamecult/vili /srv/vili-idunn/releases [Install] WantedBy=multi-user.target From 112a7a5ebb78ab5ff7ab92624defb6cb99805d1b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 00:15:16 +0200 Subject: [PATCH 159/196] Idunn unit: Vili's roots are optional so a missing target cannot stop Idunn systemd refuses to start a unit whose ReadWritePaths names a missing path, and none of Vili's three roots exist on Yggdrasil yet. Idunn's startup must not depend on a managed target, so the Vili line takes the "-" prefix. Co-Authored-By: Claude Opus 5.5 --- deploy/idunn-yggdrasil.service | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index c63c1ed..1c45594 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -104,8 +104,11 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web -# Vili (motion worker): runtime bundle, state, and releases. -ReadWritePaths=/etc/gamecult/vili/runtime /var/lib/gamecult/vili /srv/vili-idunn/releases +# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a +# target whose roots are not created yet cannot stop Idunn itself from starting +# (Idunn startup never depends on a managed target). A root created after Idunn +# starts is writable only from Idunn's next restart. +ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases [Install] WantedBy=multi-user.target From 69eb3e6d6f8b077a6cb6774fcf3c2fda5dbcb639 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 14:51:10 +0200 Subject: [PATCH 160/196] deploy-provenance map: isolation credential evidence (body facts I1-I8) Imagination for cut isolation-credential-evidence: systemd 257 prints LoadCredential as [unprintable] for every unit, so observe_unit's comparison never passes a target with no extra credential and proves nothing otherwise; the rule moves to the delivered credential directory. Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-map.md | 85 +++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md index 195867d..5ec7052 100644 --- a/docs/deploy-provenance-map.md +++ b/docs/deploy-provenance-map.md @@ -199,3 +199,88 @@ on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 terminfo links (T13). If a tree holds a link the T14 rule refuses, the job prints it and stops, and the link is reported to Self, not deleted. + +## Cut `isolation-credential-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, +at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It +serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, +first deviation): every `idunn up vili` is refused before fencing. + +### Body facts (probed read-only on Yggdrasil, 2026-10-24) + +- I1. `idunn up vili` runs 6 and 7 failed with + `systemd workload isolation properties differ from the Idunn contract`, the + message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, + only one differs on the failed Vili unit: `LoadCredential`. Idunn expects + `""` because Vili has no service credential besides the presence identity + (`service_credential_names` empty); systemd shows `[unprintable]`. Every + other conjunct matches, InaccessiblePaths included (one value across all + five idunn-vili/odin units). +- I2. systemd 257.13 `systemctl show -p LoadCredential` prints + `LoadCredential=[unprintable]` for every unit, whatever the credentials: + `ssh.service` and `systemd-journald.service` (not DynamicUser, zero + LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin + unit (one). The display form carries no information, so the conjunct was + vacuous whenever credentials existed and always false when none did. Odin + and StreamPixels passed only because each carries a regular credential. +- I3. The exact value is readable: D-Bus + `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) + answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 + "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. +- I4. What the workload can actually read is the delivered directory. On the + host, `/run/credentials/` holds a directory only for units with credentials + (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID + root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` + (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` + descriptor, never a credential. +- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn + deployment transaction authority"), with its twin clause in + `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit + touched it. No ruling or finding in any campaign concerns workload isolation + verification; gamecult-chat's foreign-workloads cut touched + `CREDENTIALS_DIRECTORY` reservation, not this check. +- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both + MessagePack encodings, compact (positional) included + (`persisted_systemd_observations_decode_as_the_systemd_variant`, + `:9915-9930`), so deleting the `load_credential` field shifts every later + slot of a live incumbent's record. The field is also in the drift list + (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed + workload: changing what the field holds without leaving that list would make + the next Idunn refuse to stop the live Odin and StreamPixels incumbents. +- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 + properties for a Vili unit (presence identity only) and the Odin unit + (presence identity plus one service credential), captured 2026-10-24 into + this Imagination's scratchpad `fixtures/`. +- I8. The new rule holds for every live systemd incumbent: inside each + MainPID root the delivered entries equal its binding's `secret_files` keys + minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, + StreamPixels web 1; key names compared, no values read). An Idunn restart onto + this cut re-observes them (`:3955-3970`) and passes. + +### Decision + +A property systemd cannot display does not belong in the comparison. The +invariant it reached for, that the workload holds exactly the credentials Idunn +delivered, is observed where it is decided: the entries of +`/run/credentials/` in the MainPID's root equal Idunn's service +credential names, absent directory equal to none. That covers +`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and +needs no new program. The D-Bus read (I3) is exact but sees only one of those +properties and adds a second systemd client and parser; rejected. + +### Authority + +Owner: `observe_unit` decides the credential rule, from the delivered +directory. `LoadCredential`'s display form is no longer read or compared; the +`load_credential` field is a retired wire slot, written empty, out of the drift +list, kept only so persisted records decode. Forbidden writers: the +`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping +the slot waits on the next schema revision of the observation. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). After it, Vili's Hands reruns from step 4 of +`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. From def0b7837892c31388856b336556689ad0102e95 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 14:53:15 +0200 Subject: [PATCH 161/196] Fixtures: systemd 257 show captures and pre-cut observation records Real systemctl show output (Vili presence-only, Odin with one service credential) and the two MessagePack encodings of a pre-cut observation, as inputs for the credential-evidence cut. Nothing reads them yet. Co-Authored-By: Claude Sonnet 5.5 --- tests/fixtures/README.md | 16 +++++++++++ .../pre-cut-systemd-observation.compact.hex | 1 + .../pre-cut-systemd-observation.named.hex | 1 + ...n-presence-and-one-service-credential.show | 28 +++++++++++++++++++ .../vili-presence-identity-only.show | 28 +++++++++++++++++++ 5 files changed, 74 insertions(+) create mode 100644 tests/fixtures/pre-cut-systemd-observation.compact.hex create mode 100644 tests/fixtures/pre-cut-systemd-observation.named.hex create mode 100644 tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show create mode 100644 tests/fixtures/systemd-257/vili-presence-identity-only.show diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 70060bf..b9f2637 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -10,3 +10,19 @@ It is checked in because the TypeScript runtime emits a camelCase MessagePack map while Rust `DatabaseEntry` derives tuple payloads. Idunn must prove it can decode the producer's real wire shape rather than round-tripping a fixture it serialized itself. + +# systemd 257 `systemctl show` captures + +`systemd-257/*.show` are the verbatim output of systemd 257.13 for the 27 +properties `show_unit` requested at `112a7a5`, captured read-only on Yggdrasil +on 2026-10-24: `vili-presence-identity-only.show` (a failed idunn-vili unit whose +only credential is the presence identity) and +`odin-presence-and-one-service-credential.show` (the live Odin unit). systemd 257 +prints `LoadCredential=[unprintable]` in both. Tests read these bytes; do not edit. + +# Pre-cut systemd observation records + +`pre-cut-systemd-observation.{compact,named}.hex` are the two MessagePack +encodings of a `SystemdWorkloadObservation` with `load_credential` set to +`[unprintable]`, produced by the code at `112a7a5` (the form live incumbents +persisted). They pin that later Idunns still decode such records. diff --git a/tests/fixtures/pre-cut-systemd-observation.compact.hex b/tests/fixtures/pre-cut-systemd-observation.compact.hex new file mode 100644 index 0000000..e7d09fb --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.compact.hex @@ -0,0 +1 @@ +dc0033d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365aa4964756e6e2074657374a3696e7601a465786563a26e6fa56d69786564c3a6753631303030a6753631303030a0a0a0c3c3a9696e76697369626c65a3616c6cc3a430303037a0ad5b756e7072696e7461626c655dcd10920194cdee48cdee48cdee48cdee4894cdee48cdee48cdee48cdee4891cdee480000000000c391010708b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d3232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323290d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190 \ No newline at end of file diff --git a/tests/fixtures/pre-cut-systemd-observation.named.hex b/tests/fixtures/pre-cut-systemd-observation.named.hex new file mode 100644 index 0000000..8898adf --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.named.hex @@ -0,0 +1 @@ +de0033a4756e6974d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365b0756e69745f6465736372697074696f6eaa4964756e6e2074657374ad696e766f636174696f6e5f6964a3696e76d923657865635f6d61696e5f73746172745f74696d657374616d705f6d6f6e6f746f6e696301ac736572766963655f74797065a465786563ae726573746172745f706f6c696379a26e6fa96b696c6c5f6d6f6465a56d69786564ac64796e616d69635f75736572c3ac73797374656d645f75736572a6753631303030ad73797374656d645f67726f7570a6753631303030b4737570706c656d656e746172795f67726f757073a0b76361706162696c6974795f626f756e64696e675f736574a0b4616d6269656e745f6361706162696c6974696573a0ae707269766174655f6d6f756e7473c3ac707269766174655f70696473c3ac70726f746563745f70726f63a9696e76697369626c65ab70726f635f737562736574a3616c6cb16e6f5f6e65775f70726976696c65676573c3a5756d61736ba430303037b2696e61636365737369626c655f7061746873a0af6c6f61645f63726564656e7469616cad5b756e7072696e7461626c655da86d61696e5f706964cd1092b270726f636573735f73746172745f74696d6501ac70726f636573735f7569647394cdee48cdee48cdee48cdee48ac70726f636573735f6769647394cdee48cdee48cdee48cdee48ae70726f636573735f67726f75707391cdee48b770726f636573735f6361705f696e686572697461626c6500b570726f636573735f6361705f7065726d697474656400b570726f636573735f6361705f65666665637469766500b470726f636573735f6361705f626f756e64696e6700b370726f636573735f6361705f616d6269656e7400b970726f636573735f6e6f5f6e65775f70726976696c65676573c3b670726f636573735f6e616d6573706163655f706964739101b26d6f756e745f6e616d6573706163655f696407b07069645f6e616d6573706163655f696408aa65786563757461626c65b52f6f70742f746573742f62696e2f73657276696365b165786563757461626c655f64657669636501b065786563757461626c655f696e6f646501b165786563757461626c655f736861323536d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131b372756e74696d655f696e7374616e63655f6964d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161b1776f726b696e675f6469726563746f7279a92f6f70742f74657374ae72756e74696d655f62756e646c65a92f72756e2f74657374b3636f6d6d616e645f6c696e655f736861323536d9477368613235362d32323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232b1656e7669726f6e6d656e745f6e616d657390bb656e7669726f6e6d656e745f636f6e74726163745f736861323536d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333ad636f6e74726f6c5f67726f7570d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365b563726564656e7469616c735f6469726563746f7279c0bc706172656e745f6f6e6c795f66696c655f64657363726970746f727390bd61637469766174696f6e5f7369676e65725f6964656e746974795f6964aa61637469766174696f6ebc61637469766174696f6e5f7369676e65725f7075626c69635f6b6579dc00200101010101010101010101010101010101010101010101010101010101010101b3736572766963655f63726564656e7469616c7390 \ No newline at end of file diff --git a/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show new file mode 100644 index 0000000..21e4573 --- /dev/null +++ b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=2571 +OpenFile=/run/idunn/activation-credentials/sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/odin/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=20630000 +ControlGroup=/system.slice/idunn-odin-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.service +UMask=0007 +WorkingDirectory=/srv/odin/releases/sha256-e5c7d0b2b9c0a8ffba368aeb08abdbe37ae0d17484a785e60a66ec4a62b2f18c +CapabilityBoundingSet= +AmbientCapabilities= +User=_duc23369f1b8b167db +Group=odin +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn odin sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8 +LoadState=loaded +ActiveState=active +SubState=running +InvocationID=ada48921043449f39dacc87b14a1bf70 diff --git a/tests/fixtures/systemd-257/vili-presence-identity-only.show b/tests/fixtures/systemd-257/vili-presence-identity-only.show new file mode 100644 index 0000000..fc78fd1 --- /dev/null +++ b/tests/fixtures/systemd-257/vili-presence-identity-only.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=0 +OpenFile=/run/idunn/activation-credentials/sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/vili/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=520083420000 +ControlGroup= +UMask=0007 +WorkingDirectory=/srv/vili-idunn/releases/sha256-aeaee4e1669742572c45a3f26bc86893c44b2a4149520975eb7d8a28aa931906 +CapabilityBoundingSet= +AmbientCapabilities= +User=_du8c06c563aac6d205 +Group=vili-state +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn vili sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041 +LoadState=loaded +ActiveState=failed +SubState=failed +InvocationID=8f7a349f2f8f49fe87ac5d89239260b4 From ae6a1bea557105633b53a1bf113f0e1f01522d5c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 14:54:15 +0200 Subject: [PATCH 162/196] Idunn: judge workload credentials from the delivered directory, not LoadCredential systemd 257 displays LoadCredential as [unprintable] for every unit, so the conjunct was always false for a unit with only the presence identity and vacuous for one with credentials. Delete the show argument, the conjunct and its twin clause; the isolation conjunction becomes require_systemd_isolation_contract. The credential rule is now require_delivered_credentials: entry names of /run/credentials/ in the MainPID's root equal Idunn's launch names. load_credential stays a retired wire slot (written empty, out of the drift list) so persisted records decode. Not yet built or verified. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 298 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 253 insertions(+), 45 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 0eb0928..43af330 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -470,6 +470,9 @@ pub struct SystemdWorkloadObservation { pub no_new_privileges: bool, pub umask: String, pub inaccessible_paths: String, + /// Retired wire slot. systemd 257 displays `LoadCredential` as `[unprintable]` for + /// every unit, so it was never evidence: written empty, never compared. Kept so + /// compact (positional) MessagePack records of earlier Idunns still decode. pub load_credential: String, pub main_pid: u32, pub process_start_time: u64, @@ -2459,7 +2462,6 @@ fn changed_systemd_workload_identity_fields( no_new_privileges, umask, inaccessible_paths, - load_credential, main_pid, process_start_time, process_uids, @@ -2859,7 +2861,6 @@ impl SystemdTransientWorkloadDriver { OsString::from("--property=NoNewPrivileges"), OsString::from("--property=UMask"), OsString::from("--property=InaccessiblePaths"), - OsString::from("--property=LoadCredential"), OsString::from("--property=OpenFile"), OsString::from("--property=WorkingDirectory"), OsString::from("--property=ControlGroup"), @@ -2971,38 +2972,7 @@ impl SystemdTransientWorkloadDriver { let umask = required_systemd_property(&values, "UMask")?.to_owned(); let inaccessible_paths = required_systemd_property(&values, "InaccessiblePaths")?.to_owned(); - let load_credential = required_systemd_property(&values, "LoadCredential")?.to_owned(); - ensure!( - service_type == "exec" - && restart_policy == "no" - && kill_mode == "mixed" - // `User` is deliberately not asserted here. Idunn never emits a - // User property, and systemd reports the identity it *allocated* - // for a DynamicUser unit -- the unit name, or a generated `_du…` - // when that is too long -- so the property is never empty for a - // running workload and cannot tell an allocated identity from a - // pinned one. The invariant it was reaching for, that the workload - // does not run as a persistent named account, is carried by - // `dynamic_user` below and by Idunn never passing User at all. - && dynamic_user - && supplementary_groups.is_empty() - && capability_bounding_set.is_empty() - && ambient_capabilities.is_empty() - && private_mounts - && private_pids - && protect_proc == "invisible" - && proc_subset == "all" - && no_new_privileges - && umask == "0007" - && inaccessible_paths == self.credential_root.display().to_string() - && load_credential - == if service_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }, - "systemd workload isolation properties differ from the Idunn contract" - ); + require_systemd_isolation_contract(&values, &self.credential_root)?; let working_directory = PathBuf::from(required_systemd_property(&values, "WorkingDirectory")?); ensure!( @@ -3144,6 +3114,11 @@ impl SystemdTransientWorkloadDriver { ); Some(path) }; + require_delivered_credentials( + &process_root.join("root"), + unit, + service_credential_names, + )?; let activation_descriptor = &parent_only_file_descriptors[0]; ensure!( activation_descriptor.fd_name == IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME @@ -3251,7 +3226,7 @@ impl SystemdTransientWorkloadDriver { no_new_privileges, umask, inaccessible_paths, - load_credential, + load_credential: String::new(), main_pid, process_start_time, process_uids: process_security.uids, @@ -3465,12 +3440,7 @@ impl SystemdTransientWorkloadDriver { // workload. The same applies to `Group` when no fixed state group is // bound. In that case the numeric primary GID must match the allocated // UID; a fixed state group still requires both its exact name and GID. - let expected_load_credential = if regular_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }; - let launch_contract: [(bool, &str); 27] = [ + let launch_contract: [(bool, &str); 26] = [ ( observation.unit_description == expected_description, "unit description", @@ -3510,10 +3480,6 @@ impl SystemdTransientWorkloadDriver { observation.inaccessible_paths == self.credential_root.display().to_string(), "inaccessible paths", ), - ( - observation.load_credential == expected_load_credential, - "load credential", - ), ( observation.working_directory == installed, "working directory", @@ -7269,6 +7235,44 @@ fn path_inside_process_root(process_root: &Path, absolute_path: &Path) -> Result Ok(process_root.join(absolute_path.strip_prefix(Path::new("/"))?)) } +/// The workload holds exactly the credentials Idunn delivered: the entries of +/// `/run/credentials/` in the MainPID's root (`process_root` is that root) +/// equal `expected`, and an absent directory is the empty set. Only entry names +/// are read; a credential is never opened here. The message names no credential. +fn require_delivered_credentials( + process_root: &Path, + unit: &str, + expected: &[String], +) -> Result<()> { + let directory = path_inside_process_root( + process_root, + &Path::new("/run/credentials").join(unit), + )?; + let mut delivered = Vec::new(); + match fs::read_dir(&directory) { + Ok(entries) => { + for entry in entries { + let name = entry + .context("reading a delivered credential entry")? + .file_name() + .into_string() + .map_err(|_| anyhow!("delivered credential name is not UTF-8"))?; + delivered.push(name); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).context("reading the delivered credential directory"); + } + } + delivered.sort(); + ensure!( + delivered == expected, + "systemd delivered credentials that differ from the Idunn launch" + ); + Ok(()) +} + fn linux_process_security( status_path: &Path, expected_host_pid: u32, @@ -7484,6 +7488,56 @@ fn parse_systemd_boolean(values: &BTreeMap, name: &str) -> Resul } } +/// The isolation properties of a transient workload unit, judged from `systemctl +/// show`. `LoadCredential` is absent on purpose: systemd 257 displays it as +/// `[unprintable]` whatever the unit holds, so what the workload was given is +/// judged by `require_delivered_credentials` from the delivered directory. +fn require_systemd_isolation_contract( + values: &BTreeMap, + credential_root: &Path, +) -> Result<()> { + let service_type = required_systemd_property(values, "Type")?; + let restart_policy = required_systemd_property(values, "Restart")?; + let kill_mode = required_systemd_property(values, "KillMode")?; + let dynamic_user = parse_systemd_boolean(values, "DynamicUser")?; + let supplementary_groups = systemd_property(values, "SupplementaryGroups")?; + let capability_bounding_set = systemd_property(values, "CapabilityBoundingSet")?; + let ambient_capabilities = systemd_property(values, "AmbientCapabilities")?; + let private_mounts = parse_systemd_boolean(values, "PrivateMounts")?; + let private_pids = parse_systemd_boolean(values, "PrivatePIDs")?; + let protect_proc = required_systemd_property(values, "ProtectProc")?; + let proc_subset = required_systemd_property(values, "ProcSubset")?; + let no_new_privileges = parse_systemd_boolean(values, "NoNewPrivileges")?; + let umask = required_systemd_property(values, "UMask")?; + let inaccessible_paths = required_systemd_property(values, "InaccessiblePaths")?; + ensure!( + service_type == "exec" + && restart_policy == "no" + && kill_mode == "mixed" + // `User` is deliberately not asserted here. Idunn never emits a + // User property, and systemd reports the identity it *allocated* + // for a DynamicUser unit -- the unit name, or a generated `_du…` + // when that is too long -- so the property is never empty for a + // running workload and cannot tell an allocated identity from a + // pinned one. The invariant it was reaching for, that the workload + // does not run as a persistent named account, is carried by + // `dynamic_user` below and by Idunn never passing User at all. + && dynamic_user + && supplementary_groups.is_empty() + && capability_bounding_set.is_empty() + && ambient_capabilities.is_empty() + && private_mounts + && private_pids + && protect_proc == "invisible" + && proc_subset == "all" + && no_new_privileges + && umask == "0007" + && inaccessible_paths == credential_root.display().to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + Ok(()) +} + fn read_process_environment(path: &Path) -> Result>> { let bytes = fs::read(path) .with_context(|| format!("reading process environment {}", path.display()))?; @@ -9968,6 +10022,160 @@ processes 9 ); } + #[cfg(unix)] + fn fixture_show_properties(text: &str) -> Result> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let shown = temp.path().join("shown"); + std::fs::write(&shown, text)?; + let systemctl = temp.path().join("systemctl"); + std::fs::write( + &systemctl, + format!("#!/bin/sh\ncat '{}'\nexit 0\n", shown.display()), + )?; + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; + let driver = SystemdTransientWorkloadDriver { + systemctl_program: systemctl, + ..Default::default() + }; + Ok(driver + .show_unit("idunn-test.service")? + .context("fixture unit was not found")? + .properties) + } + + /// systemd 257 prints `LoadCredential=[unprintable]` for every unit, so the + /// isolation contract must admit the real output of a unit with no service + /// credential as well as one with one. The inputs are real captures, not a + /// re-spelling. + #[cfg(unix)] + #[test] + fn systemd_257_isolation_contract_admits_a_presence_only_target_and_one_with_service_credentials( + ) -> Result<()> { + let root = Path::new("/run/idunn/activation-credentials"); + let vili = include_str!("../tests/fixtures/systemd-257/vili-presence-identity-only.show"); + let odin = include_str!( + "../tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show" + ); + for text in [vili, odin] { + assert!(text.contains("LoadCredential=[unprintable]")); + require_systemd_isolation_contract(&fixture_show_properties(text)?, root)?; + } + let default_proc = vili.replace("ProtectProc=invisible", "ProtectProc=default"); + assert_ne!(default_proc, vili); + let other_path = vili.replace( + "/run/idunn/activation-credentials", + "/run/idunn/elsewhere", + ); + assert_ne!(other_path, vili); + for text in [default_proc, other_path] { + let error = + require_systemd_isolation_contract(&fixture_show_properties(&text)?, root) + .expect_err("a drifted isolation property must be refused"); + assert_eq!( + error.to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn delivered_credentials_are_exactly_the_launch_names() -> Result<()> { + let unit = "idunn-test.service"; + let names = |list: &[&str]| list.iter().map(|name| name.to_string()).collect::>(); + // `Some(entries)` creates the unit's directory in a fresh process root. + let check = |delivered: Option<&[&str]>, expected: &[&str]| -> Result<()> { + let temp = tempfile::tempdir()?; + if let Some(entries) = delivered { + let directory = temp.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + for entry in entries { + std::fs::write(directory.join(entry), "canary-credential-contents")?; + } + } + let result = require_delivered_credentials(temp.path(), unit, &names(expected)); + if let Err(error) = &result { + let text = format!("{error:#}"); + for canary in ["STOWAWAY", "ODIN_TOPOLOGY_IDENTITY", "canary-credential"] { + assert!(!text.contains(canary), "error echoed {canary}"); + } + assert!(!text.contains(&*temp.path().to_string_lossy())); + } + result + }; + check(None, &[])?; + check(Some(&[]), &[])?; + check(Some(&["ODIN_TOPOLOGY_IDENTITY"]), &["ODIN_TOPOLOGY_IDENTITY"])?; + check(Some(&["B", "A"]), &["A", "B"])?; + for (delivered, expected) in [ + (None, &["ODIN_TOPOLOGY_IDENTITY"][..]), + (Some(&[][..]), &["ODIN_TOPOLOGY_IDENTITY"][..]), + ( + Some(&["ODIN_TOPOLOGY_IDENTITY", "STOWAWAY"][..]), + &["ODIN_TOPOLOGY_IDENTITY"][..], + ), + (Some(&["STOWAWAY"][..]), &[][..]), + (Some(&["A"][..]), &["B"][..]), + (Some(&["A"][..]), &["A", "B"][..]), + ] { + let error = check(delivered, expected) + .expect_err("delivered credentials differing from the launch must be refused"); + assert_eq!( + error.to_string(), + "systemd delivered credentials that differ from the Idunn launch" + ); + } + // The rule reads the process root it is given, never the host's /run. + let temp = tempfile::tempdir()?; + let outside = tempfile::tempdir()?; + let directory = outside.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + std::fs::write(directory.join("A"), "")?; + require_delivered_credentials(temp.path(), unit, &[])?; + assert!(require_delivered_credentials(temp.path(), unit, &names(&["A"])).is_err()); + Ok(()) + } + + /// Live Odin and StreamPixels hold records persisted by the Idunn that + /// compared `LoadCredential`: `[unprintable]` in the retired slot. The bytes + /// are that Idunn's own output (tests/fixtures/pre-cut-*); they must decode + /// under both encodings and read as no drift against a new observation, or + /// `stop` refuses to stop the incumbents. + #[test] + fn an_incumbent_recorded_with_the_display_form_is_not_identity_drift() -> Result<()> { + let decode_hex = |text: &str| -> Vec { + (0..text.trim().len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text.trim()[at..at + 2], 16).unwrap()) + .collect() + }; + let mut recorded = systemd_audit_observation("no", 61_000); + recorded.load_credential = "[unprintable]".into(); + let observed = systemd_audit_observation("no", 61_000); + assert!(observed.load_credential.is_empty()); + for hex in [ + include_str!("../tests/fixtures/pre-cut-systemd-observation.compact.hex"), + include_str!("../tests/fixtures/pre-cut-systemd-observation.named.hex"), + ] { + let decoded: WorkloadObservation = rmp_serde::from_slice(&decode_hex(hex))?; + assert_eq!(decoded, WorkloadObservation::Systemd(recorded.clone())); + let WorkloadObservation::Systemd(prior) = decoded else { + unreachable!() + }; + assert!(changed_systemd_workload_identity_fields(&observed, &prior).is_empty()); + } + let mut drifted = observed.clone(); + drifted.main_pid += 1; + assert_eq!( + changed_systemd_workload_identity_fields(&drifted, &recorded), + vec!["main_pid"] + ); + Ok(()) + } + #[test] fn host_isolation_is_two_live_distinct_processes_on_one_host() { let live = |pid: u32, created: u64| { From dfdc226c2544150ce7b4975e1137a875c329034a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 15:14:15 +0200 Subject: [PATCH 163/196] Idunn credential rule: an unlistable credential directory is a refusal cargo-mutants showed that treating every read_dir error as the empty set survived. Test the non-NotFound case (the unit path is a file). Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 43af330..7f87896 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -10128,6 +10128,14 @@ processes 9 "systemd delivered credentials that differ from the Idunn launch" ); } + // A directory that cannot be listed is a refusal, never the empty set. + let temp = tempfile::tempdir()?; + let parent = temp.path().join("run/credentials"); + std::fs::create_dir_all(&parent)?; + std::fs::write(parent.join(unit), "")?; + let error = require_delivered_credentials(temp.path(), unit, &[]) + .expect_err("an unreadable credential directory must be refused"); + assert!(!format!("{error:#}").contains(&*temp.path().to_string_lossy())); // The rule reads the process root it is given, never the host's /run. let temp = tempfile::tempdir()?; let outside = tempfile::tempdir()?; From 78086705a05742ca94cf5308aea2f268720dd0da Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:04:37 +0100 Subject: [PATCH 164/196] deploy-provenance map: runner failure evidence (body facts R1-R8) Imagination for cut runner-failure-evidence: the docker runner discards stdout, so a step that reports on stdout fails with an empty message; the tail moves to a root-only typed evidence store read by idunn status. Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-map.md | 108 ++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md index 5ec7052..95c0eee 100644 --- a/docs/deploy-provenance-map.md +++ b/docs/deploy-provenance-map.md @@ -284,3 +284,111 @@ Idunn's body changes only through gamecult-ops `scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and bindings unchanged). After it, Vili's Hands reruns from step 4 of `motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. + +## Cut `runner-failure-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at +`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area +Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, +transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 +UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and +nobody could tell which recipe step failed. + +### Body facts (source read at `804ed75`; host read-only, 2026-10-25) + +- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by + `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and + `Command::output()`: stdout and stderr are each buffered whole in the + daemon's memory, unbounded. On a non-zero exit the error is + `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The + container is gone the moment it exits, so `docker ps -a` and `docker logs` + hold nothing. +- R2. The empty message: the failing process wrote nothing to stderr, and + stdout, the only stream that carried its report, is discarded at + `:2182-2193`. The same interpolation echoes whatever a step does write to + stderr into `last_error`, `pre_fencing_abort.error` and + `completion.error` (`bounded_detail`, 2048 bytes), and from there into + `idunn up`'s `deployment failed:` line and `idunn status`. +- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops + over `declaration.steps` and calls `run_in_workspace` with argv only; + `step.id` is used only in the environment precondition. External inputs + (`curl` in the runner, `:2247-2258`) fail through the same helper with the + same anonymous message. +- R4. Which step failed on 2026-10-25, inferred from the failed transaction's + staging root, which survives (926 MiB, + `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, + `node_modules/.package-lock.json` was written at 00:51:24 (so + `install-vili-dependencies`, `npm ci`, finished), nothing outside + `node_modules` is newer, and `release/vili` (the next step's output) is + absent; the python workspace holds only fetched inputs. The failing step was + `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test + reporter writes to stdout: exactly the stream R1 discards. Inference from + residue, not from output. +- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) + sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which + persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal + on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only + provider-exclusion lines. +- R6. Stores. `control.cc` (root 0640) holds commands, transactions + (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and + expiry requests; `ControlSnapshot::read` refuses any other type + (`:3420`). Records must re-encode byte-exactly (`decode_record`, + `:3717-3727`), so a new transaction field is a schema bump and a v4 lift + (`:4100-4110`). Terminal transactions move verbatim to `history.cc` + (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. + `incidents.cc` lives in the published projection directory and by its own + doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, + because a type added there stops continuity for every target on a rollback. +- R7. The daemon runs as root (`User=root`). Runner secrets reach the + container as read-only file mounts named by env (`:2041-2047`), so any text + a step derives from them can land in its output. +- R8. Staging residue: 475 transaction roots, 55 GiB under + `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces + because `materialize` returns before `remove_tree_inside`. + +### Decision + +The failure becomes a typed value at the runner, `RunnerStepFailure`: the +subject (recipe step or external input, by id), the runner id, the exit code or +signal, and the last 8 KiB of stdout and of stderr with each stream's total +byte count. Its `Display` carries the subject and status, never output. +Capture streams both pipes through bounded tails, so a verbose build no longer +buffers whole in the daemon. + +The tail lives in a new private store, `runner-evidence.cc`, beside +`control.cc` (mode 0600), one record per transaction +(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the +transaction record: that is a v5 bump whose rollback makes every older Idunn +refuse its control store (R6), the scar `incident.rs` already names. Not in +`incidents.cc`: published, and free text is barred there. Not in the journal: +journal readers are a wider set than root, so the journal line names the +transaction, the subject, the status and the store, and no output. + +No redaction. Matching secret bytes would make the daemon read every secret, +and still misses any encoding, prefix or transform of one; a claimed redaction +that leaks is worse than a stated rule. The rule instead is the reader set: the +store's only readers (root) can already read every secret the runner was +given. + +Keeping the failed container (dropping `--rm`) is worse: its output then lives +in Docker's json-file log, unbounded, untyped, outside Idunn's state and +readable by the docker group; containers accumulate with no owner to reap them; +`docker system prune` erases the evidence; and nothing ties a container to the +transaction in `idunn status`. + +### Authority + +Owner: the runner decides what failed and keeps the bounded tail; the engine's +pre-fence failure path is the one writer of `runner-evidence.cc`. +`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the +failure's `Display`. Forbidden writers: the `stderr` interpolation in +`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the +store needs no external owner; staging roots (R8) stay with +`ghostlight-verse:follow_up:idunn-generation-retention`. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). The next failed Sealing then shows its step and tail on +`sudo idunn status --command `. From ef82355bab8926362bde39e544afd3230a393488 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:11:14 +0100 Subject: [PATCH 165/196] Runner failure: a typed, bounded step failure kept in runner-evidence.cc DockerRunnerDriver::docker no longer buffers whole streams or interpolates stderr into the error. It pipes both streams through an 8 KiB tail and returns RunnerStepFailure (subject, runner, status, tails); the engine's pre-fence path is the one writer of runner-evidence.cc (0600, newest 32) and journals a line that names the store and no output. idunn status --command renders the record with control bytes escaped. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 71 ++++++++++- src/drivers.rs | 167 ++++++++++++++++++++++-- src/lib.rs | 1 + src/runner_evidence.rs | 283 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 508 insertions(+), 14 deletions(-) create mode 100644 src/runner_evidence.rs diff --git a/src/control_plane.rs b/src/control_plane.rs index 9597d43..b045e0a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -42,11 +42,12 @@ use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, RunnerStepFailure, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; +use crate::runner_evidence::{self, RunnerFailureRecord, runner_evidence_store_path}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -4877,6 +4878,24 @@ fn status( } if let Some(abort) = &transaction.pre_fencing_abort { writeln!(out, " pre-fencing abort {abort:?}")?; + // The step that failed and its output tail live in their + // own private store. An unreadable store costs its lines. + match runner_evidence::for_transaction( + &runner_evidence_store_path(store_path), + &transaction.transaction_id, + ) { + Ok(Some(record)) => { + for line in runner_evidence::render(&record) { + writeln!(out, "{line}")?; + } + } + Ok(None) => {} + Err(error) => writeln!( + out, + " runner evidence unreadable: {}", + error.root_cause() + )?, + } } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -5374,6 +5393,33 @@ impl Engine { IncidentStore::new(&self.options.incident_store) } + /// A failed runner step keeps its output tail in the runner evidence + /// store and says so on the journal, naming no output. The store write is + /// best effort: its failure is named in the line and never stops the + /// abort that follows. + fn keep_runner_failure(&self, transaction: &DeploymentTransaction, error: &anyhow::Error) { + let Some(failure) = error.downcast_ref::() else { + return; + }; + let path = runner_evidence_store_path(&self.options.state_store); + let recorded = now_millis().and_then(|now| { + runner_evidence::record( + &path, + &RunnerFailureRecord::from_failure( + &transaction.transaction_id, + &transaction.command_id, + &transaction.target, + failure, + now, + ), + ) + }); + eprintln!( + "{}", + runner_failure_journal_line(transaction, failure, &path, &recorded) + ); + } + /// Settle the outcome of one incident operation. A failure never leaves /// this function: it is traced once per episode, under the operation and /// the subject it was decided for (`*` for a pass over the whole store), @@ -6315,6 +6361,7 @@ impl Engine { if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() { + self.keep_runner_failure(&latest.value, &error); self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -10641,6 +10688,28 @@ const DETAIL_EMPTY: &str = "(no detail)"; /// only judge, and this output always satisfies it. NUL is dropped, empty /// input gets a fixed placeholder, and an over-long value keeps its head and /// its tail (where a build log's error is) cut on char boundaries. +/// The journal line for a failed runner step: the transaction, the subject and +/// status (the failure's `Display`), and where the evidence is. Never output. +fn runner_failure_journal_line( + transaction: &DeploymentTransaction, + failure: &RunnerStepFailure, + store_path: &Path, + recorded: &Result<()>, +) -> String { + let head = format!( + "Idunn transaction {} (command {}, target {}) {failure}", + transaction.transaction_id, transaction.command_id, transaction.target + ); + match recorded { + Ok(()) => format!( + "{head}; evidence in {}; read it with idunn status --command {}", + store_path.display(), + transaction.command_id + ), + Err(error) => format!("{head}; evidence was not kept: {error:#}"), + } +} + fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { diff --git a/src/drivers.rs b/src/drivers.rs index 7f87896..5c22429 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -2152,6 +2152,115 @@ pub(crate) fn docker_run_args( Ok(args) } +/// Bytes of each output stream that a failed runner step keeps. The daemon +/// never holds more than this per stream while the step runs. +pub(crate) const RUNNER_OUTPUT_TAIL_BYTES: usize = 8192; + +/// What a runner container was running: a recipe step or an external input, +/// by the id the recipe gave it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RunnerSubject { + Step(String), + ExternalInput(String), +} + +impl RunnerSubject { + pub(crate) fn id(&self) -> &str { + match self { + Self::Step(id) | Self::ExternalInput(id) => id, + } + } + + fn noun(&self) -> &'static str { + match self { + Self::Step(_) => "step", + Self::ExternalInput(_) => "external input", + } + } +} + +/// A runner container that did not exit successfully: what failed, how, and +/// the last `RUNNER_OUTPUT_TAIL_BYTES` of each stream with its total size. +/// The tails are evidence for `runner-evidence.cc`. `Display` and `Debug` +/// carry none of them, so no error chain, record or log line renders output. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RunnerStepFailure { + pub(crate) subject: RunnerSubject, + pub(crate) runner: String, + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stdout_total_bytes: u64, + pub(crate) stderr_tail: Vec, + pub(crate) stderr_total_bytes: u64, +} + +impl RunnerStepFailure { + /// The status as an operator reads it. + pub(crate) fn status(&self) -> String { + runner_status(self.exit_code, self.signal) + } +} + +/// How a container ended, as an operator reads it. +pub(crate) fn runner_status(exit_code: Option, signal: Option) -> String { + match (exit_code, signal) { + (Some(code), _) => format!("exit status {code}"), + (None, Some(signal)) => format!("signal {signal}"), + (None, None) => "an unknown status".to_owned(), + } +} + +impl std::fmt::Display for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "runner {} {} on runner {} exited with {}; its output tail is kept in runner-evidence.cc", + self.subject.noun(), + self.subject.id(), + self.runner, + self.status() + ) + } +} + +impl std::fmt::Debug for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerStepFailure") + .field("subject", &self.subject) + .field("runner", &self.runner) + .field("exit_code", &self.exit_code) + .field("signal", &self.signal) + .field("stdout_total_bytes", &self.stdout_total_bytes) + .field("stderr_total_bytes", &self.stderr_total_bytes) + .finish_non_exhaustive() + } +} + +impl std::error::Error for RunnerStepFailure {} + +/// Read `reader` to its end keeping only the last `cap` bytes and counting +/// every byte, so a verbose step never buffers whole in the daemon. +fn drain_tail(mut reader: impl Read, cap: usize) -> std::io::Result<(Vec, u64)> { + let mut tail = Vec::new(); + let mut total = 0u64; + let mut chunk = [0u8; 8192]; + loop { + let read = match reader.read(&mut chunk) { + Ok(0) => return Ok((tail, total)), + Ok(read) => read, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + total += read as u64; + tail.extend_from_slice(&chunk[..read]); + if tail.len() > cap { + tail.drain(..tail.len() - cap); + } + } +} + /// Docker is only a runner substrate. Recipe argv stays an argv vector, the /// operator binding selects the exact image/network/mount affordances, and the /// driver returns complete materialization receipts rather than build truth by @@ -2170,7 +2279,7 @@ impl Default for DockerRunnerDriver { } impl DockerRunnerDriver { - fn docker(&self, args: I) -> Result + fn docker(&self, subject: RunnerSubject, runner_id: &str, args: I) -> Result<()> where I: IntoIterator, S: AsRef, @@ -2179,23 +2288,50 @@ impl DockerRunnerDriver { self.docker_program.is_absolute(), "Docker runner program is not absolute" ); - let output = Command::new(&self.docker_program) + let mut child = Command::new(&self.docker_program) .args(args) .stdin(Stdio::null()) - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .with_context(|| format!("starting Docker runner {}", self.docker_program.display()))?; - if !output.status.success() { - bail!( - "Docker runner exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); + let stdout = child.stdout.take().context("Docker runner has no stdout pipe")?; + let stderr = child.stderr.take().context("Docker runner has no stderr pipe")?; + let stdout = thread::spawn(move || drain_tail(stdout, RUNNER_OUTPUT_TAIL_BYTES)); + let stderr = thread::spawn(move || drain_tail(stderr, RUNNER_OUTPUT_TAIL_BYTES)); + let status = child.wait().context("waiting for the Docker runner")?; + let (stdout_tail, stdout_total_bytes) = stdout + .join() + .map_err(|_| anyhow!("the runner stdout reader panicked"))? + .context("reading the runner stdout")?; + let (stderr_tail, stderr_total_bytes) = stderr + .join() + .map_err(|_| anyhow!("the runner stderr reader panicked"))? + .context("reading the runner stderr")?; + if status.success() { + return Ok(()); } - Ok(output) + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal = None; + Err(RunnerStepFailure { + subject, + runner: runner_id.to_owned(), + exit_code: status.code(), + signal, + stdout_tail, + stdout_total_bytes, + stderr_tail, + stderr_total_bytes, + } + .into()) } fn run_in_workspace( &self, + step_id: &str, + runner_id: &str, runner: &DockerRunnerBinding, workspace: &Path, working_directory: &Path, @@ -2212,8 +2348,7 @@ impl DockerRunnerDriver { build_machine_id_file(workspace)?; let spec = ContainerSpec::for_step(runner, required_environment, source_stamp)?; let args = docker_run_args(&spec, workspace, working_directory, argv)?; - self.docker(args)?; - Ok(()) + self.docker(RunnerSubject::Step(step_id.to_owned()), runner_id, args) } fn materialize_external_input( @@ -2255,7 +2390,11 @@ impl DockerRunnerDriver { input.url.clone(), ]; let args = docker_run_args(&spec, workspace, Path::new("."), &curl_argv)?; - self.docker(args)?; + self.docker( + RunnerSubject::ExternalInput(input.id.clone()), + &input.runner, + args, + )?; let bytes = fs::read(&destination).with_context(|| { format!( "reading materialized external input {}", @@ -2377,6 +2516,8 @@ impl RunnerPort for DockerRunnerDriver { ); } self.run_in_workspace( + &step.id, + &step.runner, runner, workspace, &step.working_directory, diff --git a/src/lib.rs b/src/lib.rs index e71a17f..4f365a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,4 +7,5 @@ pub mod host_actuator; #[cfg(unix)] pub mod launcher; mod incident; +mod runner_evidence; pub mod provisioning; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs new file mode 100644 index 0000000..6e305d4 --- /dev/null +++ b/src/runner_evidence.rs @@ -0,0 +1,283 @@ +//! The output tail of a failed runner step. +//! +//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the +//! last bytes of a failed runner container's stdout and stderr, one record per +//! transaction. The engine's pre-fence failure path is its only writer, and +//! `idunn status --command` its only reader. It is never `control.cc`, which +//! refuses any type it does not know, and never `incidents.cc`, which is +//! published and carries no free text. +//! +//! The tail is raw runner output. It is kept unredacted and readable by root +//! alone (mode 0600): root can already read every secret the runner was given. +//! It never enters an error, a transaction record or the journal; the render +//! below escapes control characters because the status reader is a terminal. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + TryCompareExchangeSnapshotOutcome, +}; +use serde::{Deserialize, Serialize}; + +use crate::control_plane::{decode_record, require_id, typed_envelope}; +use crate::drivers::{ + LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject, + exchange_behind_private_lock, runner_status, +}; + +const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1"; + +/// Records kept; the writer drops the oldest past this, so the store needs no +/// other owner. +pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32; + +/// `runner-evidence.cc`, beside the control store. +pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf { + state_store.with_file_name("runner-evidence.cc") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum SubjectKind { + Step, + ExternalInput, +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")] +pub(crate) struct RunnerFailureRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) transaction_id: String, + #[cultcache(key = 2)] + pub(crate) command_id: String, + #[cultcache(key = 3)] + pub(crate) target: String, + #[cultcache(key = 4)] + pub(crate) subject_kind: SubjectKind, + #[cultcache(key = 5)] + pub(crate) subject_id: String, + #[cultcache(key = 6)] + pub(crate) runner: String, + #[cultcache(key = 7)] + pub(crate) exit_code: Option, + #[cultcache(key = 8)] + pub(crate) signal: Option, + #[cultcache(key = 9)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 10)] + pub(crate) stdout_total_bytes: u64, + #[cultcache(key = 11)] + pub(crate) stderr_tail: Vec, + #[cultcache(key = 12)] + pub(crate) stderr_total_bytes: u64, + #[cultcache(key = 13)] + pub(crate) recorded_at_unix_millis: u64, +} + +/// Tails stay out of `Debug`, so a failed assertion or a logged record never +/// prints runner output. +impl std::fmt::Debug for RunnerFailureRecord { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerFailureRecord") + .field("transaction_id", &self.transaction_id) + .field("subject_id", &self.subject_id) + .finish_non_exhaustive() + } +} + +impl RunnerFailureRecord { + pub(crate) fn from_failure( + transaction_id: &str, + command_id: &str, + target: &str, + failure: &RunnerStepFailure, + now: u64, + ) -> Self { + let (subject_kind, subject_id) = match &failure.subject { + RunnerSubject::Step(id) => (SubjectKind::Step, id), + RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id), + }; + Self { + schema_version: RUNNER_FAILURE_SCHEMA.to_owned(), + transaction_id: transaction_id.to_owned(), + command_id: command_id.to_owned(), + target: target.to_owned(), + subject_kind, + subject_id: subject_id.clone(), + runner: failure.runner.clone(), + exit_code: failure.exit_code, + signal: failure.signal, + stdout_tail: failure.stdout_tail.clone(), + stdout_total_bytes: failure.stdout_total_bytes, + stderr_tail: failure.stderr_tail.clone(), + stderr_total_bytes: failure.stderr_total_bytes, + recorded_at_unix_millis: now, + } + } + + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == RUNNER_FAILURE_SCHEMA, + "runner failure schema is unsupported" + ); + require_id(&self.transaction_id, "runner failure transaction")?; + require_id(&self.command_id, "runner failure command")?; + require_id(&self.target, "runner failure target")?; + require_id(&self.subject_id, "runner failure subject")?; + require_id(&self.runner, "runner failure runner")?; + for (tail, total, stream) in [ + (&self.stdout_tail, self.stdout_total_bytes, "stdout"), + (&self.stderr_tail, self.stderr_total_bytes, "stderr"), + ] { + ensure!( + tail.len() <= RUNNER_OUTPUT_TAIL_BYTES, + "runner failure {stream} tail exceeds its bound" + ); + ensure!( + tail.len() as u64 <= total, + "runner failure {stream} tail exceeds its stream" + ); + } + Ok(()) + } + + fn envelope(&self) -> Result { + self.validate()?; + typed_envelope( + &self.transaction_id, + Self::TYPE, + RUNNER_FAILURE_SCHEMA, + self, + self.recorded_at_unix_millis, + ) + } +} + +/// One read of the store: the envelopes exactly as stored, and the records +/// decoded from them in the same order. +fn snapshot(path: &Path) -> Result<(Vec, Vec)> { + let envelopes = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading the runner evidence store")?; + let mut records = Vec::new(); + for envelope in &envelopes { + ensure!( + envelope.r#type == RunnerFailureRecord::TYPE + && envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA), + "the runner evidence store holds a document that is not a runner failure" + ); + let record: RunnerFailureRecord = decode_record(envelope)?; + record.validate()?; + ensure!( + envelope.key == record.transaction_id, + "runner evidence store key differs from the record's transaction" + ); + records.push(record); + } + Ok((envelopes, records)) +} + +/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the +/// oldest records go in the same exchange. The file ends at mode 0600. +pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result<()> { + let (envelopes, records) = snapshot(path)?; + let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes + .iter() + .zip(&records) + .filter(|(_, stored)| stored.transaction_id != record.transaction_id) + .map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone())) + .collect(); + kept.push((record.recorded_at_unix_millis, record.envelope()?)); + // Stable: records of one instant keep their stored order, so the record + // being written survives a tie. + kept.sort_by_key(|(recorded_at, _)| *recorded_at); + let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS); + let replacements: Vec<_> = kept + .into_iter() + .skip(excess) + .map(|(_, envelope)| envelope) + .collect(); + match exchange_behind_private_lock(path, &envelopes, &replacements)? { + TryCompareExchangeSnapshotOutcome::Exchanged => {} + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("the runner evidence store changed while it was being written") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + return Err(LockContended { + store: "runner evidence store", + } + .into()); + } + } + keep_private(path) +} + +#[cfg(unix)] +fn keep_private(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("making the runner evidence store private") +} + +#[cfg(not(unix))] +fn keep_private(_path: &Path) -> Result<()> { + Ok(()) +} + +/// The one reader. An absent store holds nothing. +pub(crate) fn for_transaction( + path: &Path, + transaction_id: &str, +) -> Result> { + Ok(snapshot(path)? + .1 + .into_iter() + .find(|stored| stored.transaction_id == transaction_id)) +} + +/// The record as `idunn status --command` prints it. The store keeps raw +/// bytes; every control character except newline and tab is escaped here, so +/// build output cannot drive the operator's terminal. +pub(crate) fn render(record: &RunnerFailureRecord) -> Vec { + let noun = match record.subject_kind { + SubjectKind::Step => "step", + SubjectKind::ExternalInput => "external input", + }; + let mut lines = vec![ + format!(" failed {noun} {}", escape(&record.subject_id)), + format!(" runner {}", escape(&record.runner)), + format!(" status {}", runner_status(record.exit_code, record.signal)), + ]; + for (stream, tail, total) in [ + ("stdout", &record.stdout_tail, record.stdout_total_bytes), + ("stderr", &record.stderr_tail, record.stderr_total_bytes), + ] { + lines.push(format!( + " {stream} (last {} of {total} bytes)", + tail.len() + )); + let text = escape(&String::from_utf8_lossy(tail)); + let text = text.strip_suffix('\n').unwrap_or(&text); + if !text.is_empty() { + lines.extend(text.split('\n').map(|line| format!(" | {line}"))); + } + } + lines +} + +fn escape(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + if character.is_control() && character != '\n' && character != '\t' { + escaped.push_str(&format!("\\u{{{:x}}}", u32::from(character))); + } else { + escaped.push(character); + } + } + escaped +} From 22bd904ed9d4ffdfacfdbe5d303971a40aa03b46 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:16:03 +0100 Subject: [PATCH 166/196] Runner failure tests: named step, bounded tail, never echoed, private store Drive the real DockerRunnerDriver over a fake docker program, through the engine's Sealing path: the second step is named with its exit status, both tails are kept in a 0600 store capped at 32, no error, control.cc or journal line carries the output, status escapes control bytes, an unwritable store does not stop the abort, and control.cc's record bytes equal those captured at base 804ed75. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 313 +++++++++++++++++++++++++++++++++++++++-- src/drivers.rs | 197 ++++++++++++++++++++++++++ src/runner_evidence.rs | 146 +++++++++++++++++++ 3 files changed, 647 insertions(+), 9 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index b045e0a..d927152 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5397,10 +5397,12 @@ impl Engine { /// store and says so on the journal, naming no output. The store write is /// best effort: its failure is named in the line and never stops the /// abort that follows. - fn keep_runner_failure(&self, transaction: &DeploymentTransaction, error: &anyhow::Error) { - let Some(failure) = error.downcast_ref::() else { - return; - }; + fn keep_runner_failure( + &self, + transaction: &DeploymentTransaction, + error: &anyhow::Error, + ) -> Option { + let failure = error.downcast_ref::()?; let path = runner_evidence_store_path(&self.options.state_store); let recorded = now_millis().and_then(|now| { runner_evidence::record( @@ -5414,10 +5416,7 @@ impl Engine { ), ) }); - eprintln!( - "{}", - runner_failure_journal_line(transaction, failure, &path, &recorded) - ); + Some(runner_failure_journal_line(transaction, failure, &path, &recorded)) } /// Settle the outcome of one incident operation. A failure never leaves @@ -6361,7 +6360,9 @@ impl Engine { if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() { - self.keep_runner_failure(&latest.value, &error); + if let Some(line) = self.keep_runner_failure(&latest.value, &error) { + eprintln!("{line}"); + } self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -24726,4 +24727,298 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } } + + // BEGIN runner-failure-evidence tests + // A failed runner step is named, its output tail is kept in + // runner-evidence.cc, and no output reaches an error, control.cc or the + // journal line. The runner is the real DockerRunnerDriver driving a fake + // docker program, through the engine's Sealing path. + + const RUNNER_CANARY: &str = "CANARY-7c41e0d9"; + + /// A source that freezes a fixed tree, so a deployment reaches the runner. + struct TreeSource { + recipe: String, + root: PathBuf, + } + + impl SourcePort for TreeSource { + fn resolve( + &self, + _: &OperatorBinding, + _: &str, + _: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result { + Ok(FrozenSourceReceipt { + transaction_id: transaction_id.to_owned(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: format!("sha256-{}", "a".repeat(64)), + }) + } + + fn observe_frozen( + &self, + plan: &CompiledDeploymentPlan, + receipt: &FrozenSourceReceipt, + ) -> Result { + Ok(crate::drivers::FrozenSource::fixture( + receipt.clone(), + plan.source.clone(), + plan.recipe_blob.clone(), + self.root.clone(), + )) + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// The unrouted recipe with two steps ahead of its build, and no external + /// input: the second step, `test-daemon`, is the one the fake docker fails. + fn two_step_recipe() -> String { + let recipe = unrouted_recipe(); + let (head, rest) = recipe + .split_once("[[external_inputs]]") + .expect("the recipe declares an external input"); + let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts"); + let head = head.replacen( + "[[steps]]\nid = \"build\"", + "[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\ + [[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\ + [[steps]]\nid = \"build\"", + 1, + ); + format!("{head}[[artifacts]]{tail}") + } + + /// A Sealing world whose runner is the real driver over a fake docker: the + /// first call succeeds, the second prints the canary and an escape + /// sequence on both streams and exits 3. + #[cfg(unix)] + fn failing_runner_world() -> Result<(EngineFixture, TempDir)> { + use std::os::unix::fs::PermissionsExt; + let tools = TempDir::new()?; + let docker = tools.path().join("docker"); + std::fs::write( + &docker, + format!( + "#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n" + ), + )?; + std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?; + let recipe = two_step_recipe(); + let mut world = sealing_world(&recipe)?; + let tree = tools.path().join("tree"); + std::fs::create_dir_all(&tree)?; + std::fs::write(tree.join("README"), b"source")?; + world.engine.source = Arc::new(TreeSource { recipe, root: tree }); + world.engine.docker_runner = DockerRunnerDriver { docker_program: docker }; + Ok((world, tools)) + } + + /// Scheduler ticks until the deployment ends (or the budget is spent). + fn tick_until_ended(world: &EngineFixture) -> Result { + for _ in 0..24 { + world.engine.resume_backoff.lock().unwrap().clear(); + let _ = world.engine.run_scheduler_tick(); + if latest(world)?.completion.is_some() { + break; + } + } + latest(world) + } + + fn printed_status(world: &EngineFixture) -> Result { + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?; + Ok(String::from_utf8(printed)?) + } + + fn canary_failure(stdout: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject: crate::drivers::RunnerSubject::Step("test-daemon".into()), + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stdout.to_vec(), + stderr_total_bytes: stdout.len() as u64, + } + } + + #[cfg(unix)] + #[test] + fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error); + }; + // Named: the second step, on its runner, with the exit status. + assert!( + error.contains("runner step test-daemon on runner rust exited with exit status 3"), + "{error}" + ); + // Never echoed: no error text, no control.cc byte, carries the output. + let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; + for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); + } + let control = std::fs::read(&world.state_store)?; + assert!( + !control.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), + "runner output reached control.cc" + ); + // Kept: in the private store, tails of both streams, and on status. + let evidence = runner_evidence_store_path(&world.state_store); + assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("no evidence for the failed transaction")?; + assert_eq!(record.subject_id, "test-daemon"); + assert_eq!(record.exit_code, Some(3)); + assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY)); + let printed = printed_status(&world)?; + assert!(printed.contains("failed step test-daemon"), "{printed}"); + assert!(printed.contains(RUNNER_CANARY), "{printed}"); + assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte"); + Ok(()) + } + + #[test] + fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> { + let world = EngineFixture::new()?; + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let line = world + .engine + .keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing")) + .context("a runner failure produced no journal line")?; + assert!(!line.contains(RUNNER_CANARY), "{line}"); + for needle in [ + transaction.transaction_id.as_str(), + "up-ghostlight", + "ghostlight", + "runner step test-daemon on runner rust exited with exit status 3", + "runner-evidence.cc", + "idunn status --command up-ghostlight", + ] { + assert!(line.contains(needle), "{needle} missing from {line}"); + } + // Not a runner failure: nothing is kept and nothing is said. + assert!( + world + .engine + .keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure")) + .is_none() + ); + // The line a failed write produces says so, with no output. + let unwritable = runner_failure_journal_line( + &transaction, + &failure, + Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"), + &Err(anyhow::anyhow!("disk full")), + ); + assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}"); + assert!(!unwritable.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + // A directory where the store should be: every write to it fails. + std::fs::create_dir(runner_evidence_store_path(&world.state_store))?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b""))) + .context("no journal line")?; + assert!(line.contains("evidence was not kept"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?; + let printed = printed_status(&world)?; + assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}"); + assert!(printed.contains("target service phase"), "{printed}"); + assert!(printed.contains("pre-fencing abort"), "{printed}"); + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + // The store a rolled-back Idunn reads: it refuses any foreign type. + ControlSnapshot::read(&world.state_store)?; + let envelopes = + SingleFileMessagePackBackingStore::new(&world.state_store).pull_all_read_only_snapshot()?; + assert!(!envelopes.is_empty()); + for envelope in &envelopes { + assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); + if envelope.r#type == DeploymentTransaction::TYPE { + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + } + } + Ok(()) + } + + /// The records control.cc holds re-encode byte for byte. The hex was + /// captured at base 804ed75, before this cut touched anything: a runner + /// failure adds no field, no type and no schema to the control store. + #[test] + fn control_records_encode_exactly_as_they_did_at_base() -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?; + transaction.transaction_id = "tx-pinned".into(); + transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into()); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "runner step test-daemon on runner rust exited with exit status 3".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); + let command_hex = hex(&command_envelope(&command, 100)?.payload); + let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload); + assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed"); + assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed"); + Ok(()) + } + + const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; + const PINNED_TRANSACTION_HEX: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; + // END runner-failure-evidence tests } diff --git a/src/drivers.rs b/src/drivers.rs index 5c22429..1c8d067 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -157,6 +157,24 @@ pub struct FrozenSourceReceipt { pub snapshot_sha256: String, } +#[cfg(test)] +impl FrozenSource { + /// A frozen source over an existing tree, for tests that drive a runner. + pub(crate) fn fixture( + receipt: FrozenSourceReceipt, + facts: SourceSelectionFacts, + recipe_bytes: Vec, + root: PathBuf, + ) -> Self { + Self { + receipt, + facts, + recipe_bytes, + root, + } + } +} + impl FrozenSourceReceipt { pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> { plan.validate()?; @@ -16372,4 +16390,183 @@ processes 9 ); Ok(()) } + + // BEGIN runner-failure-evidence tests + // The runner decides what failed and what it kept: these drive the real + // DockerRunnerDriver over a fake docker program, never the capture alone. + + const RUNNER_CANARY: &str = "CANARY-5d2e91b7"; + + #[cfg(unix)] + fn fake_docker(dir: &Path, body: &str) -> Result { + use std::os::unix::fs::PermissionsExt; + let program = dir.join("docker"); + fs::write(&program, format!("#!/bin/sh\n{body}\n"))?; + fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?; + Ok(DockerRunnerDriver { + docker_program: program, + }) + } + + /// One recipe step through the real `run_in_workspace` path. + #[cfg(unix)] + fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> { + let workspace = dir.join("workspace"); + fs::create_dir_all(&workspace)?; + driver.run_in_workspace( + step_id, + "rust", + &fixture_docker_runner_binding(), + &workspace, + Path::new("."), + &["cargo".to_owned(), "test".to_owned()], + &std::collections::BTreeSet::new(), + ("IDUNN_STAMP", "rev"), + ) + } + + #[cfg(unix)] + fn failure_of(result: Result<()>) -> Result { + let error = result.expect_err("the step did not fail"); + error + .downcast_ref::() + .cloned() + .with_context(|| format!("the failure is not typed: {error:#}")) + } + + #[cfg(unix)] + #[test] + fn a_failed_step_is_named_with_both_tails() -> Result<()> { + let temp = tempfile::tempdir()?; + // The first step passes, the second fails: the count is the fake's. + let driver = fake_docker( + temp.path(), + &format!( + "n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\ + echo 'stderr report' >&2\nexit 3" + ), + )?; + run_step(&driver, temp.path(), "first")?; + let failure = failure_of(run_step(&driver, temp.path(), "second"))?; + assert_eq!(failure.subject, RunnerSubject::Step("second".into())); + assert_eq!(failure.runner, "rust"); + assert_eq!(failure.exit_code, Some(3)); + assert_eq!(failure.signal, None); + assert!( + failure + .stdout_tail + .ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()), + "stdout was dropped: {:?}", + String::from_utf8_lossy(&failure.stdout_tail) + ); + assert_eq!(failure.stderr_tail, b"stderr report\n"); + assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "kill -9 $$")?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert_eq!((failure.exit_code, failure.signal), (None, Some(9))); + assert!(failure.to_string().contains("exited with signal 9"), "{failure}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_external_input_is_named_as_one() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?; + let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?; + let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?; + let input_id = declaration.external_inputs[0].id.clone(); + let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]); + fs::create_dir_all(&workspaces["rust"])?; + let error = driver + .materialize_external_input(&declaration, &binding, &workspaces, &input_id) + .expect_err("the download did not fail"); + let failure = error + .downcast_ref::() + .with_context(|| format!("the failure is not typed: {error:#}"))?; + assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone())); + assert_eq!(failure.exit_code, Some(22)); + assert!( + failure.to_string().contains(&format!("runner external input {input_id} on runner rust")), + "{failure}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn runner_output_never_reaches_the_error_text() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY)); + let error = anyhow::Error::new(failure.clone()).context("sealing"); + for text in [ + format!("{error:#}"), + format!("{error:?}"), + format!("{failure}"), + format!("{failure:?}"), + ] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { + let temp = tempfile::tempdir()?; + let data = temp.path().join("data"); + for size in [ + 0, + 1, + RUNNER_OUTPUT_TAIL_BYTES - 1, + RUNNER_OUTPUT_TAIL_BYTES, + RUNNER_OUTPUT_TAIL_BYTES + 1, + 1024 * 1024, + ] { + // Distinct neighbours, so a shifted or head-keeping tail differs. + let bytes: Vec = (0..size).map(|index| (index % 251) as u8).collect(); + fs::write(&data, &bytes)?; + let driver = fake_docker( + temp.path(), + &format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..]; + assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes"); + assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes"); + assert_eq!(failure.stdout_total_bytes, size as u64); + assert_eq!(failure.stderr_total_bytes, size as u64); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> { + let temp = tempfile::tempdir()?; + run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?; + let missing = DockerRunnerDriver { + docker_program: temp.path().join("no-such-docker"), + }; + let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran"); + assert!(error.downcast_ref::().is_none()); + assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); + Ok(()) + } + // END runner-failure-evidence tests } diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index 6e305d4..9eeab4c 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -281,3 +281,149 @@ fn escape(text: &str) -> String { } escaped } + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject, + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stderr.to_vec(), + stderr_total_bytes: stderr.len() as u64, + } + } + + fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { + RunnerFailureRecord::from_failure( + transaction, + "up-service", + "service", + &failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"), + now, + ) + } + + #[test] + fn the_evidence_store_keeps_the_newest_32() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + for index in 0..=RUNNER_EVIDENCE_RECORDS { + record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; + } + let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); + assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS); + assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); + assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); + assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); + // Rewriting a resident transaction replaces it and drops nothing. + let mut again = record_at("tx-05", 5000); + again.stdout_tail = b"rewritten\n".to_vec(); + again.stdout_total_bytes = 10; + record(&path, &again)?; + let kept = snapshot(&path)?.1; + assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS); + assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); + let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; + assert_eq!(stored.stdout_tail, b"rewritten\n"); + assert_eq!(for_transaction(&path, "tx-00")?, None); + Ok(()) + } + + #[test] + fn a_record_round_trips_byte_for_byte() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let written = RunnerFailureRecord::from_failure( + "tx-a", + "up-service", + "service", + &failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""), + 77, + ); + record(&path, &written)?; + assert_eq!(for_transaction(&path, "tx-a")?, Some(written)); + assert_eq!(for_transaction(&path, "tx-b")?, None); + Ok(()) + } + + #[test] + fn the_evidence_store_is_private() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // Whatever the umask, the mode is the store's own rule, on every write. + for index in 0..2 { + record(&path, &record_at(&format!("tx-{index}"), 10 + index))?; + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn a_tail_over_its_bound_or_its_stream_is_refused() { + let mut oversize = record_at("tx-a", 1); + oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1]; + oversize.stdout_total_bytes = u64::MAX; + assert!(oversize.validate().is_err()); + let mut impossible = record_at("tx-a", 1); + impossible.stderr_total_bytes = 1; + assert!(impossible.validate().is_err()); + let mut exact = record_at("tx-a", 1); + exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES]; + exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64; + assert!(exact.validate().is_ok()); + } + + #[test] + fn status_renders_the_tail_with_control_bytes_escaped() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec(); + stored.stdout_total_bytes = 5000; + stored.stderr_tail = b"bad\r\x07".to_vec(); + stored.stderr_total_bytes = 6; + let lines = render(&stored); + let text = lines.join("\n"); + assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); + assert!(!text.contains('\u{7}')); + assert_eq!( + lines, + vec![ + " failed step build", + " runner rust", + " status exit status 3", + " stdout (last 30 of 5000 bytes)", + " | first", + " | \\u{1b}[2Jsecond\\u{0}end\tTab", + " | last", + " stderr (last 5 of 6 bytes)", + " | bad\\u{d}\\u{7}", + ] + ); + } + + #[test] + fn an_empty_tail_renders_only_its_header() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = Vec::new(); + stored.stdout_total_bytes = 0; + let lines = render(&stored); + assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); + assert!(!lines.iter().any(|line| line == " | ")); + } + + #[test] + fn a_foreign_or_corrupt_store_is_an_error_not_an_empty_store() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 not a store")?; + assert!(for_transaction(&path, "tx-a").is_err()); + assert!(record(&path, &record_at("tx-a", 1)).is_err()); + Ok(()) + } +} From 7ef31ee69966b978eb69284023ea507a3a89087c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:18:06 +0100 Subject: [PATCH 167/196] Runner failure engine tests: declare readiness, read both control stores Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d927152..f7093c9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -24784,7 +24784,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// The unrouted recipe with two steps ahead of its build, and no external /// input: the second step, `test-daemon`, is the one the fake docker fails. fn two_step_recipe() -> String { - let recipe = unrouted_recipe(); + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); let (head, rest) = recipe .split_once("[[external_inputs]]") .expect("the recipe declares an external input"); @@ -24876,11 +24879,16 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); } - let control = std::fs::read(&world.state_store)?; - assert!( - !control.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), - "runner output reached control.cc" - ); + // The terminal transaction has moved to history.cc: both stores. + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + let bytes = std::fs::read(&store)?; + assert!(!bytes.is_empty(), "{} is empty", store.display()); + assert!( + !bytes.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), + "runner output reached {}", + store.display() + ); + } // Kept: in the private store, tails of both streams, and on status. let evidence = runner_evidence_store_path(&world.state_store); assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); @@ -24976,15 +24984,17 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide tick_until_ended(&world)?; // The store a rolled-back Idunn reads: it refuses any foreign type. ControlSnapshot::read(&world.state_store)?; - let envelopes = - SingleFileMessagePackBackingStore::new(&world.state_store).pull_all_read_only_snapshot()?; - assert!(!envelopes.is_empty()); - for envelope in &envelopes { - assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); - if envelope.r#type == DeploymentTransaction::TYPE { - assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + let mut transactions = 0; + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + for envelope in SingleFileMessagePackBackingStore::new(&store).pull_all_read_only_snapshot()? { + assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); + if envelope.r#type == DeploymentTransaction::TYPE { + transactions += 1; + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + } } } + assert_eq!(transactions, 1, "the failed transaction is in neither store"); Ok(()) } From 54a75ef9b09bd6523c665e153311bc104442d29f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:36:34 +0100 Subject: [PATCH 168/196] Runner failure regression: a stdout-only report is kept with an empty stderr The vili failure: node --test wrote only to stdout, stderr was empty, the step exited 1. Pin the cap at the literal 32 and the Debug impls that keep tails out of text. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 48 ++++++++++++++++++++++++++++++++++++++++++ src/runner_evidence.rs | 16 +++++++++++--- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 1c8d067..fdd4d3a 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -16568,5 +16568,53 @@ processes 9 assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); Ok(()) } + + /// The 2026-10-25 vili failure: `node --test` reported on stdout alone, + /// stderr stayed empty (the runner's /tmp is a noexec tmpfs) and the step + /// exited 1. The old capture dropped stdout and printed an empty message. + #[cfg(unix)] + #[test] + fn a_step_that_reports_only_on_stdout_keeps_that_report_and_its_name() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'not ok 1 - daemon {RUNNER_CANARY}'\necho '# fail 1'\nexit 1"), + )?; + let error = run_step(&driver, temp.path(), "test-daemon").expect_err("the step passed"); + // The error names the step and carries no output. + let text = format!("{error:#}"); + assert!(text.contains("runner step test-daemon on runner rust exited with exit status 1"), "{text}"); + assert!(!text.contains(RUNNER_CANARY) && !text.contains("not ok"), "{text}"); + // The evidence keeps the stdout tail; the empty stderr is a recorded 0. + let failure = error + .downcast_ref::() + .context("the failure is not typed")?; + let store = temp.path().join("runner-evidence.cc"); + let record = crate::runner_evidence::RunnerFailureRecord::from_failure( + "tx-vili", + "up-vili", + "vili", + failure, + 1, + ); + crate::runner_evidence::record(&store, &record)?; + let kept = crate::runner_evidence::for_transaction(&store, "tx-vili")?.context("no evidence kept")?; + assert_eq!(kept.subject_id, "test-daemon"); + assert_eq!(kept.exit_code, Some(1)); + assert_eq!( + String::from_utf8_lossy(&kept.stdout_tail), + format!("not ok 1 - daemon {RUNNER_CANARY}\n# fail 1\n") + ); + assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); + assert!(kept.stderr_tail.is_empty()); + assert_eq!(kept.stderr_total_bytes, 0); + let rendered = crate::runner_evidence::render(&kept).join("\n"); + assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); + assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); + // Debug names the failure and prints no tail. + let debug = format!("{failure:?}"); + assert!(debug.starts_with("RunnerStepFailure") && !debug.contains("not ok"), "{debug}"); + Ok(()) + } // END runner-failure-evidence tests } diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index 9eeab4c..d77961f 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -314,11 +314,12 @@ mod tests { fn the_evidence_store_keeps_the_newest_32() -> Result<()> { let temp = tempfile::tempdir()?; let path = runner_evidence_store_path(&temp.path().join("control.cc")); - for index in 0..=RUNNER_EVIDENCE_RECORDS { + // The literal, not the constant: the spec's bound is 32. + for index in 0..=32 { record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; } let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); - assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS); + assert_eq!(kept.len(), 32); assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); @@ -328,7 +329,7 @@ mod tests { again.stdout_total_bytes = 10; record(&path, &again)?; let kept = snapshot(&path)?.1; - assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS); + assert_eq!(kept.len(), 32); assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; assert_eq!(stored.stdout_tail, b"rewritten\n"); @@ -407,6 +408,15 @@ mod tests { ); } + #[test] + fn debug_names_the_record_and_prints_no_tail() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"SECRET-OUTPUT".to_vec(); + stored.stdout_total_bytes = 13; + let debug = format!("{stored:?}"); + assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}"); + } + #[test] fn an_empty_tail_renders_only_its_header() { let mut stored = record_at("tx-a", 1); From 134c1fb5a25215763c80ef0fd3b99b96b0fd3b91 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 03:15:45 +0100 Subject: [PATCH 169/196] drain_tail tests: a scripted reader pins the Interrupted guard An interrupted read is retried with no byte lost, a persistent read error ends the drain instead of spinning, and a non-interrupted error after data is returned. Kills the three Interrupted-guard mutants (false, !=, true). Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index fdd4d3a..e0e5667 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -16525,6 +16525,65 @@ processes 9 Ok(()) } + /// A reader that answers from a script, so a read error the OS rarely + /// produces (an interrupted call) is produced on demand. + struct ScriptedReader(Vec>>); + + impl Read for ScriptedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.0.is_empty() { + return Ok(0); + } + let bytes = self.0.remove(0)?; + buffer[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + + struct BrokenReader; + + impl Read for BrokenReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("persistent read failure")) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_loses_no_byte() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + Ok(b"cd".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + ]); + let drained = drain_tail(reader, 3).expect("an interrupted read ended the drain"); + assert_eq!(drained, (b"bcd".to_vec(), 4)); + } + + #[test] + fn a_persistent_read_error_ends_the_drain_instead_of_spinning() { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(drain_tail(BrokenReader, 8).is_err()); + }); + assert_eq!( + receiver.recv_timeout(Duration::from_secs(5)), + Ok(true), + "a persistent read error spun or was swallowed" + ); + } + + #[test] + fn a_non_interrupted_read_error_after_data_is_returned_not_retried() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::BrokenPipe)), + Ok(b"cd".to_vec()), + ]); + let error = drain_tail(reader, 8).expect_err("a broken pipe was retried past"); + assert_eq!(error.kind(), ErrorKind::BrokenPipe); + } + #[cfg(unix)] #[test] fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { From 4a68df520fad6e764931a047b9aa27ad74c31e46 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 03:15:45 +0100 Subject: [PATCH 170/196] Runner evidence: set an undecodable store aside; escape bidi and pin C1 in status The writer moves a store it cannot decode (corrupt, or a newer schema) to runner-evidence.unreadable.cc, root-only, one slot replaced by the next, and opens a fresh store; the journal names the file and none of its content. A store that cannot be read at all is left in place and the write fails as before. Status escapes the bidi controls (U+061C, U+200E/F, U+202A-E, U+2066-9) in tails, subject and runner, and a test pins U+0080-U+009F. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 57 +++++++++-- src/runner_evidence.rs | 211 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 254 insertions(+), 14 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index f7093c9..dbe5096 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -47,7 +47,9 @@ use crate::drivers::{ WriteLeasePort, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; -use crate::runner_evidence::{self, RunnerFailureRecord, runner_evidence_store_path}; +use crate::runner_evidence::{ + self, Kept, RunnerFailureRecord, runner_evidence_store_path, unreadable_store_path, +}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -10695,18 +10697,27 @@ fn runner_failure_journal_line( transaction: &DeploymentTransaction, failure: &RunnerStepFailure, store_path: &Path, - recorded: &Result<()>, + recorded: &Result, ) -> String { let head = format!( "Idunn transaction {} (command {}, target {}) {failure}", transaction.transaction_id, transaction.command_id, transaction.target ); match recorded { - Ok(()) => format!( - "{head}; evidence in {}; read it with idunn status --command {}", - store_path.display(), - transaction.command_id - ), + Ok(kept) => { + let aside = match kept { + Kept::Stored => String::new(), + Kept::AfterSettingAsideAnUnreadableStore => format!( + "; an undecodable evidence store was set aside as {}", + unreadable_store_path(store_path).display() + ), + }; + format!( + "{head}; evidence in {}{aside}; read it with idunn status --command {}", + store_path.display(), + transaction.command_id + ) + } Err(error) => format!("{head}; evidence was not kept: {error:#}"), } } @@ -24963,6 +24974,38 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + #[cfg(unix)] + #[test] + fn an_undecodable_evidence_store_does_not_stop_the_next_failure_keeping_its_evidence() -> Result<()> { + let damage = format!("\u{c1} not a store {RUNNER_CANARY}-in-the-damage"); + let (world, _tools) = failing_runner_world()?; + let evidence = runner_evidence_store_path(&world.state_store); + std::fs::write(&evidence, damage.as_bytes())?; + let ended = tick_until_ended(&world)?; + // The transaction's own outcome is the runner failure, not the store. + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the damaged store changed the outcome: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + assert!(!error.contains("damage"), "{error}"); + // The damage is kept aside; the failure's evidence is in a fresh store. + assert_eq!(std::fs::read(unreadable_store_path(&evidence))?, damage.as_bytes()); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("the failure kept no evidence beside a damaged store")?; + assert_eq!(record.subject_id, "test-daemon"); + assert!(printed_status(&world)?.contains("failed step test-daemon")); + // The journal says it, naming the file and none of its content. + std::fs::write(&evidence, damage.as_bytes())?; + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"x"))) + .context("no journal line")?; + assert!(line.contains("an undecodable evidence store was set aside as"), "{line}"); + assert!(line.contains("runner-evidence.unreadable.cc"), "{line}"); + assert!(!line.contains("damage") && !line.contains("not a store"), "{line}"); + Ok(()) + } + #[cfg(unix)] #[test] fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index d77961f..cb08e0a 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -182,10 +182,44 @@ fn snapshot(path: &Path) -> Result<(Vec, Vec PathBuf { + path.with_file_name("runner-evidence.unreadable.cc") +} + +/// An undecodable store (a corrupt file, or one a newer Idunn wrote) must not +/// stop later failures from keeping their evidence. Move it aside, root-only, +/// over the previous one. Nothing of its content is named: a decode error can +/// quote stored values. A store that cannot even be read is left in place. +fn set_aside_unreadable(path: &Path) -> Result<()> { + std::fs::read(path).context("the runner evidence store could not be read")?; + let aside = unreadable_store_path(path); + std::fs::rename(path, &aside) + .context("the runner evidence store is undecodable and could not be set aside")?; + keep_private(&aside) +} + /// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the -/// oldest records go in the same exchange. The file ends at mode 0600. -pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result<()> { - let (envelopes, records) = snapshot(path)?; +/// oldest records go in the same exchange. The file ends at mode 0600. A store +/// that cannot be decoded is set aside first and the record opens a new one. +pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { + let (envelopes, records, how) = match snapshot(path) { + Ok((envelopes, records)) => (envelopes, records, Kept::Stored), + Err(_) => { + set_aside_unreadable(path)?; + (Vec::new(), Vec::new(), Kept::AfterSettingAsideAnUnreadableStore) + } + }; let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes .iter() .zip(&records) @@ -214,7 +248,8 @@ pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result<()> { .into()); } } - keep_private(path) + keep_private(path)?; + Ok(how) } #[cfg(unix)] @@ -270,10 +305,22 @@ pub(crate) fn render(record: &RunnerFailureRecord) -> Vec { lines } +/// Format characters that reorder what a terminal shows: the set the mind's +/// Title rule refuses. +fn is_bidi_control(character: char) -> bool { + matches!( + character, + '\u{61c}' | '\u{200e}' | '\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}' + ) +} + fn escape(text: &str) -> String { let mut escaped = String::with_capacity(text.len()); for character in text.chars() { - if character.is_control() && character != '\n' && character != '\t' { + if (character.is_control() || is_bidi_control(character)) + && character != '\n' + && character != '\t' + { escaped.push_str(&format!("\\u{{{:x}}}", u32::from(character))); } else { escaped.push(character); @@ -428,12 +475,162 @@ mod tests { } #[test] - fn a_foreign_or_corrupt_store_is_an_error_not_an_empty_store() -> Result<()> { + fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> { let temp = tempfile::tempdir()?; let path = runner_evidence_store_path(&temp.path().join("control.cc")); std::fs::write(&path, b"\xc1 not a store")?; assert!(for_transaction(&path, "tx-a").is_err()); - assert!(record(&path, &record_at("tx-a", 1)).is_err()); Ok(()) } + + fn files_beside(path: &Path) -> Result> { + let mut names: Vec = std::fs::read_dir(path.parent().context("no parent")?)? + .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned())) + .collect::>()?; + names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock")); + names.sort(); + Ok(names) + } + + /// A store a newer Idunn wrote: well-formed, but a schema this one does not know. + fn write_future_store(path: &Path) -> Result<()> { + let mut future = record_at("tx-future", 1); + future.schema_version = "idunn.runner_failure.v2".into(); + let envelope = typed_envelope( + &future.transaction_id, + RunnerFailureRecord::TYPE, + "idunn.runner_failure.v2", + &future, + 1, + )?; + let outcome = exchange_behind_private_lock(path, &[], &[envelope])?; + ensure!( + matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged), + "the future store was not written" + ); + Ok(()) + } + + #[test] + fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damage = b"\xc1 not a store \x00 damage"; + std::fs::write(&path, damage)?; + let kept = record(&path, &record_at("tx-a", 5))?; + assert_eq!(kept, Kept::AfterSettingAsideAnUnreadableStore); + // Kept, not deleted: the damage is where the journal says, root-only. + let aside = unreadable_store_path(&path); + assert_eq!(std::fs::read(&aside)?, damage); + assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600); + // The failure's evidence is in a fresh, private store. + assert_eq!( + for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id), + Some("tx-a".to_owned()) + ); + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + // A healthy store is not set aside again. + assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored); + assert_eq!(std::fs::read(&aside)?, damage); + Ok(()) + } + + #[test] + fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + write_future_store(&path)?; + let before = std::fs::read(&path)?; + assert!(for_transaction(&path, "tx-future").is_err()); + assert_eq!( + record(&path, &record_at("tx-a", 5))?, + Kept::AfterSettingAsideAnUnreadableStore + ); + assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); + assert!(for_transaction(&path, "tx-a")?.is_some()); + assert_eq!(for_transaction(&path, "tx-future")?, None); + Ok(()) + } + + #[test] + fn at_most_one_unreadable_store_is_kept() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"]; + for (index, damage) in damages.into_iter().enumerate() { + std::fs::write(&path, damage)?; + record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?; + assert_eq!( + std::fs::read(unreadable_store_path(&path))?, + damage, + "the newest damage is the one kept" + ); + } + assert_eq!( + files_beside(&path)?, + vec![ + "runner-evidence.cc".to_owned(), + "runner-evidence.unreadable.cc".to_owned() + ] + ); + Ok(()) + } + + #[test] + fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // A directory where the store should be: unreadable, not undecodable. + std::fs::create_dir(&path)?; + assert!(record(&path, &record_at("tx-a", 1)).is_err()); + assert!(path.is_dir()); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn status_escapes_bidi_controls_in_tails_and_in_names() { + for bidi in [ + '\u{61c}', '\u{200e}', '\u{200f}', '\u{202a}', '\u{202b}', '\u{202c}', '\u{202d}', + '\u{202e}', '\u{2066}', '\u{2067}', '\u{2068}', '\u{2069}', + ] { + let mut stored = record_at("tx-a", 1); + let tail = format!("ok {bidi}gpj.exe\n"); + stored.stdout_tail = tail.clone().into_bytes(); + stored.stdout_total_bytes = tail.len() as u64; + stored.stderr_tail = tail.clone().into_bytes(); + stored.stderr_total_bytes = tail.len() as u64; + stored.subject_id = format!("step{bidi}x"); + stored.runner = format!("rust{bidi}"); + let text = render(&stored).join("\n"); + let escaped = format!("\\u{{{:x}}}", bidi as u32); + assert!(!text.contains(bidi), "U+{:04X} reached the terminal", bidi as u32); + // Escaped in every place it can appear: two tails, the subject, the runner. + assert_eq!(text.matches(&escaped).count(), 4, "{text}"); + } + // Format characters either side of the set pass. + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = "caf\u{e9} \u{200d}\u{202f}\u{2065}\u{206a}\n".as_bytes().to_vec(); + stored.stdout_total_bytes = stored.stdout_tail.len() as u64; + let text = render(&stored).join("\n"); + assert!(text.contains("caf\u{e9} \u{200d}\u{202f}\u{2065}\u{206a}"), "{text}"); + } + + #[test] + fn status_escapes_the_c1_controls() { + for code in 0x80u32..=0x9f { + let character = char::from_u32(code).unwrap(); + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = format!("a{character}b\n").into_bytes(); + stored.stdout_total_bytes = stored.stdout_tail.len() as u64; + let text = render(&stored).join("\n"); + assert!(!text.contains(character), "U+{code:04X} reached the terminal"); + assert!(text.contains(&format!("a\\u{{{code:x}}}b")), "U+{code:04X}: {text}"); + } + // U+00A0 is not a control character and passes. + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = "a\u{a0}b\n".as_bytes().to_vec(); + stored.stdout_total_bytes = stored.stdout_tail.len() as u64; + assert!(render(&stored).join("\n").contains("a\u{a0}b")); + } } From 549f4be09328933d7e5dc55edaead1d7f474f6dc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 07:16:09 +0100 Subject: [PATCH 171/196] Runner evidence: one allowlist sink for status, quarantine only on a decode failure, a displaced set-aside is journaled Status writes everything it prints from the evidence store, including the unreadable-store line, through one function (emit): printable ASCII and tab pass, every other code point is a \u{..} escape and every invalid UTF-8 byte a \x.. escape. The bidi and C1 class lists are deleted. The writer reads the store through a seam. An I/O error leaves the store in place, keeps no evidence and names its kind; the store is set aside only when bytes that were read still read the same on a second read and do not decode. A set-aside that displaces an earlier one says so on the journal (last-written time, no content). Tests pin a failed set-aside, its no-echo, the I/O seam and the replacement. Not yet built or run; verified in the next job. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 124 +++++++++++++-- src/runner_evidence.rs | 351 ++++++++++++++++++++++++++++++----------- 2 files changed, 367 insertions(+), 108 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index dbe5096..e43a7ac 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -4882,22 +4882,11 @@ fn status( writeln!(out, " pre-fencing abort {abort:?}")?; // The step that failed and its output tail live in their // own private store. An unreadable store costs its lines. - match runner_evidence::for_transaction( + runner_evidence::show_status( + out, &runner_evidence_store_path(store_path), &transaction.transaction_id, - ) { - Ok(Some(record)) => { - for line in runner_evidence::render(&record) { - writeln!(out, "{line}")?; - } - } - Ok(None) => {} - Err(error) => writeln!( - out, - " runner evidence unreadable: {}", - error.root_cause() - )?, - } + )?; } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -10707,9 +10696,15 @@ fn runner_failure_journal_line( Ok(kept) => { let aside = match kept { Kept::Stored => String::new(), - Kept::AfterSettingAsideAnUnreadableStore => format!( - "; an undecodable evidence store was set aside as {}", - unreadable_store_path(store_path).display() + Kept::SetAside { replaced } => format!( + "; an undecodable evidence store was set aside as {}{}", + unreadable_store_path(store_path).display(), + match replaced { + Some(last_written) => format!( + ", replacing an earlier set-aside store last written at unix millis {last_written}" + ), + None => String::new(), + } ), }; format!( @@ -25020,6 +25015,101 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// Everything status prints from the evidence store goes through one + /// allowlist, so a hostile build log reaches the terminal as visible + /// escapes whatever its class. + #[cfg(unix)] + #[test] + fn status_prints_hostile_runner_output_as_visible_escapes_only() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let hostile: Vec<(&str, &str)> = vec![ + ("\u{1b}[2J", "\\u{1b}[2J"), + ("\u{7}", "\\u{7}"), + ("\r", "\\u{d}"), + ("\u{202e}", "\\u{202e}"), + ("\u{2066}", "\\u{2066}"), + ("\u{85}", "\\u{85}"), + ("\u{e0041}", "\\u{e0041}"), + ("\u{200b}", "\\u{200b}"), + ("\u{feff}", "\\u{feff}"), + ]; + let mut tail: Vec = Vec::new(); + for &(raw, _) in &hostile { + tail.extend_from_slice(b"a"); + tail.extend_from_slice(raw.as_bytes()); + tail.extend_from_slice(b"b\n"); + } + tail.extend_from_slice(b"bad \xff\xc1 bytes\n"); + runner_evidence::record( + &runner_evidence_store_path(&world.state_store), + &RunnerFailureRecord::from_failure( + &ended.transaction_id, + &ended.command_id, + &ended.target, + &canary_failure(&tail), + 9, + ), + )?; + let printed = printed_status(&world)?; + for &(raw, escaped) in &hostile { + assert!(!printed.contains(raw), "{escaped} reached the terminal raw"); + assert!(printed.contains(&format!("a{escaped}b")), "{escaped} missing: {printed}"); + } + assert!(printed.contains("bad \\xff\\xc1 bytes"), "{printed}"); + let section = printed.find(" failed step").context("no evidence section")?; + assert!(printed[section..].is_ascii(), "non-ASCII reached the terminal: {printed}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn status_prints_the_unreadable_store_line_escaped() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + let evidence = runner_evidence_store_path(&world.state_store); + // The decode error of this store quotes the stored subject kind. + let mut bytes = std::fs::read(&evidence)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker in the evidence store")?; + bytes[at..at + 4].copy_from_slice(b"\x1b[2J"); + std::fs::write(&evidence, bytes)?; + let printed = printed_status(&world)?; + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + assert!(!printed.contains('\u{1b}'), "an escape byte reached the terminal"); + assert!(printed.contains("\\u{1b}[2J"), "{printed}"); + Ok(()) + } + + #[test] + fn the_journal_line_names_a_displaced_set_aside_store_by_time_only() -> Result<()> { + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let path = Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"); + let first = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: None }), + ); + assert!(first.contains("was set aside as"), "{first}"); + assert!(!first.contains("replacing"), "{first}"); + let second = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: Some(1234) }), + ); + assert!( + second.contains("replacing an earlier set-aside store last written at unix millis 1234"), + "{second}" + ); + assert!(!second.contains(RUNNER_CANARY)); + Ok(()) + } + #[cfg(unix)] #[test] fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index cb08e0a..9e9e5ee 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -9,8 +9,9 @@ //! //! The tail is raw runner output. It is kept unredacted and readable by root //! alone (mode 0600): root can already read every secret the runner was given. -//! It never enters an error, a transaction record or the journal; the render -//! below escapes control characters because the status reader is a terminal. +//! It never enters an error, a transaction record or the journal. The status +//! reader is a terminal, so everything it prints from this store goes through +//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped. use std::path::{Path, PathBuf}; @@ -186,9 +187,11 @@ fn snapshot(path: &Path) -> Result<(Vec, Vec }, } /// Where an undecodable store is kept. One slot: the next one replaces it, so @@ -197,27 +200,62 @@ pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf { path.with_file_name("runner-evidence.unreadable.cc") } -/// An undecodable store (a corrupt file, or one a newer Idunn wrote) must not -/// stop later failures from keeping their evidence. Move it aside, root-only, -/// over the previous one. Nothing of its content is named: a decode error can -/// quote stored values. A store that cannot even be read is left in place. -fn set_aside_unreadable(path: &Path) -> Result<()> { - std::fs::read(path).context("the runner evidence store could not be read")?; +/// Moves the store over the previous set-aside one, root-only, and reports +/// what it displaced. Nothing of the store's content is named: a decode error +/// can quote stored values, and none is carried here. +fn set_aside_unreadable(path: &Path) -> Result> { let aside = unreadable_store_path(path); + let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| { + earlier + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |since| since.as_millis() as u64) + }); std::fs::rename(path, &aside) .context("the runner evidence store is undecodable and could not be set aside")?; - keep_private(&aside) + keep_private(&aside)?; + Ok(replaced) +} + +/// How the writer reads the store's bytes; tests inject an I/O error here. +type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result>; + +fn unreadable_fault(error: &std::io::Error) -> anyhow::Error { + anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind()) } /// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the -/// oldest records go in the same exchange. The file ends at mode 0600. A store -/// that cannot be decoded is set aside first and the record opens a new one. +/// oldest records go in the same exchange. The file ends at mode 0600. +/// +/// A store is set aside only when bytes that were read do not decode. An I/O +/// error leaves the store in place, keeps no evidence for that failure, and +/// the error names its kind for the journal. pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { + record_reading(path, record, &|path| std::fs::read(path)) +} + +fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result { + let first = match read(path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(unreadable_fault(&error)), + }; let (envelopes, records, how) = match snapshot(path) { Ok((envelopes, records)) => (envelopes, records, Kept::Stored), Err(_) => { - set_aside_unreadable(path)?; - (Vec::new(), Vec::new(), Kept::AfterSettingAsideAnUnreadableStore) + let Some(first) = first else { + bail!("the runner evidence store could not be decoded"); + }; + // Undecodable only if the same bytes are still there to read: a + // second read that fails, or finds other bytes, is not a verdict. + match read(path) { + Ok(second) if second == first => {} + Ok(_) => bail!("the runner evidence store changed while it was being read"), + Err(error) => return Err(unreadable_fault(&error)), + } + let replaced = set_aside_unreadable(path)?; + (Vec::new(), Vec::new(), Kept::SetAside { replaced }) } }; let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes @@ -275,58 +313,81 @@ pub(crate) fn for_transaction( .find(|stored| stored.transaction_id == transaction_id)) } -/// The record as `idunn status --command` prints it. The store keeps raw -/// bytes; every control character except newline and tab is escaped here, so -/// build output cannot drive the operator's terminal. -pub(crate) fn render(record: &RunnerFailureRecord) -> Vec { +/// The record as `idunn status --command` shows it, as raw lines. They are +/// not text yet: `show_status` is the only caller, and `emit` the only +/// writer. +fn render(record: &RunnerFailureRecord) -> Vec> { let noun = match record.subject_kind { SubjectKind::Step => "step", SubjectKind::ExternalInput => "external input", }; - let mut lines = vec![ - format!(" failed {noun} {}", escape(&record.subject_id)), - format!(" runner {}", escape(&record.runner)), - format!(" status {}", runner_status(record.exit_code, record.signal)), + let mut lines: Vec> = vec![ + format!(" failed {noun} {}", record.subject_id).into_bytes(), + format!(" runner {}", record.runner).into_bytes(), + format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(), ]; for (stream, tail, total) in [ ("stdout", &record.stdout_tail, record.stdout_total_bytes), ("stderr", &record.stderr_tail, record.stderr_total_bytes), ] { - lines.push(format!( - " {stream} (last {} of {total} bytes)", - tail.len() - )); - let text = escape(&String::from_utf8_lossy(tail)); - let text = text.strip_suffix('\n').unwrap_or(&text); + lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes()); + let text = tail.strip_suffix(b"\n").unwrap_or(tail); if !text.is_empty() { - lines.extend(text.split('\n').map(|line| format!(" | {line}"))); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); } } lines } -/// Format characters that reorder what a terminal shows: the set the mind's -/// Title rule refuses. -fn is_bidi_control(character: char) -> bool { - matches!( - character, - '\u{61c}' | '\u{200e}' | '\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}' - ) +/// What `idunn status --command` prints for a transaction's runner evidence: +/// its record, or one line naming why the store is unreadable. Nothing is +/// written when the transaction has no record. +pub(crate) fn show_status( + out: &mut impl std::io::Write, + store: &Path, + transaction_id: &str, +) -> std::io::Result<()> { + let lines = match for_transaction(store, transaction_id) { + Ok(Some(record)) => render(&record), + Ok(None) => return Ok(()), + Err(error) => { + vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()] + } + }; + emit(out, &lines) } -fn escape(text: &str) -> String { - let mut escaped = String::with_capacity(text.len()); - for character in text.chars() { - if (character.is_control() || is_bidi_control(character)) - && character != '\n' - && character != '\t' - { - escaped.push_str(&format!("\\u{{{:x}}}", u32::from(character))); - } else { - escaped.push(character); +/// The one sink: every byte the status read prints from the evidence store, +/// or from an error about it, is written here and nowhere else. An allowlist: +/// printable ASCII and tab pass; every other code point is a visible +/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline +/// inside a line is escaped too: this function writes the only ones. +fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { + for line in lines { + out.write_all(escape(line).as_bytes())?; + out.write_all(b"\n")?; + } + Ok(()) +} + +fn escape(bytes: &[u8]) -> String { + let mut shown = String::with_capacity(bytes.len()); + for chunk in bytes.utf8_chunks() { + for character in chunk.valid().chars() { + match character { + ' '..='~' | '\t' => shown.push(character), + _ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))), + } + } + for byte in chunk.invalid() { + shown.push_str(&format!("\\x{byte:02x}")); } } - escaped + shown } #[cfg(all(test, unix))] @@ -347,6 +408,17 @@ mod tests { } } + /// What status writes for a record, through the one sink. + fn shown(record: &RunnerFailureRecord) -> String { + let mut written = Vec::new(); + emit(&mut written, &render(record)).expect("a Vec takes any write"); + String::from_utf8(written).expect("the sink writes ASCII only") + } + + fn shown_lines(record: &RunnerFailureRecord) -> Vec { + shown(record).lines().map(str::to_owned).collect() + } + fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { RunnerFailureRecord::from_failure( transaction, @@ -435,7 +507,7 @@ mod tests { stored.stdout_total_bytes = 5000; stored.stderr_tail = b"bad\r\x07".to_vec(); stored.stderr_total_bytes = 6; - let lines = render(&stored); + let lines = shown_lines(&stored); let text = lines.join("\n"); assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); assert!(!text.contains('\u{7}')); @@ -469,7 +541,7 @@ mod tests { let mut stored = record_at("tx-a", 1); stored.stdout_tail = Vec::new(); stored.stdout_total_bytes = 0; - let lines = render(&stored); + let lines = shown_lines(&stored); assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); assert!(!lines.iter().any(|line| line == " | ")); } @@ -518,7 +590,7 @@ mod tests { let damage = b"\xc1 not a store \x00 damage"; std::fs::write(&path, damage)?; let kept = record(&path, &record_at("tx-a", 5))?; - assert_eq!(kept, Kept::AfterSettingAsideAnUnreadableStore); + assert_eq!(kept, Kept::SetAside { replaced: None }); // Kept, not deleted: the damage is where the journal says, root-only. let aside = unreadable_store_path(&path); assert_eq!(std::fs::read(&aside)?, damage); @@ -544,7 +616,7 @@ mod tests { assert!(for_transaction(&path, "tx-future").is_err()); assert_eq!( record(&path, &record_at("tx-a", 5))?, - Kept::AfterSettingAsideAnUnreadableStore + Kept::SetAside { replaced: None } ); assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); assert!(for_transaction(&path, "tx-a")?.is_some()); @@ -589,48 +661,145 @@ mod tests { } #[test] - fn status_escapes_bidi_controls_in_tails_and_in_names() { - for bidi in [ - '\u{61c}', '\u{200e}', '\u{200f}', '\u{202a}', '\u{202b}', '\u{202c}', '\u{202d}', - '\u{202e}', '\u{2066}', '\u{2067}', '\u{2068}', '\u{2069}', - ] { - let mut stored = record_at("tx-a", 1); - let tail = format!("ok {bidi}gpj.exe\n"); - stored.stdout_tail = tail.clone().into_bytes(); - stored.stdout_total_bytes = tail.len() as u64; - stored.stderr_tail = tail.clone().into_bytes(); - stored.stderr_total_bytes = tail.len() as u64; - stored.subject_id = format!("step{bidi}x"); - stored.runner = format!("rust{bidi}"); - let text = render(&stored).join("\n"); - let escaped = format!("\\u{{{:x}}}", bidi as u32); - assert!(!text.contains(bidi), "U+{:04X} reached the terminal", bidi as u32); - // Escaped in every place it can appear: two tails, the subject, the runner. - assert_eq!(text.matches(&escaped).count(), 4, "{text}"); + fn every_code_point_but_printable_ascii_is_a_visible_escape() { + // The allowlist, exhaustively: no class list decides anything. + for code in 0..=0x10ffffu32 { + let Some(character) = char::from_u32(code) else { + continue; + }; + let text = character.to_string(); + let shown = escape(text.as_bytes()); + if matches!(character, ' '..='~' | '\t') { + assert_eq!(shown, text, "U+{code:04X} should pass"); + } else { + assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}"); + } } - // Format characters either side of the set pass. - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = "caf\u{e9} \u{200d}\u{202f}\u{2065}\u{206a}\n".as_bytes().to_vec(); - stored.stdout_total_bytes = stored.stdout_tail.len() as u64; - let text = render(&stored).join("\n"); - assert!(text.contains("caf\u{e9} \u{200d}\u{202f}\u{2065}\u{206a}"), "{text}"); + // Invalid UTF-8 is escaped byte by byte; valid text round it passes. + assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82"); + assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}"); } #[test] - fn status_escapes_the_c1_controls() { - for code in 0x80u32..=0x9f { - let character = char::from_u32(code).unwrap(); - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = format!("a{character}b\n").into_bytes(); - stored.stdout_total_bytes = stored.stdout_tail.len() as u64; - let text = render(&stored).join("\n"); - assert!(!text.contains(character), "U+{code:04X} reached the terminal"); - assert!(text.contains(&format!("a\\u{{{code:x}}}b")), "U+{code:04X}: {text}"); - } - // U+00A0 is not a control character and passes. + fn a_newline_inside_a_line_cannot_forge_a_line() { let mut stored = record_at("tx-a", 1); - stored.stdout_tail = "a\u{a0}b\n".as_bytes().to_vec(); - stored.stdout_total_bytes = stored.stdout_tail.len() as u64; - assert!(render(&stored).join("\n").contains("a\u{a0}b")); + stored.subject_id = "build\n status exit status 0".into(); + let lines = shown_lines(&stored); + assert_eq!(lines[0], " failed step build\\u{a} status exit status 0"); + assert_eq!( + lines.iter().filter(|line| line.starts_with(" status ")).count(), + 1 + ); + } + + /// A store whose first record is well-formed except that its subject kind + /// holds `marker`: the decode error quotes it. + fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> { + record(path, &record_at("tx-a", 1))?; + let mut bytes = std::fs::read(path)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker")?; + bytes[at..at + 4].copy_from_slice(marker); + std::fs::write(path, bytes)?; + Ok(()) + } + + #[test] + fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"\x1b[2J")?; + let mut written = Vec::new(); + show_status(&mut written, &path, "tx-a")?; + assert!(!written.contains(&0x1b), "an escape byte reached the terminal"); + assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n'))); + let text = String::from_utf8(written)?; + assert!(text.starts_with(" runner evidence unreadable: "), "{text}"); + assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}"); + Ok(()) + } + + #[test] + fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"CNRY")?; + let damage = std::fs::read(&path)?; + // Sanity: this damage decodes to an error that quotes the canary. + assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY")); + // A non-empty directory where the aside goes: rename fails even as root. + let aside = unreadable_store_path(&path); + std::fs::create_dir(&aside)?; + std::fs::write(aside.join("occupied"), b"x")?; + let error = record(&path, &record_at("tx-b", 2)).unwrap_err(); + for text in [format!("{error:#}"), format!("{error:?}")] { + assert!(text.contains("could not be set aside"), "{text}"); + assert!(!text.contains("CNRY"), "stored content reached the error: {text}"); + } + // The failure cost nothing: the store is as it was, with no evidence + // written over it. + assert_eq!(std::fs::read(&path)?, damage); + assert!(aside.join("occupied").exists()); + Ok(()) + } + + #[test] + fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + record(&path, &record_at("tx-a", 1))?; + let before = std::fs::read(&path)?; + let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied); + // The first read fails on a healthy store. + let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, before, "the store changed"); + assert!(!unreadable_store_path(&path).exists()); + assert!(for_transaction(&path, "tx-b")?.is_none()); + // The second read, the one that confirms the bytes do not decode, fails. + std::fs::write(&path, b"\xc1 not a store")?; + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| { + calls.set(calls.get() + 1); + if calls.get() == 2 { + return Err(denied()); + } + std::fs::read(path) + }; + let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + // Bytes that changed between the reads are no verdict either. + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| -> std::io::Result> { + calls.set(calls.get() + 1); + let mut bytes = std::fs::read(path)?; + bytes.push(calls.get() as u8); + Ok(bytes) + }; + assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err()); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 first")?; + assert_eq!( + record(&path, &record_at("tx-a", 1))?, + Kept::SetAside { replaced: None } + ); + std::fs::write(&path, b"\xc1 second")?; + let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else { + bail!("a displaced set-aside store went unreported"); + }; + assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second"); + Ok(()) } } From 1518cfb13c46a1cad8982dc3c2b3d515fe65a885 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 07:18:51 +0100 Subject: [PATCH 172/196] drivers test: read the kept evidence through show_status, the one status sink Fixes the test build broken by 549f4be (render is private now). Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/drivers.rs b/src/drivers.rs index e0e5667..382bd23 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -16667,7 +16667,9 @@ processes 9 assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); assert!(kept.stderr_tail.is_empty()); assert_eq!(kept.stderr_total_bytes, 0); - let rendered = crate::runner_evidence::render(&kept).join("\n"); + let mut shown = Vec::new(); + crate::runner_evidence::show_status(&mut shown, &store, "tx-vili")?; + let rendered = String::from_utf8(shown)?; assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); // Debug names the failure and prints no tail. From 024f2d7a09ec642e63f696899190b20c0d7840fe Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 29 Oct 2026 22:54:26 +0100 Subject: [PATCH 173/196] Workload: incarnation units are collected when they end, so a failed candidate no longer stays loaded and marks the host degraded Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 382bd23..3ea1145 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -605,7 +605,9 @@ pub trait WorkloadPort { /// /// Idunn launches candidates as transient units with `Restart=no`, so a /// unit that has failed, or that systemd no longer knows, will not come - /// back on its own. Past the fence every error is otherwise treated as + /// back on its own. Those units are collected once they fail + /// (`INCARNATION_UNIT_LIFECYCLE`), so a failed candidate usually reads as + /// forgotten rather than failed. Past the fence every error is otherwise treated as /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; @@ -2574,6 +2576,13 @@ impl RunnerPort for DockerRunnerDriver { } } +/// How every incarnation unit lives and dies. It runs once (`Restart=no`) +/// and systemd forgets it once it is inactive or failed (`--collect`): the +/// outcome is Idunn's to record in its own store, and a failed transient unit +/// left loaded is only clutter that marks the host degraded. +const INCARNATION_UNIT_LIFECYCLE: [&str; 3] = + ["--collect", "--property=Type=exec", "--property=Restart=no"]; + /// systemd owns process execution. This driver lowers one validated launch /// contract into a transient unit and then proves the native process and /// executable that systemd actually started. It does not decide admission, @@ -3764,8 +3773,9 @@ impl SystemdTransientWorkloadDriver { OsString::from("--no-ask-password"), OsString::from("--expand-environment=no"), OsString::from(format!("--unit={unit}")), - OsString::from("--property=Type=exec"), - OsString::from("--property=Restart=no"), + ]; + args.extend(INCARNATION_UNIT_LIFECYCLE.map(OsString::from)); + args.extend([ OsString::from("--property=KillMode=mixed"), OsString::from("--property=DynamicUser=yes"), OsString::from("--property=SupplementaryGroups="), @@ -3801,7 +3811,7 @@ impl SystemdTransientWorkloadDriver { OsString::from(format!("--working-directory={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", bundle.display())), - ]; + ]); for open_file in parent_only_open_files { args.push(OsString::from(format!("--property=OpenFile={open_file}"))); } @@ -9942,6 +9952,16 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// An incarnation unit runs once and is collected when it ends, so a + /// failed candidate does not stay loaded and mark the host degraded. + /// Collection is safe because a forgotten unit already reads as + /// permanently stopped (the test below). + #[test] + fn incarnation_units_run_once_and_are_collected() { + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--collect")); + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--property=Restart=no")); + } + /// Admission audit, claim 3: `is_permanently_stopped` now lets the /// Isolation gate skip `prove_isolation` against the incumbent. Enumerate /// what it answers `true` for, using a fake `systemctl show` that replays From e50a721dbca3324db6b9a31b4eac9746385289c9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:01 +0100 Subject: [PATCH 174/196] Steward orders: typed package, unit and deploy orders from one steward, in a sibling root unit idunn orders serve takes orders over a root-only socket from the steward uid, records order/start/outcome/post in its own orders.cc, and posts acceptance and result to #operator as a retried projection. Unit orders may act on any unit except the compiled always-refused set (idunn-*, mind-self@*, mind-persona@*), matched on a unit's Id and every name, and for stop and restart on the units systemd stops with it. A refusal names a field and a code, never the value. A started order is never run twice. Deploy orders commit through the same primitive as idunn up. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 1 + Cargo.toml | 1 + src/control_plane.rs | 112 +- src/lib.rs | 2 + src/runner_evidence.rs | 4 +- src/steward_orders.rs | 2932 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 3036 insertions(+), 16 deletions(-) create mode 100644 src/steward_orders.rs diff --git a/Cargo.lock b/Cargo.lock index dfc3087..44c9457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "libc", "rmp-serde", "serde", + "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index dcaca2e..ee797ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } +serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/src/control_plane.rs b/src/control_plane.rs index e43a7ac..cd49073 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -2934,6 +2934,10 @@ enum Command { recipe: PathBuf, binding: Option, }, + #[cfg(unix)] + OrdersServe(crate::steward_orders::ServeOptions), + #[cfg(unix)] + Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { @@ -2973,6 +2977,10 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + #[cfg(unix)] + Command::OrdersServe(options) => crate::steward_orders::serve(options), + #[cfg(unix)] + Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -3025,6 +3033,10 @@ fn parse(args: impl Iterator) -> Result { "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), + #[cfg(unix)] + "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), + #[cfg(unix)] + "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -4730,20 +4742,21 @@ fn resolve_selector( Ok(targets) } -fn submit( +/// The one commit primitive for a deployment command: `idunn up` and a steward +/// deploy order both write theirs here. Returns the commit time. +pub(crate) fn commit_deployment_command( store_path: &Path, + command_id: &str, selector: &str, requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { +) -> Result { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), + command_id: command_id.into(), kind: CommandKind::Deploy, selector: selector.into(), requested_by: requested_by.into(), @@ -4761,7 +4774,33 @@ fn submit( )?, "deployment command id collided" ); - println!("{}", command.command_id); + Ok(now) +} + +/// A command's present state as `idunn status --command` derives it, for the +/// steward order poster: `queued` and `running` are not terminal. +#[cfg(unix)] +pub(crate) fn command_state( + state_store: &Path, + command_id: &str, +) -> Result<(&'static str, String)> { + let snapshot = ControlSnapshot::read(state_store)?; + let archived = read_history_transactions(state_store); + let mut transactions = snapshot.transaction_for_command(command_id); + transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); + Ok(derived_command_status(&transactions)) +} + +fn submit( + store_path: &Path, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + let command_id = format!("up-{}", Uuid::new_v4()); + let now = commit_deployment_command(store_path, &command_id, selector, requested_by)?; + println!("{command_id}"); if !wait { return Ok(()); } @@ -4772,10 +4811,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command.command_id), + .any(|stored| stored.value.command_id == command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command.command_id); + let transactions = snapshot.transaction_for_command(&command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4800,7 +4839,7 @@ fn submit( } } -fn status( +pub(crate) fn status( out: &mut impl Write, store_path: &Path, incident_store: &Path, @@ -10592,7 +10631,13 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -fn now_millis() -> Result { +/// The control store `idunn up` and `idunn serve` use unless told otherwise. +#[cfg(unix)] +pub(crate) fn default_state_store() -> PathBuf { + RuntimeOptions::default().state_store +} + +pub(crate) fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -10612,12 +10657,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -fn string_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -fn path_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -10633,7 +10678,7 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -fn require_selector(value: &str) -> Result<()> { +pub(crate) fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } @@ -10757,7 +10802,10 @@ fn usage() -> &'static str { idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\ + idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ + idunn order --reason TEXT\n\ + idunn order show ID | idunn order list\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -11969,9 +12017,45 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["order", "run", "sh", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], + vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], + vec!["orders", "serve", "--allow-units", "nginx.service"], + vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } + // The declarative order forms parse, and only as order commands. + #[cfg(unix)] + for arguments in [ + vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r"], + vec!["order", "list"], + vec!["orders", "serve"], + ] { + let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); + assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); + } + } + + #[test] + fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + commit_deployment_command(&store, "up-one", "vili", "steward:eureka order ord-1")?; + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.commands.len(), 1); + let command = &snapshot.commands[0].value; + assert_eq!(command.command_id, "up-one"); + assert_eq!(command.selector, "vili"); + assert_eq!(command.requested_by, "steward:eureka order ord-1"); + assert!(commit_deployment_command(&store, "up-one", "other", "x").is_err()); + assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); + Ok(()) } #[test] diff --git a/src/lib.rs b/src/lib.rs index 4f365a3..b5499a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,3 +9,5 @@ pub mod launcher; mod incident; mod runner_evidence; pub mod provisioning; +#[cfg(unix)] +mod steward_orders; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index 9e9e5ee..30dd1d2 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -366,7 +366,7 @@ pub(crate) fn show_status( /// printable ASCII and tab pass; every other code point is a visible /// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline /// inside a line is escaped too: this function writes the only ones. -fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { +pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { for line in lines { out.write_all(escape(line).as_bytes())?; out.write_all(b"\n")?; @@ -374,7 +374,7 @@ fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> Ok(()) } -fn escape(bytes: &[u8]) -> String { +pub(crate) fn escape(bytes: &[u8]) -> String { let mut shown = String::with_capacity(bytes.len()); for chunk in bytes.utf8_chunks() { for character in chunk.valid().chars() { diff --git a/src/steward_orders.rs b/src/steward_orders.rs new file mode 100644 index 0000000..2aded30 --- /dev/null +++ b/src/steward_orders.rs @@ -0,0 +1,2932 @@ +//! Typed orders from Idunn's steward. +//! +//! `idunn orders serve` is a second root unit from the same binary as +//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints +//! order ids, decides acceptance and refusal, actuates, and records outcomes. +//! The survival daemon takes no part: `control.cc` never learns an order type. +//! +//! An order is a closed enum, never a command string: install packages, act on +//! a systemd unit, or deploy a target through the same commit primitive +//! `idunn up` uses. Every unit is allowed except the always-refused set, which +//! is compiled in here and read from nowhere: no binding field, record or flag +//! holds it. The `#operator` post is a retried projection of the store. +//! +//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the +//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and +//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE +//! `mind-control check`; START record before actuation, at most once; RUN; +//! RESULT outcome; POST accepted and result. +//! +//! No refusal or error here carries a value the issuer supplied: it names the +//! field and a code. A refused unit that a record names was resolved by +//! systemd and passed the unit grammar. + +use std::collections::{BTreeSet, VecDeque}; +use std::fmt; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::control_plane::{ + command_state, commit_deployment_command, decode_record, default_state_store, now_millis, + path_value, require_id, require_selector, string_value, typed_envelope, +}; +use crate::runner_evidence::{emit, escape}; + +const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; +const START_SCHEMA: &str = "idunn.steward_order_start.v1"; +const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; +const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; + +const REQUEST_FRAME_LIMIT: usize = 16 * 1024; +const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; +const TAIL_BYTES: usize = 8 * 1024; +const REASON_LIMIT: usize = 300; +const PACKAGE_LIMIT: usize = 16; +const UNIT_LIMIT: usize = 200; +const CLOSURE_LIMIT: usize = 512; +const LIST_LIMIT: usize = 50; +const UNIT_SUFFIXES: [&str; 6] = [".service", ".socket", ".timer", ".path", ".mount", ".target"]; + +/// The always-refused set: Idunn's own units and the minds' own. Matched on +/// every name a unit answers to, ASCII case-insensitively (systemd units are +/// case-sensitive, so the fold only widens the refusal). +fn is_refused_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + ["idunn-", "mind-self@", "mind-persona@"] + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +// ---------------------------------------------------------------- grammar -- + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Code { + Empty, + TooLong, + TooMany, + BadCharacter, + BadShape, + Malformed, + Unauthorized, + Unknown, + Unavailable, +} + +/// A refusal names the field and a code. It never carries the offending value. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Refusal { + pub(crate) field: String, + pub(crate) code: Code, +} + +impl Refusal { + fn new(field: &'static str, code: Code) -> Self { + Self { + field: field.to_owned(), + code, + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{} refused: {:?}", self.field, self.code) + } +} + +impl std::error::Error for Refusal {} + +fn validate_package(spec: &str) -> Result<(), Refusal> { + const FIELD: &str = "package"; + if spec.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if spec.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let (name, version) = match spec.split_once('=') { + Some((name, version)) => (name, Some(version)), + None => (spec, None), + }; + let bytes = name.as_bytes(); + if bytes.len() < 2 { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); + let rest = bytes[1..].iter().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') + }); + if !head || !rest { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + // apt reads `name-` as a removal and `name.deb` as a local file. + if name.ends_with('-') || name.ends_with(".deb") { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if let Some(version) = version { + if version.is_empty() { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if !version.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') + }) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + } + Ok(()) +} + +fn validate_unit(unit: &str) -> Result<(), Refusal> { + const FIELD: &str = "unit"; + if unit.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if unit.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let allowed = unit.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') + }); + if !allowed || unit.starts_with('-') { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + match UNIT_SUFFIXES + .iter() + .find_map(|suffix| unit.strip_suffix(suffix)) + { + Some(stem) if !stem.is_empty() => Ok(()), + _ => Err(Refusal::new(FIELD, Code::BadShape)), + } +} + +fn validate_reason(reason: &str) -> Result<(), Refusal> { + const FIELD: &str = "reason"; + if reason.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if reason.len() > REASON_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Verb { + Start, + Stop, + Restart, + Reload, +} + +impl Verb { + fn as_str(self) -> &'static str { + match self { + Verb::Start => "start", + Verb::Stop => "stop", + Verb::Restart => "restart", + Verb::Reload => "reload", + } + } + + fn parse(word: &str) -> Option { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + .into_iter() + .find(|verb| verb.as_str() == word) + } + + /// Verbs systemd propagates down the dependency graph. + fn propagates(self) -> bool { + matches!(self, Verb::Stop | Verb::Restart) + } +} + +/// The closed set of things an order can ask for. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Operation { + PackageInstall { packages: Vec }, + UnitAction { verb: Verb, unit: String }, + Deploy { selector: String }, +} + +/// One fixed argv for `systemd-run`, built from the typed operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Step { + pub(crate) argv: Vec, + pub(crate) environment: Vec<(String, String)>, +} + +impl Operation { + fn validate(&self) -> Result<(), Refusal> { + match self { + Operation::PackageInstall { packages } => { + if packages.is_empty() { + return Err(Refusal::new("package", Code::Empty)); + } + if packages.len() > PACKAGE_LIMIT { + return Err(Refusal::new("package", Code::TooMany)); + } + packages.iter().try_for_each(|package| validate_package(package)) + } + Operation::UnitAction { unit, .. } => validate_unit(unit), + Operation::Deploy { selector } => require_selector(selector) + .map_err(|_| Refusal::new("selector", Code::BadShape)), + } + } + + fn steps(&self) -> Vec { + let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); + let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; + match self { + Operation::PackageInstall { packages } => { + let mut install: Vec = words(&["apt-get", "install", "-y", "-q", "--"]); + install.extend(packages.iter().cloned()); + vec![ + Step { + argv: words(&["apt-get", "update", "-q"]), + environment: apt_environment.clone(), + }, + Step { + argv: install, + environment: apt_environment, + }, + ] + } + Operation::UnitAction { verb, unit } => vec![Step { + argv: vec![ + "systemctl".into(), + verb.as_str().into(), + "--".into(), + unit.clone(), + ], + environment: Vec::new(), + }], + Operation::Deploy { .. } => Vec::new(), + } + } + + fn describe(&self) -> String { + match self { + Operation::PackageInstall { packages } => { + format!("package-install {}", packages.join(" ")) + } + Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), + Operation::Deploy { selector } => format!("deploy {selector}"), + } + } +} + +// ---------------------------------------------------------------- records -- + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] +pub(crate) struct OrderRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) principal: String, + #[cultcache(key = 3)] + pub(crate) issuer_uid: u32, + #[cultcache(key = 4)] + pub(crate) issuer_unit: String, + #[cultcache(key = 5)] + pub(crate) issuer_invocation: String, + #[cultcache(key = 6)] + pub(crate) issued_at_unix_millis: u64, + #[cultcache(key = 7)] + pub(crate) operation: Operation, + #[cultcache(key = 8)] + pub(crate) reason: String, +} + +impl fmt::Debug for OrderRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OrderRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_start", + schema = "idunn.steward_order_start.v1" +)] +pub(crate) struct StartRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) started_at_unix_millis: u64, + /// The transient unit name, or the deployment command id. + #[cultcache(key = 3)] + pub(crate) actuation: String, +} + +impl fmt::Debug for StartRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StartRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Via { + Name, + Alias, + Propagation, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Refused { + AlwaysRefused { unit: String, via: Via }, + Braked, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Outcome { + Ran, + Failed { + exit_code: Option, + signal: Option, + }, + Refused(Refused), + Lost, + Submitted { command_id: String }, +} + +impl Outcome { + fn describe(&self) -> String { + match self { + Outcome::Ran => "ran (exit 0)".into(), + Outcome::Failed { + exit_code: Some(code), + .. + } => format!("failed (exit {code})"), + Outcome::Failed { + signal: Some(signal), + .. + } => format!("failed (signal {signal})"), + Outcome::Failed { .. } => "failed (not run to an exit)".into(), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( + "refused: always-refused unit {} reached by {}", + escape(unit.as_bytes()), + match via { + Via::Name => "name", + Via::Alias => "alias", + Via::Propagation => "propagation", + } + ), + Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Lost => "lost: the orders service restarted mid-run".into(), + Outcome::Submitted { command_id } => { + format!("submitted deployment {}", escape(command_id.as_bytes())) + } + } + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_outcome", + schema = "idunn.steward_order_outcome.v1" +)] +pub(crate) struct OutcomeRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) ended_at_unix_millis: u64, + #[cultcache(key = 3)] + pub(crate) outcome: Outcome, + #[cultcache(key = 4)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 5)] + pub(crate) stderr_tail: Vec, +} + +/// Tails stay out of `Debug`: they are raw program output. +impl fmt::Debug for OutcomeRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutcomeRecord") + .field("order_id", &self.order_id) + .field("outcome", &self.outcome) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum PostKind { + Accepted, + Result, +} + +impl PostKind { + fn as_str(self) -> &'static str { + match self { + PostKind::Accepted => "accepted", + PostKind::Result => "result", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_post", + schema = "idunn.steward_order_post.v1" +)] +pub(crate) struct PostRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `:`. + #[cultcache(key = 1)] + pub(crate) post_key: String, + #[cultcache(key = 2)] + pub(crate) event_id: String, + #[cultcache(key = 3)] + pub(crate) posted_at_unix_millis: u64, +} + +// ------------------------------------------------------------------ store -- + +/// `orders.cc`: root-only, written by `idunn orders serve` alone. +pub(crate) struct OrderStore { + directory: PathBuf, +} + +#[derive(Default)] +struct Ledger { + orders: Vec, + starts: Vec, + outcomes: Vec, + posts: Vec, +} + +impl Ledger { + fn start_of(&self, order_id: &str) -> Option<&StartRecord> { + self.starts.iter().find(|start| start.order_id == order_id) + } + + fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { + self.outcomes + .iter() + .find(|outcome| outcome.order_id == order_id) + } + + fn posted(&self, order_id: &str, kind: PostKind) -> bool { + let key = post_key(order_id, kind); + self.posts.iter().any(|post| post.post_key == key) + } +} + +fn post_key(order_id: &str, kind: PostKind) -> String { + format!("{order_id}:{}", kind.as_str()) +} + +impl OrderStore { + pub(crate) fn new(directory: &Path) -> Self { + Self { + directory: directory.to_owned(), + } + } + + fn path(&self) -> PathBuf { + self.directory.join("orders.cc") + } + + fn prepare(&self) -> Result<()> { + std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; + std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) + .context("making the orders directory private") + } + + /// Insert a record whose (type, key) is absent. `false` means it was there. + fn put(&self, key: &str, schema: &str, value: &T) -> Result { + self.prepare()?; + let path = self.path(); + let now = now_millis()?; + let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: T::TYPE.into(), + key: key.into(), + current: None, + }], + &[typed_envelope(key, T::TYPE, schema, value, now)?], + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .context("making the orders store private")?; + Ok(inserted) + } + + fn ledger(&self) -> Result { + let path = self.path(); + let mut ledger = Ledger::default(); + if !path.exists() { + return Ok(ledger); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .context("reading the orders store")?; + for envelope in &envelopes { + let schema = envelope.schema_id.as_deref(); + let key = envelope.key.as_str(); + match envelope.r#type.as_str() { + OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { + let record: OrderRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an order is stored under another key"); + ledger.orders.push(record); + } + StartRecord::TYPE if schema == Some(START_SCHEMA) => { + let record: StartRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "a start is stored under another key"); + ledger.starts.push(record); + } + OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { + let record: OutcomeRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an outcome is stored under another key"); + ledger.outcomes.push(record); + } + PostRecord::TYPE if schema == Some(POST_SCHEMA) => { + let record: PostRecord = decode_record(envelope)?; + ensure!(record.post_key == key, "a post is stored under another key"); + ledger.posts.push(record); + } + _ => bail!("the orders store holds a document that is not an order record"), + } + } + ledger + .orders + .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); + Ok(ledger) + } +} + +// ------------------------------------------------------------------ ports -- + +/// What `systemctl show` says about one unit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct UnitFacts { + pub(crate) id: String, + pub(crate) names: Vec, + pub(crate) required_by: Vec, + pub(crate) bound_by: Vec, + pub(crate) consists_of: Vec, +} + +impl UnitFacts { + fn identities(&self) -> impl Iterator { + std::iter::once(&self.id).chain(self.names.iter()) + } + + /// The units systemd stops with this one. + fn dependents(&self) -> impl Iterator { + self.required_by + .iter() + .chain(self.bound_by.iter()) + .chain(self.consists_of.iter()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RunReport { + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stderr_tail: Vec, +} + +pub(crate) trait Systemd: Send + Sync { + fn show(&self, unit: &str) -> Result; + fn invocation(&self, unit: &str) -> Option; + /// Run one step as the transient unit `unit`, to its end. + fn run(&self, unit: &str, step: &Step) -> Result; +} + +pub(crate) struct Peer { + pub(crate) uid: u32, + pub(crate) issuer_unit: String, +} + +pub(crate) trait PeerCredentials: Send + Sync { + fn peer(&self, stream: &UnixStream) -> Result; +} + +pub(crate) trait BrakeCheck: Send + Sync { + /// `true` only when the instance's brake check exits 0. + fn clear(&self, principal: &str) -> bool; +} + +pub(crate) trait Homeserver: Send + Sync { + /// Idempotent by `txn_id`; returns the event id. + fn post(&self, txn_id: &str, text: &str) -> Result; +} + +pub(crate) struct Ctx { + pub(crate) store: OrderStore, + pub(crate) state_store: PathBuf, + pub(crate) steward_uid: u32, + pub(crate) principal: String, + pub(crate) systemd: Arc, + pub(crate) peers: Arc, + pub(crate) brake: Arc, + pub(crate) homeserver: Arc, +} + +// ------------------------------------------------------------------ scope -- + +fn refused(unit: &str, via: Via) -> Refused { + Refused::AlwaysRefused { + unit: unit.to_owned(), + via, + } +} + +/// Would `verb` on `unit` reach an always-refused unit: by its name, by an +/// alias it resolves from, or, for the verbs systemd propagates, by the units +/// systemd stops with it. Resolution reads systemd before anything runs. +fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { + if is_refused_name(unit) { + return Ok(Some(refused(unit, Via::Name))); + } + let facts = systemd.show(unit)?; + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Alias))); + } + if !verb.propagates() { + return Ok(None); + } + let mut seen: BTreeSet = facts.identities().cloned().collect(); + seen.insert(unit.to_owned()); + let mut queue: VecDeque = facts.dependents().cloned().collect(); + while let Some(next) = queue.pop_front() { + if is_refused_name(&next) { + return Ok(Some(refused(&next, Via::Propagation))); + } + if !seen.insert(next.clone()) { + continue; + } + ensure!( + seen.len() <= CLOSURE_LIMIT, + "the units a stop would reach are too many to check" + ); + let facts = systemd.show(&next)?; + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Propagation))); + } + seen.extend(facts.identities().cloned()); + queue.extend(facts.dependents().cloned()); + } + Ok(None) +} + +// -------------------------------------------------------------- execution -- + +fn tail(bytes: &[u8]) -> Vec { + bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() +} + +fn conclude( + ctx: &Ctx, + order_id: &str, + outcome: Outcome, + stdout_tail: &[u8], + stderr_tail: &[u8], +) -> Result<()> { + ctx.store.put( + order_id, + OUTCOME_SCHEMA, + &OutcomeRecord { + schema_version: OUTCOME_SCHEMA.into(), + order_id: order_id.into(), + ended_at_unix_millis: now_millis()?, + outcome, + stdout_tail: tail(stdout_tail), + stderr_tail: tail(stderr_tail), + }, + )?; + Ok(()) +} + +fn order_uuid(order_id: &str) -> &str { + order_id.strip_prefix("ord-").unwrap_or(order_id) +} + +pub(crate) fn requester(principal: &str, order_id: &str) -> String { + format!("steward:{principal} order {order_id}") +} + +/// Run one accepted order. A start record is written before anything is +/// actuated and an order that has one is never run again. +pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { + let ledger = ctx.store.ledger()?; + let order = ledger + .orders + .iter() + .find(|order| order.order_id == order_id) + .context("the order is not in the store")?; + if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { + return Ok(()); + } + if order.operation.validate().is_err() { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the stored order no longer satisfies the grammar", + ); + } + if let Operation::UnitAction { verb, unit } = &order.operation { + match refusal_for(ctx.systemd.as_ref(), unit, *verb) { + Ok(None) => {} + Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), + Err(_) => { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the unit could not be resolved, so it was not acted on", + ); + } + } + } + if !ctx.brake.clear(&order.principal) { + return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); + } + let uuid = order_uuid(order_id); + let command_id = format!("up-{uuid}"); + let steps = order.operation.steps(); + let actuation = match &order.operation { + Operation::Deploy { .. } => command_id.clone(), + _ => format!("idunn-order-{uuid}"), + }; + let started = StartRecord { + schema_version: START_SCHEMA.into(), + order_id: order_id.into(), + started_at_unix_millis: now_millis()?, + actuation, + }; + if !ctx.store.put(order_id, START_SCHEMA, &started)? { + return Ok(()); + } + if let Operation::Deploy { selector } = &order.operation { + let outcome = match commit_deployment_command( + &ctx.state_store, + &command_id, + selector, + &requester(&order.principal, order_id), + ) { + Ok(_) => Outcome::Submitted { command_id }, + Err(_) => Outcome::Failed { + exit_code: None, + signal: None, + }, + }; + return conclude(ctx, order_id, outcome, b"", b""); + } + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let mut outcome = Outcome::Ran; + for (index, step) in steps.iter().enumerate() { + let unit = if steps.len() == 1 { + format!("idunn-order-{uuid}") + } else { + format!("idunn-order-{uuid}-{}", index + 1) + }; + match ctx.systemd.run(&unit, step) { + Ok(report) => { + stdout = report.stdout_tail; + stderr = report.stderr_tail; + if report.exit_code != Some(0) { + outcome = Outcome::Failed { + exit_code: report.exit_code, + signal: report.signal, + }; + break; + } + } + Err(_) => { + outcome = Outcome::Failed { + exit_code: None, + signal: None, + }; + stderr = b"the transient unit could not be started".to_vec(); + break; + } + } + } + conclude(ctx, order_id, outcome, &stdout, &stderr) +} + +/// On service start: an order with a start record and no outcome ran, or began +/// to, in a life that ended. It is Lost and never run again. Returns the +/// orders that never started, oldest first. +pub(crate) fn recover(ctx: &Ctx) -> Result> { + let ledger = ctx.store.ledger()?; + for start in &ledger.starts { + if ledger.outcome_of(&start.order_id).is_none() { + conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; + } + } + Ok(ledger + .orders + .iter() + .filter(|order| { + ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() + }) + .map(|order| order.order_id.clone()) + .collect()) +} + +// ---------------------------------------------------------------- posting -- + +struct Due { + order_id: String, + kind: PostKind, + text: String, +} + +fn quoted(reason: &str) -> String { + format!("\"{}\"", escape(reason.as_bytes())) +} + +fn accepted_text(order: &OrderRecord) -> String { + format!( + "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", + order.order_id, + escape(order.operation.describe().as_bytes()), + escape(order.issuer_unit.as_bytes()), + escape(order.issuer_invocation.as_bytes()), + quoted(&order.reason) + ) +} + +/// A deploy's command has finished once its state is neither queued nor running. +fn deploy_finished(state: &str) -> bool { + !matches!(state, "queued" | "running") +} + +fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { + let mut due = Vec::new(); + for order in &ledger.orders { + let id = &order.order_id; + if !ledger.posted(id, PostKind::Accepted) { + due.push(Due { + order_id: id.clone(), + kind: PostKind::Accepted, + text: accepted_text(order), + }); + } + if ledger.posted(id, PostKind::Result) { + continue; + } + let Some(record) = ledger.outcome_of(id) else { + continue; + }; + let mut line = record.outcome.describe(); + if let Outcome::Submitted { command_id } = &record.outcome { + match command_state(&ctx.state_store, command_id) { + Ok((state, detail)) if deploy_finished(state) => { + line = format!("{line}: {state} {}", escape(detail.as_bytes())); + } + _ => continue, + } + } + due.push(Due { + order_id: id.clone(), + kind: PostKind::Result, + text: format!( + "Steward order {id}: {line}\n{}", + escape(order.operation.describe().as_bytes()) + ), + }); + } + due +} + +/// Post what is due, in order, stopping at the first failure. Returns how many +/// went out. +pub(crate) fn post_pass(ctx: &Ctx) -> Result { + let ledger = ctx.store.ledger()?; + let mut sent = 0; + for due in due_posts(ctx, &ledger) { + let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); + let event_id = ctx.homeserver.post(&transaction, &due.text)?; + ctx.store.put( + &post_key(&due.order_id, due.kind), + POST_SCHEMA, + &PostRecord { + schema_version: POST_SCHEMA.into(), + post_key: post_key(&due.order_id, due.kind), + event_id, + posted_at_unix_millis: now_millis()?, + }, + )?; + sent += 1; + } + Ok(sent) +} + +// ----------------------------------------------------------------- frames -- + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct OrderRequest { + pub(crate) operation: Operation, + pub(crate) reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ShowRequest { + pub(crate) id: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Request { + Order(OrderRequest), + Show(ShowRequest), + List, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderView { + pub(crate) order: OrderRecord, + pub(crate) start: Option, + pub(crate) outcome: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderLine { + pub(crate) order_id: String, + pub(crate) status: String, + pub(crate) summary: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Response { + Accepted { order_id: String }, + Refused(Refusal), + Show(Box), + List(Vec), +} + +fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { + let bytes = rmp_serde::to_vec_named(value)?; + stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; + stream.write_all(&bytes)?; + Ok(stream.flush()?) +} + +fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { + let mut length = [0u8; 4]; + stream.read_exact(&mut length)?; + let length = u32::from_be_bytes(length) as usize; + ensure!(length <= limit, "a frame exceeds its limit"); + let mut bytes = vec![0u8; length]; + stream.read_exact(&mut bytes)?; + Ok(bytes) +} + +fn status_word(view: &OrderView) -> &'static str { + match (&view.outcome, &view.start) { + (Some(record), _) => match record.outcome { + Outcome::Ran => "ran", + Outcome::Failed { .. } => "failed", + Outcome::Refused(_) => "refused", + Outcome::Lost => "lost", + Outcome::Submitted { .. } => "submitted", + }, + (None, Some(_)) => "running", + (None, None) => "accepted", + } +} + +fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { + OrderView { + order: order.clone(), + start: ledger.start_of(&order.order_id).cloned(), + outcome: ledger.outcome_of(&order.order_id).cloned(), + } +} + +fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { + write_frame(stream, &response) +} + +fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { + answer(stream, Response::Refused(Refusal::new(field, code))) +} + +/// Serve one connection. The peer is authenticated before its frame is read. +pub(crate) fn handle( + ctx: &Ctx, + stream: &mut UnixStream, + queue: &mpsc::Sender, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + let peer = ctx.peers.peer(stream)?; + if peer.uid != ctx.steward_uid { + eprintln!("steward socket: refused a peer that is not the steward"); + return refuse(stream, "peer", Code::Unauthorized); + } + let request = read_frame(stream, REQUEST_FRAME_LIMIT) + .ok() + .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); + let Some(request) = request else { + eprintln!("steward socket: refused a malformed frame"); + return refuse(stream, "frame", Code::Malformed); + }; + match request { + Request::Order(request) => { + let checked = + validate_reason(&request.reason).and_then(|()| request.operation.validate()); + if let Err(refusal) = checked { + eprintln!("steward socket: refused an order at the grammar"); + return answer(stream, Response::Refused(refusal)); + } + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: format!("ord-{}", Uuid::new_v4()), + principal: ctx.principal.clone(), + issuer_uid: peer.uid, + issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), + issuer_unit: peer.issuer_unit, + issued_at_unix_millis: now_millis()?, + operation: request.operation, + reason: request.reason, + }; + match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { + Ok(true) => {} + _ => return refuse(stream, "store", Code::Unavailable), + } + queue + .send(order.order_id.clone()) + .map_err(|_| anyhow!("the executor has stopped"))?; + answer( + stream, + Response::Accepted { + order_id: order.order_id, + }, + ) + } + Request::Show(request) => { + if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) + { + return refuse(stream, "id", Code::BadShape); + } + let ledger = ctx.store.ledger()?; + match ledger.orders.iter().find(|order| order.order_id == request.id) { + Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), + None => refuse(stream, "id", Code::Unknown), + } + } + Request::List => { + let ledger = ctx.store.ledger()?; + let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); + let lines = ledger + .orders + .iter() + .skip(skip) + .map(|order| { + let view = view_of(&ledger, order); + OrderLine { + order_id: order.order_id.clone(), + status: status_word(&view).into(), + summary: order.operation.describe(), + } + }) + .collect(); + answer(stream, Response::List(lines)) + } + } +} + +// ------------------------------------------------------------- real ports -- + +/// `systemctl` and `systemd-run`, found by name. +pub(crate) struct HostSystemd; + +fn parse_names(value: &str) -> Vec { + value.split_whitespace().map(str::to_owned).collect() +} + +fn parse_show(output: &str) -> Result { + let mut facts = UnitFacts::default(); + for line in output.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + match key { + "Id" => facts.id = value.to_owned(), + "Names" => facts.names = parse_names(value), + "RequiredBy" => facts.required_by = parse_names(value), + "BoundBy" => facts.bound_by = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), + _ => {} + } + } + ensure!(!facts.id.is_empty(), "systemctl named no unit"); + Ok(facts) +} + +impl Systemd for HostSystemd { + fn show(&self, unit: &str) -> Result { + let output = Command::new("systemctl") + .args([ + "show", + "--no-pager", + "--property=Id,Names,RequiredBy,BoundBy,ConsistsOf", + "--", + unit, + ]) + .stdin(Stdio::null()) + .output() + .context("running systemctl show")?; + ensure!(output.status.success(), "systemctl show failed"); + parse_show(&String::from_utf8_lossy(&output.stdout)) + } + + fn invocation(&self, unit: &str) -> Option { + let output = Command::new("systemctl") + .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) + .arg(unit) + .stdin(Stdio::null()) + .output() + .ok()?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + (output.status.success() && !value.is_empty()).then_some(value) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + let mut command = Command::new("systemd-run"); + command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); + command.args(["-p", "RuntimeMaxSec=15min"]); + for (name, value) in &step.environment { + command.arg(format!("--setenv={name}={value}")); + } + let output = command + .arg("--") + .args(&step.argv) + .stdin(Stdio::null()) + .output() + .context("running systemd-run")?; + Ok(RunReport { + exit_code: output.status.code(), + signal: output.status.signal(), + stdout_tail: tail(&output.stdout), + stderr_tail: tail(&output.stderr), + }) + } +} + +/// The last `.service` or `.scope` in a process's cgroup path names its issuer. +fn unit_from_cgroup(text: &str) -> String { + let line = text + .lines() + .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) + .unwrap_or_default(); + let path = line.rsplit_once(':').map_or("", |(_, path)| path); + path.rsplit('/') + .find(|segment| segment.ends_with(".service") || segment.ends_with(".scope")) + .filter(|segment| segment.len() <= UNIT_LIMIT) + .map_or_else(|| "unknown".to_owned(), str::to_owned) +} + +/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. +pub(crate) struct KernelPeers; + +impl PeerCredentials for KernelPeers { + fn peer(&self, stream: &UnixStream) -> Result { + use std::os::fd::AsRawFd; + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = std::mem::size_of::() as libc::socklen_t; + // SAFETY: the descriptor is a live socket and the pointer and length + // describe `credentials`, which outlives the call. + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&mut credentials as *mut libc::ucred).cast(), + &mut length, + ) + }; + ensure!( + status == 0 && length as usize == std::mem::size_of::(), + "peer credentials are unavailable" + ); + let cgroup = + std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); + Ok(Peer { + uid: credentials.uid, + issuer_unit: unit_from_cgroup(&cgroup), + }) + } +} + +/// `mind-control --instance check`; a missing binary is a held brake. +pub(crate) struct MindControl { + pub(crate) program: PathBuf, +} + +impl BrakeCheck for MindControl { + fn clear(&self, principal: &str) -> bool { + Command::new(&self.program) + .args(["--instance", principal, "check"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} + +/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is +/// never chunked. The token is read from its credential file for each post. +pub(crate) struct Synapse { + pub(crate) address: std::net::SocketAddr, + pub(crate) room: String, + pub(crate) token_file: PathBuf, +} + +fn percent_encode(text: &str) -> String { + let mut encoded = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +impl Homeserver for Synapse { + fn post(&self, txn_id: &str, text: &str) -> Result { + let token = std::fs::read_to_string(&self.token_file) + .context("the homeserver token is unreadable")?; + let token = token.trim(); + ensure!( + !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), + "the homeserver token is not a bearer token" + ); + let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; + let head = format!( + "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ + Host: {}\r\nAuthorization: Bearer {token}\r\n\ + Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", + percent_encode(&self.room), + self.address, + body.len() + ); + let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) + .context("the homeserver is unreachable")?; + socket.set_read_timeout(Some(Duration::from_secs(10)))?; + socket.set_write_timeout(Some(Duration::from_secs(10)))?; + socket.write_all(head.as_bytes())?; + socket.write_all(&body)?; + let mut reply = Vec::new(); + socket + .take(RESPONSE_FRAME_LIMIT as u64) + .read_to_end(&mut reply) + .context("the homeserver reply was cut short")?; + let split = reply + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("the homeserver reply has no body")?; + let status = String::from_utf8_lossy(&reply[..split]); + ensure!( + status.split_whitespace().nth(1) == Some("200"), + "the homeserver refused the post" + ); + #[derive(Deserialize)] + struct Sent { + event_id: String, + } + let sent: Sent = serde_json::from_slice(&reply[split + 4..]) + .map_err(|_| anyhow!("the homeserver reply has no event id"))?; + Ok(sent.event_id) + } +} + +// ---------------------------------------------------------------- binding -- + +/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. +/// It lists no unit of any kind, and any field outside this set is refused. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct StewardBinding { + pub(crate) principal: String, + pub(crate) user: String, + pub(crate) mind_control: PathBuf, + pub(crate) room: String, + pub(crate) homeserver: String, + pub(crate) token_credential: String, +} + +impl StewardBinding { + pub(crate) fn parse(text: &str) -> Result { + let binding: Self = toml::from_str(text) + .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; + binding.validate()?; + Ok(binding) + } + + fn validate(&self) -> Result<()> { + require_id(&self.principal, "steward principal")?; + ensure!( + !self.principal.starts_with('-') && !self.principal.contains('/'), + "steward principal is invalid" + ); + let user_ok = !self.user.is_empty() + && self.user.len() <= 32 + && self.user.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) + && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); + ensure!(user_ok && self.user != "root", "steward user is invalid or root"); + ensure!( + self.mind_control.is_absolute(), + "mind_control must be an absolute path" + ); + let room_ok = self.room.starts_with('!') + && self.room.len() <= 255 + && self.room.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') + }); + ensure!(room_ok, "room must be a Matrix room id"); + self.address()?; + let credential_ok = !self.token_credential.is_empty() + && !self.token_credential.starts_with('.') + && self.token_credential.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') + }); + ensure!(credential_ok, "token_credential is not a credential name"); + Ok(()) + } + + fn address(&self) -> Result { + let address: std::net::SocketAddr = self + .homeserver + .strip_prefix("http://") + .and_then(|rest| rest.parse().ok()) + .context("homeserver must be http://:")?; + ensure!(address.ip().is_loopback(), "homeserver must be loopback"); + Ok(address) + } +} + +/// Root-admitted: owned by root and writable by no one else. +fn admitted(owner: u32, mode: u32) -> bool { + owner == 0 && mode & 0o022 == 0 +} + +fn load_binding(path: &Path) -> Result { + let metadata = std::fs::metadata(path).context("reading the steward binding")?; + ensure!( + metadata.is_file() && admitted(metadata.uid(), metadata.mode()), + "the steward binding is not a root-owned file closed to writes" + ); + let text = std::fs::read_to_string(path).context("reading the steward binding")?; + StewardBinding::parse(&text) +} + +fn user_uid(name: &str) -> Result { + let name = std::ffi::CString::new(name)?; + // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in + // `buffer`, which outlives every read of them below (none are read). + let (status, found, uid) = unsafe { + let mut passwd: libc::passwd = std::mem::zeroed(); + let mut buffer = vec![0 as libc::c_char; 4096]; + let mut result: *mut libc::passwd = std::ptr::null_mut(); + let status = libc::getpwnam_r( + name.as_ptr(), + &mut passwd, + buffer.as_mut_ptr(), + buffer.len(), + &mut result, + ); + (status, !result.is_null(), passwd.pw_uid) + }; + ensure!(status == 0 && found, "the steward user is unknown"); + ensure!(uid != 0, "the steward user resolves to root"); + Ok(uid) +} + +// ------------------------------------------------------------------ serve -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ServeOptions { + pub(crate) state_store: PathBuf, + pub(crate) store: PathBuf, + pub(crate) steward_binding: PathBuf, + pub(crate) socket: PathBuf, +} + +const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; +const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; +const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; + +pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { + ensure!( + args.next().as_deref() == Some("serve"), + "idunn orders takes the subcommand serve" + ); + let mut options = ServeOptions { + state_store: default_state_store(), + store: DEFAULT_STORE.into(), + steward_binding: DEFAULT_BINDING.into(), + socket: DEFAULT_SOCKET.into(), + }; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, + "--store" => options.store = path_value(&mut args, "--store")?, + "--steward-binding" => { + options.steward_binding = path_value(&mut args, "--steward-binding")? + } + "--socket" => options.socket = path_value(&mut args, "--socket")?, + _ => bail!("an orders serve option is not part of the grammar"), + } + } + Ok(options) +} + +/// Run a thread whose end ends the service: systemd restarts it, and the +/// start recovery settles what it left. +fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { + struct ExitOnDrop(&'static str); + impl Drop for ExitOnDrop { + fn drop(&mut self) { + eprintln!("steward orders: the {} thread ended", self.0); + std::process::exit(1); + } + } + thread::Builder::new() + .name(name.into()) + .spawn(move || { + let _guard = ExitOnDrop(name); + work(); + }) + .expect("a thread can be spawned"); +} + +fn bind_socket(path: &Path) -> Result { + if let Ok(metadata) = std::fs::symlink_metadata(path) { + use std::os::unix::fs::FileTypeExt; + ensure!( + metadata.file_type().is_socket(), + "the socket path holds something that is not a socket" + ); + std::fs::remove_file(path).context("removing a stale socket")?; + } + let listener = UnixListener::bind(path).context("binding the steward socket")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) + .context("setting the steward socket mode")?; + Ok(listener) +} + +pub(crate) fn serve(options: ServeOptions) -> Result<()> { + let binding = load_binding(&options.steward_binding)?; + let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") + .map(PathBuf::from) + .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; + let ctx = Arc::new(Ctx { + store: OrderStore::new(&options.store), + state_store: options.state_store, + steward_uid: user_uid(&binding.user)?, + principal: binding.principal.clone(), + systemd: Arc::new(HostSystemd), + peers: Arc::new(KernelPeers), + brake: Arc::new(MindControl { + program: binding.mind_control.clone(), + }), + homeserver: Arc::new(Synapse { + address: binding.address()?, + room: binding.room.clone(), + token_file: credentials.join(&binding.token_credential), + }), + }); + ctx.store.prepare()?; + let pending = recover(&ctx)?; + let listener = bind_socket(&options.socket)?; + let (queue, orders) = mpsc::channel::(); + for order_id in pending { + queue.send(order_id)?; + } + let executor = Arc::clone(&ctx); + vital("executor", move || { + for order_id in orders { + if let Err(error) = execute(&executor, &order_id) { + eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); + } + } + }); + let poster = Arc::clone(&ctx); + vital("poster", move || { + let mut failures = 0u32; + loop { + let pause = match post_pass(&poster) { + Ok(_) => { + failures = 0; + 2 + } + Err(error) => { + failures = failures.saturating_add(1); + eprintln!("steward orders: posting is delayed: {error:#}"); + (2u64 << failures.min(5)).min(60) + } + }; + thread::sleep(Duration::from_secs(pause)); + } + }); + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + if let Err(error) = handle(&ctx, &mut stream, &queue) { + eprintln!("steward socket: a connection ended in error: {error:#}"); + } + } + Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), + } + } + bail!("the steward socket stopped accepting") +} + +// ----------------------------------------------------------------- client -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OrderAction { + Issue { operation: Operation, reason: String }, + Show { id: String }, + List, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct OrderCommand { + pub(crate) socket: PathBuf, + pub(crate) action: OrderAction, +} + +pub(crate) fn parse_order(mut args: impl Iterator) -> Result { + let kind = args.next().context("idunn order requires an operation")?; + let mut socket = PathBuf::from(DEFAULT_SOCKET); + let mut reason = None; + let mut positional = Vec::new(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--socket" => socket = path_value(&mut args, "--socket")?, + "--reason" => reason = Some(string_value(&mut args, "--reason")?), + word if word.starts_with('-') => bail!("an order option is not part of the grammar"), + _ => positional.push(argument), + } + } + let action = match kind.as_str() { + "list" => { + ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); + OrderAction::List + } + "show" => { + ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); + OrderAction::Show { + id: positional.remove(0), + } + } + "package-install" | "unit" | "deploy" => { + let operation = match kind.as_str() { + "package-install" => Operation::PackageInstall { + packages: positional, + }, + "unit" => { + ensure!(positional.len() == 2, "unit takes a verb and a unit"); + let verb = Verb::parse(&positional[0]) + .context("the unit verb is not start, stop, restart or reload")?; + Operation::UnitAction { + verb, + unit: positional.remove(1), + } + } + _ => { + ensure!(positional.len() == 1, "deploy takes one selector"); + Operation::Deploy { + selector: positional.remove(0), + } + } + }; + let reason = reason.context("an order requires --reason")?; + validate_reason(&reason)?; + operation.validate()?; + OrderAction::Issue { operation, reason } + } + _ => bail!("the order operation is not part of the grammar"), + }; + Ok(OrderCommand { socket, action }) +} + +fn call(socket: &Path, request: &Request) -> Result { + let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + write_frame(&mut stream, request)?; + let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; + rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) +} + +fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); + let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); +} + +/// Raw lines for the terminal; `emit` is their only way out. +fn render_view(view: &OrderView) -> Vec> { + let order = &view.order; + let mut lines: Vec> = vec![ + format!("order {}", order.order_id).into_bytes(), + format!("status {}", status_word(view)).into_bytes(), + format!("operation {}", order.operation.describe()).into_bytes(), + format!( + "issuer {} (invocation {}, uid {})", + order.issuer_unit, order.issuer_invocation, order.issuer_uid + ) + .into_bytes(), + format!("reason {}", quoted(&order.reason)).into_bytes(), + ]; + if let Some(start) = &view.start { + lines.push(format!("started {}", start.actuation).into_bytes()); + } + if let Some(record) = &view.outcome { + lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); + render_tail(&mut lines, "stdout", &record.stdout_tail); + render_tail(&mut lines, "stderr", &record.stderr_tail); + } + lines +} + +pub(crate) fn run_order(command: OrderCommand) -> Result<()> { + let request = match command.action { + OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), + OrderAction::Show { id } => Request::Show(ShowRequest { id }), + OrderAction::List => Request::List, + }; + let mut out = std::io::stdout().lock(); + match call(&command.socket, &request)? { + Response::Accepted { order_id } => writeln!(out, "{order_id}")?, + Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), + Response::Show(view) => emit(&mut out, &render_view(&view))?, + Response::List(lines) => { + let lines: Vec> = lines + .iter() + .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) + .collect(); + emit(&mut out, &lines)? + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tempfile::TempDir; + + use super::*; + + const STEWARD: u32 = 1000; + const CANARY: &str = "CANARY-7f3a91"; + + struct FakeSystemd { + units: Mutex>, + runs: Mutex>, + report: Mutex, + } + + impl Systemd for FakeSystemd { + fn show(&self, unit: &str) -> Result { + let known = self.units.lock().unwrap().get(unit).cloned(); + Ok(known.unwrap_or_else(|| UnitFacts { + id: unit.into(), + names: vec![unit.into()], + ..UnitFacts::default() + })) + } + + fn invocation(&self, _unit: &str) -> Option { + Some("invocation-1".into()) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + self.runs.lock().unwrap().push((unit.into(), step.clone())); + Ok(self.report.lock().unwrap().clone()) + } + } + + struct FakePeers(u32); + + impl PeerCredentials for FakePeers { + fn peer(&self, _stream: &UnixStream) -> Result { + Ok(Peer { + uid: self.0, + issuer_unit: "mind-eureka-self.service".into(), + }) + } + } + + struct FakeBrake(AtomicBool); + + impl BrakeCheck for FakeBrake { + fn clear(&self, _principal: &str) -> bool { + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct FakeHome { + failing: AtomicBool, + sent: Mutex>, + } + + impl Homeserver for FakeHome { + fn post(&self, txn_id: &str, text: &str) -> Result { + ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); + let mut sent = self.sent.lock().unwrap(); + sent.push((txn_id.into(), text.into())); + Ok(format!("$event{}", sent.len())) + } + } + + struct Harness { + directory: TempDir, + ctx: Ctx, + systemd: Arc, + brake: Arc, + home: Arc, + queue: mpsc::Sender, + _orders: mpsc::Receiver, + } + + fn harness_for(peer_uid: u32) -> Harness { + let directory = TempDir::new().unwrap(); + let systemd = Arc::new(FakeSystemd { + units: Mutex::new(BTreeMap::new()), + runs: Mutex::new(Vec::new()), + report: Mutex::new(RunReport { + exit_code: Some(0), + ..RunReport::default() + }), + }); + let brake = Arc::new(FakeBrake(AtomicBool::new(true))); + let home = Arc::new(FakeHome::default()); + let (queue, orders) = mpsc::channel(); + let ctx = Ctx { + store: OrderStore::new(&directory.path().join("orders")), + state_store: directory.path().join("state").join("control.cc"), + steward_uid: STEWARD, + principal: "eureka".into(), + systemd: systemd.clone(), + peers: Arc::new(FakePeers(peer_uid)), + brake: brake.clone(), + homeserver: home.clone(), + }; + Harness { + directory, + ctx, + systemd, + brake, + home, + queue, + _orders: orders, + } + } + + fn harness() -> Harness { + harness_for(STEWARD) + } + + impl Harness { + fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { + let (mut client, mut server) = UnixStream::pair().unwrap(); + write(&mut client); + handle(&self.ctx, &mut server, &self.queue).unwrap(); + let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); + rmp_serde::from_slice(&bytes).unwrap() + } + + fn ask(&self, request: &Request) -> Response { + self.exchange(|client| write_frame(client, request).unwrap()) + } + + fn issue(&self, operation: Operation) -> Response { + self.ask(&Request::Order(OrderRequest { + operation, + reason: "probe".into(), + })) + } + + fn issue_unit(&self, verb: Verb, unit: &str) -> Response { + self.issue(Operation::UnitAction { + verb, + unit: unit.into(), + }) + } + + fn accept(&self, response: Response) -> String { + match response { + Response::Accepted { order_id } => order_id, + other => panic!("expected an accepted order, got {other:?}"), + } + } + + fn outcome(&self, order_id: &str) -> Outcome { + self.ctx + .store + .ledger() + .unwrap() + .outcome_of(order_id) + .expect("the order has an outcome") + .outcome + .clone() + } + + fn runs(&self) -> Vec<(String, Step)> { + self.systemd.runs.lock().unwrap().clone() + } + + fn describe_unit(&self, name: &str, facts: UnitFacts) { + self.systemd.units.lock().unwrap().insert(name.into(), facts); + } + } + + fn all_verbs() -> [Verb; 4] { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + } + + #[test] + fn foreign_uid_is_refused_at_the_door() { + let h = harness_for(STEWARD + 1); + let response = h.issue_unit(Verb::Restart, "nginx.service"); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + let response = h.ask(&Request::List); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); + assert!(h.runs().is_empty()); + } + + #[derive(Serialize)] + struct ForgedBody { + operation: Operation, + reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + } + + #[derive(Serialize)] + struct ForgedFrame { + order: ForgedBody, + } + + #[test] + fn order_id_and_requester_are_minted_not_declared() { + let h = harness(); + let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { + let frame = ForgedFrame { + order: ForgedBody { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into(), + }, + reason: "probe".into(), + order_id: order_id.map(str::to_owned), + requested_by: requested_by.map(str::to_owned), + command: command.map(str::to_owned), + }, + }; + h.exchange(|client| write_frame(client, &frame).unwrap()) + }; + // The same frame without the extra field is a good order. + let honest = h.accept(forge(None, None, None)); + for forged in [ + forge(Some("ord-declared"), None, None), + forge(None, Some("steward:someone-else order x"), None), + forge(None, None, Some("reboot-now")), + ] { + assert_eq!( + forged, + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + } + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); + let order = &ledger.orders[0]; + assert_eq!(order.order_id, honest); + let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); + assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); + assert_eq!(order.issuer_unit, "mind-eureka-self.service"); + assert_eq!(order.issuer_invocation, "invocation-1"); + assert_eq!(order.principal, "eureka"); + assert_eq!(order.issuer_uid, STEWARD); + assert_eq!( + requester(&order.principal, &order.order_id), + format!("steward:eureka order {}", order.order_id) + ); + } + + #[test] + fn package_names_outside_grammar_are_refused() { + let h = harness(); + let bad = [ + "-y", + "--allow-unauthenticated", + "-", + "a", + "", + "Foo", + "a/b", + "/tmp/x", + "./x", + "x.deb", + "foo.deb", + "foo bar", + "foo;reboot", + "foo$(id)", + "foo`id`", + "foo|bar", + "foo&bar", + "foo\nbar", + "foo*", + "foo?", + "pkg:amd64", + "pkg/stable", + "libc6-", + "foo=", + "foo=1 2", + "foo=1;2", + "foo=1=2", + "fo\u{ff0f}o", + "f\u{43e}o", + ]; + for name in bad { + let response = h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into(), name.into()], + }); + assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); + } + let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); + assert_eq!( + h.issue(Operation::PackageInstall { packages: many }), + Response::Refused(Refusal::new("package", Code::TooMany)) + ); + assert_eq!( + h.issue(Operation::PackageInstall { packages: vec![] }), + Response::Refused(Refusal::new("package", Code::Empty)) + ); + assert!(!h.ctx.store.path().exists(), "no refusal left a record"); + + let good = [ + "git-lfs", + "matrix-synapse-py3", + "libnss-systemd", + "bubblewrap", + "socat", + "g++", + "libstdc++6", + "python3.11", + "nginx=1.24.0-2ubuntu7", + "foo=1:2.3-4+b1~rc1", + ]; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: good.iter().map(|name| (*name).to_owned()).collect(), + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + let runs = h.runs(); + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); + let mut install = vec!["apt-get", "install", "-y", "-q", "--"]; + install.extend(good); + assert_eq!(runs[1].1.argv, install); + for (unit, step) in &runs { + assert!(unit.starts_with("idunn-order-")); + assert_eq!( + step.environment, + [("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + ); + } + assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + } + + #[test] + fn any_unit_outside_the_refused_set_runs() { + let h = harness(); + let units = [ + "wg-quick@wg0.service", + "nginx.service", + "matrix-synapse.service", + "ssh.socket", + "huginn@eureka.service", + "claude-connector.service", + "persona-matrix-mouth.service", + "mind-heartbeat.service", + "apt-daily.timer", + "multi-user.target", + // Near misses: the prefix is the rule, not a substring or a stem. + "midunn-x.service", + "not-idunn-x.service", + "idunn.service", + "mind-self.service", + "mind-selfx@a.service", + "mind-persona.service", + "mind-personal@a.service", + ]; + for verb in all_verbs() { + for unit in units { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); + let (transient, step) = h.runs().pop().unwrap(); + assert!(transient.starts_with("idunn-order-")); + assert_eq!(step.argv, ["systemctl", verb.as_str(), "--", unit]); + } + } + assert_eq!(h.runs().len(), units.len() * 4); + } + + #[test] + fn refused_set_is_refused_by_name_alias_and_propagation() { + let h = harness(); + h.describe_unit( + "backup-name.service", + UnitFacts { + id: "idunn-yggdrasil.service".into(), + names: vec!["idunn-yggdrasil.service".into(), "backup-name.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "quiet-alias.service", + UnitFacts { + id: "quiet-alias.service".into(), + names: vec!["quiet-alias.service".into(), "mind-persona@void.service".into()], + ..UnitFacts::default() + }, + ); + // fuse <- wrapper <- (bound by) mind-self@eureka + h.describe_unit( + "fuse.service", + UnitFacts { + id: "fuse.service".into(), + names: vec!["fuse.service".into()], + required_by: vec!["wrapper.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "wrapper.service", + UnitFacts { + id: "wrapper.service".into(), + names: vec!["wrapper.service".into()], + bound_by: vec!["mind-self@eureka.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "stack.target", + UnitFacts { + id: "stack.target".into(), + names: vec!["stack.target".into()], + consists_of: vec!["mind-persona@void.service".into()], + ..UnitFacts::default() + }, + ); + // A cycle that reaches an alias of a refused unit on its second hop. + h.describe_unit( + "loop-a.service", + UnitFacts { + id: "loop-a.service".into(), + names: vec!["loop-a.service".into()], + required_by: vec!["loop-b.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "loop-b.service", + UnitFacts { + id: "loop-b.service".into(), + names: vec!["loop-b.service".into()], + required_by: vec!["loop-a.service".into(), "hidden.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "hidden.service", + UnitFacts { + id: "idunn-orders.service".into(), + names: vec!["idunn-orders.service".into(), "hidden.service".into()], + ..UnitFacts::default() + }, + ); + + let direct = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-x.service", + "idunn-order-3f2c.service", + "idunn-yggdrasil.timer", + "idunn-x.socket", + "idunn-.service", + "idunn-yggdrasil.service.service", + "idunn-x@y.service", + "mind-self@eureka.service", + "mind-persona@void.service", + "mind-self@.service", + "mind-persona@a@b.service", + // Case folds: a lookalike is still refused, never run. + "IDUNN-yggdrasil.service", + "Idunn-Orders.service", + "MIND-SELF@eureka.service", + "Mind-Persona@Void.service", + ]; + for verb in all_verbs() { + for unit in direct { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + match h.outcome(&id) { + Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { + assert_eq!(hit, unit, "{verb:?} {unit}"); + assert_eq!(via, Via::Name); + } + other => panic!("{verb:?} {unit} was {other:?}"), + } + } + } + for verb in all_verbs() { + for (unit, hit) in [ + ("backup-name.service", "idunn-yggdrasil.service"), + ("quiet-alias.service", "mind-persona@void.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Alias + }), + "{verb:?} {unit}" + ); + } + } + for verb in [Verb::Stop, Verb::Restart] { + for (unit, hit) in [ + ("fuse.service", "mind-self@eureka.service"), + ("stack.target", "mind-persona@void.service"), + ("loop-a.service", "idunn-orders.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Propagation + }), + "{verb:?} {unit}" + ); + } + } + assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); + // start and reload propagate nothing, so the same units are not refused. + for verb in [Verb::Start, Verb::Reload] { + let id = h.accept(h.issue_unit(verb, "fuse.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + } + // Every refusal is recorded and has a result post due. + let ledger = h.ctx.store.ledger().unwrap(); + let refusals = ledger + .outcomes + .iter() + .filter(|record| matches!(record.outcome, Outcome::Refused(_))) + .count(); + assert_eq!(refusals, direct.len() * 4 + 8 + 6); + let due = due_posts(&h.ctx, &ledger); + let results = due + .iter() + .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) + .count(); + assert_eq!(results, refusals); + // A refused order never started. + assert!(ledger.starts.len() == 2); + } + + #[test] + fn globs_slices_and_scopes_are_not_units() { + let h = harness(); + let bad = [ + "idunn*", + "idunn-*.service", + "idunn-?ggdrasil.service", + "mind-?elf@eureka.service", + "mind-self@*.service", + "mind-[s]elf@eureka.service", + "system.slice", + "user.slice", + "session-1.scope", + "init.scope", + "idunn-yggdrasil", + "mind-self@eureka", + "nginx", + ".service", + "", + "-idunn-x.service", + "-x.service", + "/etc/systemd/system/idunn-yggdrasil.service", + "../idunn-yggdrasil.service", + "idunn-yggdrasil.service\n", + "idunn-yggdrasil.service ", + " idunn-yggdrasil.service", + "idunn-yggdrasil.service\0", + "idunn-yggdrasil.service;reboot", + "idunn\\x2dyggdrasil.service", + "idunn%2dyggdrasil.service", + // Unicode lookalikes of a refused name. + "\u{131}dunn-yggdrasil.service", + "idunn\u{2010}yggdrasil.service", + "idunn\u{ff0d}yggdrasil.service", + "idunn-yggdrasil.service\u{200b}", + "\u{200b}idunn-yggdrasil.service", + "idunn-yggdrasil\u{2024}service", + "mind-self\u{ff20}eureka.service", + ]; + for unit in bad { + for verb in all_verbs() { + assert!( + matches!(h.issue_unit(verb, unit), Response::Refused(_)), + "{unit:?} was not refused by the grammar" + ); + } + } + let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); + assert_eq!( + h.issue_unit(Verb::Start, &long), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); + } + + #[test] + fn binding_with_a_unit_list_is_refused() { + let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ + mind_control = \"/usr/local/bin/mind-control\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ + token_credential = \"idunn-matrix-token\"\n"; + let binding = StewardBinding::parse(valid).unwrap(); + assert_eq!(binding.principal, "eureka"); + assert_eq!(binding.address().unwrap().port(), 8008); + for extra in [ + "units = [\"nginx.service\"]", + "allow = [\"nginx.service\"]", + "deny = [\"idunn-*\"]", + "denylist = [\"idunn-*\"]", + "refused = [\"idunn-*\"]", + "unit = \"nginx.service\"", + "[units]\nnginx = true", + ] { + assert!( + StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), + "{extra} loaded" + ); + } + let swap = |from: &str, to: &str| valid.replace(from, to); + for broken in [ + swap("user = \"mind-eureka-self\"", "user = \"root\""), + swap("user = \"mind-eureka-self\"", "user = \"\""), + swap("/usr/local/bin/mind-control", "mind-control"), + swap("!abc:example.org", "#alias:example.org"), + swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), + swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), + swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), + swap("idunn-matrix-token", "../token"), + swap("principal = \"eureka\"\n", ""), + ] { + assert!(StewardBinding::parse(&broken).is_err()); + } + assert!(admitted(0, 0o100600)); + assert!(admitted(0, 0o100644)); + assert!(!admitted(0, 0o100664)); + assert!(!admitted(0, 0o100666)); + assert!(!admitted(0, 0o100602)); + assert!(!admitted(1000, 0o100600)); + } + + #[test] + fn held_brake_refuses_at_start() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); + assert!(h.runs().is_empty()); + // The deploy path is gated the same way and writes no command. + let deploy = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &deploy).unwrap(); + assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); + assert!(!h.ctx.state_store.exists()); + // A brake that cannot be asked is a held brake. + let directory = TempDir::new().unwrap(); + assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); + } + + #[test] + fn brake_check_is_the_exit_status_of_mind_control_check() { + let directory = TempDir::new().unwrap(); + let program = |name: &str, body: &str| { + let path = directory.path().join(name); + std::fs::write(&path, body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + MindControl { program: path } + }; + let recorder = directory.path().join("argv"); + let clear = program( + "clear", + &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), + ); + assert!(clear.clear("eureka")); + assert_eq!( + std::fs::read_to_string(&recorder).unwrap().trim(), + "--instance eureka check" + ); + assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); + } + + #[test] + fn started_order_is_never_rerun() { + let h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The first order began and its service died: a start with no outcome. + h.ctx + .store + .put( + &id, + START_SCHEMA, + &StartRecord { + schema_version: START_SCHEMA.into(), + order_id: id.clone(), + started_at_unix_millis: 1, + actuation: format!("idunn-order-{}", order_uuid(&id)), + }, + ) + .unwrap(); + let pending = recover(&h.ctx).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Lost); + assert_eq!(pending, [waiting.clone()], "an order that never started waits"); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); + // Recovery is idempotent, and a finished order is not run again. + assert_eq!(recover(&h.ctx).unwrap(), [waiting.clone()]); + execute(&h.ctx, &waiting).unwrap(); + execute(&h.ctx, &waiting).unwrap(); + assert_eq!(h.runs().len(), 1); + assert!(recover(&h.ctx).unwrap().is_empty()); + } + + #[test] + fn deploy_order_uses_the_up_primitive() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let command_id = format!("up-{}", order_uuid(&id)); + assert_eq!( + h.outcome(&id), + Outcome::Submitted { + command_id: command_id.clone() + } + ); + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); + assert!(h.runs().is_empty(), "a deploy runs no transient unit"); + let mut shown = Vec::new(); + crate::control_plane::status( + &mut shown, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + Some(&command_id), + ) + .unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); + assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); + // Re-executing writes no second command. + execute(&h.ctx, &id).unwrap(); + let mut again = Vec::new(); + crate::control_plane::status( + &mut again, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + None, + ) + .unwrap(); + assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); + // The result post waits for the transactions: queued is not finished. + let due = due_posts(&h.ctx, &ledger); + assert_eq!(due.len(), 1); + assert_eq!(due[0].kind, PostKind::Accepted); + assert!(!deploy_finished("queued") && !deploy_finished("running")); + assert!(deploy_finished("succeeded") && deploy_finished("failed")); + assert!(deploy_finished("complete")); + } + + #[test] + fn chat_outage_delays_posts_not_orders() { + let h = harness(); + h.home.failing.store(true, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + assert_eq!(h.runs().len(), 1); + assert!(post_pass(&h.ctx).is_err()); + assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); + h.home.failing.store(false, Ordering::SeqCst); + assert_eq!(post_pass(&h.ctx).unwrap(), 2); + assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); + let sent = h.home.sent.lock().unwrap().clone(); + assert_eq!(sent.len(), 2); + assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); + assert_eq!(sent[1].0, format!("idunn-{id}-result")); + assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); + assert!(sent[0].1.contains("mind-eureka-self.service")); + assert!(sent[0].1.contains("reason: \"probe\"")); + assert!(sent[1].1.contains("ran (exit 0)")); + } + + #[test] + fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { + let h = harness(); + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(100), + signal: None, + stdout_tail: vec![b'o'; 3 * TAIL_BYTES], + stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), + }; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into()], + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Failed { + exit_code: Some(100), + signal: None + } + ); + assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); + let ledger = h.ctx.store.ledger().unwrap(); + let record = ledger.outcome_of(&id).unwrap(); + assert_eq!(record.stdout_tail.len(), TAIL_BYTES); + assert!(!format!("{record:?}").contains("secret-looking")); + // Neither the post nor the journal text carries a tail. + let results: Vec<_> = due_posts(&h.ctx, &ledger) + .into_iter() + .filter(|due| due.kind == PostKind::Result) + .collect(); + assert_eq!(results.len(), 1); + assert!(results[0].text.contains("failed (exit 100)")); + assert!(!results[0].text.contains("secret-looking")); + // The steward reads them through the escape sink and nothing raw. + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("expected a show") + }; + let mut shown = Vec::new(); + emit(&mut shown, &render_view(&view)).unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert!(shown.contains("secret-looking")); + assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); + assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); + assert!(shown.contains("status failed")); + // A show of an unknown or malformed id names the field only. + assert_eq!( + h.ask(&Request::Show(ShowRequest { + id: format!("ord-{}", Uuid::new_v4()) + })), + Response::Refused(Refusal::new("id", Code::Unknown)) + ); + assert_eq!( + h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), + Response::Refused(Refusal::new("id", Code::BadShape)) + ); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("expected a list") + }; + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].status, "failed"); + } + + #[test] + fn a_refusal_or_error_never_echoes_an_input_value() { + let h = harness(); + let mut seen: Vec> = Vec::new(); + let mut note = |response: Response| { + let bytes = rmp_serde::to_vec_named(&response).unwrap(); + if let Response::Refused(refusal) = &response { + seen.push(refusal.to_string().into_bytes()); + } + seen.push(bytes); + }; + for unit in [ + format!("{CANARY}*.service"), + format!("{CANARY}/../idunn-orders.service"), + format!("idunn-{CANARY};x.service"), + format!("{CANARY}"), + ] { + note(h.issue_unit(Verb::Restart, &unit)); + } + note(h.issue(Operation::PackageInstall { + packages: vec![format!("{CANARY}/x")], + })); + note(h.issue(Operation::PackageInstall { + packages: vec![format!("pkg={CANARY} 1")], + })); + note(h.issue(Operation::Deploy { + selector: format!("{CANARY} bad"), + })); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: format!("{CANARY}\n"), + }))); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: CANARY.repeat(40), + }))); + note(h.ask(&Request::Show(ShowRequest { + id: format!("ord-{CANARY}"), + }))); + // A frame whose own keys and values are the canary, and plain garbage. + let named: std::collections::BTreeMap> = + [( + CANARY.to_owned(), + [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), + )] + .into_iter() + .collect(); + note(h.exchange(|client| write_frame(client, &named).unwrap())); + note(h.exchange(|client| { + let garbage = format!("{CANARY}{CANARY}"); + client + .write_all(&(garbage.len() as u32).to_be_bytes()) + .unwrap(); + client.write_all(garbage.as_bytes()).unwrap(); + })); + note(h.exchange(|client| { + client.write_all(&u32::MAX.to_be_bytes()).unwrap(); + })); + // The client grammar fails the same way. + let mut messages = Vec::new(); + let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); + for line in [ + words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), + words(&["unit", CANARY, "nginx.service", "--reason", "r"]), + words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), + words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), + words(&[CANARY]), + words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), + words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), + words(&["show"]), + ] { + messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); + } + for line in [ + words(&[CANARY]), + words(&["serve", &format!("--{CANARY}")]), + words(&["serve", "--store"]), + ] { + messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); + } + for bytes in &seen { + assert!( + !String::from_utf8_lossy(bytes).contains(CANARY), + "a response carried an input value" + ); + } + for message in &messages { + assert!(!message.contains(CANARY), "an error carried an input value: {message}"); + } + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn cgroup_names_the_issuer_unit() { + assert_eq!( + unit_from_cgroup("0::/system.slice/claude-connector.service\n"), + "claude-connector.service" + ); + assert_eq!( + unit_from_cgroup("0::/system.slice/huginn@eureka.service/sub/leaf\n"), + "huginn@eureka.service" + ); + assert_eq!( + unit_from_cgroup("0::/user.slice/user-1000.slice/session-3.scope\n"), + "session-3.scope" + ); + assert_eq!( + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.service\n"), + "a.service" + ); + assert_eq!(unit_from_cgroup("0::/\n"), "unknown"); + assert_eq!(unit_from_cgroup(""), "unknown"); + } + + #[test] + fn systemctl_show_output_is_read_into_facts() { + let facts = parse_show( + "Id=a.service\nNames=a.service b.service\nRequiredBy=c.service\n\ + BoundBy=\nConsistsOf=d.target e.service\nOther=x\n", + ) + .unwrap(); + assert_eq!(facts.id, "a.service"); + assert_eq!(facts.names, ["a.service", "b.service"]); + assert_eq!(facts.required_by, ["c.service"]); + assert!(facts.bound_by.is_empty()); + assert_eq!(facts.consists_of, ["d.target", "e.service"]); + assert!(parse_show("Names=a.service\n").is_err()); + } + + #[test] + fn the_store_refuses_a_second_record_and_a_foreign_document() { + let h = harness(); + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: "ord-x".into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }; + assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert_eq!( + std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); + SingleFileMessagePackBackingStore::new(&h.ctx.store.path()) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: "idunn.other".into(), + key: "k".into(), + current: None, + }], + &[foreign], + ) + .unwrap(); + assert!(h.ctx.store.ledger().is_err()); + } + + #[test] + fn synapse_post_is_one_idempotent_put_on_loopback() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (sent, requests) = mpsc::channel(); + let server = thread::spawn(move || { + for reply in [ + "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", + "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", + ] { + let (mut socket, _) = listener.accept().unwrap(); + let mut received = Vec::new(); + let mut chunk = [0u8; 1024]; + while !received.windows(4).any(|window| window == b"\r\n\r\n") + || !String::from_utf8_lossy(&received).contains("\"body\"") + { + let count = socket.read(&mut chunk).unwrap(); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); + socket.write_all(reply.as_bytes()).unwrap(); + } + }); + let synapse = Synapse { + address, + room: "!room:example.org".into(), + token_file, + }; + assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); + let request = requests.recv().unwrap(); + assert!(request.starts_with( + "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" + )); + assert!(request.contains("Authorization: Bearer tok-123\r\n")); + assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); + let error = synapse.post("txn-2", "x").unwrap_err(); + assert!(!format!("{error:#}").contains("tok-123")); + server.join().unwrap(); + assert!( + Synapse { + address, + room: "!r:e".into(), + token_file: directory.path().join("absent") + } + .post("t", "x") + .is_err() + ); + } + + #[test] + fn grammar_limits_are_exact_at_their_edges() { + let h = harness(); + let order = |operation: Operation, reason: &str| { + h.ask(&Request::Order(OrderRequest { + operation, + reason: reason.into(), + })) + }; + let packages = |names: Vec| Operation::PackageInstall { packages: names }; + let unit = |name: String| Operation::UnitAction { + verb: Verb::Start, + unit: name, + }; + // Package: two characters is the shortest name, 200 bytes the longest. + assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); + assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); + assert_eq!( + order(packages(vec!["a".repeat(201)]), "r"), + Response::Refused(Refusal::new("package", Code::TooLong)) + ); + assert_eq!( + order(packages(vec!["a".into()]), "r"), + Response::Refused(Refusal::new("package", Code::BadShape)) + ); + let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); + assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); + // Unit: 200 bytes with its suffix is the longest. + let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); + assert_eq!(longest.len(), UNIT_LIMIT); + assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); + let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); + assert_eq!( + order(unit(too_long), "r"), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + // Reason: 300 bytes is the longest, and it must be there. + assert!(matches!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), + Response::Accepted { .. } + )); + assert_eq!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), + Response::Refused(Refusal::new("reason", Code::TooLong)) + ); + assert_eq!( + order(unit("nginx.service".into()), ""), + Response::Refused(Refusal::new("reason", Code::Empty)) + ); + assert_eq!( + order(unit("nginx.service".into()), "caf\u{e9}"), + Response::Refused(Refusal::new("reason", Code::BadCharacter)) + ); + } + + #[test] + fn a_concluded_order_is_never_run_later() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + // The brake is released by someone else; the refused order stays refused. + h.brake.0.store(true, Ordering::SeqCst); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty()); + let ledger = h.ctx.store.ledger().unwrap(); + assert!(ledger.start_of(&id).is_none()); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + } + + #[test] + fn the_orders_cli_has_no_command_string() { + let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); + let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); + assert_eq!( + parsed.action, + OrderAction::Issue { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into() + }, + reason: "why".into() + } + ); + assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); + assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); + assert!(parse(&["show", "ord-x"]).is_ok()); + assert!(parse(&["list"]).is_ok()); + for refused in [ + vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], + vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], + vec!["run", "sh", "--reason", "r"], + vec!["exec", "--reason", "r"], + vec!["unit", "enable", "nginx.service", "--reason", "r"], + vec!["unit", "restart", "--reason", "r"], + vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], + vec!["unit", "restart", "nginx.service"], + vec!["unit", "restart", "nginx*", "--reason", "r"], + vec!["package-install", "--reason", "r"], + vec!["package-install", "-y", "x", "--reason", "r"], + vec!["deploy", "--reason", "r"], + vec!["deploy", "a", "b", "--reason", "r"], + vec!["list", "extra"], + vec!["show"], + vec![], + ] { + assert!(parse(&refused).is_err(), "{refused:?} parsed"); + } + } +} From 1a3634e04696bfaee40b8c4e4c317c2202e6357f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:02 +0100 Subject: [PATCH 175/196] Document idunn order and add the idunn-orders unit Co-Authored-By: Claude Sonnet 5.5 --- README.md | 29 ++++++++++++++++++++++ deploy/idunn-orders.service | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 deploy/idunn-orders.service diff --git a/README.md b/README.md index a5c48bc..6af4527 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,35 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. +### Steward orders + +A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), +takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. +It has its own store, `orders.cc`, and its own binding, `steward.toml`; the +survival daemon takes no order. There is still no command string: + +```bash +idunn order package-install [=version]... --reason TEXT # apt-get update, then install +idunn order unit --reason TEXT +idunn order deploy --reason TEXT # the same commit as `idunn up` +idunn order show # status and output tails +idunn order list +``` + +A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' +own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and +read from no configuration. A name, an alias that resolves to one of them, or a +`stop` or `restart` whose dependents systemd would stop with it is refused, +recorded and posted. Slices, scopes and globs are not units an order can name. + +An order runs once, after the steward's uid is checked and the instance's +brake check (`mind-control --instance check`) exits 0; an order +that had started when the service died is recorded as lost and never run +again. No order releases a brake. Acceptance and result are posted to +`#operator` as a retried projection of `orders.cc`: a chat outage delays the +posts and not the orders. A refusal names the field and a code, never the value +it refused. + ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service new file mode 100644 index 0000000..cd3c46b --- /dev/null +++ b/deploy/idunn-orders.service @@ -0,0 +1,48 @@ +[Unit] +Description=Idunn steward orders: typed package, unit and deploy orders from the steward +# Deliberately no After= or Requires= on any managed target, Synapse or +# idunn-yggdrasil: orders are how an operator repairs those, so none of them +# may be a condition of this unit starting. Posts are a retried projection and +# wait for Synapse on their own. + +[Service] +Type=simple + +# Root, from the same binary as idunn-yggdrasil, with its own store and socket. +# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: +# no order code runs in `idunn serve`. +# +# The unit name matches idunn-*, which is in the always-refused set, so no +# order can stop or restart this service. +User=root +# The socket is created root:idunn-steward 0660 (UMask below). The steward user +# is the only peer the service answers: it checks SO_PEERCRED on every +# connection, so the group is a second door held shut, not the lock. +Group=idunn-steward +ExecStart=/usr/local/bin/idunn orders serve \ + --store /var/lib/gamecult/idunn-orders \ + --state-store /var/lib/gamecult/idunn/control.cc \ + --steward-binding /etc/gamecult/idunn/steward.toml \ + --socket /run/idunn-orders/steward.sock +Restart=always +RestartSec=5 +TimeoutStopSec=20 +UMask=0117 + +RuntimeDirectory=idunn-orders +RuntimeDirectoryMode=0750 + +# The Synapse access token of @idunn, copied from a root-only file, and read +# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. +LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token + +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +# Its own order store, and the control store a deploy order commits a command +# to. Transient units started with systemd-run run in the manager's namespace, +# so packages and units are not limited by this list. +ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn + +[Install] +WantedBy=multi-user.target From 7d5cd85728b0d9e7dad801f48b15b358a0bd6bc3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:12 +0100 Subject: [PATCH 176/196] Reword a test comment so the no-brake-release tripwire reads clean Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 2aded30..31a2d06 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2880,7 +2880,7 @@ mod tests { let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); execute(&h.ctx, &id).unwrap(); assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - // The brake is released by someone else; the refused order stays refused. + // Someone else clears the brake; the refused order stays refused. h.brake.0.store(true, Ordering::SeqCst); execute(&h.ctx, &id).unwrap(); assert!(h.runs().is_empty()); From ec6222e2a114d3ebf5287bb326d0b5edc5ea907f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:49:26 +0100 Subject: [PATCH 177/196] Steward unit orders: refuse by the systemd transaction, install exactly the named packages An order is refused when the closure systemd would build for it reaches an always-refused unit: a start pulls Requires/Requisite/BindsTo/Wants/Upholds and stops Conflicts/ConflictedBy, a stop takes down RequiredBy/BoundBy/RequisiteOf/ ConsistsOf/PropagatesStopTo/StopPropagatedFrom, a restart does both. A refused unit already running is left alone by a start. Names are unquoted from systemctl show; a unit systemd cannot show fails the order. Package installs run apt-get --no-remove with ?exact-name() per package and a final dpkg -s step that fails unless every named package is installed. Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 676 +++++++++++++++++++++++++++++++++--------- 1 file changed, 544 insertions(+), 132 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 31a2d06..ce95337 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -213,11 +213,6 @@ impl Verb { .into_iter() .find(|verb| verb.as_str() == word) } - - /// Verbs systemd propagates down the dependency graph. - fn propagates(self) -> bool { - matches!(self, Verb::Stop | Verb::Restart) - } } /// The closed set of things an order can ask for. @@ -259,8 +254,21 @@ impl Operation { let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; match self { Operation::PackageInstall { packages } => { - let mut install: Vec = words(&["apt-get", "install", "-y", "-q", "--"]); - install.extend(packages.iter().cloned()); + // Never a removal, never a pattern: each name selects exactly + // itself, and apt aborts rather than displace an installed package. + let mut install: Vec = + words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); + let mut present: Vec = words(&["dpkg", "-s", "--"]); + for package in packages { + let (name, version) = package.split_once('=').unwrap_or((package, "")); + install.push(match version { + "" => format!("?exact-name({name})"), + version => format!("?exact-name({name})={version}"), + }); + present.push(name.to_owned()); + } + // An exact name that matches nothing installs nothing and exits 0, + // so the last step fails unless every named package is installed. vec![ Step { argv: words(&["apt-get", "update", "-q"]), @@ -270,6 +278,10 @@ impl Operation { argv: install, environment: apt_environment, }, + Step { + argv: present, + environment: Vec::new(), + }, ] } Operation::UnitAction { verb, unit } => vec![Step { @@ -595,14 +607,45 @@ impl OrderStore { // ------------------------------------------------------------------ ports -- +/// Which way systemd moves along a dependency edge when an order is given. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Edge { + /// Units a start brings up with this one. + Pulls, + /// Units a start stops: the unit's `Conflicts=` and the units that conflict with it. + Conflicts, + /// Units a stop takes down with this one. + Stops, +} + +/// The `systemctl show` properties the closure reads, and the way each one +/// moves. This is the only list: the request and the parse both read it. +const EDGES: [(&str, Edge); 13] = [ + ("Requires", Edge::Pulls), + ("Requisite", Edge::Pulls), + ("BindsTo", Edge::Pulls), + ("Wants", Edge::Pulls), + ("Upholds", Edge::Pulls), + ("Conflicts", Edge::Conflicts), + ("ConflictedBy", Edge::Conflicts), + ("RequiredBy", Edge::Stops), + ("BoundBy", Edge::Stops), + ("RequisiteOf", Edge::Stops), + ("ConsistsOf", Edge::Stops), + ("PropagatesStopTo", Edge::Stops), + ("StopPropagatedFrom", Edge::Stops), +]; + /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { pub(crate) id: String, pub(crate) names: Vec, - pub(crate) required_by: Vec, - pub(crate) bound_by: Vec, - pub(crate) consists_of: Vec, + pub(crate) not_found: bool, + pub(crate) running: bool, + pub(crate) pulls: Vec, + pub(crate) conflicts: Vec, + pub(crate) stops: Vec, } impl UnitFacts { @@ -610,12 +653,20 @@ impl UnitFacts { std::iter::once(&self.id).chain(self.names.iter()) } - /// The units systemd stops with this one. - fn dependents(&self) -> impl Iterator { - self.required_by - .iter() - .chain(self.bound_by.iter()) - .chain(self.consists_of.iter()) + fn along(&self, edge: Edge) -> &[String] { + match edge { + Edge::Pulls => &self.pulls, + Edge::Conflicts => &self.conflicts, + Edge::Stops => &self.stops, + } + } + + fn along_mut(&mut self, edge: Edge) -> &mut Vec { + match edge { + Edge::Pulls => &mut self.pulls, + Edge::Conflicts => &mut self.conflicts, + Edge::Stops => &mut self.stops, + } } } @@ -673,40 +724,95 @@ fn refused(unit: &str, via: Via) -> Refused { } } +/// The units systemd would touch for one order, read from `systemctl show`. +struct Transaction<'a> { + systemd: &'a dyn Systemd, + facts: BTreeMap, +} + +impl Transaction<'_> { + fn facts(&mut self, unit: &str) -> Result<&UnitFacts> { + if !self.facts.contains_key(unit) { + ensure!( + self.facts.len() < CLOSURE_LIMIT, + "the units an order would reach are too many to check" + ); + let facts = self.systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + self.facts.insert(unit.to_owned(), facts); + } + Ok(&self.facts[unit]) + } + + /// Every unit reached from `seeds` along `edge`, seeds included. + fn reach(&mut self, seeds: &BTreeSet, edge: Edge) -> Result> { + let mut reached = BTreeSet::new(); + let mut queue: VecDeque = seeds.iter().cloned().collect(); + while let Some(next) = queue.pop_front() { + if !reached.insert(next.clone()) { + continue; + } + queue.extend(self.facts(&next)?.along(edge).iter().cloned()); + } + Ok(reached) + } + + /// Every unit a start of `started` stops through `Conflicts=`, and what + /// that stop takes down with it. + fn conflicted(&mut self, started: &BTreeSet) -> Result> { + let mut seeds = BTreeSet::new(); + for unit in started { + seeds.extend(self.facts(unit)?.along(Edge::Conflicts).iter().cloned()); + } + self.reach(&seeds, Edge::Stops) + } +} + /// Would `verb` on `unit` reach an always-refused unit: by its name, by an -/// alias it resolves from, or, for the verbs systemd propagates, by the units -/// systemd stops with it. Resolution reads systemd before anything runs. +/// alias it resolves from, or through the transaction systemd builds for it. +/// A start pulls in what the unit requires or wants and stops what conflicts; +/// a stop takes down what depends on the unit; a restart does both. A unit +/// the transaction would stop is refused when it is refused; a unit it would +/// start is refused when it is refused and not running. Every unit in the +/// closure must be one systemd can show. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); } - let facts = systemd.show(unit)?; - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + let mut transaction = Transaction { + systemd, + facts: BTreeMap::new(), + }; + if let Some(name) = transaction + .facts(unit)? + .identities() + .find(|name| is_refused_name(name)) + { return Ok(Some(refused(name, Via::Alias))); } - if !verb.propagates() { - return Ok(None); + let root = BTreeSet::from([unit.to_owned()]); + let mut stopped = BTreeSet::new(); + let mut started = BTreeSet::new(); + match verb { + Verb::Reload => {} + Verb::Start => started = transaction.reach(&root, Edge::Pulls)?, + Verb::Stop => stopped = transaction.reach(&root, Edge::Stops)?, + Verb::Restart => { + stopped = transaction.reach(&root, Edge::Stops)?; + started = transaction.reach(&stopped, Edge::Pulls)?; + } } - let mut seen: BTreeSet = facts.identities().cloned().collect(); - seen.insert(unit.to_owned()); - let mut queue: VecDeque = facts.dependents().cloned().collect(); - while let Some(next) = queue.pop_front() { - if is_refused_name(&next) { - return Ok(Some(refused(&next, Via::Propagation))); + stopped.extend(transaction.conflicted(&started)?); + for (units, only_if_stopped) in [(&stopped, false), (&started, true)] { + for member in units { + let facts = transaction.facts(member)?; + if only_if_stopped && facts.running { + continue; + } + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Propagation))); + } } - if !seen.insert(next.clone()) { - continue; - } - ensure!( - seen.len() <= CLOSURE_LIMIT, - "the units a stop would reach are too many to check" - ); - let facts = systemd.show(&next)?; - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { - return Ok(Some(refused(name, Via::Propagation))); - } - seen.extend(facts.identities().cloned()); - queue.extend(facts.dependents().cloned()); } Ok(None) } @@ -1148,8 +1254,39 @@ pub(crate) fn handle( /// `systemctl` and `systemd-run`, found by name. pub(crate) struct HostSystemd; +/// The names in one `systemctl show` list value. systemd quotes a name that +/// holds a backslash and doubles the backslash inside the quotes. fn parse_names(value: &str) -> Vec { - value.split_whitespace().map(str::to_owned).collect() + let mut names = Vec::new(); + let mut current = String::new(); + let (mut quoted, mut open, mut escaped) = (false, false, false); + for character in value.chars() { + match character { + _ if escaped => { + current.push(character); + escaped = false; + } + '\\' if quoted => escaped = true, + '"' => { + quoted = !quoted; + open = true; + } + _ if character.is_whitespace() && !quoted => { + if open { + names.push(std::mem::take(&mut current)); + open = false; + } + } + _ => { + current.push(character); + open = true; + } + } + } + if open { + names.push(current); + } + names } fn parse_show(output: &str) -> Result { @@ -1161,10 +1298,15 @@ fn parse_show(output: &str) -> Result { match key { "Id" => facts.id = value.to_owned(), "Names" => facts.names = parse_names(value), - "RequiredBy" => facts.required_by = parse_names(value), - "BoundBy" => facts.bound_by = parse_names(value), - "ConsistsOf" => facts.consists_of = parse_names(value), - _ => {} + "LoadState" => facts.not_found = value == "not-found", + "ActiveState" => { + facts.running = matches!(value, "active" | "activating" | "reloading"); + } + _ => { + if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { + facts.along_mut(*edge).extend(parse_names(value)); + } + } } } ensure!(!facts.id.is_empty(), "systemctl named no unit"); @@ -1173,14 +1315,15 @@ fn parse_show(output: &str) -> Result { impl Systemd for HostSystemd { fn show(&self, unit: &str) -> Result { + let properties = ["Id", "Names", "LoadState", "ActiveState"] + .into_iter() + .chain(EDGES.iter().map(|(property, _)| *property)) + .collect::>() + .join(","); let output = Command::new("systemctl") - .args([ - "show", - "--no-pager", - "--property=Id,Names,RequiredBy,BoundBy,ConsistsOf", - "--", - unit, - ]) + .args(["show", "--no-pager"]) + .arg(format!("--property={properties}")) + .args(["--", unit]) .stdin(Stdio::null()) .output() .context("running systemctl show")?; @@ -1750,8 +1893,23 @@ mod tests { const STEWARD: u32 = 1000; const CANARY: &str = "CANARY-7f3a91"; + /// What `systemctl show` prints for a loaded, active unit with no edges, + /// plus the given properties (a given key replaces its default). + fn show_text(id: &str, properties: &[(&str, &str)]) -> String { + let mut text = format!("Id={id}\n"); + for (key, default) in [("Names", id), ("LoadState", "loaded"), ("ActiveState", "active")] { + if !properties.iter().any(|(given, _)| *given == key) { + text += &format!("{key}={default}\n"); + } + } + for (key, value) in properties { + text += &format!("{key}={value}\n"); + } + text + } + struct FakeSystemd { - units: Mutex>, + units: Mutex>, runs: Mutex>, report: Mutex, } @@ -1759,11 +1917,7 @@ mod tests { impl Systemd for FakeSystemd { fn show(&self, unit: &str) -> Result { let known = self.units.lock().unwrap().get(unit).cloned(); - Ok(known.unwrap_or_else(|| UnitFacts { - id: unit.into(), - names: vec![unit.into()], - ..UnitFacts::default() - })) + parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) } fn invocation(&self, _unit: &str) -> Option { @@ -1907,8 +2061,13 @@ mod tests { self.systemd.runs.lock().unwrap().clone() } - fn describe_unit(&self, name: &str, facts: UnitFacts) { - self.systemd.units.lock().unwrap().insert(name.into(), facts); + /// Tell the fake `systemctl show` what it prints for `name`. + fn describe_unit(&self, name: &str, text: String) { + self.systemd.units.lock().unwrap().insert(name.into(), text); + } + + fn describe(&self, name: &str, properties: &[(&str, &str)]) { + self.describe_unit(name, show_text(name, properties)); } } @@ -2065,19 +2224,52 @@ mod tests { execute(&h.ctx, &id).unwrap(); assert_eq!(h.outcome(&id), Outcome::Ran); let runs = h.runs(); - assert_eq!(runs.len(), 2); + assert_eq!(runs.len(), 3); assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); - let mut install = vec!["apt-get", "install", "-y", "-q", "--"]; - install.extend(good); + // Each name selects exactly itself and apt may remove nothing. + let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; + install.extend(["--no-remove".to_owned(), "--".into()]); + install.extend(good.iter().map(|spec| match spec.split_once('=') { + Some((name, version)) => format!("?exact-name({name})={version}"), + None => format!("?exact-name({spec})"), + })); assert_eq!(runs[1].1.argv, install); + assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); + assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); + // The last step fails unless every named package is installed. + let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; + present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); + assert_eq!(runs[2].1.argv, present); for (unit, step) in &runs { assert!(unit.starts_with("idunn-order-")); + let apt = step.argv[0] == "apt-get"; assert_eq!( step.environment, - [("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + if apt { + vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + } else { + Vec::new() + } ); } assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); + + // A package whose name apt would read as a regex or a conflict is + // exactly itself and never a removal. + for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { + let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); + assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); + } + // A step that failed ends the order: the absent package is a failure. + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(1), + ..RunReport::default() + }; + let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); + execute(&h.ctx, &absent).unwrap(); + assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); + assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); } #[test] @@ -2121,74 +2313,28 @@ mod tests { let h = harness(); h.describe_unit( "backup-name.service", - UnitFacts { - id: "idunn-yggdrasil.service".into(), - names: vec!["idunn-yggdrasil.service".into(), "backup-name.service".into()], - ..UnitFacts::default() - }, + show_text( + "idunn-yggdrasil.service", + &[("Names", "idunn-yggdrasil.service backup-name.service")], + ), ); - h.describe_unit( + h.describe( "quiet-alias.service", - UnitFacts { - id: "quiet-alias.service".into(), - names: vec!["quiet-alias.service".into(), "mind-persona@void.service".into()], - ..UnitFacts::default() - }, + &[("Names", "quiet-alias.service mind-persona@void.service")], ); // fuse <- wrapper <- (bound by) mind-self@eureka - h.describe_unit( - "fuse.service", - UnitFacts { - id: "fuse.service".into(), - names: vec!["fuse.service".into()], - required_by: vec!["wrapper.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "wrapper.service", - UnitFacts { - id: "wrapper.service".into(), - names: vec!["wrapper.service".into()], - bound_by: vec!["mind-self@eureka.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "stack.target", - UnitFacts { - id: "stack.target".into(), - names: vec!["stack.target".into()], - consists_of: vec!["mind-persona@void.service".into()], - ..UnitFacts::default() - }, - ); + h.describe("fuse.service", &[("RequiredBy", "wrapper.service")]); + h.describe("wrapper.service", &[("BoundBy", "mind-self@eureka.service")]); + h.describe("stack.target", &[("ConsistsOf", "mind-persona@void.service")]); // A cycle that reaches an alias of a refused unit on its second hop. - h.describe_unit( - "loop-a.service", - UnitFacts { - id: "loop-a.service".into(), - names: vec!["loop-a.service".into()], - required_by: vec!["loop-b.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "loop-b.service", - UnitFacts { - id: "loop-b.service".into(), - names: vec!["loop-b.service".into()], - required_by: vec!["loop-a.service".into(), "hidden.service".into()], - ..UnitFacts::default() - }, - ); + h.describe("loop-a.service", &[("RequiredBy", "loop-b.service")]); + h.describe("loop-b.service", &[("RequiredBy", "loop-a.service hidden.service")]); h.describe_unit( "hidden.service", - UnitFacts { - id: "idunn-orders.service".into(), - names: vec!["idunn-orders.service".into(), "hidden.service".into()], - ..UnitFacts::default() - }, + show_text( + "idunn-orders.service", + &[("Names", "idunn-orders.service hidden.service")], + ), ); let direct = [ @@ -2260,7 +2406,7 @@ mod tests { } } assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); - // start and reload propagate nothing, so the same units are not refused. + // start and reload take nothing down, so the same units are not refused. for verb in [Verb::Start, Verb::Reload] { let id = h.accept(h.issue_unit(verb, "fuse.service")); execute(&h.ctx, &id).unwrap(); @@ -2284,6 +2430,248 @@ mod tests { assert!(ledger.starts.len() == 2); } + /// Yggdrasil's `systemctl show` shapes (systemd 257, 2026-11-01), cut down + /// to the units an order can reach: seven refused units all conflict with + /// shutdown.target, and the shutdown services require it. + fn yggdrasil(h: &Harness, idunn_running: bool) { + let state = if idunn_running { "active" } else { "inactive" }; + let refused = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-e0ada640.service", + "mind-self@eureka.service", + "mind-persona@void.service", + ]; + h.describe( + "shutdown.target", + &[ + ("ActiveState", "inactive"), + ( + "RequiredBy", + "systemd-soft-reboot.service systemd-reboot.service systemd-poweroff.service", + ), + ( + "ConflictedBy", + "nginx.service idunn-yggdrasil.service idunn-orders.service \ + idunn-vili-e0ada640.service mind-self@eureka.service mind-persona@void.service", + ), + ], + ); + for refused in refused { + h.describe( + refused, + &[ + ("Requires", "sysinit.target system.slice -.mount"), + ("Wants", "tmp.mount network-online.target"), + ("Conflicts", "shutdown.target"), + ("ActiveState", if refused == "idunn-yggdrasil.service" { state } else { "inactive" }), + ], + ); + } + h.describe("nginx.service", &[("Wants", "network-online.target"), ("Conflicts", "shutdown.target")]); + for (service, target) in [ + ("systemd-poweroff.service", "poweroff.target"), + ("systemd-reboot.service", "reboot.target"), + ("systemd-soft-reboot.service", "soft-reboot.target"), + ] { + h.describe( + service, + &[ + ("Requires", "shutdown.target final.target system.slice umount.target"), + ("RequiredBy", target), + ("ActiveState", "inactive"), + ], + ); + } + for (target, service) in [ + ("poweroff.target", "systemd-poweroff.service"), + ("reboot.target", "systemd-reboot.service"), + ("soft-reboot.target", "systemd-soft-reboot.service"), + ] { + h.describe(target, &[("Requires", service), ("ActiveState", "inactive")]); + } + h.describe( + "umount.target", + &[ + ("ActiveState", "inactive"), + ("RequiredBy", "systemd-poweroff.service systemd-reboot.service"), + ("ConflictedBy", "boot-efi.mount boot.mount \"blockdev@dev-vda2.target\""), + ], + ); + h.describe( + "multi-user.target", + &[ + ("Requires", "basic.target"), + ("Wants", "cron.service ssh.service idunn-yggdrasil.service"), + ("Conflicts", "shutdown.target rescue.target rescue.service"), + ], + ); + h.describe( + "voidbot.service", + &[ + ("Requires", "docker.service -.mount sysinit.target system.slice"), + ("Wants", "idunn-yggdrasil.service network-online.target"), + ("Conflicts", "shutdown.target"), + ], + ); + } + + fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + h.outcome(&id) + } + + fn hit(outcome: &Outcome) -> Option<&str> { + match outcome { + Outcome::Refused(Refused::AlwaysRefused { unit, via: Via::Propagation }) => Some(unit.as_str()), + _ => None, + } + } + + #[test] + fn a_transaction_that_reaches_a_refused_unit_is_refused() { + let h = harness(); + yggdrasil(&h, true); + // The targets and services that power the host down reach every refused + // unit through Conflicts=, however many requirements away. + for (verb, unit) in [ + (Verb::Start, "shutdown.target"), + (Verb::Start, "poweroff.target"), + (Verb::Restart, "poweroff.target"), + (Verb::Start, "reboot.target"), + (Verb::Restart, "reboot.target"), + (Verb::Start, "soft-reboot.target"), + (Verb::Start, "systemd-soft-reboot.service"), + (Verb::Restart, "systemd-poweroff.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert!(hit(&outcome).is_some(), "{verb:?} {unit} was {outcome:?}"); + } + assert!(h.runs().is_empty(), "no refused order ran anything"); + // A stop of shutdown.target takes down what requires it and runs no start. + let outcome = outcome_of_unit(&h, Verb::Stop, "shutdown.target"); + assert_eq!(outcome, Outcome::Ran); + + // A start that wants a refused unit that is stopped would start it. + h.describe("idunn-yggdrasil.service", &[("ActiveState", "inactive"), ("Conflicts", "shutdown.target")]); + for (verb, unit) in [ + (Verb::Start, "multi-user.target"), + (Verb::Restart, "multi-user.target"), + (Verb::Start, "voidbot.service"), + (Verb::Restart, "voidbot.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert_eq!(hit(&outcome), Some("idunn-yggdrasil.service"), "{verb:?} {unit}"); + } + // The same orders start nothing while the refused unit runs. + h.describe("idunn-yggdrasil.service", &[("ActiveState", "active"), ("Conflicts", "shutdown.target")]); + for (verb, unit) in [ + (Verb::Start, "multi-user.target"), + (Verb::Restart, "voidbot.service"), + (Verb::Start, "voidbot.service"), + ] { + assert_eq!(outcome_of_unit(&h, verb, unit), Outcome::Ran, "{verb:?} {unit}"); + } + // A unit that wants only what is not refused runs, and so does a target + // that unmounts nothing a refused unit requires. + for verb in [Verb::Start, Verb::Restart, Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "nginx.service"), Outcome::Ran); + } + assert_eq!(outcome_of_unit(&h, Verb::Start, "umount.target"), Outcome::Ran); + } + + #[test] + fn the_closure_follows_every_edge_and_every_hop() { + let h = harness(); + // Start: three hops of Wants/Requires/Requisite/BindsTo/Upholds to a + // stopped refused unit. + h.describe("hop-1.service", &[("Wants", "hop-2.service")]); + h.describe("hop-2.service", &[("Requires", "hop-3.service")]); + h.describe("hop-3.service", &[("Requisite", "hop-4.service")]); + h.describe("hop-4.service", &[("BindsTo", "hop-5.service")]); + h.describe("hop-5.service", &[("Upholds", "idunn-orders.service")]); + h.describe("idunn-orders.service", &[("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "hop-1.service"); + assert_eq!(hit(&outcome), Some("idunn-orders.service")); + // Conflicts of a pulled-in unit stop it and what requires it: the + // conflict, then two hops of reverse requirement, to a refused unit. + h.describe("rival.service", &[("Conflicts", "victim.service")]); + h.describe("victim.service", &[("RequiredBy", "mid.service")]); + h.describe("mid.service", &[("BoundBy", "mind-self@eureka.service")]); + h.describe("pulls-rival.service", &[("Requires", "rival.service")]); + for unit in ["rival.service", "pulls-rival.service"] { + let outcome = outcome_of_unit(&h, Verb::Start, unit); + assert_eq!(hit(&outcome), Some("mind-self@eureka.service"), "{unit}"); + } + // The reverse Conflicts edge: a unit conflicted by a refused one. + h.describe("incumbent.service", &[("ConflictedBy", "mind-persona@void.service")]); + let outcome = outcome_of_unit(&h, Verb::Start, "incumbent.service"); + assert_eq!(hit(&outcome), Some("mind-persona@void.service")); + // Stop and restart: every stop edge, one at a time. + for (property, reached) in [ + ("RequiredBy", "idunn-vili-x.service"), + ("BoundBy", "idunn-vili-x.service"), + ("RequisiteOf", "idunn-vili-x.service"), + ("ConsistsOf", "idunn-vili-x.service"), + ("PropagatesStopTo", "mind-persona@void.service"), + ("StopPropagatedFrom", "mind-persona@void.service"), + ] { + let name = format!("edge-{property}.service"); + h.describe(&name, &[(property, reached)]); + for verb in [Verb::Stop, Verb::Restart] { + let outcome = outcome_of_unit(&h, verb, &name); + assert_eq!(hit(&outcome), Some(reached), "{verb:?} {property}"); + } + } + // A restart starts what its stopped dependents pull in. + h.describe("base.service", &[("RequiredBy", "dependent.service")]); + h.describe("dependent.service", &[("Wants", "idunn-orders.service")]); + let outcome = outcome_of_unit(&h, Verb::Restart, "base.service"); + assert_eq!(hit(&outcome), Some("idunn-orders.service")); + assert_eq!(outcome_of_unit(&h, Verb::Stop, "base.service"), Outcome::Ran); + assert_eq!(outcome_of_unit(&h, Verb::Reload, "base.service"), Outcome::Ran); + } + + #[test] + fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { + let h = harness(); + // systemctl show quotes a name with a backslash and doubles the backslash. + h.describe( + "huginn.service", + &[("RequiredBy", "\"mind-self@foo\\\\x2dbar.service\"")], + ); + let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); + assert_eq!(hit(&outcome), Some("mind-self@foo\\x2dbar.service")); + h.describe("quiet.service", &[("Wants", "\"idunn-a\\\\x2db.service\" other.service")]); + h.describe("idunn-a\\x2db.service", &[("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); + assert_eq!(hit(&outcome), Some("idunn-a\\x2db.service")); + assert!(h.runs().is_empty()); + } + + #[test] + fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { + let h = harness(); + h.describe("ghost.service", &[("LoadState", "not-found")]); + h.describe("pulls-ghost.service", &[("Wants", "ghost.service")]); + h.describe("needed-by-ghost.service", &[("RequiredBy", "ghost.service")]); + for (verb, unit) in [ + (Verb::Start, "ghost.service"), + (Verb::Start, "pulls-ghost.service"), + (Verb::Stop, "needed-by-ghost.service"), + (Verb::Restart, "needed-by-ghost.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert!( + matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), + "{verb:?} {unit}" + ); + } + assert!(h.runs().is_empty()); + } + #[test] fn globs_slices_and_scopes_are_not_units() { let h = harness(); @@ -2715,16 +3103,40 @@ mod tests { #[test] fn systemctl_show_output_is_read_into_facts() { let facts = parse_show( - "Id=a.service\nNames=a.service b.service\nRequiredBy=c.service\n\ - BoundBy=\nConsistsOf=d.target e.service\nOther=x\n", + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=inactive\n\ + Requires=r.service \"q\\\\x2dr.slice\" -.mount\nRequisite=t.service\nWants=w.service\n\ + BindsTo=b.service\nUpholds=u.service\nConflicts=x.target\nConflictedBy=y.service\n\ + RequiredBy=c.service\nBoundBy=\nRequisiteOf=ro.service\nConsistsOf=d.target e.service\n\ + PropagatesStopTo=p.service\nStopPropagatedFrom=f.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); assert_eq!(facts.names, ["a.service", "b.service"]); - assert_eq!(facts.required_by, ["c.service"]); - assert!(facts.bound_by.is_empty()); - assert_eq!(facts.consists_of, ["d.target", "e.service"]); + assert!(!facts.not_found && !facts.running); + // A quoted name holds one backslash, not two and not a quote mark. + assert_eq!( + facts.pulls, + ["r.service", "q\\x2dr.slice", "-.mount", "t.service", "w.service", "b.service", "u.service"] + ); + assert_eq!(facts.conflicts, ["x.target", "y.service"]); + assert_eq!( + facts.stops, + ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] + ); assert!(parse_show("Names=a.service\n").is_err()); + let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); + assert!(gone.not_found); + for (state, running) in [ + ("active", true), + ("activating", true), + ("reloading", true), + ("inactive", false), + ("deactivating", false), + ("failed", false), + ] { + let facts = parse_show(&format!("Id=a.service\nActiveState={state}\n")).unwrap(); + assert_eq!(facts.running, running, "{state}"); + } } #[test] From cea17c06a79f0e02dc88990805d1b4be10c31798 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:49:36 +0100 Subject: [PATCH 178/196] README: a unit order is refused by the systemd transaction; a package order installs only what it names Co-Authored-By: Claude Sonnet 5.5 --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6af4527..2bf6675 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ It has its own store, `orders.cc`, and its own binding, `steward.toml`; the survival daemon takes no order. There is still no command string: ```bash -idunn order package-install [=version]... --reason TEXT # apt-get update, then install +idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them idunn order unit --reason TEXT idunn order deploy --reason TEXT # the same commit as `idunn up` idunn order show # status and output tails @@ -75,9 +75,20 @@ idunn order list A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and -read from no configuration. A name, an alias that resolves to one of them, or a -`stop` or `restart` whose dependents systemd would stop with it is refused, -recorded and posted. Slices, scopes and globs are not units an order can name. +read from no configuration. An order is refused, recorded and posted when its +name, an alias it resolves from, or the transaction systemd would build for it +reaches one of them: a start pulls in what the unit requires or wants (a +refused unit that is already running is left alone) and stops what conflicts +with it, a stop takes down what depends on it, and a restart does both. The +transaction is read from `systemctl show`, so `start poweroff.target` is refused +because every refused unit conflicts with `shutdown.target`, not because +targets are banned. A unit `systemctl show` cannot resolve fails the order. +Slices, scopes and globs are not units an order can name. + +A package order installs the packages it names and nothing else: apt runs with +`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` +match themselves, and a last step fails the order unless every named package +is installed. An order runs once, after the steward's uid is checked and the instance's brake check (`mind-control --instance check`) exits 0; an order From be3b95bc2161239bea16d414d4b9511120ce6d9f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:50:20 +0100 Subject: [PATCH 179/196] Import BTreeMap for the transaction facts cache Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index ce95337..127e0e1 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,7 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::{BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; From 27ca875194c0126a09fc0e12d611e36a102989d4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:53:41 +0100 Subject: [PATCH 180/196] Steward orders tests: clear four clippy hits in the module Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 127e0e1..fdcaafa 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2836,11 +2836,11 @@ mod tests { .unwrap(); let pending = recover(&h.ctx).unwrap(); assert_eq!(h.outcome(&id), Outcome::Lost); - assert_eq!(pending, [waiting.clone()], "an order that never started waits"); + assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); execute(&h.ctx, &id).unwrap(); assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); // Recovery is idempotent, and a finished order is not run again. - assert_eq!(recover(&h.ctx).unwrap(), [waiting.clone()]); + assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); execute(&h.ctx, &waiting).unwrap(); execute(&h.ctx, &waiting).unwrap(); assert_eq!(h.runs().len(), 1); @@ -2995,7 +2995,7 @@ mod tests { format!("{CANARY}*.service"), format!("{CANARY}/../idunn-orders.service"), format!("idunn-{CANARY};x.service"), - format!("{CANARY}"), + CANARY.to_string(), ] { note(h.issue_unit(Verb::Restart, &unit)); } @@ -3162,7 +3162,7 @@ mod tests { 0o600 ); let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); - SingleFileMessagePackBackingStore::new(&h.ctx.store.path()) + SingleFileMessagePackBackingStore::new(h.ctx.store.path()) .compare_exchange( &[CultCacheExpectedEnvelope { r#type: "idunn.other".into(), From ca439a29ff394f51511eef095a1da234d06a6b62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 04:17:12 +0100 Subject: [PATCH 181/196] Pin the systemctl show property list and unquoted backslashes Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index fdcaafa..44f28db 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -1289,6 +1289,15 @@ fn parse_names(value: &str) -> Vec { names } +/// The properties one `systemctl show` is asked for: identity, state, edges. +fn show_properties() -> String { + ["Id", "Names", "LoadState", "ActiveState"] + .into_iter() + .chain(EDGES.iter().map(|(property, _)| *property)) + .collect::>() + .join(",") +} + fn parse_show(output: &str) -> Result { let mut facts = UnitFacts::default(); for line in output.lines() { @@ -1315,14 +1324,9 @@ fn parse_show(output: &str) -> Result { impl Systemd for HostSystemd { fn show(&self, unit: &str) -> Result { - let properties = ["Id", "Names", "LoadState", "ActiveState"] - .into_iter() - .chain(EDGES.iter().map(|(property, _)| *property)) - .collect::>() - .join(","); let output = Command::new("systemctl") .args(["show", "--no-pager"]) - .arg(format!("--property={properties}")) + .arg(format!("--property={}", show_properties())) .args(["--", unit]) .stdin(Stdio::null()) .output() @@ -3124,6 +3128,17 @@ mod tests { ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] ); assert!(parse_show("Names=a.service\n").is_err()); + // The request asks for exactly the properties the parse reads. + assert_eq!( + show_properties(), + "Id,Names,LoadState,ActiveState,Requires,Requisite,BindsTo,Wants,Upholds,\ + Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ + PropagatesStopTo,StopPropagatedFrom" + ); + // A backslash outside quotes is a backslash; inside, it escapes one character. + assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); + assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); + assert!(parse_names(" ").is_empty()); let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); for (state, running) in [ From 56edad95c1b211e982b1032a09fb29b74fa28931 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:03:17 +0100 Subject: [PATCH 182/196] Steward orders: one steward gate for fresh and recovered orders; the orders unit leaks no group or umask; root reads; kernel-attested issuer; end-to-end serve test execute refuses, with a record and a post, an order whose principal or issuing uid is not the steward bound now, before grammar, scope and brake; the brake is asked for the bound principal only. idunn-orders.service drops Group= and UMask=: bind_socket gives the socket to the steward uid at 0600 and serve sets Idunn's own umask, so a deploy order writes control.cc 0640. Root may show and list, never order. The issuer unit is the first unit under system.slice, never a name the peer chose below it. Synapse waits for a reply within one deadline. due_posts takes the deploy-state reader so a finished deploy's post is tested. Tests: the gate both ways, root, per-order posts, deploy result, intake and Synapse timeouts, parse_serve, user_uid, socket and store modes, and the whole service as root with a fake systemctl, systemd-run and homeserver. Co-Authored-By: Claude Sonnet 5.5 --- deploy/idunn-orders.service | 11 +- src/steward_orders.rs | 657 ++++++++++++++++++++++++++++++++++-- 2 files changed, 629 insertions(+), 39 deletions(-) diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service index cd3c46b..f2681c2 100644 --- a/deploy/idunn-orders.service +++ b/deploy/idunn-orders.service @@ -15,10 +15,10 @@ Type=simple # The unit name matches idunn-*, which is in the always-refused set, so no # order can stop or restart this service. User=root -# The socket is created root:idunn-steward 0660 (UMask below). The steward user -# is the only peer the service answers: it checks SO_PEERCRED on every -# connection, so the group is a second door held shut, not the lock. -Group=idunn-steward +# No Group= and no UMask=: the service gives its socket to the steward user +# (mode 0600) itself and sets Idunn's own umask for the files it writes, so +# nothing here reaches control.cc. The steward user is the only peer the +# service answers: it checks SO_PEERCRED on every connection. ExecStart=/usr/local/bin/idunn orders serve \ --store /var/lib/gamecult/idunn-orders \ --state-store /var/lib/gamecult/idunn/control.cc \ @@ -27,10 +27,9 @@ ExecStart=/usr/local/bin/idunn orders serve \ Restart=always RestartSec=5 TimeoutStopSec=20 -UMask=0117 RuntimeDirectory=idunn-orders -RuntimeDirectoryMode=0750 +RuntimeDirectoryMode=0755 # The Synapse access token of @idunn, copied from a root-only file, and read # through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 44f28db..e707307 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -381,6 +381,8 @@ pub(crate) enum Via { pub(crate) enum Refused { AlwaysRefused { unit: String, via: Via }, Braked, + /// The order's principal or issuing uid is not the steward bound now. + NotSteward, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -419,6 +421,9 @@ impl Outcome { } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Refused(Refused::NotSteward) => { + "refused: the issuer is not the current steward".into() + } Outcome::Lost => "lost: the orders service restarted mid-run".into(), Outcome::Submitted { command_id } => { format!("submitted deployment {}", escape(command_id.as_bytes())) @@ -865,6 +870,11 @@ pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { return Ok(()); } + // One gate for a fresh order and a recovered one: who may order is what the + // binding says now, not what it said when the order was accepted. + if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { + return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); + } if order.operation.validate().is_err() { return conclude( ctx, @@ -895,7 +905,7 @@ pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { } } } - if !ctx.brake.clear(&order.principal) { + if !ctx.brake.clear(&ctx.principal) { return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); } let uuid = order_uuid(order_id); @@ -1010,7 +1020,12 @@ fn deploy_finished(state: &str) -> bool { !matches!(state, "queued" | "running") } -fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { +/// What is owed to the room. `deploy_state` is a deployment command's derived +/// state and detail, read from the control store by the caller. +fn due_posts( + ledger: &Ledger, + deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, +) -> Vec { let mut due = Vec::new(); for order in &ledger.orders { let id = &order.order_id; @@ -1029,7 +1044,7 @@ fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { }; let mut line = record.outcome.describe(); if let Outcome::Submitted { command_id } = &record.outcome { - match command_state(&ctx.state_store, command_id) { + match deploy_state(command_id) { Ok((state, detail)) if deploy_finished(state) => { line = format!("{line}: {state} {}", escape(detail.as_bytes())); } @@ -1053,7 +1068,8 @@ fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { pub(crate) fn post_pass(ctx: &Ctx) -> Result { let ledger = ctx.store.ledger()?; let mut sent = 0; - for due in due_posts(ctx, &ledger) { + let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); + for due in due_posts(&ledger, &deploy_state) { let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); let event_id = ctx.homeserver.post(&transaction, &due.text)?; ctx.store.put( @@ -1173,7 +1189,9 @@ pub(crate) fn handle( stream.set_read_timeout(Some(Duration::from_secs(5)))?; stream.set_write_timeout(Some(Duration::from_secs(5)))?; let peer = ctx.peers.peer(stream)?; - if peer.uid != ctx.steward_uid { + // The steward may order and read; root, the operator, may only read. + let steward = peer.uid == ctx.steward_uid; + if !steward && peer.uid != 0 { eprintln!("steward socket: refused a peer that is not the steward"); return refuse(stream, "peer", Code::Unauthorized); } @@ -1185,6 +1203,10 @@ pub(crate) fn handle( return refuse(stream, "frame", Code::Malformed); }; match request { + Request::Order(_) if !steward => { + eprintln!("steward socket: refused an order from a peer that is not the steward"); + refuse(stream, "peer", Code::Unauthorized) + } Request::Order(request) => { let checked = validate_reason(&request.reason).and_then(|()| request.operation.validate()); @@ -1368,17 +1390,27 @@ impl Systemd for HostSystemd { } } -/// The last `.service` or `.scope` in a process's cgroup path names its issuer. +/// A system service or scope names the issuer: the first unit under +/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices +/// above it are the manager's, and what a delegated subtree adds below it is +/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. fn unit_from_cgroup(text: &str) -> String { let line = text .lines() .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) .unwrap_or_default(); let path = line.rsplit_once(':').map_or("", |(_, path)| path); - path.rsplit('/') - .find(|segment| segment.ends_with(".service") || segment.ends_with(".scope")) - .filter(|segment| segment.len() <= UNIT_LIMIT) - .map_or_else(|| "unknown".to_owned(), str::to_owned) + let unit = path.strip_prefix("/system.slice/").and_then(|under| { + for segment in under.split('/') { + if segment.ends_with(".slice") { + continue; + } + let unit = segment.ends_with(".service") || segment.ends_with(".scope"); + return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); + } + None + }); + unit.map_or_else(|| "unknown".to_owned(), str::to_owned) } /// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. @@ -1440,6 +1472,8 @@ pub(crate) struct Synapse { pub(crate) address: std::net::SocketAddr, pub(crate) room: String, pub(crate) token_file: PathBuf, + /// The whole wait for a reply, however slowly it drips. + pub(crate) reply_within: Duration, } fn percent_encode(text: &str) -> String { @@ -1474,15 +1508,26 @@ impl Homeserver for Synapse { ); let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) .context("the homeserver is unreachable")?; - socket.set_read_timeout(Some(Duration::from_secs(10)))?; - socket.set_write_timeout(Some(Duration::from_secs(10)))?; + socket.set_write_timeout(Some(self.reply_within))?; socket.write_all(head.as_bytes())?; socket.write_all(&body)?; + let deadline = std::time::Instant::now() + self.reply_within; let mut reply = Vec::new(); - socket - .take(RESPONSE_FRAME_LIMIT as u64) - .read_to_end(&mut reply) - .context("the homeserver reply was cut short")?; + let mut chunk = [0u8; 4096]; + while reply.len() < RESPONSE_FRAME_LIMIT { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|left| !left.is_zero()) + .context("the homeserver reply was too slow")?; + socket.set_read_timeout(Some(remaining))?; + let count = socket + .read(&mut chunk) + .context("the homeserver reply was cut short")?; + if count == 0 { + break; + } + reply.extend_from_slice(&chunk[..count]); + } let split = reply .windows(4) .position(|window| window == b"\r\n\r\n") @@ -1664,7 +1709,9 @@ fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { .expect("a thread can be spawned"); } -fn bind_socket(path: &Path) -> Result { +/// The socket belongs to the steward and no one else may open it; the peer +/// credential check on every connection stays the lock. +fn bind_socket(path: &Path, steward_uid: u32) -> Result { if let Ok(metadata) = std::fs::symlink_metadata(path) { use std::os::unix::fs::FileTypeExt; ensure!( @@ -1674,12 +1721,20 @@ fn bind_socket(path: &Path) -> Result { std::fs::remove_file(path).context("removing a stale socket")?; } let listener = UnixListener::bind(path).context("binding the steward socket")?; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) + std::os::unix::fs::chown(path, Some(steward_uid), None) + .context("giving the steward socket to the steward")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) .context("setting the steward socket mode")?; Ok(listener) } +/// Files this service creates in Idunn's own stores (a deploy order writes +/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. +const IDUNN_UMASK: libc::mode_t = 0o027; + pub(crate) fn serve(options: ServeOptions) -> Result<()> { + // SAFETY: umask only sets the process file-mode mask; it cannot fail. + unsafe { libc::umask(IDUNN_UMASK) }; let binding = load_binding(&options.steward_binding)?; let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") .map(PathBuf::from) @@ -1698,11 +1753,12 @@ pub(crate) fn serve(options: ServeOptions) -> Result<()> { address: binding.address()?, room: binding.room.clone(), token_file: credentials.join(&binding.token_credential), + reply_within: Duration::from_secs(10), }), }); ctx.store.prepare()?; let pending = recover(&ctx)?; - let listener = bind_socket(&options.socket)?; + let listener = bind_socket(&options.socket, ctx.steward_uid)?; let (queue, orders) = mpsc::channel::(); for order_id in pending { queue.send(order_id)?; @@ -1945,10 +2001,12 @@ mod tests { } } - struct FakeBrake(AtomicBool); + /// Whether the brake is clear, and the principals it was asked about. + struct FakeBrake(AtomicBool, Mutex>); impl BrakeCheck for FakeBrake { - fn clear(&self, _principal: &str) -> bool { + fn clear(&self, principal: &str) -> bool { + self.1.lock().unwrap().push(principal.into()); self.0.load(Ordering::SeqCst) } } @@ -1988,7 +2046,7 @@ mod tests { ..RunReport::default() }), }); - let brake = Arc::new(FakeBrake(AtomicBool::new(true))); + let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); let home = Arc::new(FakeHome::default()); let (queue, orders) = mpsc::channel(); let ctx = Ctx { @@ -2424,7 +2482,7 @@ mod tests { .filter(|record| matches!(record.outcome, Outcome::Refused(_))) .count(); assert_eq!(refusals, direct.len() * 4 + 8 + 6); - let due = due_posts(&h.ctx, &ledger); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); let results = due .iter() .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) @@ -2891,7 +2949,7 @@ mod tests { .unwrap(); assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); // The result post waits for the transactions: queued is not finished. - let due = due_posts(&h.ctx, &ledger); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); assert_eq!(due.len(), 1); assert_eq!(due[0].kind, PostKind::Accepted); assert!(!deploy_finished("queued") && !deploy_finished("running")); @@ -2948,7 +3006,7 @@ mod tests { assert_eq!(record.stdout_tail.len(), TAIL_BYTES); assert!(!format!("{record:?}").contains("secret-looking")); // Neither the post nor the journal text carries a tail. - let results: Vec<_> = due_posts(&h.ctx, &ledger) + let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) .into_iter() .filter(|due| due.kind == PostKind::Result) .collect(); @@ -3088,20 +3146,30 @@ mod tests { unit_from_cgroup("0::/system.slice/claude-connector.service\n"), "claude-connector.service" ); + // What a delegated subtree adds below the unit is the peer's to name. assert_eq!( - unit_from_cgroup("0::/system.slice/huginn@eureka.service/sub/leaf\n"), + unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), "huginn@eureka.service" ); assert_eq!( - unit_from_cgroup("0::/user.slice/user-1000.slice/session-3.scope\n"), - "session-3.scope" + unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), + "getty@tty1.service" ); assert_eq!( - unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.service\n"), - "a.service" + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), + "a.scope" ); - assert_eq!(unit_from_cgroup("0::/\n"), "unknown"); - assert_eq!(unit_from_cgroup(""), "unknown"); + // A user manager's units, and paths no manager made, name no one. + for path in [ + "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", + "0::/user.slice/user-1000.slice/session-3.scope\n", + "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", + "0::/system.slice/\n", + "0::/\n", + "", + ] { + assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); + } } #[test] @@ -3223,6 +3291,7 @@ mod tests { address, room: "!room:example.org".into(), token_file, + reply_within: Duration::from_secs(10), }; assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); let request = requests.recv().unwrap(); @@ -3238,7 +3307,8 @@ mod tests { Synapse { address, room: "!r:e".into(), - token_file: directory.path().join("absent") + token_file: directory.path().join("absent"), + reply_within: Duration::from_secs(10), } .post("t", "x") .is_err() @@ -3356,4 +3426,525 @@ mod tests { assert!(parse(&refused).is_err(), "{refused:?} parsed"); } } + + #[test] + fn an_order_runs_only_under_the_steward_bound_now() { + let mut h = harness(); + let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The binding changes while both wait: another uid, then another + // principal with the old uid. Each is refused with a record, before + // the brake is asked and before anything runs. + h.ctx.steward_uid = STEWARD + 7; + execute(&h.ctx, &first).unwrap(); + h.ctx.steward_uid = STEWARD; + h.ctx.principal = "void".into(); + execute(&h.ctx, &second).unwrap(); + for id in [&first, &second] { + assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); + assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); + } + assert!(h.runs().is_empty()); + assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); + // The refusal is posted like any other. + let ledger = h.ctx.store.ledger().unwrap(); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert!(due.iter().any(|due| due.kind == PostKind::Result + && due.text.contains("refused: the issuer is not the current steward"))); + // The other direction: an order the bound steward issued after the + // rebinding, and a recovered one under an unchanged binding, run and + // are gated by the bound principal's brake. + let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); + let resumed = harness(); + let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); + assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&resumed.ctx, &waiting).unwrap(); + execute(&h.ctx, &third).unwrap(); + assert_eq!(resumed.outcome(&waiting), Outcome::Ran); + assert_eq!(h.outcome(&third), Outcome::Ran); + assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); + assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); + } + + #[test] + fn root_reads_orders_and_never_issues_one() { + let mut h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + h.ctx.peers = Arc::new(FakePeers(0)); + let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); + assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("root could not read an order"); + }; + assert_eq!(view.order.order_id, id); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("root could not list orders"); + }; + assert_eq!(lines.len(), 1, "root's refused order left no record"); + h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); + assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); + } + + #[test] + fn every_order_posts_once_in_its_turn() { + let h = harness(); + let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] + .into_iter() + .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) + .collect(); + for id in &ids { + execute(&h.ctx, id).unwrap(); + } + assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); + assert_eq!(post_pass(&h.ctx).unwrap(), 0); + let sent = h.home.sent.lock().unwrap().clone(); + let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); + keys.sort(); + let mut expected: Vec = ids + .iter() + .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) + .collect(); + expected.sort(); + assert_eq!(keys, expected); + // An order accepted later is posted by the next pass alone. + let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + execute(&h.ctx, &late).unwrap(); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + } + + #[test] + fn a_deploys_result_is_posted_once_its_command_has_finished() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let ledger = h.ctx.store.ledger().unwrap(); + let results = |state: Option<(&'static str, &str)>| { + let owed = due_posts(&ledger, &|_| match state { + Some((state, detail)) => Ok((state, detail.to_owned())), + None => Err(anyhow!("the command cannot be read")), + }); + owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() + }; + assert!(results(Some(("queued", ""))).is_empty()); + assert!(results(Some(("running", "vili:Sealing"))).is_empty()); + assert!(results(None).is_empty()); + for state in ["succeeded", "failed"] { + let owed = results(Some((state, "vili"))); + assert_eq!(owed.len(), 1, "{state}"); + assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); + assert!(owed[0].text.contains("submitted deployment up-")); + } + } + + #[test] + fn intake_gives_up_on_a_peer_that_sends_no_frame() { + let h = harness(); + let (mut client, mut server) = UnixStream::pair().unwrap(); + // Half a length prefix, then nothing: the connection must end by itself. + client.write_all(&[0, 0]).unwrap(); + let (done, ended) = mpsc::channel(); + let worker = thread::spawn(move || { + let result = handle(&h.ctx, &mut server, &h.queue); + let _ = done.send(result.is_ok()); + h + }); + let answered = ended + .recv_timeout(Duration::from_secs(20)) + .expect("intake never gave up on a silent peer"); + assert!(answered, "a silent peer is refused as malformed, not left as an error"); + let h = worker.join().unwrap(); + assert_eq!( + rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn serve_options_are_read_from_every_flag() { + let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); + let defaults = parse(&["serve"]).unwrap(); + assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); + assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); + assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); + let given = parse(&[ + "serve", + "--state-store", + "/a/control.cc", + "--store", + "/b", + "--steward-binding", + "/c.toml", + "--socket", + "/d.sock", + ]) + .unwrap(); + assert_eq!( + given, + ServeOptions { + state_store: "/a/control.cc".into(), + store: "/b".into(), + steward_binding: "/c.toml".into(), + socket: "/d.sock".into(), + } + ); + assert!(parse(&["serve", "--store"]).is_err()); + assert!(parse(&["serve", "--other", "/x"]).is_err()); + assert!(parse(&["start"]).is_err()); + } + + #[test] + fn the_steward_user_is_a_real_user_and_never_root() { + assert!(user_uid("root").is_err()); + assert!(user_uid("no-such-user-here").is_err()); + assert!(user_uid("bad\0name").is_err()); + assert_eq!(user_uid("nobody").unwrap(), 65534); + } + + #[test] + fn the_orders_directory_is_made_private() { + let directory = TempDir::new().unwrap(); + let store = OrderStore::new(&directory.path().join("orders")); + store.prepare().unwrap(); + let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + #[test] + fn the_socket_belongs_to_the_steward_alone() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("steward.sock"); + let me = std::fs::metadata(directory.path()).unwrap().uid(); + let _listener = bind_socket(&path, me).unwrap(); + let metadata = std::fs::metadata(&path).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); + // A stale socket is replaced; anything else in the way is not. + let _again = bind_socket(&path, me).unwrap(); + let other = directory.path().join("other"); + std::fs::write(&other, "x").unwrap(); + assert!(bind_socket(&other, me).is_err()); + } + + #[test] + fn synapse_gives_up_on_a_reply_that_never_ends() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + thread::spawn(move || { + // The first connection stays silent; the second drips a byte at a time. + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + thread::spawn(move || { + let mut chunk = [0u8; 4096]; + let _ = socket.read(&mut chunk); + for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { + if index == 0 || socket.write_all(&[*byte]).is_err() { + break; + } + thread::sleep(Duration::from_millis(40)); + } + thread::sleep(Duration::from_secs(5)); + }); + } + }); + let synapse = Synapse { + address, + room: "!r:e".into(), + token_file, + reply_within: Duration::from_millis(600), + }; + for _ in 0..2 { + let started = std::time::Instant::now(); + let error = synapse.post("t", "x").unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error:#}"); + assert!(!format!("{error:#}").contains("tok-123")); + } + } + + /// The whole service, as root, against fake `systemctl`, `systemd-run` + /// and a loopback homeserver, with the steward connecting from another + /// uid. It runs only where the test process is root (the verify container). + #[test] + fn serve_end_to_end_as_root() { + // SAFETY: geteuid reads the process credentials. + if unsafe { libc::geteuid() } != 0 { + eprintln!("serve_end_to_end_as_root: skipped, not root"); + return; + } + let directory = TempDir::new().unwrap(); + let root = directory.path().to_owned(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let bin = root.join("bin"); + std::fs::create_dir(&bin).unwrap(); + let systemctl = format!( + "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ + case \"$*\" in *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + root.display() + ); + let run = format!( + "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", + root.display() + ); + for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { + let path = bin.join(name); + std::fs::write(&path, text).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let credentials = root.join("cred"); + std::fs::create_dir(&credentials).unwrap(); + std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (posted, posts) = mpsc::channel::(); + thread::spawn(move || { + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + let mut received = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + let text = String::from_utf8_lossy(&received).into_owned(); + if let Some(split) = text.find("\r\n\r\n") { + let length = text + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if received.len() >= split + 4 + length { + break; + } + } + let count = socket.read(&mut chunk).unwrap_or(0); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); + let _ = socket.write_all( + format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), + ); + } + }); + let binding = root.join("steward.toml"); + std::fs::write( + &binding, + format!( + "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ + token_credential = \"idunn-matrix-token\"\n" + ), + ) + .unwrap(); + std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); + let run_directory = root.join("run"); + std::fs::create_dir(&run_directory).unwrap(); + std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let socket = run_directory.join("steward.sock"); + let store = root.join("orders"); + let control = root.join("state").join("control.cc"); + // SAFETY: the only test that reads these variables is this one; PATH + // keeps its old entries. + unsafe { + std::env::set_var( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), + ); + std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); + } + let options = ServeOptions { + state_store: control.clone(), + store: store.clone(), + steward_binding: binding, + socket: socket.clone(), + }; + // The unit's umask must not decide the modes of what the service writes. + // SAFETY: umask only sets the process file-mode mask. + let before = unsafe { libc::umask(0o117) }; + let (ended, serve_end) = mpsc::channel::(); + thread::spawn(move || { + let result = serve(options); + let _ = ended.send(format!("{result:?}")); + }); + for _ in 0..100 { + if socket.exists() || serve_end.try_recv().is_ok() { + break; + } + thread::sleep(Duration::from_millis(100)); + } + // SAFETY: as above. + unsafe { libc::umask(before) }; + assert!(socket.exists(), "serve never bound its socket"); + let metadata = std::fs::metadata(&socket).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); + // Root is not the steward: it may read, it may not order. + let unit = |name: &str| { + Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: name.into(), + }, + reason: "end to end".into(), + }) + }; + let as_root = |request: &Request| { + let mut stream = UnixStream::connect(&socket).unwrap(); + write_frame(&mut stream, request).unwrap(); + rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + }; + assert_eq!( + as_root(&unit("nginx.service")), + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert_eq!(as_root(&Request::List), Response::List(Vec::new())); + let orders = OrderStore::new(&store); + let finished = |order_id: &str| { + for _ in 0..200 { + if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { + return record.clone(); + } + thread::sleep(Duration::from_millis(100)); + } + panic!("{order_id} never concluded"); + }; + // The steward is uid 65534 with gid 0: a service that read the gid for + // the uid would take it for root. + let first = steward_asks(&socket, &unit("nginx.service")); + let Response::Accepted { order_id: first } = first else { + panic!("the steward was not accepted: {first:?}"); + }; + assert_eq!(finished(&first).outcome, Outcome::Ran); + std::fs::write(root.join("fail"), "").unwrap(); + let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { + panic!("the second order was not accepted"); + }; + let failed = finished(&second); + assert_eq!( + failed.outcome, + Outcome::Failed { + exit_code: Some(3), + signal: None + } + ); + assert_eq!(failed.stderr_tail, b"boom\n"); + let deploy = Request::Order(OrderRequest { + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "end to end".into(), + }); + let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { + panic!("the deploy was not accepted"); + }; + assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); + let ledger = orders.ledger().unwrap(); + let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); + assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // What the service called, exactly. + let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); + assert!( + run_log.contains(&format!( + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart -- nginx.service", + order_uuid(&first) + )), + "{run_log}" + ); + let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); + assert!( + systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), + "{systemctl_log}" + ); + // Every record is posted; the deploy's result waits for its command. + let mut seen = Vec::new(); + while seen.len() < 5 { + seen.push(posts.recv_timeout(Duration::from_secs(30)).expect("five posts within 30 s")); + } + for (id, kinds) in [ + (&first, ["accepted", "result"].as_slice()), + (&second, &["accepted", "result"]), + (&third, &["accepted"]), + ] { + for kind in kinds { + let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); + assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); + } + } + // Modes: the store is private, and the deploy wrote control.cc under + // Idunn's umask, not the 0117 the process was started with. + let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; + assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); + assert_eq!(mode(&control), 0o640); + } + + /// One request as the steward: uid 65534 and gid 0, from a forked child + /// that only makes system calls. + fn steward_asks(socket: &Path, request: &Request) -> Response { + let body = rmp_serde::to_vec_named(request).unwrap(); + let mut frame = (body.len() as u32).to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + // SAFETY: plain data. + let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { + address.sun_path[index] = *byte as libc::c_char; + } + let mut fds = [0i32; 2]; + // SAFETY: `fds` holds two descriptors. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + // SAFETY: the child allocates nothing and ends with _exit. + let pid = unsafe { libc::fork() }; + if pid == 0 { + // SAFETY: only system calls on buffers made before the fork. + unsafe { + libc::close(fds[0]); + if libc::setgroups(0, std::ptr::null()) != 0 + || libc::setgid(0) != 0 + || libc::setuid(65534) != 0 + { + libc::_exit(2); + } + let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if libc::connect( + fd, + (&address as *const libc::sockaddr_un).cast(), + std::mem::size_of::() as libc::socklen_t, + ) != 0 + { + libc::_exit(3); + } + let mut offset = 0usize; + while offset < frame.len() { + let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); + if count <= 0 { + libc::_exit(4); + } + offset += count as usize; + } + let mut buffer = [0u8; 4096]; + loop { + let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); + if count <= 0 { + break; + } + libc::write(fds[1], buffer.as_ptr().cast(), count as usize); + } + libc::_exit(0); + } + } + // SAFETY: the parent's copy of the write end. + unsafe { libc::close(fds[1]) }; + let mut reply = Vec::new(); + // SAFETY: `fds[0]` is an open descriptor this function now owns. + let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; + pipe.read_to_end(&mut reply).unwrap(); + let mut status = 0; + // SAFETY: `pid` is our child. + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); + rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + } } From 572f9e6aa424661c36abf7e876a467a08d033eb8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:31:34 +0100 Subject: [PATCH 183/196] Steward orders: the end-to-end test keeps serve's umask until it has read control.cc Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index e707307..3f0d229 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3776,8 +3776,6 @@ mod tests { } thread::sleep(Duration::from_millis(100)); } - // SAFETY: as above. - unsafe { libc::umask(before) }; assert!(socket.exists(), "serve never bound its socket"); let metadata = std::fs::metadata(&socket).unwrap(); assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); @@ -3877,7 +3875,10 @@ mod tests { // Idunn's umask, not the 0117 the process was started with. let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); - assert_eq!(mode(&control), 0o640); + let control_mode = mode(&control); + // SAFETY: umask only sets the process file-mode mask. + unsafe { libc::umask(before) }; + assert_eq!(control_mode, 0o640); } /// One request as the steward: uid 65534 and gid 0, from a forked child From 7e15bf1a7b5e9003fd44025dd9e5fae05f18b2b3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:33:23 +0100 Subject: [PATCH 184/196] Steward orders: the Synapse deadline test fails instead of hanging when the deadline is gone Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 3f0d229..261cb24 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3658,11 +3658,20 @@ mod tests { token_file, reply_within: Duration::from_millis(600), }; + let synapse = Arc::new(synapse); for _ in 0..2 { let started = std::time::Instant::now(); - let error = synapse.post("t", "x").unwrap_err(); - assert!(started.elapsed() < Duration::from_secs(3), "{error:#}"); - assert!(!format!("{error:#}").contains("tok-123")); + let (done, ended) = mpsc::channel(); + let poster = Arc::clone(&synapse); + thread::spawn(move || { + let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); + }); + let error = ended + .recv_timeout(Duration::from_secs(4)) + .expect("the post never gave up") + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error}"); + assert!(!error.contains("tok-123")); } } From 8a1beb5c873c96b2741ffb64e7cfb958adbdcd4f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:35:30 +0100 Subject: [PATCH 185/196] README: an order is gated by the steward bound at execution; the socket is the steward's; root reads Co-Authored-By: Claude Sonnet 5.5 --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2bf6675..5240106 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,12 @@ match themselves, and a last step fails the order unless every named package is installed. An order runs once, after the steward's uid is checked and the instance's -brake check (`mind-control --instance check`) exits 0; an order -that had started when the service died is recorded as lost and never run -again. No order releases a brake. Acceptance and result are posted to +brake check (`mind-control --instance check`) exits 0. The check +that the order's principal and uid are the bound steward's runs again at every +execution, so an order accepted before `steward.toml` changed is refused and +recorded, not run. The socket belongs to the steward user alone (mode 0600); +root may `show` and `list` orders but never issue one. An order that had +started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to `#operator` as a retried projection of `orders.cc`: a chat outage delays the posts and not the orders. A refusal names the field and a code, never the value it refused. From 9653d270e5d47b6bc7b17a6680666e5529b26e07 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:16:04 +0100 Subject: [PATCH 186/196] Steward orders: pin the ledger schema guards, invocation ids and post backoff; tests fail fast instead of waiting Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 91 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 261cb24..1394891 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -1690,6 +1690,16 @@ pub(crate) fn parse_serve(mut args: impl Iterator) -> Result u64 { + if failures == 0 { + 2 + } else { + (2u64 << failures.min(5)).min(60) + } +} + /// Run a thread whose end ends the service: systemd restarts it, and the /// start recovery settles what it left. fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { @@ -1775,18 +1785,14 @@ pub(crate) fn serve(options: ServeOptions) -> Result<()> { vital("poster", move || { let mut failures = 0u32; loop { - let pause = match post_pass(&poster) { - Ok(_) => { - failures = 0; - 2 - } + match post_pass(&poster) { + Ok(_) => failures = 0, Err(error) => { failures = failures.saturating_add(1); eprintln!("steward orders: posting is delayed: {error:#}"); - (2u64 << failures.min(5)).min(60) } - }; - thread::sleep(Duration::from_secs(pause)); + } + thread::sleep(Duration::from_secs(post_pause(failures))); } }); for stream in listener.incoming() { @@ -2077,6 +2083,7 @@ mod tests { impl Harness { fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); write(&mut client); handle(&self.ctx, &mut server, &self.queue).unwrap(); let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); @@ -3543,6 +3550,7 @@ mod tests { fn intake_gives_up_on_a_peer_that_sends_no_frame() { let h = harness(); let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); // Half a length prefix, then nothing: the connection must end by itself. client.write_all(&[0, 0]).unwrap(); let (done, ended) = mpsc::channel(); @@ -3563,6 +3571,63 @@ mod tests { assert!(!h.ctx.store.path().exists()); } + #[test] + fn a_record_under_another_schema_is_not_read() { + let put = |h: &Harness, schema: &str, key: &str| match schema { + _ if schema.contains("start") => h.ctx.store.put( + key, + schema, + &StartRecord { + schema_version: schema.into(), + order_id: key.into(), + started_at_unix_millis: 1, + actuation: "idunn-order-x".into(), + }, + ), + _ if schema.contains("outcome") => h.ctx.store.put( + key, + schema, + &OutcomeRecord { + schema_version: schema.into(), + order_id: key.into(), + ended_at_unix_millis: 1, + outcome: Outcome::Lost, + stdout_tail: Vec::new(), + stderr_tail: Vec::new(), + }, + ), + _ => h.ctx.store.put( + key, + schema, + &PostRecord { + schema_version: schema.into(), + post_key: key.into(), + event_id: "$e".into(), + posted_at_unix_millis: 1, + }, + ), + }; + for (current, key) in [ + (START_SCHEMA, "ord-a"), + (OUTCOME_SCHEMA, "ord-a"), + (POST_SCHEMA, "ord-a:accepted"), + ] { + let h = harness(); + assert!(put(&h, current, key).unwrap()); + assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); + let h = harness(); + let other = current.replace(".v1", ".v0"); + assert!(put(&h, &other, key).unwrap()); + assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); + } + } + + #[test] + fn posts_are_retried_after_a_widening_pause() { + let pauses: Vec = (0..9).map(post_pause).collect(); + assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); + } + #[test] fn serve_options_are_read_from_every_flag() { let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); @@ -3692,7 +3757,7 @@ mod tests { std::fs::create_dir(&bin).unwrap(); let systemctl = format!( "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ - case \"$*\" in *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", root.display() ); let run = format!( @@ -3800,6 +3865,7 @@ mod tests { }; let as_root = |request: &Request| { let mut stream = UnixStream::connect(&socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); write_frame(&mut stream, request).unwrap(); rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() }; @@ -3851,6 +3917,10 @@ mod tests { let ledger = orders.ledger().unwrap(); let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // An invocation id is what a succeeding systemctl prints, if anything. + assert_eq!(HostSystemd.invocation("quiet.service"), None); + assert_eq!(HostSystemd.invocation("broken.service"), None); + assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); // What the service called, exactly. let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); assert!( @@ -3868,7 +3938,7 @@ mod tests { // Every record is posted; the deploy's result waits for its command. let mut seen = Vec::new(); while seen.len() < 5 { - seen.push(posts.recv_timeout(Duration::from_secs(30)).expect("five posts within 30 s")); + seen.push(posts.recv_timeout(Duration::from_secs(15)).expect("five posts within 15 s")); } for (id, kinds) in [ (&first, ["accepted", "result"].as_slice()), @@ -3911,6 +3981,7 @@ mod tests { // SAFETY: only system calls on buffers made before the fork. unsafe { libc::close(fds[0]); + libc::alarm(10); if libc::setgroups(0, std::ptr::null()) != 0 || libc::setgid(0) != 0 || libc::setuid(65534) != 0 From caacbb4bbaa4de31ffa63f76c1d4eec53a5080ce Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:35:31 +0100 Subject: [PATCH 187/196] Steward orders: the end-to-end test waits 10 s for its posts Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 1394891..688edc8 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3938,7 +3938,7 @@ mod tests { // Every record is posted; the deploy's result waits for its command. let mut seen = Vec::new(); while seen.len() < 5 { - seen.push(posts.recv_timeout(Duration::from_secs(15)).expect("five posts within 15 s")); + seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); } for (id, kinds) in [ (&first, ["accepted", "result"].as_slice()), From d9710c988a0a09adb74bb7f3ae2580ed881c9ef0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:35:43 +0100 Subject: [PATCH 188/196] Steward orders: the ledger reads an order only under its own schema too Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 688edc8..1d72e0e 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3573,8 +3573,25 @@ mod tests { #[test] fn a_record_under_another_schema_is_not_read() { - let put = |h: &Harness, schema: &str, key: &str| match schema { - _ if schema.contains("start") => h.ctx.store.put( + let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { + "order" => h.ctx.store.put( + key, + schema, + &OrderRecord { + schema_version: schema.into(), + order_id: key.into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }, + ), + "start" => h.ctx.store.put( key, schema, &StartRecord { @@ -3584,7 +3601,7 @@ mod tests { actuation: "idunn-order-x".into(), }, ), - _ if schema.contains("outcome") => h.ctx.store.put( + "outcome" => h.ctx.store.put( key, schema, &OutcomeRecord { @@ -3607,17 +3624,18 @@ mod tests { }, ), }; - for (current, key) in [ - (START_SCHEMA, "ord-a"), - (OUTCOME_SCHEMA, "ord-a"), - (POST_SCHEMA, "ord-a:accepted"), + for (kind, current, key) in [ + ("order", ORDER_SCHEMA, "ord-a"), + ("start", START_SCHEMA, "ord-a"), + ("outcome", OUTCOME_SCHEMA, "ord-a"), + ("post", POST_SCHEMA, "ord-a:accepted"), ] { let h = harness(); - assert!(put(&h, current, key).unwrap()); + assert!(put(&h, kind, current, key).unwrap()); assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); let h = harness(); let other = current.replace(".v1", ".v0"); - assert!(put(&h, &other, key).unwrap()); + assert!(put(&h, kind, &other, key).unwrap()); assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); } } From d29bf93be60fdf08412d43299f28e0fbe40016b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:26:29 +0100 Subject: [PATCH 189/196] Steward orders: refuse a start or stop whose unit ends by emergency action or isolates; boot and recovery units are never started from stopped refusal_for now reads SuccessAction, FailureAction, StartLimitAction and JobTimeoutAction (each mapped to the target it runs, so reboot, poweroff, halt, kexec, soft-reboot and exit, with -force and -immediate) and OnFailure and OnSuccess as units a touched unit fires, followed through what they pull in and what those fire. An action it does not know fails the order closed. A stopped DefaultDependencies=no unit in the transaction is refused (Via:: EarlyBoot): rescue.service ends by isolating default.target, which no edge describes. Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 175 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 1d72e0e..b9d64b7 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -374,6 +374,10 @@ pub(crate) enum Via { Name, Alias, Propagation, + /// A unit the order would start from a stopped state that is boot or + /// recovery machinery (`DefaultDependencies=no`): such a unit can end by + /// isolating a target, which no dependency edge describes. + EarlyBoot, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -418,6 +422,7 @@ impl Outcome { Via::Name => "name", Via::Alias => "alias", Via::Propagation => "propagation", + Via::EarlyBoot => "early-boot", } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), @@ -641,6 +646,32 @@ const EDGES: [(&str, Edge); 13] = [ ("StopPropagatedFrom", Edge::Stops), ]; +/// The properties naming what systemd does when a unit ends or its job times +/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). +const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; + +/// The properties naming units systemd starts when a unit ends. +const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; + +/// The target an emergency action runs, `None` for `none`. An action this +/// check does not know is an error: the unit is not checked, so not run. +fn action_target(action: &str) -> Result> { + let base = ["-force", "-immediate"] + .iter() + .find_map(|suffix| action.strip_suffix(suffix)) + .unwrap_or(action); + Ok(match base { + "" | "none" => None, + "reboot" => Some("reboot.target"), + "poweroff" => Some("poweroff.target"), + "halt" => Some("halt.target"), + "kexec" => Some("kexec.target"), + "soft-reboot" => Some("soft-reboot.target"), + "exit" => Some("exit.target"), + _ => bail!("a unit has an emergency action this check does not know"), + }) +} + /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { @@ -648,6 +679,11 @@ pub(crate) struct UnitFacts { pub(crate) names: Vec, pub(crate) not_found: bool, pub(crate) running: bool, + /// `DefaultDependencies=no`: boot, recovery and shutdown machinery. + pub(crate) early_boot: bool, + /// What systemd starts when this unit ends or times out: the targets of + /// its emergency actions and its `OnFailure=` and `OnSuccess=` units. + pub(crate) fires: Vec, pub(crate) pulls: Vec, pub(crate) conflicts: Vec, pub(crate) stops: Vec, @@ -762,6 +798,27 @@ impl Transaction<'_> { Ok(reached) } + /// Every unit the emergency actions and `OnFailure=`/`OnSuccess=` of + /// `touched` would start, with what each pulls in, and in turn what those + /// that are not running would set off. A touched unit ends, so it fires. + fn fired(&mut self, touched: &BTreeSet) -> Result> { + let mut fired = BTreeSet::new(); + let mut seen = BTreeSet::new(); + let mut queue: VecDeque = touched.iter().cloned().collect(); + while let Some(unit) = queue.pop_front() { + if !seen.insert(unit.clone()) { + continue; + } + let seeds: BTreeSet = self.facts(&unit)?.fires.iter().cloned().collect(); + for reached in self.reach(&seeds, Edge::Pulls)? { + if fired.insert(reached.clone()) && !self.facts(&reached)?.running { + queue.push_back(reached); + } + } + } + Ok(fired) + } + /// Every unit a start of `started` stops through `Conflicts=`, and what /// that stop takes down with it. fn conflicted(&mut self, started: &BTreeSet) -> Result> { @@ -807,6 +864,13 @@ fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result Result Vec { /// The properties one `systemctl show` is asked for: identity, state, edges. fn show_properties() -> String { - ["Id", "Names", "LoadState", "ActiveState"] + ["Id", "Names", "LoadState", "ActiveState", "DefaultDependencies"] .into_iter() + .chain(ACTIONS) + .chain(ON_END) .chain(EDGES.iter().map(|(property, _)| *property)) .collect::>() .join(",") @@ -1333,6 +1402,9 @@ fn parse_show(output: &str) -> Result { "ActiveState" => { facts.running = matches!(value, "active" | "activating" | "reloading"); } + "DefaultDependencies" => facts.early_boot = value == "no", + _ if ACTIONS.contains(&key) => facts.fires.extend(action_target(value)?.map(str::to_owned)), + _ if ON_END.contains(&key) => facts.fires.extend(parse_names(value)), _ => { if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { facts.along_mut(*edge).extend(parse_names(value)); @@ -2719,6 +2791,107 @@ mod tests { assert!(h.runs().is_empty()); } + #[test] + fn a_unit_whose_end_reboots_the_host_is_refused_by_its_properties() { + let h = harness(); + yggdrasil(&h, true); + // Each emergency action property alone, in each of systemd's spellings, + // on a unit no name list knows: every verb that starts or stops it. + for (property, action) in [ + ("SuccessAction", "reboot"), + ("FailureAction", "reboot"), + ("StartLimitAction", "poweroff-force"), + ("JobTimeoutAction", "reboot-immediate"), + ("SuccessAction", "poweroff"), + ] { + let name = format!("storagetm-{property}.service"); + h.describe(&name, &[(property, action), ("ActiveState", "inactive")]); + for verb in [Verb::Start, Verb::Restart, Verb::Stop] { + let outcome = outcome_of_unit(&h, verb, &name); + assert!(hit(&outcome).is_some(), "{verb:?} {property}={action} {outcome:?}"); + } + } + // OnFailure= and OnSuccess= start the units they name, and what those + // set off in turn. + for property in ["OnFailure", "OnSuccess"] { + let name = format!("handled-{property}.service"); + h.describe(&name, &[(property, "reboot.target"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, &name); + assert!(hit(&outcome).is_some(), "{property} {outcome:?}"); + } + h.describe("on-failure-chain.service", &[("OnFailure", "handler.service"), ("ActiveState", "inactive")]); + h.describe("handler.service", &[("FailureAction", "reboot"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "on-failure-chain.service"); + assert!(hit(&outcome).is_some(), "{outcome:?}"); + // A unit a start pulls in fires its action too. + h.describe("pulls-storagetm.service", &[("Wants", "storagetm.service")]); + h.describe("storagetm.service", &[("SuccessAction", "reboot"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "pulls-storagetm.service"); + assert!(hit(&outcome).is_some(), "{outcome:?}"); + // A running unit the order does not stop is not ended by it, and an + // action of none ends nothing. + h.describe("pulls-running.service", &[("Wants", "running-action.service")]); + h.describe("running-action.service", &[("SuccessAction", "reboot")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "pulls-running.service"), Outcome::Ran); + h.describe("quiet-actions.service", &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", "")]); + for verb in [Verb::Start, Verb::Restart, Verb::Stop] { + assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); + } + // An action this check does not know is not checked, so it does not run. + h.describe("odd-action.service", &[("SuccessAction", "implode")]); + let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); + execute(&h.ctx, &id).unwrap(); + assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); + assert!(h.runs().is_empty()); + } + + #[test] + fn a_stopped_boot_or_recovery_unit_is_never_started() { + let h = harness(); + yggdrasil(&h, true); + // rescue.service ends by isolating default.target, which stops every + // unit outside it; no edge says so, its DefaultDependencies=no does. + h.describe( + "rescue.service", + &[("DefaultDependencies", "no"), ("ActiveState", "inactive")], + ); + h.describe("rescue.target", &[("Requires", "rescue.service"), ("ActiveState", "inactive")]); + h.describe("pulls-rescue.service", &[("Wants", "rescue.target")]); + for (verb, unit) in [ + (Verb::Start, "rescue.service"), + (Verb::Restart, "rescue.service"), + (Verb::Start, "rescue.target"), + (Verb::Restart, "rescue.target"), + (Verb::Start, "pulls-rescue.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert!( + matches!( + outcome, + Outcome::Refused(Refused::AlwaysRefused { ref unit, via: Via::EarlyBoot }) + if unit == "rescue.service" + ), + "{verb:?} {unit} was {outcome:?}" + ); + } + assert!(h.runs().is_empty()); + // Stopping it, or starting one whose defaults are on, is an ordinary order. + assert_eq!(outcome_of_unit(&h, Verb::Stop, "rescue.service"), Outcome::Ran); + h.describe("ordinary.service", &[("DefaultDependencies", "yes"), ("ActiveState", "inactive")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "ordinary.service"), Outcome::Ran); + // A boot unit that is already running is left alone by a start. + h.describe("journald.service", &[("DefaultDependencies", "no")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); + } + + #[test] + fn show_asks_for_every_property_the_refusal_reads() { + let asked = show_properties(); + for property in ["DefaultDependencies", "SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction", "OnFailure", "OnSuccess"] { + assert!(asked.split(',').any(|asked| asked == property), "{property}"); + } + } + #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); From 631fb7836b6e875fcbfa64fcc01b0ea78a4a08fd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:29:13 +0100 Subject: [PATCH 190/196] Steward orders: pin the new show properties; the unknown-action probe counts runs instead of expecting none Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index b9d64b7..884afda 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2838,11 +2838,12 @@ mod tests { assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); } // An action this check does not know is not checked, so it does not run. + let ran = h.runs().len(); h.describe("odd-action.service", &[("SuccessAction", "implode")]); let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); execute(&h.ctx, &id).unwrap(); assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); - assert!(h.runs().is_empty()); + assert_eq!(h.runs().len(), ran); } #[test] @@ -2884,14 +2885,6 @@ mod tests { assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); } - #[test] - fn show_asks_for_every_property_the_refusal_reads() { - let asked = show_properties(); - for property in ["DefaultDependencies", "SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction", "OnFailure", "OnSuccess"] { - assert!(asked.split(',').any(|asked| asked == property), "{property}"); - } - } - #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); @@ -3379,8 +3372,9 @@ mod tests { // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), - "Id,Names,LoadState,ActiveState,Requires,Requisite,BindsTo,Wants,Upholds,\ - Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ + "Id,Names,LoadState,ActiveState,DefaultDependencies,SuccessAction,FailureAction,\ + StartLimitAction,JobTimeoutAction,OnFailure,OnSuccess,Requires,Requisite,BindsTo,\ + Wants,Upholds,Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ PropagatesStopTo,StopPropagatedFrom" ); // A backslash outside quotes is a backslash; inside, it escapes one character. From 879b76a8425df76ba18f17a78fff488e87e1bc0a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:50:23 +0100 Subject: [PATCH 191/196] Steward orders: pin every emergency action spelling to its target Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 884afda..69eea41 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2846,6 +2846,31 @@ mod tests { assert_eq!(h.runs().len(), ran); } + #[test] + fn every_emergency_action_runs_the_target_it_names() { + for (action, target) in [ + ("none", None), + ("", None), + ("reboot", Some("reboot.target")), + ("reboot-force", Some("reboot.target")), + ("reboot-immediate", Some("reboot.target")), + ("poweroff", Some("poweroff.target")), + ("poweroff-force", Some("poweroff.target")), + ("poweroff-immediate", Some("poweroff.target")), + ("halt", Some("halt.target")), + ("halt-force", Some("halt.target")), + ("kexec", Some("kexec.target")), + ("kexec-immediate", Some("kexec.target")), + ("soft-reboot", Some("soft-reboot.target")), + ("soft-reboot-force", Some("soft-reboot.target")), + ("exit", Some("exit.target")), + ("exit-force", Some("exit.target")), + ] { + assert_eq!(action_target(action).unwrap(), target, "{action}"); + } + assert!(action_target("implode").is_err()); + } + #[test] fn a_stopped_boot_or_recovery_unit_is_never_started() { let h = harness(); From 4894ad20df0f98fa308044d82e1f9fa21468ddf7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:52:41 +0100 Subject: [PATCH 192/196] README: an order's transaction follows emergency actions and OnFailure/OnSuccess; stopped boot units are refused Co-Authored-By: Claude Opus 5.5 --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5240106..28c1e94 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,14 @@ refused unit that is already running is left alone) and stops what conflicts with it, a stop takes down what depends on it, and a restart does both. The transaction is read from `systemctl show`, so `start poweroff.target` is refused because every refused unit conflicts with `shutdown.target`, not because -targets are banned. A unit `systemctl show` cannot resolve fails the order. +targets are banned. The units a start or stop ends or times out are followed too: +the target each of `SuccessAction=`, `FailureAction=`, `StartLimitAction=` and +`JobTimeoutAction=` runs (`reboot`, `poweroff`, `halt`, `kexec`, `soft-reboot`, +`exit`, with `-force` and `-immediate`), and the units `OnFailure=` and +`OnSuccess=` name, with what each pulls in; an action it does not know fails the +order. A start refuses a stopped `DefaultDependencies=no` unit (boot and recovery +machinery, such as `rescue.service`, which ends by isolating the default +target), whatever pulls it in. A unit `systemctl show` cannot resolve fails the order. Slices, scopes and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with From a859914b0bb481ee9abeb999d73753f9313c28a0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:40:58 +0100 Subject: [PATCH 193/196] Steward orders: run every unit verb as one job and check only the named unit Delete the transaction model (Edge, EDGES, action_target, Transaction, the early-boot rule). Every unit verb runs systemctl --job-mode=ignore-requirements -- ; refusal_for reads one systemctl show of the ordered unit: its names, the host-ending names, what fires when it ends, and who is bound to it. Targets are no longer orderable. Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 806 +++++++++++------------------------------- 1 file changed, 207 insertions(+), 599 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 69eea41..d5fdcfe 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,7 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::BTreeMap; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -58,9 +58,8 @@ const TAIL_BYTES: usize = 8 * 1024; const REASON_LIMIT: usize = 300; const PACKAGE_LIMIT: usize = 16; const UNIT_LIMIT: usize = 200; -const CLOSURE_LIMIT: usize = 512; const LIST_LIMIT: usize = 50; -const UNIT_SUFFIXES: [&str; 6] = [".service", ".socket", ".timer", ".path", ".mount", ".target"]; +const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; /// The always-refused set: Idunn's own units and the minds' own. Matched on /// every name a unit answers to, ASCII case-insensitively (systemd units are @@ -72,6 +71,21 @@ fn is_refused_name(name: &str) -> bool { .any(|prefix| name.starts_with(prefix)) } +/// Units whose binaries suspend, hibernate or open recovery: no property +/// reveals it, so they are refused by name, on every name they answer to. +const HOST_NAMES: [&str; 6] = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", +]; + +fn is_host_name(name: &str) -> bool { + HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) +} + // ---------------------------------------------------------------- grammar -- #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -288,6 +302,7 @@ impl Operation { argv: vec![ "systemctl".into(), verb.as_str().into(), + "--job-mode=ignore-requirements".into(), "--".into(), unit.clone(), ], @@ -373,11 +388,9 @@ impl fmt::Debug for StartRecord { pub(crate) enum Via { Name, Alias, - Propagation, - /// A unit the order would start from a stopped state that is boot or - /// recovery machinery (`DefaultDependencies=no`): such a unit can end by - /// isolating a target, which no dependency edge describes. - EarlyBoot, + Host, + OnEnd, + Bound, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -421,8 +434,9 @@ impl Outcome { match via { Via::Name => "name", Via::Alias => "alias", - Via::Propagation => "propagation", - Via::EarlyBoot => "early-boot", + Via::Host => "host", + Via::OnEnd => "on-end", + Via::Bound => "bound", } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), @@ -617,35 +631,6 @@ impl OrderStore { // ------------------------------------------------------------------ ports -- -/// Which way systemd moves along a dependency edge when an order is given. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Edge { - /// Units a start brings up with this one. - Pulls, - /// Units a start stops: the unit's `Conflicts=` and the units that conflict with it. - Conflicts, - /// Units a stop takes down with this one. - Stops, -} - -/// The `systemctl show` properties the closure reads, and the way each one -/// moves. This is the only list: the request and the parse both read it. -const EDGES: [(&str, Edge); 13] = [ - ("Requires", Edge::Pulls), - ("Requisite", Edge::Pulls), - ("BindsTo", Edge::Pulls), - ("Wants", Edge::Pulls), - ("Upholds", Edge::Pulls), - ("Conflicts", Edge::Conflicts), - ("ConflictedBy", Edge::Conflicts), - ("RequiredBy", Edge::Stops), - ("BoundBy", Edge::Stops), - ("RequisiteOf", Edge::Stops), - ("ConsistsOf", Edge::Stops), - ("PropagatesStopTo", Edge::Stops), - ("StopPropagatedFrom", Edge::Stops), -]; - /// The properties naming what systemd does when a unit ends or its job times /// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; @@ -653,62 +638,24 @@ const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction" /// The properties naming units systemd starts when a unit ends. const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; -/// The target an emergency action runs, `None` for `none`. An action this -/// check does not know is an error: the unit is not checked, so not run. -fn action_target(action: &str) -> Result> { - let base = ["-force", "-immediate"] - .iter() - .find_map(|suffix| action.strip_suffix(suffix)) - .unwrap_or(action); - Ok(match base { - "" | "none" => None, - "reboot" => Some("reboot.target"), - "poweroff" => Some("poweroff.target"), - "halt" => Some("halt.target"), - "kexec" => Some("kexec.target"), - "soft-reboot" => Some("soft-reboot.target"), - "exit" => Some("exit.target"), - _ => bail!("a unit has an emergency action this check does not know"), - }) -} - /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { pub(crate) id: String, pub(crate) names: Vec, pub(crate) not_found: bool, - pub(crate) running: bool, - /// `DefaultDependencies=no`: boot, recovery and shutdown machinery. - pub(crate) early_boot: bool, - /// What systemd starts when this unit ends or times out: the targets of - /// its emergency actions and its `OnFailure=` and `OnSuccess=` units. - pub(crate) fires: Vec, - pub(crate) pulls: Vec, - pub(crate) conflicts: Vec, - pub(crate) stops: Vec, + /// The properties that make the unit act when it ends: each emergency + /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. + pub(crate) on_end: Vec, + /// The units bound to this one at runtime (`BoundBy=`): systemd stops + /// them when this unit stops, whatever the job mode. + pub(crate) bound_by: Vec, } impl UnitFacts { fn identities(&self) -> impl Iterator { std::iter::once(&self.id).chain(self.names.iter()) } - - fn along(&self, edge: Edge) -> &[String] { - match edge { - Edge::Pulls => &self.pulls, - Edge::Conflicts => &self.conflicts, - Edge::Stops => &self.stops, - } - } - - fn along_mut(&mut self, edge: Edge) -> &mut Vec { - match edge { - Edge::Pulls => &mut self.pulls, - Edge::Conflicts => &mut self.conflicts, - Edge::Stops => &mut self.stops, - } - } } #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -765,126 +712,31 @@ fn refused(unit: &str, via: Via) -> Refused { } } -/// The units systemd would touch for one order, read from `systemctl show`. -struct Transaction<'a> { - systemd: &'a dyn Systemd, - facts: BTreeMap, -} - -impl Transaction<'_> { - fn facts(&mut self, unit: &str) -> Result<&UnitFacts> { - if !self.facts.contains_key(unit) { - ensure!( - self.facts.len() < CLOSURE_LIMIT, - "the units an order would reach are too many to check" - ); - let facts = self.systemd.show(unit)?; - ensure!(!facts.not_found, "systemd has no such unit to check"); - self.facts.insert(unit.to_owned(), facts); - } - Ok(&self.facts[unit]) - } - - /// Every unit reached from `seeds` along `edge`, seeds included. - fn reach(&mut self, seeds: &BTreeSet, edge: Edge) -> Result> { - let mut reached = BTreeSet::new(); - let mut queue: VecDeque = seeds.iter().cloned().collect(); - while let Some(next) = queue.pop_front() { - if !reached.insert(next.clone()) { - continue; - } - queue.extend(self.facts(&next)?.along(edge).iter().cloned()); - } - Ok(reached) - } - - /// Every unit the emergency actions and `OnFailure=`/`OnSuccess=` of - /// `touched` would start, with what each pulls in, and in turn what those - /// that are not running would set off. A touched unit ends, so it fires. - fn fired(&mut self, touched: &BTreeSet) -> Result> { - let mut fired = BTreeSet::new(); - let mut seen = BTreeSet::new(); - let mut queue: VecDeque = touched.iter().cloned().collect(); - while let Some(unit) = queue.pop_front() { - if !seen.insert(unit.clone()) { - continue; - } - let seeds: BTreeSet = self.facts(&unit)?.fires.iter().cloned().collect(); - for reached in self.reach(&seeds, Edge::Pulls)? { - if fired.insert(reached.clone()) && !self.facts(&reached)?.running { - queue.push_back(reached); - } - } - } - Ok(fired) - } - - /// Every unit a start of `started` stops through `Conflicts=`, and what - /// that stop takes down with it. - fn conflicted(&mut self, started: &BTreeSet) -> Result> { - let mut seeds = BTreeSet::new(); - for unit in started { - seeds.extend(self.facts(unit)?.along(Edge::Conflicts).iter().cloned()); - } - self.reach(&seeds, Edge::Stops) - } -} - -/// Would `verb` on `unit` reach an always-refused unit: by its name, by an -/// alias it resolves from, or through the transaction systemd builds for it. -/// A start pulls in what the unit requires or wants and stops what conflicts; -/// a stop takes down what depends on the unit; a restart does both. A unit -/// the transaction would stop is refused when it is refused; a unit it would -/// start is refused when it is refused and not running. Every unit in the -/// closure must be one systemd can show. +/// Would `verb` on `unit` be refused. systemd builds the transaction and the +/// order tells it to build one job (`--job-mode=ignore-requirements`), so this +/// reads only what that job mode cannot remove from the one named unit: the +/// name it answers to, what fires when it ends, and who is bound to it. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); } - let mut transaction = Transaction { - systemd, - facts: BTreeMap::new(), - }; - if let Some(name) = transaction - .facts(unit)? - .identities() - .find(|name| is_refused_name(name)) + let facts = systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + for name in facts.identities() { + if is_refused_name(name) { + return Ok(Some(refused(name, Via::Alias))); + } + if is_host_name(name) { + return Ok(Some(refused(name, Via::Host))); + } + } + if verb != Verb::Reload && !facts.on_end.is_empty() { + return Ok(Some(refused(unit, Via::OnEnd))); + } + if matches!(verb, Verb::Stop | Verb::Restart) + && let Some(bound) = facts.bound_by.first() { - return Ok(Some(refused(name, Via::Alias))); - } - let root = BTreeSet::from([unit.to_owned()]); - let mut stopped = BTreeSet::new(); - let mut started = BTreeSet::new(); - match verb { - Verb::Reload => {} - Verb::Start => started = transaction.reach(&root, Edge::Pulls)?, - Verb::Stop => stopped = transaction.reach(&root, Edge::Stops)?, - Verb::Restart => { - stopped = transaction.reach(&root, Edge::Stops)?; - started = transaction.reach(&stopped, Edge::Pulls)?; - } - } - let mut touched = stopped.clone(); - for member in &started { - if !transaction.facts(member)?.running { - touched.insert(member.clone()); - } - } - started.extend(transaction.fired(&touched)?); - stopped.extend(transaction.conflicted(&started)?); - for (units, only_if_stopped) in [(&stopped, false), (&started, true)] { - for member in units { - let facts = transaction.facts(member)?; - if only_if_stopped && facts.running { - continue; - } - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { - return Ok(Some(refused(name, Via::Propagation))); - } - if only_if_stopped && facts.early_boot { - return Ok(Some(refused(&facts.id, Via::EarlyBoot))); - } - } + return Ok(Some(refused(bound, Via::Bound))); } Ok(None) } @@ -1378,13 +1230,14 @@ fn parse_names(value: &str) -> Vec { names } -/// The properties one `systemctl show` is asked for: identity, state, edges. +/// The properties one `systemctl show` is asked for: identity, what fires at +/// the unit's end, and who is bound to it. fn show_properties() -> String { - ["Id", "Names", "LoadState", "ActiveState", "DefaultDependencies"] + ["Id", "Names", "LoadState"] .into_iter() .chain(ACTIONS) .chain(ON_END) - .chain(EDGES.iter().map(|(property, _)| *property)) + .chain(["BoundBy"]) .collect::>() .join(",") } @@ -1399,17 +1252,16 @@ fn parse_show(output: &str) -> Result { "Id" => facts.id = value.to_owned(), "Names" => facts.names = parse_names(value), "LoadState" => facts.not_found = value == "not-found", - "ActiveState" => { - facts.running = matches!(value, "active" | "activating" | "reloading"); - } - "DefaultDependencies" => facts.early_boot = value == "no", - _ if ACTIONS.contains(&key) => facts.fires.extend(action_target(value)?.map(str::to_owned)), - _ if ON_END.contains(&key) => facts.fires.extend(parse_names(value)), - _ => { - if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { - facts.along_mut(*edge).extend(parse_names(value)); + "BoundBy" => facts.bound_by = parse_names(value), + _ if ACTIONS.contains(&key) => { + if !matches!(value, "" | "none") { + facts.on_end.push(key.to_owned()); } } + _ if ON_END.contains(&key) => { + facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); + } + _ => {} } } ensure!(!facts.id.is_empty(), "systemctl named no unit"); @@ -2031,11 +1883,11 @@ mod tests { const STEWARD: u32 = 1000; const CANARY: &str = "CANARY-7f3a91"; - /// What `systemctl show` prints for a loaded, active unit with no edges, + /// What `systemctl show` prints for a loaded unit that acts on nothing, /// plus the given properties (a given key replaces its default). fn show_text(id: &str, properties: &[(&str, &str)]) -> String { let mut text = format!("Id={id}\n"); - for (key, default) in [("Names", id), ("LoadState", "loaded"), ("ActiveState", "active")] { + for (key, default) in [("Names", id), ("LoadState", "loaded")] { if !properties.iter().any(|(given, _)| *given == key) { text += &format!("{key}={default}\n"); } @@ -2049,11 +1901,13 @@ mod tests { struct FakeSystemd { units: Mutex>, runs: Mutex>, + shows: Mutex>, report: Mutex, } impl Systemd for FakeSystemd { fn show(&self, unit: &str) -> Result { + self.shows.lock().unwrap().push(unit.into()); let known = self.units.lock().unwrap().get(unit).cloned(); parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) } @@ -2119,6 +1973,7 @@ mod tests { let systemd = Arc::new(FakeSystemd { units: Mutex::new(BTreeMap::new()), runs: Mutex::new(Vec::new()), + shows: Mutex::new(Vec::new()), report: Mutex::new(RunReport { exit_code: Some(0), ..RunReport::default() @@ -2426,7 +2281,6 @@ mod tests { "persona-matrix-mouth.service", "mind-heartbeat.service", "apt-daily.timer", - "multi-user.target", // Near misses: the prefix is the rule, not a substring or a stem. "midunn-x.service", "not-idunn-x.service", @@ -2443,14 +2297,17 @@ mod tests { assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); let (transient, step) = h.runs().pop().unwrap(); assert!(transient.starts_with("idunn-order-")); - assert_eq!(step.argv, ["systemctl", verb.as_str(), "--", unit]); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] + ); } } assert_eq!(h.runs().len(), units.len() * 4); } #[test] - fn refused_set_is_refused_by_name_alias_and_propagation() { + fn refused_set_is_refused_by_name_and_alias() { let h = harness(); h.describe_unit( "backup-name.service", @@ -2463,21 +2320,6 @@ mod tests { "quiet-alias.service", &[("Names", "quiet-alias.service mind-persona@void.service")], ); - // fuse <- wrapper <- (bound by) mind-self@eureka - h.describe("fuse.service", &[("RequiredBy", "wrapper.service")]); - h.describe("wrapper.service", &[("BoundBy", "mind-self@eureka.service")]); - h.describe("stack.target", &[("ConsistsOf", "mind-persona@void.service")]); - // A cycle that reaches an alias of a refused unit on its second hop. - h.describe("loop-a.service", &[("RequiredBy", "loop-b.service")]); - h.describe("loop-b.service", &[("RequiredBy", "loop-a.service hidden.service")]); - h.describe_unit( - "hidden.service", - show_text( - "idunn-orders.service", - &[("Names", "idunn-orders.service hidden.service")], - ), - ); - let direct = [ "idunn-yggdrasil.service", "idunn-orders.service", @@ -2528,31 +2370,7 @@ mod tests { ); } } - for verb in [Verb::Stop, Verb::Restart] { - for (unit, hit) in [ - ("fuse.service", "mind-self@eureka.service"), - ("stack.target", "mind-persona@void.service"), - ("loop-a.service", "idunn-orders.service"), - ] { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - assert_eq!( - h.outcome(&id), - Outcome::Refused(Refused::AlwaysRefused { - unit: hit.into(), - via: Via::Propagation - }), - "{verb:?} {unit}" - ); - } - } assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); - // start and reload take nothing down, so the same units are not refused. - for verb in [Verb::Start, Verb::Reload] { - let id = h.accept(h.issue_unit(verb, "fuse.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran); - } // Every refusal is recorded and has a result post due. let ledger = h.ctx.store.ledger().unwrap(); let refusals = ledger @@ -2560,7 +2378,7 @@ mod tests { .iter() .filter(|record| matches!(record.outcome, Outcome::Refused(_))) .count(); - assert_eq!(refusals, direct.len() * 4 + 8 + 6); + assert_eq!(refusals, direct.len() * 4 + 8); let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); let results = due .iter() @@ -2568,93 +2386,7 @@ mod tests { .count(); assert_eq!(results, refusals); // A refused order never started. - assert!(ledger.starts.len() == 2); - } - - /// Yggdrasil's `systemctl show` shapes (systemd 257, 2026-11-01), cut down - /// to the units an order can reach: seven refused units all conflict with - /// shutdown.target, and the shutdown services require it. - fn yggdrasil(h: &Harness, idunn_running: bool) { - let state = if idunn_running { "active" } else { "inactive" }; - let refused = [ - "idunn-yggdrasil.service", - "idunn-orders.service", - "idunn-vili-e0ada640.service", - "mind-self@eureka.service", - "mind-persona@void.service", - ]; - h.describe( - "shutdown.target", - &[ - ("ActiveState", "inactive"), - ( - "RequiredBy", - "systemd-soft-reboot.service systemd-reboot.service systemd-poweroff.service", - ), - ( - "ConflictedBy", - "nginx.service idunn-yggdrasil.service idunn-orders.service \ - idunn-vili-e0ada640.service mind-self@eureka.service mind-persona@void.service", - ), - ], - ); - for refused in refused { - h.describe( - refused, - &[ - ("Requires", "sysinit.target system.slice -.mount"), - ("Wants", "tmp.mount network-online.target"), - ("Conflicts", "shutdown.target"), - ("ActiveState", if refused == "idunn-yggdrasil.service" { state } else { "inactive" }), - ], - ); - } - h.describe("nginx.service", &[("Wants", "network-online.target"), ("Conflicts", "shutdown.target")]); - for (service, target) in [ - ("systemd-poweroff.service", "poweroff.target"), - ("systemd-reboot.service", "reboot.target"), - ("systemd-soft-reboot.service", "soft-reboot.target"), - ] { - h.describe( - service, - &[ - ("Requires", "shutdown.target final.target system.slice umount.target"), - ("RequiredBy", target), - ("ActiveState", "inactive"), - ], - ); - } - for (target, service) in [ - ("poweroff.target", "systemd-poweroff.service"), - ("reboot.target", "systemd-reboot.service"), - ("soft-reboot.target", "systemd-soft-reboot.service"), - ] { - h.describe(target, &[("Requires", service), ("ActiveState", "inactive")]); - } - h.describe( - "umount.target", - &[ - ("ActiveState", "inactive"), - ("RequiredBy", "systemd-poweroff.service systemd-reboot.service"), - ("ConflictedBy", "boot-efi.mount boot.mount \"blockdev@dev-vda2.target\""), - ], - ); - h.describe( - "multi-user.target", - &[ - ("Requires", "basic.target"), - ("Wants", "cron.service ssh.service idunn-yggdrasil.service"), - ("Conflicts", "shutdown.target rescue.target rescue.service"), - ], - ); - h.describe( - "voidbot.service", - &[ - ("Requires", "docker.service -.mount sysinit.target system.slice"), - ("Wants", "idunn-yggdrasil.service network-online.target"), - ("Conflicts", "shutdown.target"), - ], - ); + assert!(ledger.starts.is_empty()); } fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { @@ -2663,277 +2395,171 @@ mod tests { h.outcome(&id) } - fn hit(outcome: &Outcome) -> Option<&str> { + fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { match outcome { - Outcome::Refused(Refused::AlwaysRefused { unit, via: Via::Propagation }) => Some(unit.as_str()), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { + Some(unit.as_str()) + } _ => None, } } - #[test] - fn a_transaction_that_reaches_a_refused_unit_is_refused() { - let h = harness(); - yggdrasil(&h, true); - // The targets and services that power the host down reach every refused - // unit through Conflicts=, however many requirements away. - for (verb, unit) in [ - (Verb::Start, "shutdown.target"), - (Verb::Start, "poweroff.target"), - (Verb::Restart, "poweroff.target"), - (Verb::Start, "reboot.target"), - (Verb::Restart, "reboot.target"), - (Verb::Start, "soft-reboot.target"), - (Verb::Start, "systemd-soft-reboot.service"), - (Verb::Restart, "systemd-poweroff.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert!(hit(&outcome).is_some(), "{verb:?} {unit} was {outcome:?}"); - } - assert!(h.runs().is_empty(), "no refused order ran anything"); - // A stop of shutdown.target takes down what requires it and runs no start. - let outcome = outcome_of_unit(&h, Verb::Stop, "shutdown.target"); - assert_eq!(outcome, Outcome::Ran); - - // A start that wants a refused unit that is stopped would start it. - h.describe("idunn-yggdrasil.service", &[("ActiveState", "inactive"), ("Conflicts", "shutdown.target")]); - for (verb, unit) in [ - (Verb::Start, "multi-user.target"), - (Verb::Restart, "multi-user.target"), - (Verb::Start, "voidbot.service"), - (Verb::Restart, "voidbot.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert_eq!(hit(&outcome), Some("idunn-yggdrasil.service"), "{verb:?} {unit}"); - } - // The same orders start nothing while the refused unit runs. - h.describe("idunn-yggdrasil.service", &[("ActiveState", "active"), ("Conflicts", "shutdown.target")]); - for (verb, unit) in [ - (Verb::Start, "multi-user.target"), - (Verb::Restart, "voidbot.service"), - (Verb::Start, "voidbot.service"), - ] { - assert_eq!(outcome_of_unit(&h, verb, unit), Outcome::Ran, "{verb:?} {unit}"); - } - // A unit that wants only what is not refused runs, and so does a target - // that unmounts nothing a refused unit requires. - for verb in [Verb::Start, Verb::Restart, Verb::Stop, Verb::Reload] { - assert_eq!(outcome_of_unit(&h, verb, "nginx.service"), Outcome::Ran); - } - assert_eq!(outcome_of_unit(&h, Verb::Start, "umount.target"), Outcome::Ran); - } - - #[test] - fn the_closure_follows_every_edge_and_every_hop() { - let h = harness(); - // Start: three hops of Wants/Requires/Requisite/BindsTo/Upholds to a - // stopped refused unit. - h.describe("hop-1.service", &[("Wants", "hop-2.service")]); - h.describe("hop-2.service", &[("Requires", "hop-3.service")]); - h.describe("hop-3.service", &[("Requisite", "hop-4.service")]); - h.describe("hop-4.service", &[("BindsTo", "hop-5.service")]); - h.describe("hop-5.service", &[("Upholds", "idunn-orders.service")]); - h.describe("idunn-orders.service", &[("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "hop-1.service"); - assert_eq!(hit(&outcome), Some("idunn-orders.service")); - // Conflicts of a pulled-in unit stop it and what requires it: the - // conflict, then two hops of reverse requirement, to a refused unit. - h.describe("rival.service", &[("Conflicts", "victim.service")]); - h.describe("victim.service", &[("RequiredBy", "mid.service")]); - h.describe("mid.service", &[("BoundBy", "mind-self@eureka.service")]); - h.describe("pulls-rival.service", &[("Requires", "rival.service")]); - for unit in ["rival.service", "pulls-rival.service"] { - let outcome = outcome_of_unit(&h, Verb::Start, unit); - assert_eq!(hit(&outcome), Some("mind-self@eureka.service"), "{unit}"); - } - // The reverse Conflicts edge: a unit conflicted by a refused one. - h.describe("incumbent.service", &[("ConflictedBy", "mind-persona@void.service")]); - let outcome = outcome_of_unit(&h, Verb::Start, "incumbent.service"); - assert_eq!(hit(&outcome), Some("mind-persona@void.service")); - // Stop and restart: every stop edge, one at a time. - for (property, reached) in [ - ("RequiredBy", "idunn-vili-x.service"), - ("BoundBy", "idunn-vili-x.service"), - ("RequisiteOf", "idunn-vili-x.service"), - ("ConsistsOf", "idunn-vili-x.service"), - ("PropagatesStopTo", "mind-persona@void.service"), - ("StopPropagatedFrom", "mind-persona@void.service"), - ] { - let name = format!("edge-{property}.service"); - h.describe(&name, &[(property, reached)]); - for verb in [Verb::Stop, Verb::Restart] { - let outcome = outcome_of_unit(&h, verb, &name); - assert_eq!(hit(&outcome), Some(reached), "{verb:?} {property}"); - } - } - // A restart starts what its stopped dependents pull in. - h.describe("base.service", &[("RequiredBy", "dependent.service")]); - h.describe("dependent.service", &[("Wants", "idunn-orders.service")]); - let outcome = outcome_of_unit(&h, Verb::Restart, "base.service"); - assert_eq!(hit(&outcome), Some("idunn-orders.service")); - assert_eq!(outcome_of_unit(&h, Verb::Stop, "base.service"), Outcome::Ran); - assert_eq!(outcome_of_unit(&h, Verb::Reload, "base.service"), Outcome::Ran); - } - #[test] fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { let h = harness(); // systemctl show quotes a name with a backslash and doubles the backslash. h.describe( "huginn.service", - &[("RequiredBy", "\"mind-self@foo\\\\x2dbar.service\"")], + &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], ); let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); - assert_eq!(hit(&outcome), Some("mind-self@foo\\x2dbar.service")); - h.describe("quiet.service", &[("Wants", "\"idunn-a\\\\x2db.service\" other.service")]); - h.describe("idunn-a\\x2db.service", &[("ActiveState", "inactive")]); + assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); + h.describe( + "quiet.service", + &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], + ); let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); - assert_eq!(hit(&outcome), Some("idunn-a\\x2db.service")); + assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); assert!(h.runs().is_empty()); } #[test] - fn a_unit_whose_end_reboots_the_host_is_refused_by_its_properties() { + fn unit_orders_run_as_one_job() { let h = harness(); - yggdrasil(&h, true); - // Each emergency action property alone, in each of systemd's spellings, - // on a unit no name list knows: every verb that starts or stops it. - for (property, action) in [ - ("SuccessAction", "reboot"), - ("FailureAction", "reboot"), - ("StartLimitAction", "poweroff-force"), - ("JobTimeoutAction", "reboot-immediate"), - ("SuccessAction", "poweroff"), - ] { - let name = format!("storagetm-{property}.service"); - h.describe(&name, &[(property, action), ("ActiveState", "inactive")]); - for verb in [Verb::Start, Verb::Restart, Verb::Stop] { - let outcome = outcome_of_unit(&h, verb, &name); - assert!(hit(&outcome).is_some(), "{verb:?} {property}={action} {outcome:?}"); - } + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); + let (_, step) = h.runs().pop().unwrap(); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] + ); } - // OnFailure= and OnSuccess= start the units they name, and what those - // set off in turn. - for property in ["OnFailure", "OnSuccess"] { - let name = format!("handled-{property}.service"); - h.describe(&name, &[(property, "reboot.target"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, &name); - assert!(hit(&outcome).is_some(), "{property} {outcome:?}"); - } - h.describe("on-failure-chain.service", &[("OnFailure", "handler.service"), ("ActiveState", "inactive")]); - h.describe("handler.service", &[("FailureAction", "reboot"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "on-failure-chain.service"); - assert!(hit(&outcome).is_some(), "{outcome:?}"); - // A unit a start pulls in fires its action too. - h.describe("pulls-storagetm.service", &[("Wants", "storagetm.service")]); - h.describe("storagetm.service", &[("SuccessAction", "reboot"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "pulls-storagetm.service"); - assert!(hit(&outcome).is_some(), "{outcome:?}"); - // A running unit the order does not stop is not ended by it, and an - // action of none ends nothing. - h.describe("pulls-running.service", &[("Wants", "running-action.service")]); - h.describe("running-action.service", &[("SuccessAction", "reboot")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "pulls-running.service"), Outcome::Ran); - h.describe("quiet-actions.service", &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", "")]); - for verb in [Verb::Start, Verb::Restart, Verb::Stop] { - assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); - } - // An action this check does not know is not checked, so it does not run. - let ran = h.runs().len(); - h.describe("odd-action.service", &[("SuccessAction", "implode")]); - let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); - execute(&h.ctx, &id).unwrap(); - assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); - assert_eq!(h.runs().len(), ran); } #[test] - fn every_emergency_action_runs_the_target_it_names() { - for (action, target) in [ - ("none", None), - ("", None), - ("reboot", Some("reboot.target")), - ("reboot-force", Some("reboot.target")), - ("reboot-immediate", Some("reboot.target")), - ("poweroff", Some("poweroff.target")), - ("poweroff-force", Some("poweroff.target")), - ("poweroff-immediate", Some("poweroff.target")), - ("halt", Some("halt.target")), - ("halt-force", Some("halt.target")), - ("kexec", Some("kexec.target")), - ("kexec-immediate", Some("kexec.target")), - ("soft-reboot", Some("soft-reboot.target")), - ("soft-reboot-force", Some("soft-reboot.target")), - ("exit", Some("exit.target")), - ("exit-force", Some("exit.target")), - ] { - assert_eq!(action_target(action).unwrap(), target, "{action}"); - } - assert!(action_target("implode").is_err()); - } - - #[test] - fn a_stopped_boot_or_recovery_unit_is_never_started() { + fn the_rule_reads_only_the_named_unit() { let h = harness(); - yggdrasil(&h, true); - // rescue.service ends by isolating default.target, which stops every - // unit outside it; no edge says so, its DefaultDependencies=no does. + // Whatever the unit lists, no other unit is looked at. h.describe( - "rescue.service", - &[("DefaultDependencies", "no"), ("ActiveState", "inactive")], + "plain.service", + &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], ); - h.describe("rescue.target", &[("Requires", "rescue.service"), ("ActiveState", "inactive")]); - h.describe("pulls-rescue.service", &[("Wants", "rescue.target")]); - for (verb, unit) in [ - (Verb::Start, "rescue.service"), - (Verb::Restart, "rescue.service"), - (Verb::Start, "rescue.target"), - (Verb::Restart, "rescue.target"), - (Verb::Start, "pulls-rescue.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert!( - matches!( - outcome, - Outcome::Refused(Refused::AlwaysRefused { ref unit, via: Via::EarlyBoot }) - if unit == "rescue.service" - ), - "{verb:?} {unit} was {outcome:?}" + h.describe("other.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); + } + + #[test] + fn a_unit_that_acts_when_it_ends_is_refused() { + let h = harness(); + let properties = [ + ("SuccessAction", "reboot"), + ("FailureAction", "poweroff-force"), + ("StartLimitAction", "soft-reboot-immediate"), + ("JobTimeoutAction", "exit"), + ("FailureAction", "a-spelling-nobody-knows"), + ("OnFailure", "rescue.service"), + ("OnSuccess", "cleanup.service"), + ]; + for (property, value) in properties { + h.describe("ender.service", &[(property, value)]); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "ender.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "ender.service".into(), + via: Via::OnEnd + }), + "{verb:?} {property}={value}" + ); + } + assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); + } + // `none` and an empty list say the unit does nothing at its end. + h.describe( + "calm.service", + &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.runs().len(), 11); + } + + #[test] + fn host_names_are_refused() { + let h = harness(); + let names = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", + ]; + h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); + for verb in all_verbs() { + for name in names { + assert_eq!( + outcome_of_unit(&h, verb, name), + Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), + "{verb:?} {name}" + ); + } + assert_eq!( + outcome_of_unit(&h, verb, "sleepy.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "systemd-hibernate.service".into(), + via: Via::Host + }), + "{verb:?}" ); } assert!(h.runs().is_empty()); - // Stopping it, or starting one whose defaults are on, is an ordinary order. - assert_eq!(outcome_of_unit(&h, Verb::Stop, "rescue.service"), Outcome::Ran); - h.describe("ordinary.service", &[("DefaultDependencies", "yes"), ("ActiveState", "inactive")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "ordinary.service"), Outcome::Ran); - // A boot unit that is already running is left alone by a start. - h.describe("journald.service", &[("DefaultDependencies", "no")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); + assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); + } + + #[test] + fn a_unit_others_are_bound_to_only_starts() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service second.service")]); + for verb in [Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "base.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "first.service".into(), + via: Via::Bound + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Start, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); + } } #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); h.describe("ghost.service", &[("LoadState", "not-found")]); - h.describe("pulls-ghost.service", &[("Wants", "ghost.service")]); - h.describe("needed-by-ghost.service", &[("RequiredBy", "ghost.service")]); - for (verb, unit) in [ - (Verb::Start, "ghost.service"), - (Verb::Start, "pulls-ghost.service"), - (Verb::Stop, "needed-by-ghost.service"), - (Verb::Restart, "needed-by-ghost.service"), - ] { - let id = h.accept(h.issue_unit(verb, unit)); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "ghost.service")); execute(&h.ctx, &id).unwrap(); assert!( matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), - "{verb:?} {unit}" + "{verb:?}" ); } assert!(h.runs().is_empty()); } #[test] - fn globs_slices_and_scopes_are_not_units() { + fn globs_slices_scopes_and_targets_are_not_units() { let h = harness(); let bad = [ "idunn*", @@ -2943,6 +2569,8 @@ mod tests { "mind-self@*.service", "mind-[s]elf@eureka.service", "system.slice", + "poweroff.target", + "multi-user.target", "user.slice", "session-1.scope", "init.scope", @@ -3373,34 +3001,24 @@ mod tests { #[test] fn systemctl_show_output_is_read_into_facts() { let facts = parse_show( - "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=inactive\n\ - Requires=r.service \"q\\\\x2dr.slice\" -.mount\nRequisite=t.service\nWants=w.service\n\ - BindsTo=b.service\nUpholds=u.service\nConflicts=x.target\nConflictedBy=y.service\n\ - RequiredBy=c.service\nBoundBy=\nRequisiteOf=ro.service\nConsistsOf=d.target e.service\n\ - PropagatesStopTo=p.service\nStopPropagatedFrom=f.service\nOther=x\n", + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ + SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ + OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ + Requires=r.service\nConflicts=x.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); assert_eq!(facts.names, ["a.service", "b.service"]); - assert!(!facts.not_found && !facts.running); + assert!(!facts.not_found); + assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); // A quoted name holds one backslash, not two and not a quote mark. - assert_eq!( - facts.pulls, - ["r.service", "q\\x2dr.slice", "-.mount", "t.service", "w.service", "b.service", "u.service"] - ); - assert_eq!(facts.conflicts, ["x.target", "y.service"]); - assert_eq!( - facts.stops, - ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] - ); + assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); assert!(parse_show("Names=a.service\n").is_err()); // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), - "Id,Names,LoadState,ActiveState,DefaultDependencies,SuccessAction,FailureAction,\ - StartLimitAction,JobTimeoutAction,OnFailure,OnSuccess,Requires,Requisite,BindsTo,\ - Wants,Upholds,Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ - PropagatesStopTo,StopPropagatedFrom" + "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ + OnFailure,OnSuccess,BoundBy" ); // A backslash outside quotes is a backslash; inside, it escapes one character. assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); @@ -3408,17 +3026,7 @@ mod tests { assert!(parse_names(" ").is_empty()); let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); - for (state, running) in [ - ("active", true), - ("activating", true), - ("reloading", true), - ("inactive", false), - ("deactivating", false), - ("failed", false), - ] { - let facts = parse_show(&format!("Id=a.service\nActiveState={state}\n")).unwrap(); - assert_eq!(facts.running, running, "{state}"); - } + } } #[test] From 8ebe0208a65bb2191acfeec7d22ffe337795a385 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:41:56 +0100 Subject: [PATCH 194/196] Steward orders: pin the one-job rule on the real unit set and a real user manager Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 118 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index d5fdcfe..7f8ced7 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3027,7 +3027,6 @@ mod tests { let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); } - } #[test] fn the_store_refuses_a_second_record_and_a_foreign_document() { @@ -3846,4 +3845,121 @@ mod tests { assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() } + + fn host_refusal(verb: Verb, unit: &str) -> Option { + refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) + } + + #[test] + #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] + fn real_host_unit_set() { + for name in [ + "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", + "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", + "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", + "wg-quick@wg-gamecult", "cron", "apt-daily", + ] { + let unit = format!("{name}.service"); + for verb in all_verbs() { + assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); + } + } + for name in [ + "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", + "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", + "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", + "systemd-storagetm", + ] { + let unit = format!("{name}.service"); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + host_refusal(verb, &unit), + Some(refused(&unit, Via::OnEnd)), + "{verb:?} {unit}" + ); + } + } + for unit in HOST_NAMES { + for verb in all_verbs() { + assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); + } + } + for unit in [ + "systemd-networkd.service", + "user-runtime-dir@1000.service", + "user-runtime-dir@105.service", + "user-runtime-dir@978.service", + ] { + assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); + assert!( + matches!( + host_refusal(Verb::Stop, unit), + Some(Refused::AlwaysRefused { via: Via::Bound, .. }) + ), + "{unit}" + ); + } + let listing = Command::new("systemctl") + .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) + .output() + .unwrap(); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let units: Vec<&str> = listing + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|unit| validate_unit(unit).is_ok()) + .collect(); + for verb in all_verbs() { + let mut counts = BTreeMap::new(); + for unit in &units { + let key = match refusal_for(&HostSystemd, unit, verb) { + Ok(None) => "ok".to_owned(), + Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), + Ok(Some(other)) => format!("{other:?}"), + Err(_) => "unresolved".to_owned(), + }; + *counts.entry(key).or_insert(0usize) += 1; + } + println!("{verb:?} over {} units: {counts:?}", units.len()); + } + } + + #[test] + #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] + fn one_job_argv_against_a_real_manager() { + let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); + std::fs::create_dir_all(&directory).unwrap(); + let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); + let body = "[Service]\nExecStart=/bin/sleep 600\n"; + std::fs::write(directory.join(a), body).unwrap(); + std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); + let user = |args: &[&str]| { + let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); + (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) + }; + let outcome = std::panic::catch_unwind(|| { + let one_job = |verb: Verb, unit: &str| { + let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); + let mut argv = step.argv.clone(); + argv.insert(1, "--user".into()); + let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); + assert!(status.success(), "{verb:?} {unit}"); + }; + assert!(user(&["daemon-reload"]).0); + // A start of b leaves a running, though b conflicts with it. + assert!(user(&["start", a]).0); + one_job(Verb::Start, b); + assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); + // A restart of b, with both running, leaves a running. + one_job(Verb::Restart, b); + assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); + }); + user(&["stop", a, b]); + std::fs::remove_file(directory.join(a)).unwrap(); + std::fs::remove_file(directory.join(b)).unwrap(); + user(&["daemon-reload"]); + outcome.unwrap(); + } } From fb1e7b5dd9a0e9f0a77febd66cb43d5ce2f995f4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:42:52 +0100 Subject: [PATCH 195/196] Steward orders: README states the one-job rule; pin the end-to-end argv Co-Authored-By: Claude Opus 5.5 --- README.md | 35 +++++++++++++++++------------------ src/steward_orders.rs | 2 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 28c1e94..3a695a4 100644 --- a/README.md +++ b/README.md @@ -73,24 +73,23 @@ idunn order show # status and outp idunn order list ``` -A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' -own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and -read from no configuration. An order is refused, recorded and posted when its -name, an alias it resolves from, or the transaction systemd would build for it -reaches one of them: a start pulls in what the unit requires or wants (a -refused unit that is already running is left alone) and stops what conflicts -with it, a stop takes down what depends on it, and a restart does both. The -transaction is read from `systemctl show`, so `start poweroff.target` is refused -because every refused unit conflicts with `shutdown.target`, not because -targets are banned. The units a start or stop ends or times out are followed too: -the target each of `SuccessAction=`, `FailureAction=`, `StartLimitAction=` and -`JobTimeoutAction=` runs (`reboot`, `poweroff`, `halt`, `kexec`, `soft-reboot`, -`exit`, with `-force` and `-immediate`), and the units `OnFailure=` and -`OnSuccess=` name, with what each pulls in; an action it does not know fails the -order. A start refuses a stopped `DefaultDependencies=no` unit (boot and recovery -machinery, such as `rescue.service`, which ends by isolating the default -target), whatever pulls it in. A unit `systemctl show` cannot resolve fails the order. -Slices, scopes and globs are not units an order can name. +A unit order acts on exactly the unit it names: every verb runs +`systemctl --job-mode=ignore-requirements -- `, so systemd builds a +transaction of one job. It does not start what the unit requires or restart what +is part of it: order the dependency first, and order `postgresql@16-main.service` +rather than the `postgresql.service` wrapper. Targets are not orderable. A unit +order may act on any unit except Idunn's own (`idunn-*`) and the minds' own +(`mind-self@*`, `mind-persona@*`), and the units that end or take the host +(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, +`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled +into the binary and read from no configuration. An order is refused, recorded +and posted when the ordered unit, or an alias it resolves from, is one of them; +when it has an emergency action (`SuccessAction=`, `FailureAction=`, +`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` +or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another +unit is `BoundBy=` it, because systemd stops that unit with it whatever the job +mode. A unit `systemctl show` cannot resolve fails the order. Slices, scopes, +targets and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with `--no-remove`, each name is passed as an exact-name pattern so `.` and `+` diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 7f8ced7..3b40c88 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3742,7 +3742,7 @@ mod tests { let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); assert!( run_log.contains(&format!( - "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart -- nginx.service", + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", order_uuid(&first) )), "{run_log}" From 77e2295e9705c2a9eebb46f27ac40f34b7b49c2a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 23:32:30 +0100 Subject: [PATCH 196/196] Steward orders: refuse Upholds and ConsistsOf units, pin the rule's gaps, README names the residuals A unit with Upholds= is refused for start and restart (systemd would start the upheld unit in a full transaction). A wrapper with ConsistsOf= is refused for stop, restart and reload with a typed Refused::Wrapper naming the first member, so it cannot report ran while touching nothing. Tests pin empty BoundBy, a single show per refusal, no show for a refused name and each posted reason. README: the one-job rule's residual full transactions. Removes the unused BTreeMap import and the never_loop in the host actuator test. Co-Authored-By: Claude Opus 5.5 --- README.md | 15 ++++- src/host_actuator.rs | 2 +- src/steward_orders.rs | 126 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3a695a4..8c6935c 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,16 @@ idunn order show # status and outp idunn order list ``` -A unit order acts on exactly the unit it names: every verb runs +A unit order submits exactly one job, for the unit it names: every verb runs `systemctl --job-mode=ignore-requirements -- `, so systemd builds a transaction of one job. It does not start what the unit requires or restart what is part of it: order the dependency first, and order `postgresql@16-main.service` -rather than the `postgresql.service` wrapper. Targets are not orderable. A unit +rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is +not: systemd still runs full transactions of its own for `Restart=` after a +failure, for a `StopWhenUnneeded=` dependency left without a user, and for a +`Conflicts=` partner of a unit that fails, so an ordered stop of a unit +that something restarts does not last, and a start of a `Restart=always` +unit can report failure and be active seconds later. A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' own (`mind-self@*`, `mind-persona@*`), and the units that end or take the host (`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, @@ -88,7 +93,11 @@ when it has an emergency action (`SuccessAction=`, `FailureAction=`, `StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another unit is `BoundBy=` it, because systemd stops that unit with it whatever the job -mode. A unit `systemctl show` cannot resolve fails the order. Slices, scopes, +mode; for a start or restart, when it has an `Upholds=` unit, which systemd +starts in a full transaction whatever the job mode; or, for a stop, restart or +reload, when it has a `ConsistsOf=` member, a wrapper such as +`postgresql.service` whose verbs would exit 0 and touch nothing. A unit +`systemctl show` cannot resolve fails the order. Slices, scopes, targets and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with diff --git a/src/host_actuator.rs b/src/host_actuator.rs index f7bf4c7..8fd776c 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1151,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - while let Some(frame) = client.receive_once()? { + if let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 3b40c88..d4be1ee 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,6 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::BTreeMap; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -391,12 +390,16 @@ pub(crate) enum Via { Host, OnEnd, Bound, + Upholds, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub(crate) enum Refused { AlwaysRefused { unit: String, via: Via }, + /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload + /// of the wrapper exits 0 and leaves the member alone. + Wrapper { member: String }, Braked, /// The order's principal or issuing uid is not the steward bound now. NotSteward, @@ -437,8 +440,13 @@ impl Outcome { Via::Host => "host", Via::OnEnd => "on-end", Via::Bound => "bound", + Via::Upholds => "upholds", } ), + Outcome::Refused(Refused::Wrapper { member }) => format!( + "refused: wrapper unit, it only wraps {}", + escape(member.as_bytes()) + ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), Outcome::Refused(Refused::NotSteward) => { "refused: the issuer is not the current steward".into() @@ -650,6 +658,12 @@ pub(crate) struct UnitFacts { /// The units bound to this one at runtime (`BoundBy=`): systemd stops /// them when this unit stops, whatever the job mode. pub(crate) bound_by: Vec, + /// The units this one upholds (`Upholds=`): systemd starts them in a full + /// transaction whenever this unit is active, whatever the job mode. + pub(crate) upholds: Vec, + /// The units this one groups (`ConsistsOf=`): the verbs reach them only + /// through propagation, which the job mode removes. + pub(crate) consists_of: Vec, } impl UnitFacts { @@ -715,7 +729,8 @@ fn refused(unit: &str, via: Via) -> Refused { /// Would `verb` on `unit` be refused. systemd builds the transaction and the /// order tells it to build one job (`--job-mode=ignore-requirements`), so this /// reads only what that job mode cannot remove from the one named unit: the -/// name it answers to, what fires when it ends, and who is bound to it. +/// name it answers to, what fires when it ends, who is bound to it, what it upholds and +/// what it only wraps. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); @@ -738,6 +753,16 @@ fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result Vec { } /// The properties one `systemctl show` is asked for: identity, what fires at -/// the unit's end, and who is bound to it. +/// the unit's end, who is bound to it, and what it upholds or groups. fn show_properties() -> String { ["Id", "Names", "LoadState"] .into_iter() .chain(ACTIONS) .chain(ON_END) - .chain(["BoundBy"]) + .chain(["BoundBy", "Upholds", "ConsistsOf"]) .collect::>() .join(",") } @@ -1253,6 +1278,8 @@ fn parse_show(output: &str) -> Result { "Names" => facts.names = parse_names(value), "LoadState" => facts.not_found = value == "not-found", "BoundBy" => facts.bound_by = parse_names(value), + "Upholds" => facts.upholds = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), _ if ACTIONS.contains(&key) => { if !matches!(value, "" | "none") { facts.on_end.push(key.to_owned()); @@ -2543,6 +2570,91 @@ mod tests { } } + #[test] + fn a_unit_that_upholds_others_is_refused_where_it_would_start() { + let h = harness(); + h.describe("holder.service", &[("Upholds", "held.service")]); + for verb in [Verb::Start, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "holder.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "holder.service".into(), + via: Via::Upholds + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { + let h = harness(); + h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); + for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { + assert_eq!( + outcome_of_unit(&h, verb, "wrapper.service"), + Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); + } + + #[test] + fn empty_dependency_lists_name_no_one() { + let h = harness(); + // systemctl prints every property it is asked for, empty or not. + h.describe( + "alone.service", + &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service")]); + h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); + assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); + h.systemd.shows.lock().unwrap().clear(); + assert_eq!( + hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), + Some("idunn-ghost.service") + ); + assert!(h.systemd.shows.lock().unwrap().is_empty()); + } + + #[test] + fn each_refusal_is_posted_under_its_own_reason() { + for (via, word) in [ + (Via::Name, "name"), + (Via::Alias, "alias"), + (Via::Host, "host"), + (Via::OnEnd, "on-end"), + (Via::Bound, "bound"), + (Via::Upholds, "upholds"), + ] { + assert_eq!( + Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) + .describe(), + format!("refused: always-refused unit u.service reached by {word}") + ); + } + assert_eq!( + Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), + "refused: wrapper unit, it only wraps m.service" + ); + } + #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); @@ -3004,7 +3116,7 @@ mod tests { "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ - Requires=r.service\nConflicts=x.service\nOther=x\n", + Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); @@ -3013,12 +3125,14 @@ mod tests { assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); // A quoted name holds one backslash, not two and not a quote mark. assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); + assert_eq!(facts.upholds, ["u.service"]); + assert_eq!(facts.consists_of, ["m.service", "n.service"]); assert!(parse_show("Names=a.service\n").is_err()); // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ - OnFailure,OnSuccess,BoundBy" + OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" ); // A backslash outside quotes is a backslash; inside, it escapes one character. assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]);