From d91ecba759b898114332c0db02cfd60a00fd90ea Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:07:13 +0200 Subject: [PATCH 001/122] Cut 2 negative tests, failing at 6138329: restart beside a building deployment, profile freeze over a crash loop a_dead_incumbent_is_restarted_alongside_a_deployment_that_is_building and a_profile_deploy_freezes_over_a_crash_looping_target use only base API and pin the states cut 2 (handover-phase-split) ends. They fail until the predicate split and the open gates land. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 100 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e077373..d67d9d3 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16091,6 +16091,99 @@ mod tests { Ok(()) } + /// The 2026-10-18 shape at the engine: a dead incumbent with a deployment + /// building over it. Continuity restarts the incumbent beside the build + /// and neither touches the other (handover-phase-split). + #[test] + fn a_dead_incumbent_is_restarted_alongside_a_deployment_that_is_building() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + let stored_deploy = |world: &EngineFixture| -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == deploy.transaction_id) + .context("the deployment left the live set") + }; + let before = stored_deploy(&world)?; + + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1, "no restart beside the building deployment"); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 2); + assert_eq!(stored_deploy(&world)?.envelope, before.envelope); + + for _ in 0..2 { + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + let after = stored_deploy(&world)?; + assert_eq!(after.envelope, before.envelope); + assert!(after.value.pre_fencing_abort.is_none()); + } + Ok(()) + } + + /// A `profile:` command over two targets, one of them crash-looping with a + /// live continuity, freezes at once: freeze refuses only a second live + /// deployment of a target (follow-up profile-deploy-joint-gap). + #[test] + fn a_profile_deploy_freezes_over_a_crash_looping_target() -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let continuity = ControlSnapshot::read(&world.state_store)?.transactions.remove(0); + + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + for target in ["service", "other"] { + let paired = binding.replacen( + "target = \"service\"\n", + &format!("target = \"{target}\"\nprofiles = [\"pair\"]\n"), + 1, + ); + std::fs::write(world.root.join(format!("bindings/{target}.toml")), paired)?; + } + queue_deploy_of(&world, "up-pair", "profile:pair")?; + assert!(world.engine.freeze_one_queued_command()?, "the profile command stayed queued"); + + let live = ControlSnapshot::read(&world.state_store)?.transactions; + let mut frozen = live + .iter() + .filter(|stored| stored.value.command_id == "up-pair") + .map(|stored| (stored.value.ordinal, stored.value.target.as_str())) + .collect::>(); + frozen.sort(); + assert_eq!(frozen, [(0, "other"), (1, "service")]); + let kept = live + .iter() + .find(|stored| stored.value.transaction_id == continuity.value.transaction_id) + .context("the continuity left the live set")?; + assert_eq!(kept.envelope, continuity.envelope); + + // A live deployment of `service` still refuses a second command for it. + queue_deploy_of(&world, "up-service-second", "service")?; + assert!(!world.engine.freeze_one_queued_command()?); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| transaction.command_id != "up-service-second") + ); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -16923,11 +17016,16 @@ mod tests { ); std::fs::create_dir_all(world.root.join("bindings"))?; std::fs::write(world.root.join("bindings/service.toml"), binding)?; + queue_deploy_of(world, command_id, "service") + } + + /// Queue a Deploy command for `selector`; the bindings are the caller's. + fn queue_deploy_of(world: &EngineFixture, command_id: &str, selector: &str) -> Result<()> { let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), command_id: command_id.into(), kind: CommandKind::Deploy, - selector: "service".into(), + selector: selector.into(), requested_by: "operator".into(), requested_at_unix_millis: now_millis()?, }; From b4bb158707bd20ff56b4200910adfcf26a65724b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:11:36 +0200 Subject: [PATCH 002/122] Cut 2, commit 1: split the ownership predicate and the one-transaction-per-target validation owns_target_authority and blocks_new_target_mutation answered four questions with one phase-blind rule. They become is_live, owns_target (derived from the phase and takes_target_at) and drains_incumbent, each read by the consumer that asks that question: sibling order, validation, status, ports, freeze and the supervision gates. Validation now holds, per target, at most one owner, one live Deploy and one live Continuity, each refused with its own message, so a continuity beside a pre-handover deployment is readable. Freeze and the mint gate are still conservative (any live or draining transaction refuses), so no intermediate commit lets two transactions coexist unguarded. seeded_transaction mints a fresh command id when its default is taken, so a world can hold two of one kind. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 216 +++++++++++++++++++++++++++++++++---------- 1 file changed, 165 insertions(+), 51 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d67d9d3..16ea8d7 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,18 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// The phase whose entry is the handover of a target: Starting when the rollout +/// stops the incumbent first (it is stopped there by design), else Fencing. The +/// one spelling of the rule; a free function so the stop-then-start arm is +/// testable without a host-actuator fixture. +fn handover_phase(stops_incumbent_first: bool) -> DeploymentPhase { + if stops_incumbent_first { + DeploymentPhase::Starting + } else { + DeploymentPhase::Fencing + } +} + impl DeploymentTransaction { /// The one writer of `phase`. It stamps the update time and the phase /// deadline together, so a record outside Fencing..=Committing carries no @@ -1891,21 +1903,36 @@ impl DeploymentTransaction { } } - /// Complete transactions retain retryable cleanup work, but admission has - /// already transferred current-incarnation authority to AdmittedGeneration. - fn owns_target_authority(&self) -> bool { + /// The phase whose entry is the handover: the one transaction that makes + /// it takes the target's incumbent slot from continuity supervision. + fn takes_target_at(&self) -> DeploymentPhase { + handover_phase(self.rollout_stops_incumbent_first()) + } + + /// Not Complete: the transaction still holds its own candidate slot. + /// Read by sibling order inside one command, freeze (a live Deploy + /// refuses a second), the mint gate (a live Continuity refuses a second), + /// candidate-port reservation and validation. + fn is_live(&self) -> bool { self.phase != DeploymentPhase::Complete } + /// From the handover until Complete the transaction owns the target's + /// incumbent slot: it fences, leases, routes and commits. Derived from the + /// phase, never stored. Read by validation (one owner), supervision (which + /// defers to an owner), the handover CAS, a waiting deployment and status. + fn owns_target(&self) -> bool { + (self.takes_target_at()..DeploymentPhase::Complete).contains(&self.phase) + } + /// Admission has moved to the new generation at Complete, but an exact /// draining incumbent still reserves its process and candidate endpoint. - /// No later mutation for the same target may overlap that cleanup. - fn blocks_new_target_mutation(&self) -> bool { - self.owns_target_authority() - || self - .post_commit_cleanup - .as_ref() - .is_some_and(|cleanup| !cleanup.is_complete()) + /// Read by the mint gate, the handover, candidate-port reservation and + /// status: no later mutation of the target may overlap that cleanup. + fn drains_incumbent(&self) -> bool { + self.post_commit_cleanup + .as_ref() + .is_some_and(|cleanup| !cleanup.is_complete()) } /// Where what this transaction has collected disagrees with what its own @@ -3242,16 +3269,27 @@ impl ControlSnapshot { "transaction kind differs from its immutable command" ); } - let mut live_targets = BTreeSet::new(); + // Per target: one owner of the incumbent slot, one live Deploy, one + // live Continuity. A continuity may sit beside a deployment that has + // not yet taken the target (handover-phase-split). + let mut owners = BTreeSet::new(); + let mut live_deploys = BTreeSet::new(); + let mut live_continuities = BTreeSet::new(); for transaction in self .transactions .iter() - .filter(|stored| stored.value.owns_target_authority()) + .filter(|stored| stored.value.is_live()) { + let target = transaction.value.target.as_str(); ensure!( - live_targets.insert(transaction.value.target.as_str()), - "multiple transactions claim current-incarnation authority for one target" + !transaction.value.owns_target() || owners.insert(target), + "multiple transactions own the incumbent slot of one target" ); + let (seen, kind) = match transaction.value.command_kind { + CommandKind::Deploy => (&mut live_deploys, "deployments"), + CommandKind::Continuity => (&mut live_continuities, "continuity transactions"), + }; + ensure!(seen.insert(target), "multiple live {kind} for one target"); if let Some(fencing) = transaction .value .fencing @@ -3381,7 +3419,7 @@ impl ControlSnapshot { fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool { self.transactions.iter().any(|stored| { - stored.value.owns_target_authority() + stored.value.is_live() && stored.value.command_id == transaction.command_id && stored.value.ordinal < transaction.ordinal }) @@ -4608,9 +4646,10 @@ fn status( } writeln!( out, - " terminal {} owns-target {}", + " terminal {} owns-target {} drains {}", transaction.is_terminal(), - transaction.blocks_new_target_mutation() + transaction.owns_target(), + transaction.drains_incumbent() )?; } } @@ -5934,7 +5973,7 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -5995,23 +6034,19 @@ impl Engine { &anyhow::Error::new(disagreement), ); } - let blocker = snapshot.transactions.iter().find(|stored| { - stored.value.target == current.value.target - && stored.value.blocks_new_target_mutation() - }); - if blocker.is_some_and(|stored| { - let owns_from = if stored.value.rollout_stops_incumbent_first() { - // The incumbent is stopped at Starting by design; a - // continuity restart in that window would fight the - // candidate for the same host resources. - DeploymentPhase::Starting - } else { - DeploymentPhase::Fencing - }; - stored.value.phase >= owns_from && stored.value.phase < DeploymentPhase::Complete - }) { + let on_target = || { + snapshot + .transactions + .iter() + .filter(|stored| stored.value.target == current.value.target) + }; + // A transaction that owns the incumbent slot fences, stops or + // replaces this generation: nothing here touches it meanwhile. + if on_target().any(|stored| stored.value.owns_target()) { continue; } + let blocked = on_target() + .any(|stored| stored.value.is_live() || stored.value.drains_incumbent()); match self.supervise_admitted_route(&snapshot, current) { Ok(true) => { progressed = true; @@ -6056,9 +6091,7 @@ impl Engine { // the admitted one leaves the projection naming an incarnation that // no longer exists -- which the workload rightly refuses to adopt. // That record is published from observation once the process is up. - if blocker.is_none() - || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) - { + if !on_target().any(|stored| stored.value.is_live()) { let repair = (|| -> Result<()> { let topology = self.topology(); let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; @@ -6228,7 +6261,7 @@ impl Engine { // Continuity defers to any live transaction for its target and // never aborts one (handover-phase-split). - if blocker.is_some() { + if blocked { continue; } if supervision.restarts_page(now) { @@ -7994,7 +8027,7 @@ impl Engine { snapshot .transactions .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) .filter_map(|stored| stored.value.plan.as_ref()), ) { @@ -11595,8 +11628,9 @@ mod tests { }); assert!(!transaction.is_terminal()); - assert!(!transaction.owns_target_authority()); - assert!(transaction.blocks_new_target_mutation()); + assert!(!transaction.is_live()); + assert!(!transaction.owns_target()); + assert!(transaction.drains_incumbent()); Ok(()) } @@ -11835,7 +11869,7 @@ mod tests { ); assert_eq!(lifted.phase_deadline, None); assert_eq!(lifted.lease_adoption, None); - assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); + assert_eq!(lifted.is_live(), legacy.phase != DeploymentPhase::Complete); } assert_eq!( phases.iter().map(|(_, phase)| *phase).collect::>(), @@ -12457,16 +12491,23 @@ mod tests { declare_odin(&mut expected); let now = now_millis()?; + // A second seeding of one kind into one world is a second command. + let base_id = format!( + "{}-service{}", + match kind { + CommandKind::Deploy => "up", + CommandKind::Continuity => "continuity", + }, + if incumbent.is_some() { "-again" } else { "" } + ); + let known = ControlSnapshot::read(&world.state_store)?.commands; + let command_id = (1..) + .map(|n| if n == 1 { base_id.clone() } else { format!("{base_id}-{n}") }) + .find(|id| known.iter().all(|stored| stored.value.command_id != *id)) + .context("no free command id")?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!( - "{}-service{}", - match kind { - CommandKind::Deploy => "up", - CommandKind::Continuity => "continuity", - }, - if incumbent.is_some() { "-again" } else { "" } - ), + command_id, kind, selector: "service".into(), requested_by: "test".into(), @@ -14220,7 +14261,7 @@ mod tests { assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); assert_eq!(lifted.phase, archived.phase, "{name}"); assert_eq!(lifted.completion, archived.completion, "{name}"); - assert!(!lifted.owns_target_authority(), "{name}"); + assert!(!lifted.is_live(), "{name}"); } else { assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); assert!(!lifted.is_terminal(), "{name}"); @@ -16184,6 +16225,79 @@ mod tests { Ok(()) } + /// The stored transaction `id`, moved into `phase` in place. + fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { + let stored = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == id) + .context("no such live transaction")?; + let mut next = stored.value.clone(); + next.enter_phase(phase, now_millis()?); + replace_transaction(&world.state_store, &stored, &next) + } + + #[test] + fn the_handover_phase_is_starting_only_for_stop_then_start() -> Result<()> { + assert_eq!(handover_phase(true), DeploymentPhase::Starting); + assert_eq!(handover_phase(false), DeploymentPhase::Fencing); + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let mut seeded = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!(seeded.takes_target_at(), DeploymentPhase::Fencing); + // The target is owned from the handover to Complete, and not after; + // it is live until Complete. + for (phase, owns) in [ + (DeploymentPhase::Sealing, false), + (DeploymentPhase::Starting, false), + (DeploymentPhase::Warming, false), + (DeploymentPhase::Fencing, true), + (DeploymentPhase::Leasing, true), + (DeploymentPhase::AwaitingReady, true), + (DeploymentPhase::Routing, true), + (DeploymentPhase::Committing, true), + (DeploymentPhase::Complete, false), + ] { + seeded.phase = phase; + assert_eq!(seeded.owns_target(), owns, "{phase:?}"); + assert_eq!(seeded.is_live(), phase != DeploymentPhase::Complete, "{phase:?}"); + } + Ok(()) + } + + #[test] + fn the_store_holds_one_deploy_one_continuity_and_one_owner_per_target() -> Result<()> { + let seed = |first: (CommandKind, DeploymentPhase), second: (CommandKind, DeploymentPhase)| { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + for (kind, phase) in [first, second] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + if phase != DeploymentPhase::Warming { + rephase(&world, &seeded.transaction_id, phase)?; + } + } + Ok::<_, anyhow::Error>(world) + }; + use CommandKind::{Continuity, Deploy}; + use DeploymentPhase::{Fencing, Warming}; + + // A pre-handover deployment beside an owning continuity, and the + // reverse: one owner, one of each kind. + for (first, second) in [((Deploy, Warming), (Continuity, Fencing)), ((Continuity, Warming), (Deploy, Fencing))] { + let world = seed(first, second)?; + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 2); + } + let refused = |first, second| -> Result { + let world = seed(first, second)?; + Ok(format!("{:#}", ControlSnapshot::read(&world.state_store).err().context("two were accepted")?)) + }; + assert!(refused((Deploy, Fencing), (Continuity, Fencing))?.contains("own the incumbent slot")); + assert!(refused((Deploy, Warming), (Deploy, Warming))?.contains("multiple live deployments")); + assert!(refused((Continuity, Warming), (Continuity, Warming))?.contains("multiple live continuity transactions")); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -17207,7 +17321,7 @@ mod tests { )); assert!(finished.is_terminal()); // The target is released and nothing of the incumbent was touched. - assert!(!finished.blocks_new_target_mutation()); + assert!(!finished.is_live() && !finished.drains_incumbent()); assert_eq!(incumbent_envelope(&world)?, before); assert!( every_transaction(&world)? From 755306d654c3c4151f9eb3ad16d137e37d411b54 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:13:38 +0200 Subject: [PATCH 003/122] Cut 2, commit 2: one phase-entry seam and one handover CAS Engine::enter is the only writer of a move into the next phase; transition and the Sealing tail both call it. Entering the phase where a transaction takes its target (takes_target_at: Fencing, or Starting for stop-then-start) goes through Engine::take_target, one CAS that expects the transaction's own envelope, the target's admitted envelope as read (or its absence) and every other non-terminal transaction of the target, and writes only the transaction. Of two entrants from one view exactly one wins; a losing deployment waits with the owner named, a losing continuity is superseded. exact_incumbent now says what a mismatch is: a fault, not a race. Test helper: the brake release is per transaction (a world can hold two deployments). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 332 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 314 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 16ea8d7..5abe6ef 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,20 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// What `Engine::take_target` decided about one transaction's entry into the +/// phase where it takes its target. +#[derive(Debug, PartialEq, Eq)] +enum Handover { + /// The CAS wrote the entry: the transaction owns the target. + Took, + /// The CAS lost to a concurrent write; nothing was written. + Lost, + /// Another transaction owns or drains the target; nothing was written. + OwnedBy(String), + /// The admitted generation is no longer the one the transaction names. + Moved, +} + /// The phase whose entry is the handover of a target: Starting when the rollout /// stops the incumbent first (it is stopped there by design), else Fencing. The /// one spelling of the rule; a free function so the stop-then-start arm is @@ -3383,6 +3397,17 @@ impl ControlSnapshot { .find(|stored| stored.value.target == target) } + /// The transaction, other than `except`, that holds the target's + /// incumbent slot or is still draining it. The one lookup supervision, a + /// waiting deployment and the handover share. + fn target_owner(&self, target: &str, except: Option<&str>) -> Option<&Stored> { + self.transactions.iter().find(|stored| { + stored.value.target == target + && Some(stored.value.transaction_id.as_str()) != except + && (stored.value.owns_target() || stored.value.drains_incumbent()) + }) + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -6838,7 +6863,9 @@ impl Engine { })?; next.enter_phase(DeploymentPhase::Starting, now); next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) + // Starting is the handover for stop-then-start, so this write goes + // through the seam that decides it. + self.enter(current, next) } fn advance_starting(&self, current: &Stored) -> Result<()> { @@ -7798,6 +7825,10 @@ impl Engine { Ok(()) } + /// The admitted generation this transaction was proven against. A + /// pre-handover deployment is rebased onto the admitted generation before + /// any step reads its incumbent, and the handover pins it after, so a + /// mismatch here is a fault, not a race. fn exact_incumbent<'a>( &self, snapshot: &'a ControlSnapshot, @@ -7809,7 +7840,9 @@ impl Engine { (Some(expected), Some(current)) if current.value.generation_id == *expected => { Ok(Some(current)) } - _ => bail!("target incumbent changed after transaction creation"), + _ => bail!( + "admitted generation changed under a transaction that owns the target or names it as its source" + ), } } @@ -8949,14 +8982,108 @@ impl Engine { current: &Stored, next_phase: DeploymentPhase, ) -> Result<()> { - ensure!( - next_phase as u8 == current.value.phase as u8 + 1, - "deployment phase transition is not adjacent" - ); let mut next = current.value.clone(); next.enter_phase(next_phase, now_millis()?); next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) + self.enter(current, next) + } + + /// The one phase-entry seam. Every move into the next phase is written + /// here, and entering the phase where the transaction takes its target + /// (`takes_target_at`) is decided by `take_target`: that CAS is the + /// handover, and nothing else writes that entry. + fn enter( + &self, + current: &Stored, + next: DeploymentTransaction, + ) -> Result<()> { + ensure!( + next.phase as u8 == current.value.phase as u8 + 1, + "deployment phase transition is not adjacent" + ); + if next.phase != next.takes_target_at() { + return replace_transaction(&self.options.state_store, current, &next); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let target = ¤t.value.target; + match (self.take_target(&snapshot, current, &next)?, current.value.command_kind) { + // Lost to a concurrent writer: the next step reads the new world. + (Handover::Took | Handover::Lost, _) => Ok(()), + // Wait, never abort: a pre-fencing error would end the deployment. + (Handover::OwnedBy(owner), CommandKind::Deploy) => self.record_gate_wait( + current, + &format!( + "target {target} is owned by {owner}; this deployment fences what is admitted when it completes" + ), + ), + // The next step rebases onto the generation now admitted. + (Handover::Moved, CommandKind::Deploy) => Ok(()), + (Handover::OwnedBy(owner), CommandKind::Continuity) => { + bail!("continuity superseded: target {target} is owned by {owner}") + } + (Handover::Moved, CommandKind::Continuity) => { + bail!("continuity superseded: the generation it restarts is no longer admitted") + } + } + } + + /// The handover: one CAS that writes `next` (the entry into + /// `takes_target_at`) only if nothing else moved. It expects the + /// transaction's own envelope, the target's admitted envelope as read + /// (or its absence), and every other non-terminal transaction of the + /// target as read, so of two transactions entering from one view exactly + /// one succeeds. + fn take_target( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + next: &DeploymentTransaction, + ) -> Result { + let target = current.value.target.as_str(); + let id = current.value.transaction_id.as_str(); + if let Some(owner) = snapshot.target_owner(target, Some(id)) { + return Ok(Handover::OwnedBy(owner.value.transaction_id.clone())); + } + let admitted = snapshot.admitted_for(target); + if admitted.map(|stored| stored.value.generation_id.as_str()) + != current.value.incumbent_generation_id.as_deref() + { + return Ok(Handover::Moved); + } + next.validate()?; + let mut expected = vec![ + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: id.into(), + current: Some(current.envelope.clone()), + }, + CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: target.into(), + current: admitted.map(|stored| stored.envelope.clone()), + }, + ]; + expected.extend( + snapshot + .transactions + .iter() + .filter(|stored| { + stored.value.target == target + && stored.value.transaction_id != id + && !stored.value.is_terminal() + }) + .map(|stored| CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: stored.value.transaction_id.clone(), + current: Some(stored.envelope.clone()), + }), + ); + let took = SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &expected, + &[transaction_envelope(next, next.updated_at_unix_millis)?], + )?; + Ok(if took { Handover::Took } else { Handover::Lost }) } /// Every post-fencing phase ends. Before a live transaction past the @@ -12329,10 +12456,17 @@ mod tests { IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, }; - let signer = enroll_service_identity_at::( - &world.root.join("identities/brake-operator.cc"), - )?; - export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + // One operator per world, however many deployments it releases. + let signer_path = world.root.join("identities/brake-operator.cc"); + let signer = if signer_path.exists() { + open_service_identity_at::(&signer_path)? + } else { + let signer = + enroll_service_identity_at::(&signer_path)?; + export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + signer + }; + let authorization_id = format!("authorization-{transaction_id}"); let mut record = IdunnDeploymentBrakeRecord { schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), @@ -12343,7 +12477,7 @@ mod tests { reason: "test rollout".into(), observed_at_unix_millis: now, expires_at_unix_millis: Some(now + 600_000), - authorization_id: Some("authorization-test".into()), + authorization_id: Some(authorization_id.clone()), authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), authorized_release_id: Some(expected.sealed_release_id.clone()), authorized_deployment_id: Some(transaction_id.to_owned()), @@ -12363,7 +12497,7 @@ mod tests { record.validate()?; let canonical_brake_bytes = rmp_serde::to_vec(&record)?; Ok(DeploymentAuthorization { - authorization_id: "authorization-test".into(), + authorization_id, brake_sha256: sha256_id(&canonical_brake_bytes), canonical_brake_bytes, authorized_at_unix_millis: now, @@ -16225,16 +16359,178 @@ mod tests { Ok(()) } - /// The stored transaction `id`, moved into `phase` in place. - fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { - let stored = ControlSnapshot::read(&world.state_store)? + /// The live transaction `id`. + fn resident_id(world: &EngineFixture, id: &str) -> Result> { + ControlSnapshot::read(&world.state_store)? .transactions .into_iter() .find(|stored| stored.value.transaction_id == id) - .context("no such live transaction")?; + .context("no such live transaction") + } + + /// `stored` one phase on, as the seam would be asked to write it. + fn entering(stored: &Stored, phase: DeploymentPhase) -> Result { let mut next = stored.value.clone(); next.enter_phase(phase, now_millis()?); - replace_transaction(&world.state_store, &stored, &next) + Ok(next) + } + + /// Drop every live transaction but `keep`. + fn clear_other_live_transactions(world: &EngineFixture, keep: &str) -> Result<()> { + let others = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .filter(|stored| stored.value.transaction_id != keep) + .map(|stored| stored.envelope) + .collect::>(); + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged(&others)?); + Ok(()) + } + + /// The stored transaction `id`, moved into `phase` in place. + fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { + let stored = resident_id(world, id)?; + replace_transaction(&world.state_store, &stored, &entering(&stored, phase)?) + } + + /// A Deploy at `deploy_phase` and a Continuity at `continuity_phase`, both + /// over one incumbent of `service`, and that incumbent. + fn deploy_beside_continuity( + world: &EngineFixture, + deploy_phase: DeploymentPhase, + continuity_phase: DeploymentPhase, + ) -> Result<(DeploymentTransaction, DeploymentTransaction, AdmittedGeneration)> { + let incumbent = admit_incumbent(world)?; + let deploy = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let continuity = seeded_transaction( + world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + for (seeded, phase) in [(&deploy, deploy_phase), (&continuity, continuity_phase)] { + if phase != DeploymentPhase::Warming { + rephase(world, &seeded.transaction_id, phase)?; + } + } + Ok((deploy, continuity, incumbent)) + } + + #[test] + fn exactly_one_transaction_takes_a_target_from_one_view() -> Result<()> { + let take = |snapshot: &ControlSnapshot, world: &EngineFixture, id: &str| -> Result { + let stored = resident_id(world, id)?; + world.engine.take_target(snapshot, &stored, &entering(&stored, DeploymentPhase::Fencing)?) + }; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let view = ControlSnapshot::read(&world.state_store)?; + + assert_eq!(take(&view, &world, &continuity.transaction_id)?, Handover::Took); + // The second claimant read the same world: its CAS loses. + assert_eq!(take(&view, &world, &deploy.transaction_id)?, Handover::Lost); + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!(after.transactions.iter().filter(|stored| stored.value.owns_target()).count(), 1); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Warming); + // A fresh view names the owner instead of racing it. + assert_eq!( + take(&after, &world, &deploy.transaction_id)?, + Handover::OwnedBy(continuity.transaction_id.clone()) + ); + + // The reverse world: the deployment wins and the continuity loses. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let view = ControlSnapshot::read(&world.state_store)?; + assert_eq!(take(&view, &world, &deploy.transaction_id)?, Handover::Took); + assert_eq!(take(&view, &world, &continuity.transaction_id)?, Handover::Lost); + Ok(()) + } + + #[test] + fn the_handover_loses_when_the_admitted_generation_moved_after_it_was_read() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, _, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + // Only the continuity is a rival here; take it out of the guards' way. + clear_other_live_transactions(&world, &deploy.transaction_id)?; + let view = ControlSnapshot::read(&world.state_store)?; + let stored = resident_id(&world, &deploy.transaction_id)?; + + // Same generation, new envelope: a refresh landed after the read. + edit_incumbent(&world, |generation| generation.odin_publisher_sequence_cursor += 1)?; + assert_eq!( + world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, + Handover::Lost + ); + // A view that already shows another generation is Moved, not raced. + let mut other = stored.clone(); + other.value.incumbent_generation_id = Some("generation-elsewhere".into()); + assert_eq!( + world.engine.take_target( + &ControlSnapshot::read(&world.state_store)?, + &other, + &entering(&other, DeploymentPhase::Fencing)? + )?, + Handover::Moved + ); + Ok(()) + } + + #[test] + fn the_seam_makes_a_deployment_wait_and_a_continuity_step_aside() -> Result<()> { + // A deployment ready to fence while a continuity owns the target waits + // where it is, with the owner named and no abort. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let stored = resident_id(&world, &deploy.transaction_id)?; + world.engine.enter(&stored, entering(&stored, DeploymentPhase::Fencing)?)?; + let waiting = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.pre_fencing_abort.is_none()); + assert!( + waiting.last_error.as_deref().is_some_and(|note| note.contains(&continuity.transaction_id)), + "{:?}", + waiting.last_error + ); + // A deployment whose generation moved is left for its rebase. + let world_two = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy_two, _, _) = + deploy_beside_continuity(&world_two, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + clear_other_live_transactions(&world_two, &deploy_two.transaction_id)?; + let mut stored_two = resident_id(&world_two, &deploy_two.transaction_id)?; + stored_two.value.incumbent_generation_id = Some("generation-elsewhere".into()); + let before = ControlSnapshot::read(&world_two.state_store)?.transactions[0].envelope.clone(); + world_two.engine.enter(&stored_two, entering(&stored_two, DeploymentPhase::Fencing)?)?; + assert_eq!(ControlSnapshot::read(&world_two.state_store)?.transactions[0].envelope, before); + + // A continuity in the same position is superseded, by either cause. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Fencing, DeploymentPhase::Warming)?; + let stored = resident_id(&world, &continuity.transaction_id)?; + let error = world + .engine + .enter(&stored, entering(&stored, DeploymentPhase::Fencing)?) + .expect_err("a continuity entered a target another transaction owns"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + assert!(format!("{error:#}").contains(&deploy.transaction_id), "{error:#}"); + clear_other_live_transactions(&world, &continuity.transaction_id)?; + let mut stored = resident_id(&world, &continuity.transaction_id)?; + stored.value.incumbent_generation_id = Some("generation-elsewhere".into()); + let error = world + .engine + .enter(&stored, entering(&stored, DeploymentPhase::Fencing)?) + .expect_err("a continuity entered over a generation that moved"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + // Phases are entered one at a time. + let stored = resident_id(&world, &continuity.transaction_id)?; + assert!(world.engine.enter(&stored, entering(&stored, DeploymentPhase::Routing)?).is_err()); + Ok(()) } #[test] From 9139d7edb8964827ee9910f2f164bf76b93eecda Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:16:05 +0200 Subject: [PATCH 004/122] Cut 2, commit 3: rebase before the handover, wait in Warming, supersede a continuity settle_before_handover runs after the two abort dispatches and before any pre-handover step. A Deploy rebases onto the admitted generation when it differs from the one its evidence was proven against: incumbent_generation_id is rewritten, warming, route_preflight and isolation are dropped and the Odin cursor is raised, nothing else. In Warming it waits (a gate note, no error, so no abort) while another transaction owns or drains the target. A Continuity is never rebased: when its target is owned or its generation replaced it ends itself through the ordinary pre-fencing abort. Tests drive a real continuity restart to Complete beside a deployment and check the deployment rebases onto it and fences it. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 306 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 294 insertions(+), 12 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5abe6ef..1229bdd 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,11 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// The note a deployment keeps while another transaction owns its target. +fn waiting_for_owner(target: &str, owner: &str) -> String { + format!("target {target} is owned by {owner}; this deployment fences what is admitted when it completes") +} + /// What `Engine::take_target` decided about one transaction's entry into the /// phase where it takes its target. #[derive(Debug, PartialEq, Eq)] @@ -6738,6 +6743,11 @@ impl Engine { if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() { return self.advance_pre_fencing_abort(current); } + if current.value.phase < current.value.takes_target_at() + && self.settle_before_handover(current)? + { + return Ok(()); + } match current.value.phase { DeploymentPhase::Sealing => self.advance_sealing(current), DeploymentPhase::Starting => self.advance_starting(current), @@ -8988,6 +8998,92 @@ impl Engine { self.enter(current, next) } + /// Before its handover a Deploy is proven against the generation that was + /// admitted when the evidence was gathered. When another generation has + /// been admitted since (continuity restarted the incumbent), rewrite the + /// incumbent and drop exactly the evidence that names the old one: the + /// warming (its lease), the route preflight (its route) and the isolation + /// (its workload). A continuity is never rebased. Returns whether it wrote. + fn rebase_onto_admitted(&self, current: &Stored) -> Result { + let held = ¤t.value; + if held.command_kind != CommandKind::Deploy + || held.pre_fencing_abort.is_some() + || held.phase >= held.takes_target_at() + { + return Ok(false); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let admitted = snapshot.admitted_for(&held.target); + let admitted_id = admitted.map(|stored| stored.value.generation_id.as_str()); + if admitted_id == held.incumbent_generation_id.as_deref() { + return Ok(false); + } + self.persist_same_phase(current, |next| { + next.incumbent_generation_id = admitted_id.map(str::to_owned); + next.warming = None; + next.route_preflight = None; + next.isolation = None; + next.odin_publisher_sequence_cursor = next + .odin_publisher_sequence_cursor + .max(admitted.map_or(0, |stored| stored.value.odin_publisher_sequence_cursor)); + Ok(()) + })?; + Ok(true) + } + + /// What a transaction that has not yet taken its target does first, when + /// the world moved under it. `true` means this decided the step. + /// + /// A Deploy rebases onto the generation now admitted; in Warming, whose + /// steps read the incumbent an owner is fencing, it waits while another + /// transaction owns or drains the target. Both return Ok: a pre-fencing + /// error is an abort, and no deployment is aborted for continuity. A + /// Continuity restarts one generation and no other, so it ends itself + /// (through the ordinary pre-fencing abort) when its target is taken or + /// its generation replaced. + fn settle_before_handover(&self, current: &Stored) -> Result { + let held = ¤t.value; + match held.command_kind { + CommandKind::Deploy => { + if self.rebase_onto_admitted(current)? { + return Ok(true); + } + if held.phase != DeploymentPhase::Warming { + return Ok(false); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id)) + else { + return Ok(false); + }; + self.record_gate_wait( + current, + &waiting_for_owner(&held.target, &owner.value.transaction_id), + )?; + Ok(true) + } + CommandKind::Continuity => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id)) + { + bail!( + "continuity superseded: target {} is owned by {}", + held.target, + owner.value.transaction_id + ); + } + ensure!( + snapshot + .admitted_for(&held.target) + .map(|stored| stored.value.generation_id.as_str()) + == held.incumbent_generation_id.as_deref(), + "continuity superseded: the generation it restarts is no longer admitted" + ); + Ok(false) + } + } + } + /// The one phase-entry seam. Every move into the next phase is written /// here, and entering the phase where the transaction takes its target /// (`takes_target_at`) is decided by `take_target`: that CAS is the @@ -9010,12 +9106,9 @@ impl Engine { // Lost to a concurrent writer: the next step reads the new world. (Handover::Took | Handover::Lost, _) => Ok(()), // Wait, never abort: a pre-fencing error would end the deployment. - (Handover::OwnedBy(owner), CommandKind::Deploy) => self.record_gate_wait( - current, - &format!( - "target {target} is owned by {owner}; this deployment fences what is admitted when it completes" - ), - ), + (Handover::OwnedBy(owner), CommandKind::Deploy) => { + self.record_gate_wait(current, &waiting_for_owner(target, &owner)) + } // The next step rebases onto the generation now admitted. (Handover::Moved, CommandKind::Deploy) => Ok(()), (Handover::OwnedBy(owner), CommandKind::Continuity) => { @@ -12863,23 +12956,47 @@ mod tests { } } - /// One engine step at a time until `stop` holds, recording each phase - /// entered and checking after every step that the deadline on the record - /// is exactly the current post-fencing phase's, and absent outside one. + /// The transaction wherever it now lives: live, or archived once terminal. + fn latest_id(world: &EngineFixture, id: &str) -> Result { + match resident_id(world, id) { + Ok(stored) => Ok(stored.value), + Err(_) => read_history_transactions(&world.state_store) + .into_iter() + .find(|transaction| transaction.transaction_id == id) + .context("the transaction is in neither the live set nor history"), + } + } + + /// `drive_id` for the world's only transaction. fn drive( world: &EngineFixture, stop: impl Fn(&DeploymentTransaction) -> bool, + ) -> Result> { + match resident(world) { + Ok(sole) => drive_id(world, &sole.value.transaction_id, stop), + Err(_) => Ok(Vec::new()), + } + } + + /// One engine step of transaction `id` at a time until `stop` holds, + /// recording each phase entered and checking after every step that the + /// deadline on the record is exactly the current post-fencing phase's, and + /// absent outside one. + fn drive_id( + world: &EngineFixture, + id: &str, + stop: impl Fn(&DeploymentTransaction) -> bool, ) -> Result> { let mut phases = Vec::new(); for _ in 0..40 { - let Ok(current) = resident(world) else { + let Ok(current) = resident_id(world, id) else { return Ok(phases); }; if stop(¤t.value) { return Ok(phases); } world.engine.advance_transaction(¤t)?; - let after = latest(world)?; + let after = latest_id(world, id)?; match &after.phase_deadline { Some(deadline) => assert_eq!(deadline.phase, after.phase), None => assert!( @@ -16461,7 +16578,7 @@ mod tests { let stored = resident_id(&world, &deploy.transaction_id)?; // Same generation, new envelope: a refresh landed after the read. - edit_incumbent(&world, |generation| generation.odin_publisher_sequence_cursor += 1)?; + edit_incumbent(&world, |generation| generation.admitted_at_unix_millis += 1)?; assert_eq!( world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, Handover::Lost @@ -16594,6 +16711,171 @@ mod tests { Ok(()) } + /// The continuity transaction minted over `service`, whatever its state. + fn the_continuity(world: &EngineFixture) -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.command_kind == CommandKind::Continuity) + .context("no live continuity") + } + + #[test] + fn a_deployment_rebases_onto_the_generation_continuity_admitted_mid_build_and_fences_it() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let deploy_id = deploy.transaction_id.as_str(); + workload.kill(); + assert!(world.engine.supervise_one_admitted_generation()?); + let continuity_id = the_continuity(&world)?.value.transaction_id; + + // Continuity restarts the incumbent beside the build, to Complete. + workload.revive(); + drive_id(&world, &continuity_id, |t| t.phase == DeploymentPhase::Warming)?; + odin_republishes(&world, &latest_id(&world, &continuity_id)?, seeded_sequence(Some(&incumbent)) + 10)?; + drive_id(&world, &continuity_id, |t| t.is_terminal())?; + let restarted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the restart admitted nothing")? + .value + .clone(); + assert_ne!(restarted.generation_id, incumbent.generation_id); + assert_eq!(restarted.transaction_id, continuity_id); + let untouched = resident_id(&world, deploy_id)?; + assert_eq!(untouched.value.incumbent_generation_id, Some(incumbent.generation_id.clone())); + + // The deployment's next step is the rebase and nothing else. + world.engine.advance_transaction(&untouched)?; + let rebased = resident_id(&world, deploy_id)?.value; + assert_eq!(rebased.incumbent_generation_id, Some(restarted.generation_id.clone())); + assert_eq!(rebased.phase, DeploymentPhase::Warming); + assert!(rebased.warming.is_none() && rebased.route_preflight.is_none() && rebased.isolation.is_none()); + assert!(rebased.pre_fencing_abort.is_none()); + assert!(rebased.odin_publisher_sequence_cursor >= restarted.odin_publisher_sequence_cursor); + assert_eq!(rebased.expected, untouched.value.expected); + + // It then fences the generation continuity admitted, and replaces it. + odin_reports_ready(&world, &deploy, restarted.odin_publisher_sequence_cursor + 10)?; + drive_id(&world, deploy_id, |t| t.is_terminal())?; + let landed = latest_id(&world, deploy_id)?; + assert!(matches!(landed.completion, Some(TransactionCompletion::Admitted { .. })), "{landed:?}"); + assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); + assert!(matches!( + landed.post_commit_cleanup.map(|cleanup| cleanup.incumbent), + Some( + IncumbentCleanupEvidence::Pending { generation_id, .. } + | IncumbentCleanupEvidence::Complete { generation_id } + ) if generation_id == restarted.generation_id + )); + let now_admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the deployment admitted nothing")? + .value + .transaction_id + .clone(); + assert_eq!(now_admitted, deploy_id); + Ok(()) + } + + /// Kill the Warming step with the workload: if the gate did not hold the + /// deployment, running the step would be a pre-fencing error and an abort. + #[test] + fn a_deployment_waits_in_warming_while_a_continuity_owns_the_target() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Leasing)?; + workload.kill(); + let deploy_id = deploy.transaction_id.as_str(); + let before = resident_id(&world, deploy_id)?; + + for _ in 0..3 { + let stored = resident_id(&world, deploy_id)?; + world.engine.advance_transaction(&stored)?; + } + let waiting = resident_id(&world, deploy_id)?; + assert_eq!(waiting.value.phase, DeploymentPhase::Warming); + assert!(waiting.value.pre_fencing_abort.is_none()); + assert_eq!(waiting.value.warming, before.value.warming); + assert_eq!(waiting.value.isolation, before.value.isolation); + assert!( + waiting.value.last_error.as_deref().is_some_and(|note| note.contains(&continuity.transaction_id)), + "{:?}", + waiting.value.last_error + ); + + // The build overlaps: a deployment still in Sealing is not gated. + rephase(&world, deploy_id, DeploymentPhase::Sealing)?; + let sealing = resident_id(&world, deploy_id)?; + world.engine.advance_transaction(&sealing)?; + let after = resident_id(&world, deploy_id)?.value; + assert!( + after.last_error.as_deref().is_some_and(|note| !note.contains(&continuity.transaction_id)), + "Sealing was held by the owner: {:?}", + after.last_error + ); + + // The owner completes having moved the generation: the next step is the + // rebase, and with the target free the deployment goes on. + rephase(&world, deploy_id, DeploymentPhase::Warming)?; + workload.revive(); + clear_other_live_transactions(&world, deploy_id)?; + edit_incumbent(&world, |generation| generation.generation_id = "generation-restarted".into())?; + world.engine.advance_transaction(&resident_id(&world, deploy_id)?)?; + let rebased = resident_id(&world, deploy_id)?.value; + assert_eq!(rebased.incumbent_generation_id.as_deref(), Some("generation-restarted")); + assert!(rebased.warming.is_none()); + Ok(()) + } + + #[test] + fn a_continuity_whose_target_was_taken_or_generation_replaced_aborts_itself_and_is_not_rebased() -> Result<()> { + let resume = |world: &EngineFixture, id: &str| -> Result<()> { + for _ in 0..12 { + let stored = resident_id(world, id); + let Ok(stored) = stored else { break }; + world.engine.resume_candidate(&stored)?; + world.engine.resume_backoff.lock().unwrap().clear(); + } + Ok(()) + }; + // Taken: the deployment owns the target; the continuity ends itself and + // the deployment goes on, untouched. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Fencing, DeploymentPhase::Warming)?; + let owner = resident_id(&world, &deploy.transaction_id)?; + resume(&world, &continuity.transaction_id)?; + let ended = latest_id(&world, &continuity.transaction_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.envelope, owner.envelope); + + // Replaced: the generation it restarts is gone. It is not rebased. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, incumbent) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + clear_other_live_transactions(&world, &continuity.transaction_id)?; + edit_incumbent(&world, |generation| generation.generation_id = "generation-elsewhere".into())?; + resume(&world, &continuity.transaction_id)?; + let ended = latest_id(&world, &continuity.transaction_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert_eq!(ended.incumbent_generation_id, Some(incumbent.generation_id)); + let _ = deploy; + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; From 278cc5c408ed3a7ac9b1f066eec65e044c0f3826 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:20:30 +0200 Subject: [PATCH 005/122] Cut 2, commit 4: open the gates; fold in the sustained-reset test reach Freeze refuses only a second live Deploy of a target, so a profile command freezes over crash-looping targets. Supervision defers to an owner, a drain or its own live continuity, never to a deployment that has not taken the target, and the admitted Expected repair runs beside one. Cut 1's interim test (no continuity beside a pre-fencing deployment) goes with the state it asserted. Fold-in c1-sustained-reset-test-reach: the lower edge to the millisecond, the newest attempt (not the oldest) deciding the reset, and the mid-wait crossing recorded in continuity_backoff's doc and pinned. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 245 +++++++++++++++++++++++++++---------------- 1 file changed, 155 insertions(+), 90 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 1229bdd..01005b1 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1244,7 +1244,9 @@ impl TargetSupervision { /// run: the generation has lived the sustained time and no attempt, charged /// or the one re-materialization, has been minted since it was admitted. /// That rule decides first: a sustained death never began a wait, so a - /// standing anchor, however it came to stand, has no say in it. + /// standing anchor, however it came to stand, has no say in it. A wait + /// already running when the generation reaches the sustained time ends + /// there instead: an accepted shortening, always under the cap. fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { let streak = self.continuity_attempt_streak; let sustained = self @@ -5680,8 +5682,8 @@ impl Engine { let continuity_progress = self.supervise_one_admitted_generation()?; // Freezing is not gated on the others' progress: a queued command // whose target is free must not wait behind an unrelated target's - // wedge. `freeze_command` already refuses a busy target, so this is - // still at most one freeze per tick and never a second claimant. + // wedge. `freeze_command` refuses only a second live deployment of a + // target, and this freezes at most one command per tick. let froze = self.freeze_one_queued_command()?; Ok(transaction_progress || continuity_progress || froze) } @@ -5963,8 +5965,10 @@ impl Engine { Ok(false) } - /// Freeze one queued command into its transactions. `Ok(false)` means its - /// target is busy and nothing was written. + /// Freeze one queued command into its transactions. `Ok(false)` means a + /// live deployment already exists for one of its targets and nothing was + /// written. Continuity, owners and drains never refuse a freeze: a + /// deployment that has not taken its target owns nothing. fn freeze_command( &self, snapshot: &ControlSnapshot, @@ -6003,7 +6007,9 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) + .filter(|stored| { + stored.value.command_kind == CommandKind::Deploy && stored.value.is_live() + }) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -6075,8 +6081,13 @@ impl Engine { if on_target().any(|stored| stored.value.owns_target()) { continue; } - let blocked = on_target() - .any(|stored| stored.value.is_live() || stored.value.drains_incumbent()); + // Continuity defers to an owner, a drain or its own live attempt, + // never to a deployment that has not taken the target, and aborts + // nothing (handover-phase-split). + let mint_blocked = snapshot.target_owner(¤t.value.target, None).is_some() + || on_target().any(|stored| { + stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity + }); match self.supervise_admitted_route(&snapshot, current) { Ok(true) => { progressed = true; @@ -6121,28 +6132,26 @@ impl Engine { // the admitted one leaves the projection naming an incarnation that // no longer exists -- which the workload rightly refuses to adopt. // That record is published from observation once the process is up. - if !on_target().any(|stored| stored.value.is_live()) { - let repair = (|| -> Result<()> { - let topology = self.topology(); - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - if !topology.admitted_expected_projection_is_exact( - ¤t.value.expected, - &provider_anchor, - )? { - ensure!( - topology.publish_expected(¤t.value.expected, &provider_anchor)? - == current.value.expected.canonical_sha256()?, - "admitted Expected projection repair differs" - ); - } - Ok(()) - })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target + let repair = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.admitted_expected_projection_is_exact( + ¤t.value.expected, + &provider_anchor, + )? { + ensure!( + topology.publish_expected(¤t.value.expected, &provider_anchor)? + == current.value.expected.canonical_sha256()?, + "admitted Expected projection repair differs" ); } + Ok(()) + })(); + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -6289,9 +6298,7 @@ impl Engine { } let continuity_key = format!("continuity:{}", current.value.target); - // Continuity defers to any live transaction for its target and - // never aborts one (handover-phase-split). - if blocked { + if mint_blocked { continue; } if supervision.restarts_page(now) { @@ -13829,6 +13836,9 @@ mod tests { alive: std::sync::atomic::AtomicBool, rebooted: std::sync::atomic::AtomicBool, silent: std::sync::atomic::AtomicBool, + /// Runtime instances that stay alive when the rest are killed: a + /// candidate building beside a dead incumbent. + spared: std::sync::Mutex>, } impl SwitchWorkload { @@ -13837,9 +13847,15 @@ mod tests { alive: std::sync::atomic::AtomicBool::new(true), rebooted: std::sync::atomic::AtomicBool::new(false), silent: std::sync::atomic::AtomicBool::new(false), + spared: std::sync::Mutex::new(Vec::new()), }) } + /// `kill` leaves this runtime instance running. + fn spare(&self, runtime_instance_id: &str) { + self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); + } + /// The host cannot be asked: observing fails as `HostUnobservable` /// whatever the workload is doing. fn silence(&self, silent: bool) { @@ -13914,8 +13930,14 @@ mod tests { } .into()); } + let spared = match prior { + WorkloadObservation::Systemd(observed) => { + self.spared.lock().unwrap().contains(&observed.runtime_instance_id) + } + WorkloadObservation::Host(_) => false, + }; ensure!( - self.alive.load(std::sync::atomic::Ordering::SeqCst), + spared || self.alive.load(std::sync::atomic::Ordering::SeqCst), "the workload is gone" ); Ok(prior.clone()) @@ -16329,53 +16351,6 @@ mod tests { ); } - /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment - /// waiting before Fencing. Continuity never stops and never yields a - /// deployment (budget-never-stop, handover-phase-split), so the window - /// sliding must not turn the deployment into a casualty. - #[test] - fn a_deployment_before_fencing_over_a_dead_incumbent_that_paged_is_never_yielded_and_lands() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - workload.kill(); - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - world.engine.supervise_one_admitted_generation()?; - // The oldest leaves the window: base would now yield the deployment. - set_meters( - &world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - for _ in 0..3 { - let _ = world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!( - after.pre_fencing_abort.is_none(), - "the deployment was yielded: {:?}", - after.pre_fencing_abort - ); - assert_eq!( - continuity_mints(&world)?, - 0, - "continuity minted beside a pre-fencing deployment" - ); - } - workload.revive(); - odin_reports_ready(&world, &redeploy, seeded_sequence(Some(&incumbent)) + 1)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - let landed = latest(&world)?; - assert_eq!(landed.transaction_id, redeploy.transaction_id); - assert_eq!(landed.phase, DeploymentPhase::Complete); - assert!(landed.pre_fencing_abort.is_none()); - Ok(()) - } - /// The incumbent was admitted this long ago, so an anchor written "in the /// past" is not older than its generation. fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { @@ -16788,7 +16763,7 @@ mod tests { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; let (deploy, continuity, _) = - deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Leasing)?; + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; workload.kill(); let deploy_id = deploy.transaction_id.as_str(); let before = resident_id(&world, deploy_id)?; @@ -16838,7 +16813,9 @@ mod tests { for _ in 0..12 { let stored = resident_id(world, id); let Ok(stored) = stored else { break }; - world.engine.resume_candidate(&stored)?; + // The last step archives the record, which the scheduler's own + // progress read reports as an error; the record is what counts. + let _ = world.engine.resume_candidate(&stored); world.engine.resume_backoff.lock().unwrap().clear(); } Ok(()) @@ -16876,6 +16853,62 @@ mod tests { Ok(()) } + /// A dead incumbent with a deployment building beside it (its candidate + /// alive) whose Warming evidence is complete, and that deployment. + fn dead_incumbent_with_a_deployment_ready_to_fence( + workload: &SwitchWorkload, + world: &EngineFixture, + ) -> Result { + let incumbent = admit_incumbent(world)?; + age_incumbent(world, 500_000)?; + let deploy = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let candidate = deploy.activation.as_ref().context("no activation")?; + workload.spare(&candidate.runtime_instance_id); + workload.kill(); + Ok(deploy) + } + + #[test] + fn a_continuity_minted_as_a_deployment_takes_the_target_is_resolved_next_pass() -> Result<()> { + // (a) The deployment fences first in the tick, so supervision reads its + // ownership and mints nothing. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + assert!(world.engine.run_scheduler_tick()?); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + assert_eq!(continuity_mints(&world)?, 0); + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0); + + // (b) Supervision minted from the older view: the continuity starts at + // Sealing and owns nothing, the deployment takes the target, and the + // continuity is superseded at its next step, never the deployment. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + assert!(world.engine.supervise_one_admitted_generation()?); + let continuity_id = the_continuity(&world)?.value.transaction_id; + assert_eq!(resident_id(&world, &continuity_id)?.value.phase, DeploymentPhase::Sealing); + world.engine.advance_transaction(&resident_id(&world, &deploy.transaction_id)?)?; + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + world.engine.resume_backoff.lock().unwrap().clear(); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!(snapshot.transactions.iter().filter(|stored| stored.value.owns_target()).count() <= 1); + } + let ended = latest_id(&world, &continuity_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert!(latest_id(&world, &deploy.transaction_id)?.pre_fencing_abort.is_none()); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -16980,11 +17013,19 @@ mod tests { meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), ContinuityBackoff::Due { streak: 0 } ); - // A moment short of it. - assert_eq!( - meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + 10_000), - ContinuityBackoff::Begin - ); + // A moment short of it, down to the millisecond. + for short in [10_000, 1_000, 1] { + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + short), + ContinuityBackoff::Begin, + "{short} ms short" + ); + } + // The newest attempt decides, not the oldest: one before admission and + // one after it, on a generation that has lived long enough. + meters.continuity_restarts = vec![now - 800_000, now - 100_000]; + assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); + meters.continuity_restarts = vec![1_000]; // Attempts after admission failed on this generation: it has not run well. meters.continuity_restarts = vec![now - 100_000]; assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); @@ -17008,6 +17049,29 @@ mod tests { assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); } + #[test] + fn a_wait_that_crosses_the_sustained_run_ends_early_and_only_by_less_than_the_cap() { + let admitted = 50_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 7; + meters.continuity_restarts = vec![admitted - 1_000]; + // The wait began when the generation was 400 s old and runs 300 s. + meters.continuity_backoff_from_unix_millis = Some(admitted + 400_000); + assert_eq!(meters.backoff_due(admitted), Some(admitted + 700_000)); + assert_eq!( + meters.continuity_backoff(admitted + CONTINUITY_SUSTAINED_RUN_MILLIS - 1, admitted), + ContinuityBackoff::Waiting + ); + assert_eq!( + meters.continuity_backoff(admitted + CONTINUITY_SUSTAINED_RUN_MILLIS, admitted), + ContinuityBackoff::Due { streak: 0 } + ); + assert!( + CONTINUITY_SUSTAINED_RUN_MILLIS - 400_000 < CONTINUITY_BACKOFF_CEILING_MILLIS, + "the shortening is under the cap" + ); + } + #[test] fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { let workload = SwitchWorkload::new(); @@ -17057,10 +17121,10 @@ mod tests { all.iter().any(|transaction| transaction.command_id == "up-service-queued"), "the queued deployment never froze" ); - assert_eq!(continuity_mints(&world)?, 0); + // Freeze needs no gap (a deployment before its handover owns nothing), + // and continuity may restart the incumbent beside it after its wait. for _ in 0..3 { let _ = world.engine.run_scheduler_tick(); - assert_eq!(continuity_mints(&world)?, 0); let aborts = every_transaction(&world)? .into_iter() .filter_map(|transaction| transaction.pre_fencing_abort) @@ -17864,10 +17928,11 @@ mod tests { for _ in 0..4 { let _ = world.engine.supervise_one_admitted_generation()?; } - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, held.transaction_id); + let after = resident_id(&world, &held.transaction_id)?.value; assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); assert!(after.completion.is_none()); + // Continuity restarts the declared incumbent beside it, once. + assert_eq!(continuity_mints(&world)?, 1); Ok(()) } From a7bfbc2296ac5a5a42ebf9d0e599d113ead9b5cf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:22:58 +0200 Subject: [PATCH 006/122] Cut 2 tests: a launching switch workload, so a real continuity restart runs beside a deployment Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 01005b1..e2518f4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13839,6 +13839,9 @@ mod tests { /// Runtime instances that stay alive when the rest are killed: a /// candidate building beside a dead incumbent. spared: std::sync::Mutex>, + /// Whether a start launches a process (a fresh private identity) + /// instead of refusing like `StillWorkload`. + launches: std::sync::atomic::AtomicBool, } impl SwitchWorkload { @@ -13848,9 +13851,15 @@ mod tests { rebooted: std::sync::atomic::AtomicBool::new(false), silent: std::sync::atomic::AtomicBool::new(false), spared: std::sync::Mutex::new(Vec::new()), + launches: std::sync::atomic::AtomicBool::new(false), }) } + /// From now on a start launches a process distinct from every recorded one. + fn launching(&self) { + self.launches.store(true, std::sync::atomic::Ordering::SeqCst); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -13897,6 +13906,9 @@ mod tests { expected: &IdunnExpectedIncarnationRecord, launch: IdunnRuntimeActivationLaunch, ) -> Result { + if self.launches.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(launch.activation().clone()); + } StillWorkload.prepare_activation(plan, expected, launch) } fn start_prepared( @@ -13907,7 +13919,25 @@ mod tests { expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) + if !self.launches.load(std::sync::atomic::Ordering::SeqCst) { + return StillWorkload.start_prepared(plan, release, installed, expected, activation); + } + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("recorded transaction has no workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + // Another UID and other namespaces than any recorded unit. + observed.process_uids[0] += 1_000_000; + observed.pid_namespace_id += 1_000_000; + observed.mount_namespace_id += 1_000_000; + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + Ok(workload) } fn discard_prepared( &self, @@ -16704,6 +16734,7 @@ mod tests { let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; let deploy_id = deploy.transaction_id.as_str(); + workload.launching(); workload.kill(); assert!(world.engine.supervise_one_admitted_generation()?); let continuity_id = the_continuity(&world)?.value.transaction_id; From 1953bf233b3aa366eb555e113029ae73a2bec4cc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:57:31 +0200 Subject: [PATCH 007/122] Cut 2: one per-target lookup, and tests that kill the survivors it exposed ControlSnapshot::on_target is the one place a transaction is filed under a target; target_owner, the supervision gates and the handover read it. Tests: ownership is per target and names only a holder; a draining target is not restarted until its cleanup ends; the admitted Expected is repaired beside a building deployment; a live continuity alone holds back a second mint. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 109 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e2518f4..63bfe73 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3408,13 +3408,23 @@ impl ControlSnapshot { /// incumbent slot or is still draining it. The one lookup supervision, a /// waiting deployment and the handover share. fn target_owner(&self, target: &str, except: Option<&str>) -> Option<&Stored> { - self.transactions.iter().find(|stored| { - stored.value.target == target - && Some(stored.value.transaction_id.as_str()) != except + self.on_target(target).find(|stored| { + Some(stored.value.transaction_id.as_str()) != except && (stored.value.owns_target() || stored.value.drains_incumbent()) }) } + /// Every transaction of the target, in whatever state. + fn on_target<'a>( + &'a self, + target: &str, + ) -> impl Iterator> + use<'a> { + let target = target.to_owned(); + self.transactions + .iter() + .filter(move |stored| stored.value.target == target) + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -6070,22 +6080,19 @@ impl Engine { &anyhow::Error::new(disagreement), ); } - let on_target = || { - snapshot - .transactions - .iter() - .filter(|stored| stored.value.target == current.value.target) - }; // A transaction that owns the incumbent slot fences, stops or // replaces this generation: nothing here touches it meanwhile. - if on_target().any(|stored| stored.value.owns_target()) { + if snapshot + .on_target(¤t.value.target) + .any(|stored| stored.value.owns_target()) + { continue; } // Continuity defers to an owner, a drain or its own live attempt, // never to a deployment that has not taken the target, and aborts // nothing (handover-phase-split). let mint_blocked = snapshot.target_owner(¤t.value.target, None).is_some() - || on_target().any(|stored| { + || snapshot.on_target(¤t.value.target).any(|stored| { stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity }); match self.supervise_admitted_route(&snapshot, current) { @@ -9165,13 +9172,8 @@ impl Engine { ]; expected.extend( snapshot - .transactions - .iter() - .filter(|stored| { - stored.value.target == target - && stored.value.transaction_id != id - && !stored.value.is_terminal() - }) + .on_target(target) + .filter(|stored| stored.value.transaction_id != id && !stored.value.is_terminal()) .map(|stored| CultCacheExpectedEnvelope { r#type: DeploymentTransaction::TYPE.into(), key: stored.value.transaction_id.clone(), @@ -16415,6 +16417,8 @@ mod tests { assert_eq!(stored_deploy(&world)?.envelope, before.envelope); for _ in 0..2 { + // Due at once again (streak 0), so only the live attempt holds it back. + set_meters(&world, &TargetSupervision::new("service"))?; let _ = world.engine.supervise_one_admitted_generation()?; assert_eq!(continuity_mints(&world)?, 1); let after = stored_deploy(&world)?; @@ -16884,6 +16888,75 @@ mod tests { Ok(()) } + #[test] + fn ownership_is_read_per_target_and_names_only_a_transaction_that_holds_it() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let mut view = ControlSnapshot::read(&world.state_store)?; + assert_eq!(view.on_target("service").count(), 2); + assert_eq!(view.on_target("elsewhere").count(), 0); + // The pre-handover deployment holds nothing; the continuity holds the slot. + let owner = view.target_owner("service", None).context("no owner")?; + assert_eq!(owner.value.transaction_id, continuity.transaction_id); + assert!(view.target_owner("service", Some(&continuity.transaction_id)).is_none()); + assert!(view.target_owner("elsewhere", None).is_none()); + // The same records filed under another target are that target's. + for stored in &mut view.transactions { + stored.value.target = "elsewhere".into(); + } + assert_eq!(view.on_target("service").count(), 0); + assert!(view.target_owner("service", None).is_none()); + assert_eq!(view.on_target("elsewhere").count(), 2); + assert!(view.target_owner("elsewhere", None).is_some()); + let _ = deploy; + Ok(()) + } + + /// A deployment that has committed still drains the old incumbent; the + /// target is not restarted over that cleanup, and is once it ends. + #[test] + fn a_target_draining_its_old_incumbent_is_not_restarted_until_the_drain_ends() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &deploy, seeded_sequence(Some(&incumbent)) + 1)?; + drive_id(&world, &deploy.transaction_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, &deploy.transaction_id)?.value; + assert!(draining.drains_incumbent() && !draining.owns_target() && !draining.is_live()); + + age_incumbent(&world, 500_000)?; + workload.kill(); + for _ in 0..3 { + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "restarted over a draining incumbent"); + } + drive_id(&world, &deploy.transaction_id, |t| t.is_terminal())?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + Ok(()) + } + + /// A deployment that has not taken the target no longer suppresses the + /// repair of the admitted Expected (only the Expected, only when inexact). + #[test] + fn the_admitted_expected_is_repaired_beside_a_deployment_that_is_building() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let store = world.engine.options.topology_store.clone(); + if store.exists() { + std::fs::remove_file(&store)?; + } + assert_eq!(projected_under(&world, &incumbent.expected)?, Vec::::new()); + + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From dd8707b5a26755e76b39d5bbaa5d9c1f962dcac9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:14:46 +0200 Subject: [PATCH 008/122] Cut 2: pin the candidate-port reservation and the rebase guards directly candidate_ports_in_use is the one reading of which ports admitted, live and draining transactions hold. Tests pin it, select_candidate_port's routed and unrouted answers, and the three conditions under which a transaction is not rebased. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 115 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 63bfe73..0e1683e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3425,6 +3425,22 @@ impl ControlSnapshot { .filter(move |stored| stored.value.target == target) } + /// The candidate ports taken by admitted generations and by transactions + /// that are live or still draining; a finished one has released its port. + fn candidate_ports_in_use(&self) -> BTreeSet { + self.admitted + .iter() + .map(|stored| &stored.value.plan) + .chain( + self.transactions + .iter() + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) + .filter_map(|stored| stored.value.plan.as_ref()), + ) + .filter_map(|plan| plan.candidate_port) + .collect() + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -8074,24 +8090,7 @@ impl Engine { let Some(route) = &binding.route else { return Ok(None); }; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut used = BTreeSet::new(); - for plan in snapshot - .admitted - .iter() - .map(|stored| &stored.value.plan) - .chain( - snapshot - .transactions - .iter() - .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) - .filter_map(|stored| stored.value.plan.as_ref()), - ) - { - if let Some(port) = plan.candidate_port { - used.insert(port); - } - } + let used = ControlSnapshot::read(&self.options.state_store)?.candidate_ports_in_use(); (route.private_port_start..=route.private_port_end) .find(|port| !used.contains(port)) .map(Some) @@ -16957,6 +16956,86 @@ mod tests { Ok(()) } + #[test] + fn a_candidate_port_is_held_by_the_admitted_a_live_and_a_draining_transaction_only() -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let mut view = ControlSnapshot::read(&world.state_store)?; + assert!(view.candidate_ports_in_use().is_empty()); + + // Ports are set in memory: the fixture plans are unrouted. + view.admitted[0].value.plan.candidate_port = Some(18000); + for stored in &mut view.transactions { + let port = if stored.value.transaction_id == deploy.transaction_id { 18001 } else { 18002 }; + stored.value.plan.as_mut().context("no plan")?.candidate_port = Some(port); + } + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001, 18002])); + // Complete with cleanup still pending drains: its port stays held. + let finished = view + .transactions + .iter_mut() + .find(|stored| stored.value.transaction_id == continuity.transaction_id) + .context("no continuity")?; + finished.value.phase = DeploymentPhase::Complete; + finished.value.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Pending, + }); + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001, 18002])); + let finished = view + .transactions + .iter_mut() + .find(|stored| stored.value.transaction_id == continuity.transaction_id) + .context("no continuity")?; + finished.value.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Complete, + }); + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001])); + + // A routed binding gets the first free port of its range; an unrouted one none. + let routed = OperatorBinding::parse(BINDING)?; + assert_eq!(world.engine.select_candidate_port(&routed)?, Some(routed.route.as_ref().context("route")?.private_port_start)); + let (head, tail) = BINDING.split_once("[route]").context("route")?; + let unrouted = OperatorBinding::parse(&format!("{head}[brakes]{}", tail.split_once("[brakes]").context("brakes")?.1))?; + assert_eq!(world.engine.select_candidate_port(&unrouted)?, None); + Ok(()) + } + + #[test] + fn only_a_deployment_before_its_handover_is_rebased() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + // Another generation is admitted under both. + edit_incumbent(&world, |generation| generation.generation_id = "generation-restarted".into())?; + let rebase = |id: &str| -> Result { + world.engine.rebase_onto_admitted(&resident_id(&world, id)?) + }; + // A continuity restarts one generation and no other. + assert!(!rebase(&continuity.transaction_id)?); + assert_ne!( + resident_id(&world, &continuity.transaction_id)?.value.incumbent_generation_id.as_deref(), + Some("generation-restarted") + ); + // A deployment already ending is left to end. + let ending = resident_id(&world, &deploy.transaction_id)?; + let mut aborting = ending.value.clone(); + aborting.pre_fencing_abort = Some(pre_fencing_abort_intent(&aborting, "cancelled by operator")); + replace_transaction(&world.state_store, &ending, &aborting)?; + assert!(!rebase(&deploy.transaction_id)?); + let ending = resident_id(&world, &deploy.transaction_id)?; + let mut live = ending.value.clone(); + live.pre_fencing_abort = None; + replace_transaction(&world.state_store, &ending, &live)?; + // Past its handover the incumbent is pinned, whatever is admitted. + rephase(&world, &deploy.transaction_id, DeploymentPhase::Fencing)?; + assert!(!rebase(&deploy.transaction_id)?); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From 6ea0d127d8ea2242625922f083fe29e32f66d461 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:15:19 +0200 Subject: [PATCH 009/122] Cut 2 test: a finished transaction retired mid-handover is not a lost race Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0e1683e..0b20e33 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17036,6 +17036,33 @@ mod tests { Ok(()) } + #[test] + fn the_handover_does_not_wait_on_a_finished_transaction_being_retired() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + // A committed transaction left resident, finished and not yet retired. + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, seeded_sequence(None) + 1)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + let incumbent = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let view = ControlSnapshot::read(&world.state_store)?; + assert!(view.transactions.iter().any(|stored| stored.value.is_terminal())); + + // It is retired between the read and the handover; that is no race. + assert!(world.engine.retire_one_terminal_transaction()?); + let stored = resident_id(&world, &deploy.transaction_id)?; + assert_eq!( + world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, + Handover::Took + ); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From 94d3b53b55007063c60f8f5e184f1043ed6a622e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:00:49 +0200 Subject: [PATCH 010/122] Cut 2 fixes: stop-then-start handover fixture, supervision owner skip, rebase preflight, archive-as-progress - Add an engine-level stop-then-start fixture (route-less host-actuator plan) and a test that two such transactions never both enter Starting, in either order; it kills enter taking the CAS only at Fencing and takes_target_at ignoring the rollout. Discharges follow-up stop-first-handover-fixture. - Pin supervision's skip of an owned target: a fenced generation is not refreshed from Odin, and a stop-first owner's stopped incumbent is not restarted. - The rebase test now carries a route preflight, so keeping it dies. - The step that ends a transaction archives it inside its write; the progress read now treats a terminal archived record as progress, not a fault. - enter's doc no longer claims every phase move goes through it. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 364 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 333 insertions(+), 31 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0b20e33..108d5f7 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5894,11 +5894,22 @@ impl Engine { } } let after = ControlSnapshot::read(&self.options.state_store)?; - let live = after + let Some(live) = after .transactions .iter() .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while checking scheduler progress")?; + else { + // The step that ends a transaction archives it inside its own + // write, so it is gone from the live set because it finished: that + // is the step's progress. Gone from both is a fault. + ensure!( + read_history_transactions(&self.options.state_store) + .iter() + .any(|archived| archived.transaction_id == id && archived.is_terminal()), + "transaction disappeared while checking scheduler progress" + ); + return Ok(true); + }; Ok(state_advanced(¤t.value, &live.value)) } @@ -9097,10 +9108,12 @@ impl Engine { } } - /// The one phase-entry seam. Every move into the next phase is written - /// here, and entering the phase where the transaction takes its target - /// (`takes_target_at`) is decided by `take_target`: that CAS is the - /// handover, and nothing else writes that entry. + /// The seam for the forward moves that can be the handover (`transition` + /// and the end of Sealing). Entering the phase where the transaction takes + /// its target (`takes_target_at`) is decided by `take_target`: that CAS is + /// the handover, and nothing else writes that entry. The other writes + /// (Starting to Warming, completions, aborts, same-phase evidence) go to + /// `replace_transaction` directly because none of them enters that phase. fn enter( &self, current: &Stored, @@ -12605,6 +12618,27 @@ mod tests { }) } + /// `binding` with its provider trust anchor and signer pointed at this + /// world's provider identity, enrolled on first use. + fn bind_provider(world: &EngineFixture, binding: &str) -> Result { + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = if provider_path.exists() { + open_service_identity_at::(&provider_path)? + } else { + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + provider + }; + Ok(binding + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id)) + } + /// What makes a target Odin: it provides the rendezvous. fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { @@ -12657,32 +12691,19 @@ mod tests { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, }; - let provider_path = world.root.join("identities/provider.cc"); - let provider_anchor = world.root.join("identities/provider-anchor.cc"); - let provider = if provider_path.exists() { - open_service_identity_at::(&provider_path)? - } else { - let provider = - enroll_service_identity_at::(&provider_path)?; - export_service_identity_trust_anchor(&provider, &provider_anchor)?; - provider - }; - let (binding_head, binding_tail) = BINDING .split_once("[route]") .context("binding has no route table")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail - .split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ) - .replace( - "/etc/gamecult/trust/service.cc", - &provider_anchor.display().to_string(), - ) - .replace("service-runtime-signer", &provider.entry().identity_id); + let binding = bind_provider( + world, + &format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ), + )?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -13987,17 +14008,25 @@ mod tests { /// Admit a generation for "service" through the real phase machine and /// retire its transaction, leaving the world with an incumbent. fn admit_incumbent(world: &EngineFixture) -> Result { + Ok(admit_seeded_incumbent(world)?.0) + } + + /// `admit_incumbent`, and the transaction that was admitted. + fn admit_seeded_incumbent( + world: &EngineFixture, + ) -> Result<(AdmittedGeneration, DeploymentTransaction)> { let seeded = transaction_at(world, DeploymentPhase::Fencing)?; odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; drive(world, |transaction| { transaction.phase == DeploymentPhase::Complete })?; assert!(world.engine.retire_one_terminal_transaction()?); - Ok(ControlSnapshot::read(&world.state_store)? + let admitted = ControlSnapshot::read(&world.state_store)? .admitted_for("service") .context("commit wrote no admitted generation")? .value - .clone()) + .clone(); + Ok((admitted, seeded)) } #[test] @@ -16686,6 +16715,260 @@ mod tests { Ok(()) } + /// A route-less host-actuator target that rolls out stop-then-start: the + /// seeded systemd binding with its runner, workload and rollout swapped. + fn stop_first_binding(world: &EngineFixture) -> Result { + use crate::deployment_plan::tests::BINDING; + const HOST: &str = r#"[runners.rust] +driver = "host-native" +affordances = ["source-read", "artifact-write"] +allowed_programs = ["cargo"] + +[workload] +driver = "host-actuator" +host = "yggdrasil" +host_trust_anchor_store = "/etc/gamecult/trust/host-actuator.cc" +release_root = "/srv/service/releases" +runtime_root = "/run/gamecult/idunn/service" +state_root = "/var/lib/gamecult/service" + +[workload.argument_bindings] +state_root = "/var/lib/gamecult/service" + +[workload.secret_files] +GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" + +"#; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; + let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; + let (identity, rest) = rest.split_once("[route]").context("no route")?; + let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; + bind_provider( + world, + &format!("{head}{HOST}[runtime_identity]{identity}[brakes]{brakes}") + .replace("candidate-then-promote", "stop-then-start"), + ) + } + + /// The seeded transaction `id`, rewritten as the Sealing transaction of a + /// stop-then-start target: the host-actuator plan, release and Expected, + /// and none of the evidence that follows Sealing. + fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + use crate::deployment_plan::tests::{ + RECIPE, artifact_receipt, external_input_receipt, source, + }; + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let binding = stop_first_binding(world)?; + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", "GAMECULT_IDUNN_RUNTIME_BUNDLE", "GAMECULT_RUNTIME_PRESENCE_IDENTITY"]"#, + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + &format!("service-{id}"), + None, + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let mut expected = release.expected_projection(&plan)?; + declare_odin(&mut expected); + let now = now_millis()?; + if next.command_kind == CommandKind::Deploy { + next.frozen_source = Some(FrozenSourceReceipt { + transaction_id: id.into(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + next.deployment_authorization = + Some(deployment_authorization(world, &expected, id, now)?); + } + next.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + next.sealed_release = Some(release); + next.expected = Some(expected); + next.plan = Some(plan); + next.expected_publication_sha256 = None; + next.activation_publication_sha256 = None; + next.activation = None; + next.workload = None; + next.warming = None; + next.isolation = None; + next.enter_phase(DeploymentPhase::Sealing, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// An admitted incumbent of `service`, and beside it a Deploy and a + /// Continuity (their ids, in that order), both stop-then-start at Sealing. + fn stop_first_world( + workload: Arc, + ) -> Result<(EngineFixture, String, String)> { + let world = EngineFixture::with_workload(workload)?; + let incumbent = admit_incumbent(&world)?; + let mut ids = Vec::new(); + for kind in [CommandKind::Deploy, CommandKind::Continuity] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + make_stop_first(&world, &seeded.transaction_id)?; + ids.push(seeded.transaction_id); + } + let continuity = ids.pop().context("no continuity")?; + let deploy = ids.pop().context("no deploy")?; + Ok((world, deploy, continuity)) + } + + /// Stop-then-start takes the target at Starting, by the same CAS as + /// Fencing: of two such transactions at Sealing, whichever enters Starting + /// first owns the target, and the other neither enters nor aborts. + #[test] + fn two_stop_then_start_transactions_never_both_enter_starting() -> Result<()> { + use DeploymentPhase::{Sealing, Starting}; + for deploy_first in [true, false] { + let (world, deploy, continuity) = stop_first_world(Arc::new(StillWorkload))?; + let (first, second) = if deploy_first { + (deploy.clone(), continuity.clone()) + } else { + (continuity.clone(), deploy.clone()) + }; + let stored = resident_id(&world, &first)?; + assert_eq!(stored.value.takes_target_at(), Starting); + assert!(!stored.value.owns_target()); + world.engine.enter(&stored, entering(&stored, Starting)?)?; + let owner = resident_id(&world, &first)?.value; + assert_eq!(owner.phase, Starting); + assert!(owner.owns_target(), "a stop-first transaction at Starting does not own the target"); + + let rival = resident_id(&world, &second)?; + let outcome = world.engine.enter(&rival, entering(&rival, Starting)?); + if deploy_first { + let error = outcome.expect_err("a continuity entered Starting over an owner"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + assert!(format!("{error:#}").contains(&deploy), "{error:#}"); + } else { + outcome?; + let waiting = resident_id(&world, &second)?.value; + assert!( + waiting.last_error.as_deref().is_some_and(|note| note.contains(&continuity)), + "{:?}", + waiting.last_error + ); + } + // The store still reads: one owner, and the rival held at Sealing. + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!(after.transactions.iter().filter(|stored| stored.value.owns_target()).count(), 1); + let held = resident_id(&world, &second)?.value; + assert_eq!(held.phase, Sealing); + assert!(held.pre_fencing_abort.is_none()); + } + Ok(()) + } + + /// A stop-first deployment that has entered Starting has stopped the + /// incumbent by design; supervision must read it as the owner and not + /// restart what it stopped. + #[test] + fn supervision_does_not_restart_the_incumbent_a_stop_first_owner_stopped() -> Result<()> { + let workload = SwitchWorkload::new(); + let (world, deploy, _) = stop_first_world(workload.clone())?; + clear_other_live_transactions(&world, &deploy)?; + age_incumbent(&world, 500_000)?; + let stored = resident_id(&world, &deploy)?; + world.engine.enter(&stored, entering(&stored, DeploymentPhase::Starting)?)?; + workload.kill(); + + let before = incumbent_envelope(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "supervision restarted what the owner stopped"); + assert_eq!(incumbent_envelope(&world)?, before); + + // Released, the same dead incumbent is restarted. + clear_other_live_transactions(&world, "")?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + Ok(()) + } + + /// An owner fencing the incumbent is the only writer of it: supervision + /// skips the generation, so it neither refreshes it from Odin nor repairs + /// or retires around it. + #[test] + fn supervision_leaves_a_generation_its_owner_is_fencing_alone() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (incumbent, admitted) = admit_seeded_incumbent(&world)?; + let owner = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + rephase(&world, &owner.transaction_id, DeploymentPhase::Fencing)?; + // Odin has a newer reading of the incumbent than the one it holds. + odin_republishes(&world, &admitted, seeded_sequence(None) + 5)?; + + let before = incumbent_envelope(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(incumbent_envelope(&world)?, before, "supervision refreshed a generation an owner is fencing"); + + // The same world with no owner takes the reading, so the skip alone held it. + clear_other_live_transactions(&world, "")?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_ne!(incumbent_envelope(&world)?, before); + Ok(()) + } + + /// The step that ends a transaction archives it inside its own write. It + /// left the live set because it finished: progress, not a fault. + #[test] + fn the_step_that_ends_a_transaction_reports_progress_and_no_fault() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let ended = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let id = ended.transaction_id; + let mut last = None; + for _ in 0..20 { + let Ok(current) = resident_id(&world, &id) else { + break; + }; + let moved = world.engine.resume_candidate(¤t)?; + if resident_id(&world, &id).is_err() { + last = Some(moved); + break; + } + } + assert_eq!(last, Some(true), "the ending step reported no progress"); + assert!(latest_id(&world, &id)?.is_terminal()); + assert!(world.engine.fault_reports.lock().unwrap().is_empty()); + + // Through the tick, the way the scheduler runs it: no fault is kept. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let ended = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + for _ in 0..20 { + if resident_id(&world, &ended.transaction_id).is_err() { + break; + } + world.engine.resume_one_transaction()?; + } + assert!(latest_id(&world, &ended.transaction_id)?.is_terminal()); + assert!(world.engine.fault_reports.lock().unwrap().is_empty()); + Ok(()) + } + #[test] fn the_store_holds_one_deploy_one_continuity_and_one_owner_per_target() -> Result<()> { let seed = |first: (CommandKind, DeploymentPhase), second: (CommandKind, DeploymentPhase)| { @@ -16737,6 +17020,24 @@ mod tests { let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; let deploy_id = deploy.transaction_id.as_str(); + // The build recorded a route preflight against the incumbent it saw. + let building = resident_id(&world, deploy_id)?; + let mut preflighted = building.value.clone(); + preflighted.route_preflight = Some(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: building + .value + .activation + .as_ref() + .context("no activation")? + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(b"membership"), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }); + replace_transaction(&world.state_store, &building, &preflighted)?; workload.launching(); workload.kill(); assert!(world.engine.supervise_one_admitted_generation()?); @@ -16758,6 +17059,7 @@ mod tests { assert_eq!(untouched.value.incumbent_generation_id, Some(incumbent.generation_id.clone())); // The deployment's next step is the rebase and nothing else. + assert!(untouched.value.route_preflight.is_some() && untouched.value.warming.is_some()); world.engine.advance_transaction(&untouched)?; let rebased = resident_id(&world, deploy_id)?.value; assert_eq!(rebased.incumbent_generation_id, Some(restarted.generation_id.clone())); From 9ea6c719217a21cc7b7c7f9d344807b1e2031a71 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:01:59 +0200 Subject: [PATCH 011/122] Cut 2 test: the stop-first fixture builds without a network input A host-native runner has no network profile, so the host-actuator recipe drops the external input the systemd recipe fetches. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 108d5f7..3932388 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16755,7 +16755,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// and none of the evidence that follows Sealing. fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { use crate::deployment_plan::tests::{ - RECIPE, artifact_receipt, external_input_receipt, source, + RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); @@ -16769,6 +16769,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, r#"["GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", "GAMECULT_IDUNN_RUNTIME_BUNDLE", "GAMECULT_RUNTIME_PRESENCE_IDENTITY"]"#, ); + // A host-native runner has no network profile to fetch an input with. + let (before_inputs, inputs) = recipe + .split_once("[[external_inputs]]") + .context("recipe has no external inputs")?; + let recipe = format!( + "{before_inputs}[[artifacts]]{}", + inputs.split_once("[[artifacts]]").context("recipe has no artifacts")?.1 + ); let plan = compile_deployment_plan( recipe.as_bytes(), binding.as_bytes(), @@ -16781,7 +16789,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let release = SealedRelease::new( &plan, vec![artifact_receipt()], - vec![external_input_receipt()], + Vec::new(), 120, )?; let mut expected = release.expected_projection(&plan)?; From 3265ccb575eec95293c020e074ed3970c73efe48 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:15:07 +0200 Subject: [PATCH 012/122] Cut c2b test: an abort stops a candidate that holds its lease before revoking it Negative test for ghostlight-abort-lease-before-stop: the candidate holds the lease lock until stopped. Recorded failing at this commit; the fix follows. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3932388..947a031 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13864,6 +13864,11 @@ mod tests { /// Whether a start launches a process (a fresh private identity) /// instead of refusing like `StillWorkload`. launches: std::sync::atomic::AtomicBool, + /// Files whose advisory locks stand for a process that holds a lease + /// for its lifetime: `stop` is what releases them. + holds: std::sync::Mutex>, + /// How many times the process was stopped. + stops: std::sync::atomic::AtomicUsize, } impl SwitchWorkload { @@ -13874,6 +13879,8 @@ mod tests { silent: std::sync::atomic::AtomicBool::new(false), spared: std::sync::Mutex::new(Vec::new()), launches: std::sync::atomic::AtomicBool::new(false), + holds: std::sync::Mutex::new(Vec::new()), + stops: std::sync::atomic::AtomicUsize::new(0), }) } @@ -13882,6 +13889,24 @@ mod tests { self.launches.store(true, std::sync::atomic::Ordering::SeqCst); } + /// The process takes a shared lock on `store`'s authority lock, as a + /// candidate holding its lifetime write lease does, until stopped. + #[cfg(unix)] + fn hold_lease_lock(&self, store: &std::path::Path) { + use std::os::unix::io::AsRawFd; + let mut lock = store.as_os_str().to_os_string(); + lock.push(".lock"); + let file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(lock) + .unwrap(); + // SAFETY: flock on a descriptor this function owns. + assert_eq!(unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) }, 0); + self.holds.lock().unwrap().push(file); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -13995,6 +14020,8 @@ mod tests { Ok(prior.clone()) } fn stop(&self, _: &WorkloadObservation) -> Result<()> { + self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.holds.lock().unwrap().clear(); Ok(()) } fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { @@ -21881,6 +21908,34 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// ghostlight-abort-lease-before-stop. A candidate that holds its + /// lifetime write lease keeps the lease's shared lock until it is + /// stopped, so the abort stops it before revoking; revoking first + /// fails on the lock on every attempt and the abort never ends. + #[cfg(unix)] + #[test] + fn an_abort_stops_a_candidate_that_holds_its_lease_before_revoking_it() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + let lease_path = routed.world.root.join("lease/process-write-lease.cc"); + routed.workload.hold_lease_lock(&lease_path); + let driver = CultCacheWriteLeaseDriver::new("service", &lease_path); + + pass_deadline(&routed.world)?; + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + let stops = routed.workload.stops.load(Ordering::SeqCst); + let finished = routed.resume_until_terminal()?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.last_error + ); + assert!(driver.observe_empty()?, "the abort left the lease on disk"); + assert!(routed.workload.stops.load(Ordering::SeqCst) > stops, "the candidate was never stopped"); + Ok(()) + } + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word /// carries the granted lease, and that Ready records the adoption, /// sourced from Odin's topology and naming the presence Odin observed. From ffb22e368b5cfd1ad74760184387b2e52b9f2d30 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:15:47 +0200 Subject: [PATCH 013/122] Cut c2b: the post-fencing abort stops its candidate before revoking the lifetime-held lease ruling c2b-abort-stops-before-revoke; fixes follow-up ghostlight-abort-lease-before-stop. Mirrors advance_fencing. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 947a031..e745693 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -9355,8 +9355,9 @@ impl Engine { } /// Withdraw the candidate in the one order that never leaves two writers: - /// take away its route, then its write lease, then the process itself, and - /// only then reconcile the projection. + /// take away its route, then stop the process and withdraw its write lease + /// (the lease is lifetime-held, so the holder stops first), then clean up, + /// and only then reconcile the projection. fn advance_post_fencing_abort(&self, current: &Stored) -> Result<()> { let abort = required( ¤t.value.post_fencing_abort, @@ -9396,6 +9397,15 @@ impl Engine { .process_write_lease .context("leased target has no write-lease binding")? .record_path; + // The candidate holds its lifetime write lease; revoking it under + // a live holder fails on the shared lock forever. A stopped + // process writes nothing, so stop first (as advance_fencing does). + if let Some(workload) = ¤t.value.workload { + let plan = required(¤t.value.plan, "abandoned candidate plan")?; + self.workload_for(plan)?.stop(workload).context( + "stopping the abandoned candidate before revoking its lifetime-held write lease", + )?; + } let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); driver .revoke_exact(Some(lease)) From 86926c4e426f4473f5557f93e216ce94f9b7c266 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:23:59 +0200 Subject: [PATCH 014/122] Cut c2b: one derivation of every deadline; Warming, aborts and drains are timed WARMING_DEADLINE_SECONDS and PRE_FENCING_ABORT_DEADLINE_SECONDS are Idunn constants (no PhaseDeadlines field: plan ids stay). DEADLINE_PHASES is the one spelling of the deadline range. deadline_for_current_activity derives the duration of a phase, an abort from its beginning, or a drain to its retire; retime is the only non-test writer. leaves_to_operator is derived from it. Validation asks for an activity, not a post-fencing phase. Records written by this cut are forward-only (ruling c2b-forward-only-records). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 310 ++++++++++++++++++++++++++++++++++++----- src/deployment_plan.rs | 7 + 2 files changed, 283 insertions(+), 34 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e745693..c9179c6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -35,7 +35,8 @@ use crate::deployment::{ capability_compatible, }; use crate::deployment_plan::{ - CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, + CompiledDeploymentPlan, DependencyProviderAuthority, PRE_FENCING_ABORT_DEADLINE_SECONDS, + SealedRelease, WARMING_DEADLINE_SECONDS, compile_deployment_plan, }; use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, @@ -925,8 +926,15 @@ impl LeaseAdoptionEvidence { } } -/// When the transaction entered its current post-fencing phase and when that -/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. +/// The phases whose entry starts a deadline: the one spelling of it. +const DEADLINE_PHASES: std::ops::RangeInclusive = + DeploymentPhase::Warming..=DeploymentPhase::Committing; + +/// What a live record is doing now and when that must end: a deadline phase +/// from its entry, an abort from its beginning, a drain to its retire. The +/// post-fencing durations come from the plan's frozen `PhaseDeadlines`; the +/// rest are Idunn's. `DeploymentTransaction::deadline_for_current_activity` +/// is the one derivation. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct PhaseDeadline { @@ -936,7 +944,8 @@ struct PhaseDeadline { } impl PhaseDeadline { - /// `None` for a phase that has no deadline (pre-fencing and Complete). + /// `None` for a phase that has no deadline of its own (Sealing, Starting + /// and Complete). fn entering( phase: DeploymentPhase, plan: &CompiledDeploymentPlan, @@ -944,6 +953,7 @@ impl PhaseDeadline { ) -> Option { let deadlines = plan.phase_deadlines(); let seconds = match phase { + DeploymentPhase::Warming => WARMING_DEADLINE_SECONDS, DeploymentPhase::Fencing => deadlines.fencing_seconds, DeploymentPhase::Leasing => deadlines.leasing_seconds, DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, @@ -959,9 +969,12 @@ impl PhaseDeadline { } } -/// How the deadline resolver ends a post-fencing phase. +/// How the deadline resolver ends a deadline phase. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum PhaseEnd { + /// Before the fence the transaction owns only its candidate: it ends + /// through its own pre-fencing abort. + AbortBeforeFencing, /// Abort through the post-fencing path. `lease_not_adopted` names a /// granted lease with no adoption evidence. Abort { lease_not_adopted: bool }, @@ -1720,25 +1733,112 @@ fn handover_phase(stops_incumbent_first: bool) -> DeploymentPhase { } impl DeploymentTransaction { - /// The one writer of `phase`. It stamps the update time and the phase - /// deadline together, so a record outside Fencing..=Committing carries no - /// deadline by construction. No other code assigns `phase`. + /// The one writer of `phase`. It stamps the update time and, through + /// `retime`, the deadline of what the record now does. No other code + /// assigns `phase`. fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { self.phase = phase; self.updated_at_unix_millis = now; - self.phase_deadline = self - .plan - .as_ref() - .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); + self.retime(now); } - /// A live transaction past the fence that is not already being aborted: - /// the only kind whose phase a deadline, or an operator's expiry, ends. + /// Whether the abort of the pre-fence path still has cleanup to do. + fn pre_fencing_abort_pending(&self) -> bool { + self.completion.is_none() + && self + .pre_fencing_abort + .as_ref() + .is_some_and(|abort| !abort.is_complete()) + } + + /// The one derivation of every duration the record is held to. (1) A + /// pending pre-fencing abort is timed from now by Idunn's constant. (2) A + /// draining Complete is timed from its incumbent's retire time (the route + /// promotion plus the drain, or now when nothing was promoted) for the + /// plan's committing duration. (3) Any other Complete has none. (4) The + /// rest take their phase's duration from now, so a post-fencing abort gets + /// the frozen duration of the phase it aborts, counted from its beginning. + fn deadline_for_current_activity(&self, now: u64) -> Option { + if self.pre_fencing_abort_pending() { + return Some(PhaseDeadline { + phase: self.phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: now.saturating_add( + u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000, + ), + }); + } + if self.phase == DeploymentPhase::Complete { + if !matches!(self.completion, Some(TransactionCompletion::Admitted { .. })) + || !self.drains_incumbent() + { + return None; + } + let plan = self.plan.as_ref()?; + let from = match self + .routing + .as_ref() + .and_then(RoutingEvidence::promoted_at_unix_millis) + { + Some(promoted_at) => { + route_drain_deadline(promoted_at, plan.parsed_inputs().ok()?.1.rollout.drain_seconds) + .ok()? + } + None => now, + }; + return Some(PhaseDeadline { + phase: self.phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: from + .max(now) + .saturating_add(u64::from(plan.phase_deadlines().committing_seconds) * 1000), + }); + } + PhaseDeadline::entering(self.phase, self.plan.as_ref()?, now) + } + + /// Stamp the deadline of what the record does now. The only non-test + /// writer that sets one: phase entry, an abort's beginning, the commit to a + /// draining Complete and the resolver's lift call it. + fn retime(&mut self, now: u64) { + self.phase_deadline = self.deadline_for_current_activity(now); + } + + /// Whether a deadline can describe the record's current activity: a + /// deadline phase, a pending pre-fencing abort, or an admitted Complete. + fn carries_a_deadline(&self) -> bool { + DEADLINE_PHASES.contains(&self.phase) + || self.pre_fencing_abort_pending() + || (self.phase == DeploymentPhase::Complete + && matches!(self.completion, Some(TransactionCompletion::Admitted { .. }))) + } + + /// A live transaction in a deadline phase that is not already being + /// aborted: the only kind whose phase a deadline, or an operator's expiry, + /// ends. fn resolves_by_deadline(&self) -> bool { self.completion.is_none() && self.pre_fencing_abort.is_none() && self.post_fencing_abort.is_none() - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) + && DEADLINE_PHASES.contains(&self.phase) + } + + /// Whether Idunn has run out of ways to end this record and leaves it to + /// the operator: its deadline has passed in an abort that cannot finish, a + /// drain whose stop keeps failing, or a stateful deploy. Derived, never + /// stored. The step goes on retrying and the target stays owned. + fn leaves_to_operator(&self, now: u64) -> bool { + if self.is_terminal() { + return false; + } + let Some(deadline) = &self.phase_deadline else { + return false; + }; + now >= deadline.deadline_at_unix_millis + && (self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some() + || self.drains_incumbent() + || (self.resolves_by_deadline() && self.phase_end() == PhaseEnd::OperatorRequired)) } /// What an ended post-fencing phase resolves to (Q3). A candidate that was @@ -1750,6 +1850,9 @@ impl DeploymentTransaction { /// new state contract, and restoring the incumbent over that is not /// Idunn's call: it is left for the operator. fn phase_end(&self) -> PhaseEnd { + if self.phase < DeploymentPhase::Fencing { + return PhaseEnd::AbortBeforeFencing; + } let leasing = self.leasing.as_ref(); let granted = leasing.and_then(LeasingEvidence::lease).is_some(); let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); @@ -2150,11 +2253,10 @@ impl DeploymentTransaction { if let Some(deadline) = &self.phase_deadline { ensure!( deadline.phase == self.phase - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&deadline.phase) + && self.carries_a_deadline() && deadline.entered_at_unix_millis > 0 && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, - "phase deadline does not describe the current post-fencing phase" + "phase deadline does not describe the record's current activity" ); } if let Some(adoption) = &self.lease_adoption { @@ -3012,7 +3114,7 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> ); ensure!( current.value.resolves_by_deadline(), - "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" ); let now = now_millis()?; let request = ExpiryRequest { @@ -7849,6 +7951,9 @@ impl Engine { generation_id: generation.generation_id.clone(), }); complete.post_commit_cleanup = Some(post_commit_cleanup); + // enter_phase ran before the cleanup was set: a draining record + // carries its drain's deadline. + complete.retime(now); complete.validate()?; let admitted_expected = CultCacheExpectedEnvelope { @@ -9205,9 +9310,10 @@ impl Engine { /// command an operator asked to expire, is resolved as `phase_end` says. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// - /// The deadline is written only by `enter_phase`, so nothing here or in a - /// resume path extends it. A record lifted from before deadlines existed - /// has none; its phase is timed once, from the first time this sees it. + /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the + /// commit and this lift), so nothing in a step or a resume path extends one. + /// A record lifted from before deadlines existed has none; what it does is + /// timed once, from the first time this sees it. fn resolve_phase_deadline( &self, current: &Stored, @@ -9250,6 +9356,10 @@ impl Engine { self.begin_post_fencing_abort(current, error)?; Ok(Some(true)) } + PhaseEnd::AbortBeforeFencing => { + self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; + Ok(Some(true)) + } PhaseEnd::OperatorRequired => { self.report_once( &format!("deadline:{}", transaction.transaction_id), @@ -9313,8 +9423,10 @@ impl Engine { "pre-fencing abort intent is already durable" ); let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); + let now = now_millis()?; self.persist_same_phase(current, |next| { next.pre_fencing_abort = Some(abort); + next.retime(now); Ok(()) }) } @@ -9348,8 +9460,10 @@ impl Engine { "post-fencing abort intent is already durable" ); let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); + let now = now_millis()?; self.persist_same_phase(current, |next| { next.post_fencing_abort = Some(abort); + next.retime(now); Ok(()) }) } @@ -12391,11 +12505,12 @@ mod tests { } #[test] - fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { + fn phase_deadlines_follow_the_plan_and_cover_warming_through_committing() -> Result<()> { let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; let plan = &generation.plan; let defaults = plan.phase_deadlines(); for (phase, seconds) in [ + (DeploymentPhase::Warming, WARMING_DEADLINE_SECONDS), (DeploymentPhase::Fencing, defaults.fencing_seconds), (DeploymentPhase::Leasing, defaults.leasing_seconds), (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), @@ -12410,7 +12525,6 @@ mod tests { for phase in [ DeploymentPhase::Sealing, DeploymentPhase::Starting, - DeploymentPhase::Warming, DeploymentPhase::Complete, ] { assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); @@ -12467,7 +12581,7 @@ mod tests { } #[test] - fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { + fn a_deadline_must_describe_the_records_current_activity() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; let plan = transaction.plan.clone().unwrap(); let now = transaction.updated_at_unix_millis; @@ -12482,10 +12596,25 @@ mod tests { backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; assert!(backwards.validate().is_err()); // Entry is not ordered against the wall-clock stamp of later writes. - let mut entered_after_the_stamp = transaction; + let mut entered_after_the_stamp = transaction.clone(); entered_after_the_stamp.phase_deadline = PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); entered_after_the_stamp.validate()?; + + // The activity rule: Warming carries a deadline; Sealing only while + // its abort is pending; Complete only once admitted. + let mut other = transaction.clone(); + other.phase = DeploymentPhase::Warming; + assert!(other.carries_a_deadline()); + other.phase = DeploymentPhase::Sealing; + assert!(!other.carries_a_deadline(), "Sealing has no activity to time"); + other.pre_fencing_abort = Some(pre_fencing_abort_intent(&other, "refused")); + assert!(other.carries_a_deadline()); + other.pre_fencing_abort = None; + other.phase = DeploymentPhase::Complete; + assert!(!other.carries_a_deadline()); + other.completion = Some(TransactionCompletion::Admitted { generation_id: "g".into() }); + assert!(other.carries_a_deadline()); Ok(()) } @@ -13019,8 +13148,8 @@ mod tests { /// One engine step of transaction `id` at a time until `stop` holds, /// recording each phase entered and checking after every step that the - /// deadline on the record is exactly the current post-fencing phase's, and - /// absent outside one. + /// deadline on the record is exactly the current phase's, and absent + /// outside a deadline phase. fn drive_id( world: &EngineFixture, id: &str, @@ -13038,10 +13167,12 @@ mod tests { let after = latest_id(world, id)?; match &after.phase_deadline { Some(deadline) => assert_eq!(deadline.phase, after.phase), + // A waiter's Warming deadline is disarmed while another + // transaction owns the target. None => assert!( - !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&after.phase), - "a post-fencing record lost its deadline" + after.phase == DeploymentPhase::Warming + || !DEADLINE_PHASES.contains(&after.phase), + "a deadline-phase record lost its deadline" ), } if phases.last() != Some(&after.phase) { @@ -13087,15 +13218,28 @@ mod tests { } #[test] - fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { + fn a_pre_fence_abort_is_timed_from_its_beginning_and_ends_without_a_deadline() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; transaction_at(&world, DeploymentPhase::Warming)?; let warming = resident(&world)?; - assert_eq!(warming.value.phase_deadline, None); + let warming_deadline = warming.value.phase_deadline.context("Warming has no deadline")?; + assert_eq!( + warming_deadline.deadline_at_unix_millis - warming_deadline.entered_at_unix_millis, + u64::from(WARMING_DEADLINE_SECONDS) * 1000 + ); + let before = now_millis()?; world .engine .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; + let aborting = latest(&world)?; + let deadline = aborting.phase_deadline.context("the abort has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Warming); + assert!(deadline.entered_at_unix_millis >= before); + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000 + ); drive(&world, |transaction| transaction.completion.is_some())?; let finished = latest(&world)?; assert_eq!(finished.phase, DeploymentPhase::Complete); @@ -13139,6 +13283,96 @@ mod tests { Ok(()) } + #[test] + fn an_abort_is_timed_from_its_beginning_with_its_phases_duration() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Routing + })?; + // The phase's own deadline is long past when the abort begins. + let routing = resident(&world)?; + let mut stale = routing.value.clone(); + let deadline = stale.phase_deadline.as_mut().context("Routing has no deadline")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, &routing, &stale)?; + + let before = now_millis()?; + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let aborting = latest(&world)?; + let deadline = aborting.phase_deadline.context("the abort has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Routing); + assert!(deadline.entered_at_unix_millis >= before); + let routing_millis = + u64::from(aborting.plan.as_ref().unwrap().phase_deadlines().routing_seconds) * 1000; + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + routing_millis + ); + assert!(!aborting.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(aborting.leaves_to_operator(deadline.deadline_at_unix_millis)); + Ok(()) + } + + #[test] + fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + let (_, mut draining) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + let plan = draining.plan.clone().unwrap(); + let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; + let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; + assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); + draining.phase = DeploymentPhase::Complete; + draining.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-new".into(), + }); + draining.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Pending { + generation_id: "generation-old".into(), + workload: workload(1001, 40, 50), + }, + source: SourceCleanupEvidence::Complete, + }); + let promoted_at = 10_000_000_u64; + match draining.routing.as_mut() { + Some(RoutingEvidence::Promoted { promoted_at_unix_millis, .. }) => { + *promoted_at_unix_millis = promoted_at; + } + other => bail!("the fixture is not promoted: {other:?}"), + } + + // Before the retire time: the retire time plus the committing duration. + draining.retime(promoted_at + 1); + let deadline = draining.phase_deadline.context("a drain has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Complete); + assert_eq!(deadline.entered_at_unix_millis, promoted_at + 1); + assert_eq!(deadline.deadline_at_unix_millis, promoted_at + drain + committing); + assert!(!draining.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(draining.leaves_to_operator(deadline.deadline_at_unix_millis)); + + // After it: the committing duration from now. + let late = promoted_at + drain + 5_000; + draining.retime(late); + assert_eq!(draining.phase_deadline.unwrap().deadline_at_unix_millis, late + committing); + + // Nothing promoted: from now. + draining.routing = Some(RoutingEvidence::SkippedUnrouted); + draining.retime(promoted_at); + assert_eq!(draining.phase_deadline.unwrap().deadline_at_unix_millis, promoted_at + committing); + + // A finished drain, and any other Complete, carry none. + draining.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Complete { generation_id: "generation-old".into() }, + source: SourceCleanupEvidence::Complete, + }); + draining.retime(promoted_at); + assert_eq!(draining.phase_deadline, None); + Ok(()) + } + #[test] fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; @@ -16277,6 +16511,7 @@ mod tests { let stored = resident(&world)?; let mut starting = seeded.clone(); starting.phase = DeploymentPhase::Starting; + starting.phase_deadline = None; starting.expected_publication_sha256 = None; starting.activation = None; starting.workload = None; @@ -16314,6 +16549,7 @@ mod tests { let stored = resident(world)?; let mut starting = seeded.clone(); starting.phase = DeploymentPhase::Starting; + starting.phase_deadline = None; starting.expected_publication_sha256 = None; starting.activation = None; starting.workload = None; @@ -21645,8 +21881,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[test] fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + // Warming is a deadline phase: the operator can expire it, and + // the pre-fencing abort then ends it. Once aborting it cannot be + // expired again. + expire(&early.world.state_store, "continuity-service", "operator")?; + early.resume()?; + assert!(early.transaction()?.pre_fencing_abort.is_some(), "the expiry did not end Warming"); let refused = expire(&early.world.state_store, "continuity-service", "operator") - .expect_err("a transaction before the fence was expired"); + .expect_err("an aborting transaction was expired"); assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 603c9d2..1363c48 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -475,6 +475,13 @@ pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300; pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120; pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120; +/// Idunn's own deadlines, not binding-declared: a new `PhaseDeadlines` field +/// would change every v3 plan id (`recomputed_plan_id` re-encodes the plan). +/// Warming is timed for every transaction, owner or not. +pub const WARMING_DEADLINE_SECONDS: u32 = 600; +/// A pre-fencing abort is timed from its beginning. +pub const PRE_FENCING_ABORT_DEADLINE_SECONDS: u32 = 300; + /// The resolved per-phase deadline durations a plan carries. Entering a /// post-fencing phase stamps its deadline from them, and the deadline /// resolver ends the phase once it passes. From a14055115cbf0f06dbcd653ebbc71383b6ecd044 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:37:21 +0200 Subject: [PATCH 015/122] Cut c2b: owner deadlines; a waiter never waits on an owner Idunn will not finish wait_for_owner disarms a waiter Warming deadline (the owner bounds the wait) and ends the waiter through its own pre-fencing abort when the owner leaves to the operator (owner-left-to-operator); it never aborts the owner. The resolver times Warming for every transaction, lifts a deadline onto a record that needs one, skips an owned waiter, and leaves an abort or drain past its deadline to the operator (report only; the incident follows). Rulings c2b-waiter-ends-on- operator-owner, c2b-warming-deadline, c2b-abort-and-drain-deadlines. Negative tests recorded failing at 9ea6c71. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 408 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 383 insertions(+), 25 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c9179c6..d408ee6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -8372,6 +8372,49 @@ impl Engine { } } + /// A deployment waits on the transaction that owns or drains its target. + /// The owner bounds the wait, so the waiter's own Warming deadline is + /// disarmed (the resolver re-arms it when the target is free). When the + /// owner's deadline has passed in a state Idunn will not end by itself, + /// the waiter ends through its own pre-fencing abort (the caller's Err + /// path): it never aborts the owner. + fn wait_for_owner( + &self, + current: &Stored, + owner_id: &str, + ) -> Result<()> { + let held = ¤t.value; + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if snapshot.transactions.iter().any(|stored| { + stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) + }) { + bail!( + "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", + held.target + ); + } + let detail = waiting_for_owner(&held.target, owner_id); + if held.last_error.as_deref() == Some(detail.as_str()) && held.phase_deadline.is_none() { + return Ok(()); + } + let mut next = held.clone(); + next.last_error = Some(detail); + next.phase_deadline = None; + next.updated_at_unix_millis = now; + replace_transaction(&self.options.state_store, current, &next) + } + + /// Idunn has run out of ways to end this record. It stays owned and its + /// step keeps retrying; the operator is told. + fn leave_to_operator(&self, transaction: &DeploymentTransaction, cause: &str) { + let id = &transaction.transaction_id; + self.report_once( + &format!("deadline:{id}"), + format!("Idunn leaves {id} to the operator: {cause}; it keeps retrying the step"), + ); + } + fn record_gate_wait( &self, current: &Stored, @@ -9185,11 +9228,8 @@ impl Engine { else { return Ok(false); }; - self.record_gate_wait( - current, - &waiting_for_owner(&held.target, &owner.value.transaction_id), - )?; - Ok(true) + self.wait_for_owner(current, &owner.value.transaction_id) + .map(|()| true) } CommandKind::Continuity => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; @@ -9238,7 +9278,7 @@ impl Engine { (Handover::Took | Handover::Lost, _) => Ok(()), // Wait, never abort: a pre-fencing error would end the deployment. (Handover::OwnedBy(owner), CommandKind::Deploy) => { - self.record_gate_wait(current, &waiting_for_owner(target, &owner)) + self.wait_for_owner(current, &owner) } // The next step rebases onto the generation now admitted. (Handover::Moved, CommandKind::Deploy) => Ok(()), @@ -9305,33 +9345,61 @@ impl Engine { Ok(if took { Handover::Took } else { Handover::Lost }) } - /// Every post-fencing phase ends. Before a live transaction past the - /// fence runs its step, a phase whose deadline has passed, or whose - /// command an operator asked to expire, is resolved as `phase_end` says. + /// Every live record is held to a deadline. Before a transaction runs its + /// step, a deadline phase whose deadline has passed, or whose command an + /// operator asked to expire, is resolved as `phase_end` says; an abort or + /// a drain past its deadline is left to the operator and its step retries. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the - /// commit and this lift), so nothing in a step or a resume path extends one. - /// A record lifted from before deadlines existed has none; what it does is - /// timed once, from the first time this sees it. + /// commit and this lift), so nothing in a step or a resume path extends + /// one. A record with none whose activity needs one is timed once, from + /// the first time this sees it: that is also how a waiter is re-armed once + /// the target is free. fn resolve_phase_deadline( &self, current: &Stored, ) -> Result> { let transaction = ¤t.value; - if !transaction.resolves_by_deadline() { + if transaction.is_terminal() { + return Ok(None); + } + // An owned waiter is timed by its owner. + if transaction.phase < DeploymentPhase::Fencing + && transaction.pre_fencing_abort.is_none() + && ControlSnapshot::read(&self.options.state_store)? + .target_owner(&transaction.target, Some(&transaction.transaction_id)) + .is_some() + { return Ok(None); } let now = now_millis()?; let Some(deadline) = transaction.phase_deadline else { let mut next = transaction.clone(); - next.enter_phase(transaction.phase, now); + next.retime(now); if next.phase_deadline.is_none() { return Ok(None); } replace_transaction(&self.options.state_store, current, &next)?; return Ok(Some(true)); }; + if !transaction.resolves_by_deadline() { + if transaction.leaves_to_operator(now) { + let what = if transaction.drains_incumbent() { + "draining its incumbent" + } else { + "its abort" + }; + self.leave_to_operator( + transaction, + &format!( + "{what} in the {:?} phase passed its deadline at unix ms {}", + transaction.phase, deadline.deadline_at_unix_millis + ), + ); + } + return Ok(None); + } let requested = ControlSnapshot::read(&self.options.state_store)? .expiry_requests .iter() @@ -9347,6 +9415,10 @@ impl Engine { return Ok(None); }; match transaction.phase_end() { + PhaseEnd::AbortBeforeFencing => { + self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; + Ok(Some(true)) + } PhaseEnd::Abort { lease_not_adopted } => { let error = if lease_not_adopted { anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") @@ -9356,17 +9428,10 @@ impl Engine { self.begin_post_fencing_abort(current, error)?; Ok(Some(true)) } - PhaseEnd::AbortBeforeFencing => { - self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; - Ok(Some(true)) - } PhaseEnd::OperatorRequired => { - self.report_once( - &format!("deadline:{}", transaction.transaction_id), - format!( - "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", - transaction.transaction_id - ), + self.leave_to_operator( + transaction, + &format!("{cause}, and its stateful candidate was issued a write lease"), ); Ok(None) } @@ -14113,6 +14178,8 @@ mod tests { holds: std::sync::Mutex>, /// How many times the process was stopped. stops: std::sync::atomic::AtomicUsize, + /// Runtime instances whose stop fails, whatever is asked. + failing_stops: std::sync::Mutex>, } impl SwitchWorkload { @@ -14125,6 +14192,7 @@ mod tests { launches: std::sync::atomic::AtomicBool::new(false), holds: std::sync::Mutex::new(Vec::new()), stops: std::sync::atomic::AtomicUsize::new(0), + failing_stops: std::sync::Mutex::new(Vec::new()), }) } @@ -14151,6 +14219,11 @@ mod tests { self.holds.lock().unwrap().push(file); } + /// From now on stopping this runtime instance fails. + fn fail_stop_of(&self, runtime_instance_id: &str) { + self.failing_stops.lock().unwrap().push(runtime_instance_id.to_owned()); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -14263,8 +14336,14 @@ mod tests { ); Ok(prior.clone()) } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { + fn stop(&self, stopping: &WorkloadObservation) -> Result<()> { self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if let WorkloadObservation::Systemd(observed) = stopping { + ensure!( + !self.failing_stops.lock().unwrap().contains(&observed.runtime_instance_id), + "the unit would not stop" + ); + } self.holds.lock().unwrap().clear(); Ok(()) } @@ -16844,6 +16923,254 @@ mod tests { Ok((deploy, continuity, incumbent)) } + /// One scheduler pass over every live transaction, with no resume backoff + /// standing between it and the step. + fn resume_pass(world: &EngineFixture) -> Result { + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_one_transaction() + } + + /// The runtime instance whose workload `id` runs. + fn runtime_instance_of(world: &EngineFixture, id: &str) -> Result { + match resident_id(world, id)?.value.workload { + Some(WorkloadObservation::Systemd(observed)) => Ok(observed.runtime_instance_id), + other => bail!("transaction {id} has no systemd workload: {other:?}"), + } + } + + /// The deadline of transaction `id` is already behind it. + fn pass_deadline_of(world: &EngineFixture, id: &str) -> Result<()> { + let current = resident_id(world, id)?; + let mut next = current.value.clone(); + let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, ¤t, &next) + } + + /// NEGATIVE (a). A continuity that owns the target is aborting and its + /// abort step fails on every attempt. Its deadline passes: Idunn leaves it + /// to the operator and keeps it owned, and the deployment waiting behind it + /// ends itself before fencing instead of waiting forever. + #[test] + fn a_waiting_deployment_ends_when_its_owners_abort_is_left_to_the_operator() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + workload.fail_stop_of(&runtime_instance_of(&world, owner_id)?); + world + .engine + .begin_post_fencing_abort(&resident_id(&world, owner_id)?, anyhow!("candidate refused"))?; + + // Before the deadline the deployment waits on a bounded owner. + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, deploy_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); + assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); + + pass_deadline_of(&world, owner_id)?; + for _ in 0..10 { + resume_pass(&world)?; + } + let owner = resident_id(&world, owner_id)?.value; + assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); + assert!(owner.owns_target()); + let ended = latest_id(&world, deploy_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); + assert!(ended.fencing.is_none()); + Ok(()) + } + + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop + /// keeps failing. A deployment waits behind it until the drain's deadline, + /// then ends itself; the drain stays and the target stays owned. + #[test] + fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; + let drainer_id = drainer.transaction_id.as_str(); + drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, drainer_id)?.value; + assert!(draining.drains_incumbent(), "the incumbent was already retired"); + assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); + match &draining.workload { + Some(WorkloadObservation::Systemd(_)) => {} + other => bail!("no workload: {other:?}"), + } + let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = + &draining.post_commit_cleanup + else { + bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); + }; + let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; + workload.fail_stop_of(&old.runtime_instance_id); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; + let waiter_id = waiter.transaction_id.as_str(); + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, waiter_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); + assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); + + pass_deadline_of(&world, drainer_id)?; + for _ in 0..10 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + let ended = latest_id(&world, waiter_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(drainer_id), "{error}"); + Ok(()) + } + + /// A stop-then-start transaction held in Warming with a started unit, as + /// the Starting step leaves it (this world serves no host actuator to + /// start one). + fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let expected = next.expected.clone().context("no Expected")?; + let now = now_millis()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&next.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + next.expected_publication_sha256 = Some(expected.canonical_sha256()?); + next.activation_publication_sha256 = Some(activation.canonical_sha256()?); + next.activation = Some(activation); + next.workload = Some(workload); + next.enter_phase(DeploymentPhase::Warming, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting + /// and waits in Warming for an Odin word that never comes. Its Warming + /// deadline ends it, and the deployment waiting at its Sealing tail takes + /// the target. + #[test] + fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( + ) -> Result<()> { + use DeploymentPhase::{Starting, Warming}; + let (world, deploy, continuity) = stop_first_world(SwitchWorkload::new())?; + warm_stop_first(&world, &continuity)?; + let owner = resident_id(&world, &continuity)?; + assert_eq!(owner.value.phase, Warming); + assert!(owner.value.owns_target() && owner.value.phase_deadline.is_some()); + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, Starting)?)?; + let held = resident_id(&world, &deploy)?.value; + assert_eq!(held.phase, DeploymentPhase::Sealing); + assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); + + pass_deadline_of(&world, &continuity)?; + world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; + let aborting = resident_id(&world, &continuity)?.value; + let abort = aborting.pre_fencing_abort.clone().context("the deadline did not end Warming")?; + assert!(abort.error.contains("Warming phase passed its deadline"), "{}", abort.error); + // This world serves no host actuator, so the abandoned unit cannot be + // stopped here; the abort's cleanup is recorded done. + let mut cleaned = aborting.clone(); + let intent = cleaned.pre_fencing_abort.as_mut().unwrap(); + intent.candidate_cleanup = CleanupEvidence::Complete; + intent.topology_reconciliation = CleanupEvidence::Complete; + replace_transaction(&world.state_store, &resident_id(&world, &continuity)?, &cleaned)?; + for _ in 0..12 { + if resident_id(&world, &continuity).is_err() { + break; + } + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; + } + let ended = latest_id(&world, &continuity)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("Warming phase passed its deadline"), "{error}"); + + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, Starting)?)?; + let took = resident_id(&world, &deploy)?.value; + assert_eq!(took.phase, Starting); + assert!(took.owns_target() && took.pre_fencing_abort.is_none()); + Ok(()) + } + + #[test] + fn a_waiters_warming_deadline_is_disarmed_while_an_owner_holds_the_target_and_rearmed_when_it_ends( + ) -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + rephase(&world, deploy_id, DeploymentPhase::Warming)?; + assert!(resident_id(&world, deploy_id)?.value.phase_deadline.is_some()); + + resume_pass(&world)?; + let waiting = resident_id(&world, deploy_id)?.value; + assert_eq!(waiting.phase_deadline, None, "the owner does not bound the waiter"); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); + // Held for as long as the owner stands: a deadline that fell due + // meanwhile would otherwise abort it. + for _ in 0..3 { + resume_pass(&world)?; + } + assert!(resident_id(&world, deploy_id)?.value.pre_fencing_abort.is_none()); + + // The owner retires without moving the generation. The waiter's own + // entry is long past, but it is timed from now. + clear_other_live_transactions(&world, deploy_id)?; + let stored = resident_id(&world, deploy_id)?; + let mut aged = stored.value.clone(); + aged.created_at_unix_millis = 1; + aged.updated_at_unix_millis = 1; + replace_transaction(&world.state_store, &stored, &aged)?; + let retired = now_millis()?; + resume_pass(&world)?; + let rearmed = resident_id(&world, deploy_id)?.value; + let deadline = rearmed.phase_deadline.context("the waiter was not re-armed")?; + assert_eq!(deadline.phase, DeploymentPhase::Warming); + assert!(deadline.entered_at_unix_millis >= retired); + assert!(rearmed.pre_fencing_abort.is_none()); + Ok(()) + } + #[test] fn exactly_one_transaction_takes_a_target_from_one_view() -> Result<()> { let take = |snapshot: &ControlSnapshot, world: &EngineFixture, id: &str| -> Result { @@ -21876,6 +22203,37 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// A candidate no one ever answers for is ended by its Warming + /// deadline, or by the operator's expiry: it owns only itself. + #[test] + fn a_candidate_that_never_correlates_aborts_before_fencing_at_its_warming_deadline() -> Result<()> { + for by_operator in [false, true] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..3 { + routed.resume()?; + } + let waiting = routed.transaction()?; + assert!(waiting.pre_fencing_abort.is_none(), "it ended before its deadline"); + assert!(waiting.phase_deadline.is_some()); + assert_eq!(waiting.phase, DeploymentPhase::Warming); + + let cause = if by_operator { + expire(&routed.world.state_store, "continuity-service", "operator")?; + "expired by the operator" + } else { + pass_deadline(&routed.world)?; + "Warming phase passed its deadline" + }; + routed.resume()?; + let abort = routed.transaction()?.pre_fencing_abort.context("it was not aborted")?; + assert!(abort.error.contains(cause), "{}", abort.error); + let ended = routed.resume_until_terminal()?; + assert!(matches!(ended.completion, Some(TransactionCompletion::FailedBeforeFencing { .. }))); + } + Ok(()) + } + /// `idunn expire` writes a request and nothing else; the resolver ends /// the phase from it, and the spent request is deleted. #[test] From 126d710de3edb997e1a40ce2415e6ba80a920af1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:39:42 +0200 Subject: [PATCH 016/122] Cut c2b: operator-required incident, opened at the decision site and closed when spent IncidentCondition::OperatorRequired (subject: transaction id) opens in leave_to_operator, only for a record that leaves_to_operator, and closes as recovered in close_spent_operator_incidents once no transaction in the snapshot is left to the operator. Incident writes are never progress. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 90 +++++++++++++++++++++++++++++++++++++++++++- src/incident.rs | 27 ++++++++++--- 2 files changed, 110 insertions(+), 7 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d408ee6..8732e4c 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5880,6 +5880,8 @@ impl Engine { fn resume_one_transaction(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; + // Incident writes are never progress. + self.close_spent_operator_incidents(&snapshot, now_millis()?); let mut candidates = snapshot .transactions .iter() @@ -8406,9 +8408,19 @@ impl Engine { } /// Idunn has run out of ways to end this record. It stays owned and its - /// step keeps retrying; the operator is told. + /// step keeps retrying; the operator is told. The incident comes first and + /// the trace second, and only for a record that is left to the operator + /// (an operator's expiry of a stateful deploy is reported, not paged). fn leave_to_operator(&self, transaction: &DeploymentTransaction, cause: &str) { let id = &transaction.transaction_id; + let now = now_millis(); + if now.as_ref().is_ok_and(|now| transaction.leaves_to_operator(*now)) { + self.record_incident( + "open", + id, + now.and_then(|now| self.incidents().open(IncidentCondition::OperatorRequired, id, now)), + ); + } self.report_once( &format!("deadline:{id}"), format!("Idunn leaves {id} to the operator: {cause}; it keeps retrying the step"), @@ -9438,6 +9450,24 @@ impl Engine { } } + /// Close the operator-required incident of every transaction that is no + /// longer left to the operator: it ended, or its step went on. A + /// transaction that is still left keeps its incident open. + fn close_spent_operator_incidents(&self, snapshot: &ControlSnapshot, now: u64) { + let store = self.incidents(); + let close = || -> Result<()> { + for subject in store.open_subjects(IncidentCondition::OperatorRequired)? { + if !snapshot.transactions.iter().any(|stored| { + stored.value.transaction_id == subject && stored.value.leaves_to_operator(now) + }) { + store.close(IncidentCondition::OperatorRequired, &subject, CloseReason::Recovered, now)?; + } + } + Ok(()) + }; + self.record_incident("close", "*", close()); + } + /// Delete expiry requests whose command has no phase left to end: it was /// resolved, it finished, or its command is gone. fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { @@ -14219,6 +14249,11 @@ mod tests { self.holds.lock().unwrap().push(file); } + /// Every stop works again. + fn mend_stops(&self) { + self.failing_stops.lock().unwrap().clear(); + } + /// From now on stopping this runtime instance fails. fn fail_stop_of(&self, runtime_instance_id: &str) { self.failing_stops.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -16964,7 +16999,9 @@ mod tests { .engine .begin_post_fencing_abort(&resident_id(&world, owner_id)?, anyhow!("candidate refused"))?; - // Before the deadline the deployment waits on a bounded owner. + // Before the deadline the deployment waits on a bounded owner, and no + // one is paged. + assert!(world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); for _ in 0..3 { resume_pass(&world)?; } @@ -16980,6 +17017,11 @@ mod tests { let owner = resident_id(&world, owner_id)?.value; assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); assert!(owner.owns_target()); + // Exactly one incident, however many passes ran. + assert_eq!( + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [owner_id] + ); let ended = latest_id(&world, deploy_id)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); @@ -16989,6 +17031,46 @@ mod tests { Ok(()) } + #[test] + fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let id = seeded.transaction_id.as_str(); + workload.fail_stop_of(&runtime_instance_of(&world, id)?); + world.engine.begin_post_fencing_abort(&resident_id(&world, id)?, anyhow!("candidate refused"))?; + pass_deadline_of(&world, id)?; + let open = || world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired); + for _ in 0..4 { + resume_pass(&world)?; + } + assert_eq!(open()?, [id]); + let records = world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + + // Still left to the operator: still open, and not written again. + resume_pass(&world)?; + assert_eq!(world.engine.incidents().read()?, records); + + // The step succeeds and the transaction ends; the next pass closes it. + workload.mend_stops(); + for _ in 0..12 { + if resident_id(&world, id).is_err() { + break; + } + resume_pass(&world)?; + } + assert!(latest_id(&world, id)?.is_terminal()); + resume_pass(&world)?; + assert!(open()?.is_empty()); + let closed = world.engine.incidents().read()?; + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + resume_pass(&world)?; + assert_eq!(world.engine.incidents().read()?, closed, "a spent incident was written again"); + Ok(()) + } + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop /// keeps failing. A deployment waits behind it until the drain's deadline, /// then ends itself; the drain stays and the target stays owned. @@ -17037,6 +17119,10 @@ mod tests { resume_pass(&world)?; } assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + assert_eq!( + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [drainer_id] + ); let ended = latest_id(&world, waiter_id)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); diff --git a/src/incident.rs b/src/incident.rs index 1d7f095..a0941ff 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -32,6 +32,9 @@ pub(crate) const INCIDENT_RETENTION_MILLIS: u64 = 7 * 24 * 60 * 60 * 1000; #[serde(rename_all = "kebab-case")] pub(crate) enum IncidentCondition { ContinuityExhausted, + /// A transaction past its deadline in a state Idunn will not end by + /// itself. Subject: the transaction id. + OperatorRequired, } impl IncidentCondition { @@ -40,6 +43,7 @@ impl IncidentCondition { pub(crate) const fn name(self) -> &'static str { match self { Self::ContinuityExhausted => "continuity-exhausted", + Self::OperatorRequired => "operator-required", } } @@ -49,6 +53,7 @@ impl IncidentCondition { pub(crate) const fn ends_with_admission(self) -> bool { match self { Self::ContinuityExhausted => true, + Self::OperatorRequired => false, } } } @@ -271,6 +276,17 @@ impl IncidentStore { Ok(true) } + /// The subjects of every open incident of `condition`. + pub(crate) fn open_subjects(&self, condition: IncidentCondition) -> Result> { + Ok(self + .snapshot()? + .records + .into_iter() + .filter(|record| record.is_open() && record.condition == condition) + .map(|record| record.subject) + .collect()) + } + /// Close the open incident for `(condition, subject)`, if there is one. /// `Ok(true)` means this call wrote the closure. pub(crate) fn close( @@ -468,11 +484,12 @@ pub(crate) mod tests { ); // The wire spelling readers match on is the spelling the key and // status use. - let payload = rmp_serde::to_vec(&IncidentCondition::ContinuityExhausted).unwrap(); - assert_eq!( - rmp_serde::from_slice::(&payload).unwrap(), - IncidentCondition::ContinuityExhausted.name() - ); + for condition in [IncidentCondition::ContinuityExhausted, IncidentCondition::OperatorRequired] { + let payload = rmp_serde::to_vec(&condition).unwrap(); + assert_eq!(rmp_serde::from_slice::(&payload).unwrap(), condition.name()); + } + assert_eq!(IncidentCondition::OperatorRequired.name(), "operator-required"); + assert!(!IncidentCondition::OperatorRequired.ends_with_admission()); let payload = rmp_serde::to_vec(&CloseReason::NoLongerAdmitted).unwrap(); assert_eq!( rmp_serde::from_slice::(&payload).unwrap(), From 0bfadcfc717908be51933255c75b0cae0c5dd9e2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 16:10:11 +0200 Subject: [PATCH 017/122] Cut c2b test: pin the stateful-deploy arm of leaves_to_operator and the incident close predicate Survivors of cargo mutants --in-diff on 9ea6c71..126d710: the OperatorRequired clause of leaves_to_operator (now a table beside phase_end) and the close predicate (a vanished subject closes while another transaction is still left). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8732e4c..3169542 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13948,6 +13948,15 @@ mod tests { transaction.leasing = leasing.clone(); transaction.lease_adoption = lease_adoption.clone(); assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); + // Only a stateful deploy is left to the operator when its phase ends. + transaction.retime(1_000); + let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; + assert_eq!( + transaction.leaves_to_operator(due), + expected == PhaseEnd::OperatorRequired, + "{kind:?} {leasing:?} {lease_adoption:?}" + ); + assert!(!transaction.leaves_to_operator(due - 1)); let recovery = if kind == Continuity && lease_adoption.is_some() { TerminalRecovery::RestartAdmitted } else { @@ -17048,6 +17057,13 @@ mod tests { let records = world.engine.incidents().read()?; assert_eq!(records.len(), 1); + // An incident whose transaction is gone closes even while another is + // still left to the operator. + world.engine.incidents().open(IncidentCondition::OperatorRequired, "tx-vanished", now_millis()?)?; + resume_pass(&world)?; + assert_eq!(open()?, [id]); + let records = world.engine.incidents().read()?; + // Still left to the operator: still open, and not written again. resume_pass(&world)?; assert_eq!(world.engine.incidents().read()?, records); @@ -17064,8 +17080,8 @@ mod tests { resume_pass(&world)?; assert!(open()?.is_empty()); let closed = world.engine.incidents().read()?; - assert_eq!(closed.len(), 1); - assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + assert!(closed.iter().all(|record| record.close_reason == Some(CloseReason::Recovered))); + assert_eq!(closed.iter().filter(|record| record.subject == id).count(), 1); resume_pass(&world)?; assert_eq!(world.engine.incidents().read()?, closed, "a spent incident was written again"); Ok(()) From 21849eb09753cccf0286105a4ff1e91ce30f533f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:48:03 +0200 Subject: [PATCH 018/122] Cut c2b fix: a drain whose stored plan no longer parses is bounded, not unbounded deadline_for_current_activity's drain arm returned None when parsed_inputs failed (the running binary re-parses and re-admits the frozen binding), which left the drain with no deadline, unpaged, and disarmed every waiter behind it. The retire time is now the promotion plus the drain when the binding parses, and otherwise now; the frozen committing duration from it bounds the drain. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3169542..0f4385b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1775,17 +1775,20 @@ impl DeploymentTransaction { return None; } let plan = self.plan.as_ref()?; - let from = match self + // The retire time needs the binding's drain, which the running + // binary re-parses and re-admits: a plan frozen by an older binary + // may not parse now. That must not leave a drain with no deadline, + // so a retire time that cannot be derived is now, and the + // committing duration from it bounds the drain. + let from = self .routing .as_ref() .and_then(RoutingEvidence::promoted_at_unix_millis) - { - Some(promoted_at) => { - route_drain_deadline(promoted_at, plan.parsed_inputs().ok()?.1.rollout.drain_seconds) - .ok()? - } - None => now, - }; + .and_then(|promoted_at| { + let (_, binding) = plan.parsed_inputs().ok()?; + route_drain_deadline(promoted_at, binding.rollout.drain_seconds).ok() + }) + .unwrap_or(now); return Some(PhaseDeadline { phase: self.phase, entered_at_unix_millis: now, From a254a3e6a5f767201f07ee7d9892d821076cfe24 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:49:14 +0200 Subject: [PATCH 019/122] Cut c2b fix: pin the forever-waits at the waiter, the pre-fence abort deadline, the stateful incident lifecycle and the archive vanish Negative tests (a)(b)(c) are one marked harness that ages a deadline only when one exists and ends on the waiter's own record, so at the base they fail at the waiter assertion (still waiting), not at setup. (c) now runs the Odin gate-wait it names: a stop-then-start continuity on a systemd unit observes and waits, where the host-actuator step of a world with no actuator errs at once. New: a stop-first owner's wedged pre-fence abort pages and ends its waiter; a waiter's own wedged pre-fence abort pages beside a standing owner; a stateful deploy's incident is silent under an early expiry, one per episode, and closes while the record goes on; an unparsable drain plan is bounded; a transaction gone from both live set and history stays a fault. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 635 +++++++++++++++++++++++++++++++------------ 1 file changed, 467 insertions(+), 168 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0f4385b..43810c4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13417,12 +13417,24 @@ mod tests { } #[test] - fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + fn a_drain_whose_stored_plan_no_longer_parses_is_still_bounded() -> Result<()> { + let (mut draining, promoted_at) = promoted_draining_record()?; + let committing = u64::from(draining.plan.as_ref().unwrap().phase_deadlines().committing_seconds) * 1000; + // A binding the running binary refuses: the drain seconds cannot be read. + draining.plan.as_mut().unwrap().binding_blob = b"not an operator binding".to_vec(); + assert!(draining.plan.as_ref().unwrap().parsed_inputs().is_err()); + draining.retime(promoted_at + 1); + let deadline = draining.phase_deadline.context("an unparsable plan left the drain with no deadline")?; + assert_eq!(deadline.deadline_at_unix_millis, promoted_at + 1 + committing); + assert!(!draining.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(draining.leaves_to_operator(deadline.deadline_at_unix_millis)); + Ok(()) + } + + /// A Complete record draining its incumbent, whose route was promoted at + /// the returned time. + fn promoted_draining_record() -> Result<(DeploymentTransaction, u64)> { let (_, mut draining) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; - let plan = draining.plan.clone().unwrap(); - let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; - let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; - assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); draining.phase = DeploymentPhase::Complete; draining.completion = Some(TransactionCompletion::Admitted { generation_id: "generation-new".into(), @@ -13441,6 +13453,16 @@ mod tests { } other => bail!("the fixture is not promoted: {other:?}"), } + Ok((draining, promoted_at)) + } + + #[test] + fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + let (mut draining, promoted_at) = promoted_draining_record()?; + let plan = draining.plan.clone().unwrap(); + let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; + let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; + assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); // Before the retire time: the retire time plus the committing duration. draining.retime(promoted_at + 1); @@ -15789,6 +15811,104 @@ mod tests { Ok(()) } + /// A workload port whose observation deletes every live transaction from + /// the control store once armed: the step ends, and the record is simply + /// gone, archived nowhere. + struct VanishingWorkload { + store: Mutex>, + armed: std::sync::atomic::AtomicBool, + } + + impl WorkloadPort for VanishingWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + if self.armed.load(std::sync::atomic::Ordering::SeqCst) { + let path = self.store.lock().unwrap().clone().context("no store")?; + let gone = ControlSnapshot::read(&path)? + .transactions + .into_iter() + .map(|stored| stored.envelope) + .collect::>(); + assert!(SingleFileMessagePackBackingStore::new(&path).delete_batch_if_unchanged(&gone)?); + } + StillWorkload.observe(expected, activation, prior) + } + fn stop(&self, observation: &WorkloadObservation) -> Result<()> { + StillWorkload.stop(observation) + } + fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result { + StillWorkload.is_permanently_stopped(observation) + } + fn lost_to_host_reboot(&self, observation: &WorkloadObservation, at: u64) -> Result { + StillWorkload.lost_to_host_reboot(observation, at) + } + } + + /// The step that ends a transaction archives it in its own write, so a + /// record gone from the live set is finished only if history holds it. One + /// that vanished from both is a fault, never progress. + #[test] + fn a_transaction_gone_from_both_the_live_set_and_history_is_a_fault() -> Result<()> { + let workload = Arc::new(VanishingWorkload { + store: Mutex::new(None), + armed: std::sync::atomic::AtomicBool::new(false), + }); + let world = EngineFixture::with_workload(workload.clone())?; + *workload.store.lock().unwrap() = Some(world.state_store.clone()); + let live = transaction_at(&world, DeploymentPhase::Warming)?; + let id = live.transaction_id.as_str(); + // The Warming step waits for Odin; waiting again rewrites nothing, so + // the step succeeds without touching the record it will find missing. + world.engine.resume_candidate(&resident_id(&world, id)?)?; + assert!(resident_id(&world, id)?.value.last_error.is_some(), "the Warming step did not wait"); + workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); + world.engine.resume_backoff.lock().unwrap().clear(); + let error = world + .engine + .resume_candidate(&resident_id(&world, id)?) + .expect_err("a record gone from both stores was read as a finished one"); + assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); + assert!(read_history_transactions(&world.state_store).is_empty()); + Ok(()) + } + // ---- fix 2: supervision owns the Expected-only precondition ---- #[test] @@ -16970,6 +17090,13 @@ mod tests { Ok((deploy, continuity, incumbent)) } + // BEGIN c2b-forever-wait harness + // The forever-waits of cut c2b, driven through the API that exists at + // 9ea6c71 (the base): each test ends on the waiter's own record, so at the + // base it fails at the waiter assertion (still waiting), never at setup. + // Lines ending in `head-only` read what only the cut adds (the incident + // store, `leaves_to_operator`); the base probe strips them. + /// One scheduler pass over every live transaction, with no resume backoff /// standing between it and the step. fn resume_pass(world: &EngineFixture) -> Result { @@ -16995,6 +17122,32 @@ mod tests { replace_transaction(&world.state_store, ¤t, &next) } + /// The deadline of `id`, when it has one, is already behind it. A record + /// that has none is left alone: at the base nothing times it, and the + /// scenario must still run to the waiter's assertion. + fn age_deadline_of(world: &EngineFixture, id: &str) -> Result<()> { + if resident_id(world, id)?.value.phase_deadline.is_some() { + pass_deadline_of(world, id)?; + } + Ok(()) + } + + /// The deployment `waiter_id` ended before fencing, with a cause naming + /// the owner it will not wait for. + fn assert_ended_for_owner(world: &EngineFixture, waiter_id: &str, owner_id: &str) -> Result<()> { + let ended = latest_id(world, waiter_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); + assert!(ended.fencing.is_none()); + Ok(()) + } + + fn operator_required(world: &EngineFixture) -> Result> { + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) // head-only + } + /// NEGATIVE (a). A continuity that owns the target is aborting and its /// abort step fails on every attempt. Its deadline passes: Idunn leaves it /// to the operator and keeps it owned, and the deployment waiting behind it @@ -17013,36 +17166,266 @@ mod tests { // Before the deadline the deployment waits on a bounded owner, and no // one is paged. - assert!(world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); + assert!(operator_required(&world)?.is_empty()); // head-only for _ in 0..3 { resume_pass(&world)?; } let waiting = resident_id(&world, deploy_id)?.value; assert!(waiting.pre_fencing_abort.is_none()); assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); - assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); + assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); // head-only - pass_deadline_of(&world, owner_id)?; - for _ in 0..10 { + age_deadline_of(&world, owner_id)?; + for _ in 0..20 { resume_pass(&world)?; } let owner = resident_id(&world, owner_id)?.value; assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); assert!(owner.owns_target()); // Exactly one incident, however many passes ran. - assert_eq!( - world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, - [owner_id] - ); - let ended = latest_id(&world, deploy_id)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + assert_eq!(operator_required(&world)?, [owner_id]); // head-only + assert_ended_for_owner(&world, deploy_id, owner_id) + } + + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop + /// keeps failing. A deployment waits behind it until the drain's deadline, + /// then ends itself; the drain stays and the target stays owned. + #[test] + fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; + let drainer_id = drainer.transaction_id.as_str(); + drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, drainer_id)?.value; + assert!(draining.drains_incumbent(), "the incumbent was already retired"); + assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); // head-only + let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = + &draining.post_commit_cleanup + else { + bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); }; - assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); - assert!(ended.fencing.is_none()); + let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; + workload.fail_stop_of(&old.runtime_instance_id); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; + let waiter_id = waiter.transaction_id.as_str(); + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, waiter_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); + assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); + assert!(operator_required(&world)?.is_empty()); // head-only + + age_deadline_of(&world, drainer_id)?; + for _ in 0..20 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + assert_eq!(operator_required(&world)?, [drainer_id]); // head-only + assert_ended_for_owner(&world, waiter_id, drainer_id) + } + + /// A stop-then-start transaction held in Warming with a started unit, as + /// the Starting step leaves it. + fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let expected = next.expected.clone().context("no Expected")?; + let now = now_millis()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&next.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + next.expected_publication_sha256 = Some(expected.canonical_sha256()?); + next.activation_publication_sha256 = Some(activation.canonical_sha256()?); + next.activation = Some(activation); + next.workload = Some(workload); + next.enter_phase(DeploymentPhase::Warming, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// The stop-then-start binding of `stop_first_binding`, on a systemd unit + /// instead of a host actuator: its Warming step observes the unit through + /// the world's workload port and waits for Odin's word, where the + /// host-actuator step of a world that serves no actuator errs at once. + fn stop_first_systemd_binding(world: &EngineFixture) -> Result { + use crate::deployment_plan::tests::BINDING; + let host_runner = "[runners.rust]\ndriver = \"host-native\"\naffordances = [\"source-read\", \"artifact-write\"]\nallowed_programs = [\"cargo\"]\n\n"; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; + let (_, rest) = rest.split_once("[workload]").context("no workload")?; + let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; + let (identity, rest) = rest.split_once("[route]").context("no route")?; + let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; + bind_provider( + world, + &format!("{head}{host_runner}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") + .replace("candidate-then-promote", "stop-then-start"), + ) + } + + /// A stop-then-start deployment and continuity over one incumbent, on a + /// systemd unit. The continuity is held in Warming with a started unit and + /// the deployment, one `enter` later, waits at its Sealing tail on it. + fn warm_stop_first_owner( + workload: Arc, + ) -> Result<(EngineFixture, String, String)> { + let world = EngineFixture::with_workload(workload)?; + let incumbent = admit_incumbent(&world)?; + let binding = stop_first_systemd_binding(&world)?; + let mut ids = Vec::new(); + for kind in [CommandKind::Deploy, CommandKind::Continuity] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + make_stop_first(&world, &seeded.transaction_id, &binding)?; + ids.push(seeded.transaction_id); + } + let continuity = ids.pop().context("no continuity")?; + let deploy = ids.pop().context("no deploy")?; + warm_stop_first(&world, &continuity)?; + let owner = resident_id(&world, &continuity)?; + assert_eq!(owner.value.phase, DeploymentPhase::Warming); + assert!(owner.value.owns_target()); + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?)?; + let held = resident_id(&world, &deploy)?.value; + assert_eq!(held.phase, DeploymentPhase::Sealing); + assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); + Ok((world, deploy, continuity)) + } + + /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting + /// and waits in Warming for an Odin word that never comes: its step runs, + /// observes the unit and gate-waits. Its Warming deadline ends it, and the + /// deployment waiting at its Sealing tail takes the target. + #[test] + fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( + ) -> Result<()> { + let (world, deploy, continuity) = warm_stop_first_owner(SwitchWorkload::new())?; + // The gate-wait is the Warming step itself, not an error of it. + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, &continuity)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.pre_fencing_abort.is_none(), "the Warming step failed: {:?}", waiting.last_error); + assert!(waiting.last_error.is_some(), "the Warming step did not wait for Odin"); + assert!(resident_id(&world, &deploy)?.value.last_error.as_deref().is_some_and(|n| n.contains(&continuity))); + + age_deadline_of(&world, &continuity)?; + for _ in 0..12 { + if resident_id(&world, &continuity).is_err() { + break; + } + resume_pass(&world)?; + } + let ended = latest_id(&world, &continuity)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("Warming phase passed its deadline"), "{error}"); + assert!(operator_required(&world)?.is_empty()); // head-only + + // The pass that retired the owner, or the next, moves the waiter to + // Starting; stop there, before its Starting step runs. + for _ in 0..3 { + if resident_id(&world, &deploy)?.value.phase >= DeploymentPhase::Starting { + break; + } + resume_pass(&world)?; + } + let took = resident_id(&world, &deploy)?.value; + assert_eq!(took.phase, DeploymentPhase::Starting); + assert!(took.owns_target() && took.pre_fencing_abort.is_none()); Ok(()) } + /// The wedged pre-fence abort of a stop-then-start owner is paged too: the + /// waiter behind it ends, and one incident opens. + #[test] + fn a_waiting_deployment_ends_when_a_stop_first_owners_prefence_abort_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let (world, deploy, owner) = warm_stop_first_owner(workload.clone())?; + workload.fail_stop_of(&runtime_instance_of(&world, &owner)?); + world + .engine + .begin_pre_fencing_abort(&resident_id(&world, &owner)?, anyhow!("candidate refused"))?; + for _ in 0..3 { + resume_pass(&world)?; + } + // Before the abort's deadline: it waits, unpaged. + let aborting = resident_id(&world, &owner)?.value; + assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); + assert!(operator_required(&world)?.is_empty()); // head-only + assert!(resident_id(&world, &deploy)?.value.pre_fencing_abort.is_none()); + + age_deadline_of(&world, &owner)?; + for _ in 0..20 { + resume_pass(&world)?; + } + let aborting = resident_id(&world, &owner)?.value; + assert!(!aborting.is_terminal() && aborting.owns_target(), "the owner was ended"); + assert_eq!(operator_required(&world)?, [owner.as_str()]); // head-only + assert_ended_for_owner(&world, &deploy, &owner) + } + + /// A waiter whose own pre-fence abort is wedged, beside an owner it will + /// not outlast, is paged by its own deadline: the owner's standing does not + /// excuse it. + #[test] + fn a_waiters_wedged_prefence_abort_is_paged_beside_a_standing_owner() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + workload.fail_stop_of(&runtime_instance_of(&world, deploy_id)?); + world + .engine + .begin_pre_fencing_abort(&resident_id(&world, deploy_id)?, anyhow!("candidate refused"))?; + for _ in 0..3 { + resume_pass(&world)?; + } + let aborting = resident_id(&world, deploy_id)?.value; + assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); + assert!(!operator_required(&world)?.iter().any(|subject| subject == deploy_id)); // head-only + + age_deadline_of(&world, deploy_id)?; + for _ in 0..3 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, deploy_id)?.value.is_terminal()); + assert!(operator_required(&world)?.iter().any(|subject| subject == deploy_id), "{owner_id}"); // head-only + Ok(()) + } + // END c2b-forever-wait harness + #[test] fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { let workload = SwitchWorkload::new(); @@ -17090,153 +17473,6 @@ mod tests { Ok(()) } - /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop - /// keeps failing. A deployment waits behind it until the drain's deadline, - /// then ends itself; the drain stays and the target stays owned. - #[test] - fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( - ) -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; - odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; - let drainer_id = drainer.transaction_id.as_str(); - drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; - let draining = resident_id(&world, drainer_id)?.value; - assert!(draining.drains_incumbent(), "the incumbent was already retired"); - assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); - match &draining.workload { - Some(WorkloadObservation::Systemd(_)) => {} - other => bail!("no workload: {other:?}"), - } - let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = - &draining.post_commit_cleanup - else { - bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); - }; - let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; - workload.fail_stop_of(&old.runtime_instance_id); - - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("nothing admitted")? - .value - .clone(); - let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; - let waiter_id = waiter.transaction_id.as_str(); - for _ in 0..3 { - resume_pass(&world)?; - } - let waiting = resident_id(&world, waiter_id)?.value; - assert!(waiting.pre_fencing_abort.is_none()); - assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); - assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); - - pass_deadline_of(&world, drainer_id)?; - for _ in 0..10 { - resume_pass(&world)?; - } - assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); - assert_eq!( - world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, - [drainer_id] - ); - let ended = latest_id(&world, waiter_id)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("owner-left-to-operator") && error.contains(drainer_id), "{error}"); - Ok(()) - } - - /// A stop-then-start transaction held in Warming with a started unit, as - /// the Starting step leaves it (this world serves no host actuator to - /// start one). - fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { - let stored = resident_id(world, id)?; - let mut next = stored.value.clone(); - let expected = next.expected.clone().context("no Expected")?; - let now = now_millis()?; - let launch = IdunnRuntimeActivationLaunch::issue( - &expected, - runtime_instance_id(&next.transaction_id)?, - now, - &world.engine.idunn_signer, - )?; - let mut seed = Vec::new(); - let activation = launch.write_credential(&mut seed)?; - let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? - .1 - .workload - .context("no recorded workload")?; - match &mut workload { - WorkloadObservation::Systemd(observed) => { - observed.runtime_instance_id = activation.runtime_instance_id.clone(); - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), - } - next.expected_publication_sha256 = Some(expected.canonical_sha256()?); - next.activation_publication_sha256 = Some(activation.canonical_sha256()?); - next.activation = Some(activation); - next.workload = Some(workload); - next.enter_phase(DeploymentPhase::Warming, now); - replace_transaction(&world.state_store, &stored, &next) - } - - /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting - /// and waits in Warming for an Odin word that never comes. Its Warming - /// deadline ends it, and the deployment waiting at its Sealing tail takes - /// the target. - #[test] - fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( - ) -> Result<()> { - use DeploymentPhase::{Starting, Warming}; - let (world, deploy, continuity) = stop_first_world(SwitchWorkload::new())?; - warm_stop_first(&world, &continuity)?; - let owner = resident_id(&world, &continuity)?; - assert_eq!(owner.value.phase, Warming); - assert!(owner.value.owns_target() && owner.value.phase_deadline.is_some()); - let waiting = resident_id(&world, &deploy)?; - world.engine.enter(&waiting, entering(&waiting, Starting)?)?; - let held = resident_id(&world, &deploy)?.value; - assert_eq!(held.phase, DeploymentPhase::Sealing); - assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); - - pass_deadline_of(&world, &continuity)?; - world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; - let aborting = resident_id(&world, &continuity)?.value; - let abort = aborting.pre_fencing_abort.clone().context("the deadline did not end Warming")?; - assert!(abort.error.contains("Warming phase passed its deadline"), "{}", abort.error); - // This world serves no host actuator, so the abandoned unit cannot be - // stopped here; the abort's cleanup is recorded done. - let mut cleaned = aborting.clone(); - let intent = cleaned.pre_fencing_abort.as_mut().unwrap(); - intent.candidate_cleanup = CleanupEvidence::Complete; - intent.topology_reconciliation = CleanupEvidence::Complete; - replace_transaction(&world.state_store, &resident_id(&world, &continuity)?, &cleaned)?; - for _ in 0..12 { - if resident_id(&world, &continuity).is_err() { - break; - } - world.engine.resume_backoff.lock().unwrap().clear(); - world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; - } - let ended = latest_id(&world, &continuity)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("Warming phase passed its deadline"), "{error}"); - - let waiting = resident_id(&world, &deploy)?; - world.engine.enter(&waiting, entering(&waiting, Starting)?)?; - let took = resident_id(&world, &deploy)?.value; - assert_eq!(took.phase, Starting); - assert!(took.owns_target() && took.pre_fencing_abort.is_none()); - Ok(()) - } - #[test] fn a_waiters_warming_deadline_is_disarmed_while_an_owner_holds_the_target_and_rearmed_when_it_ends( ) -> Result<()> { @@ -17458,13 +17694,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// The seeded transaction `id`, rewritten as the Sealing transaction of a /// stop-then-start target: the host-actuator plan, release and Expected, /// and none of the evidence that follows Sealing. - fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + fn make_stop_first(world: &EngineFixture, id: &str, binding: &str) -> Result<()> { use crate::deployment_plan::tests::{ RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); - let binding = stop_first_binding(world)?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -17533,11 +17768,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ) -> Result<(EngineFixture, String, String)> { let world = EngineFixture::with_workload(workload)?; let incumbent = admit_incumbent(&world)?; + let binding = stop_first_binding(&world)?; let mut ids = Vec::new(); for kind in [CommandKind::Deploy, CommandKind::Continuity] { let seeded = seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id)?; + make_stop_first(&world, &seeded.transaction_id, &binding)?; ids.push(seeded.transaction_id); } let continuity = ids.pop().context("no continuity")?; @@ -22592,6 +22828,69 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// The incident of a stateful deployment left to the operator is one + /// per episode. An operator's expiry before the deadline reports the + /// hold and pages no one; the deadline opens one incident; and the + /// incident closes as soon as the record is no longer left, while the + /// transaction goes on. + #[test] + fn a_stateful_deploys_incident_is_one_per_episode_and_follows_its_deadline() -> Result<()> { + let deploy = + build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; + deploy.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let held = deploy.transaction()?; + let id = held.transaction_id.as_str(); + let open = || deploy.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired); + let now = now_millis()?; + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: held.command_id.clone(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + assert!(SingleFileMessagePackBackingStore::new(&deploy.world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: request.command_id.clone(), + current: None, + }], + &[expiry_request_envelope(&request, now)?], + )?); + + // The operator asked, the deadline has not passed: nobody is paged, + // however many passes see the request. + for _ in 0..4 { + deploy.resume()?; + } + assert!(open()?.is_empty(), "an expiry before the deadline paged the operator"); + assert!(deploy.world.engine.incidents().read()?.is_empty()); + assert!(deploy.transaction()?.post_fencing_abort.is_none(), "a leased deployment was aborted"); + + // The deadline passes: one incident, not one per pass. + pass_deadline(&deploy.world)?; + for _ in 0..4 { + deploy.resume()?; + } + assert_eq!(open()?, [id]); + assert_eq!(deploy.world.engine.incidents().read()?.len(), 1); + + // The record goes on with a deadline in the future (its step + // advanced): the incident closes while the transaction lives. + let current = resident(&deploy.world)?; + let mut next = current.value.clone(); + let due = next.phase_deadline.as_mut().context("no deadline")?; + due.entered_at_unix_millis = now_millis()?; + due.deadline_at_unix_millis = due.entered_at_unix_millis + 3_600_000; + replace_transaction(&deploy.world.state_store, ¤t, &next)?; + deploy.resume()?; + assert!(!deploy.transaction()?.is_terminal()); + assert!(open()?.is_empty(), "the incident outlived the episode"); + let records = deploy.world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + Ok(()) + } + /// B5 Soul A. The physical grant lands before `Granted` is durable. A /// continuity that dies in that window and passes its deadline is /// aborted from `Prepared`; the abort still withdraws the lease on From 8b6c539a6a1ba89e2714bb347633077e0bbaa568 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:51:02 +0200 Subject: [PATCH 020/122] Cut c2b fix: the stop-first harness owner runs on a systemd unit --- src/control_plane.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 43810c4..c8b0cd6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15897,7 +15897,8 @@ mod tests { // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. world.engine.resume_candidate(&resident_id(&world, id)?)?; - assert!(resident_id(&world, id)?.value.last_error.is_some(), "the Warming step did not wait"); + let waited = resident_id(&world, id)?.value; + assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?} {:?}", waited.phase, waited.warming); workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); world.engine.resume_backoff.lock().unwrap().clear(); let error = world @@ -17276,15 +17277,13 @@ mod tests { /// host-actuator step of a world that serves no actuator errs at once. fn stop_first_systemd_binding(world: &EngineFixture) -> Result { use crate::deployment_plan::tests::BINDING; - let host_runner = "[runners.rust]\ndriver = \"host-native\"\naffordances = [\"source-read\", \"artifact-write\"]\nallowed_programs = [\"cargo\"]\n\n"; - let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; - let (_, rest) = rest.split_once("[workload]").context("no workload")?; + let (head, rest) = BINDING.split_once("[workload]").context("no workload")?; let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; bind_provider( world, - &format!("{head}{host_runner}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") + &format!("{head}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") .replace("candidate-then-promote", "stop-then-start"), ) } @@ -17680,7 +17679,6 @@ state_root = "/var/lib/gamecult/service" GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" "#; - let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; From c6805d48a79252ed2493ac62b260e86da43c2f6a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:51:38 +0200 Subject: [PATCH 021/122] Cut c2b fix: restore stop_first_binding's runner split --- src/control_plane.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index c8b0cd6..8e43d81 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17679,6 +17679,7 @@ state_root = "/var/lib/gamecult/service" GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" "#; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; From 2ff57533b112f36e1486cae801f1dc6b82cdf564 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:53:58 +0200 Subject: [PATCH 022/122] Cut c2b fix: a test world runs a host-actuator target's workload through its own port --- src/control_plane.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8e43d81..e7a8009 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5039,6 +5039,10 @@ struct Engine { docker_runner: DockerRunnerDriver, systemd_workload: Arc, host_actuators: Option, + /// A test world that serves no host actuator runs a host-actuator + /// binding's workload through this port instead. + #[cfg(test)] + host_workload: Mutex>>, history_report: ReportOnce, /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. @@ -5140,6 +5144,8 @@ impl Engine { docker_runner: DockerRunnerDriver::default(), systemd_workload, host_actuators, + #[cfg(test)] + host_workload: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), resume_backoff: Mutex::default(), @@ -5375,9 +5381,15 @@ impl Engine { let (_, binding) = plan.parsed_inputs()?; Ok(match &binding.workload { WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), - WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { - access: self.host_access()?, - }), + WorkloadBinding::HostActuator(_) => { + #[cfg(test)] + if let Some(port) = self.host_workload.lock().expect("host workload mutex").clone() { + return Ok(port); + } + Arc::new(HostActuatorWorkloadDriver { + access: self.host_access()?, + }) + } }) } @@ -15892,7 +15904,8 @@ mod tests { }); let world = EngineFixture::with_workload(workload.clone())?; *workload.store.lock().unwrap() = Some(world.state_store.clone()); - let live = transaction_at(&world, DeploymentPhase::Warming)?; + let incumbent = admit_incumbent(&world)?; + let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Continuity, Some(&incumbent))?; let id = live.transaction_id.as_str(); // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. From 95e807661867d3f3afb4975e2fd8012246a8c768 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:55:04 +0200 Subject: [PATCH 023/122] Cut c2b fix: the stop-first owner harness uses the host-actuator world with its workload port --- src/control_plane.rs | 39 +++++++-------------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e7a8009..dacf5d9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17284,41 +17284,16 @@ mod tests { replace_transaction(&world.state_store, &stored, &next) } - /// The stop-then-start binding of `stop_first_binding`, on a systemd unit - /// instead of a host actuator: its Warming step observes the unit through - /// the world's workload port and waits for Odin's word, where the - /// host-actuator step of a world that serves no actuator errs at once. - fn stop_first_systemd_binding(world: &EngineFixture) -> Result { - use crate::deployment_plan::tests::BINDING; - let (head, rest) = BINDING.split_once("[workload]").context("no workload")?; - let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; - let (identity, rest) = rest.split_once("[route]").context("no route")?; - let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; - bind_provider( - world, - &format!("{head}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") - .replace("candidate-then-promote", "stop-then-start"), - ) - } - /// A stop-then-start deployment and continuity over one incumbent, on a /// systemd unit. The continuity is held in Warming with a started unit and /// the deployment, one `enter` later, waits at its Sealing tail on it. fn warm_stop_first_owner( workload: Arc, ) -> Result<(EngineFixture, String, String)> { - let world = EngineFixture::with_workload(workload)?; - let incumbent = admit_incumbent(&world)?; - let binding = stop_first_systemd_binding(&world)?; - let mut ids = Vec::new(); - for kind in [CommandKind::Deploy, CommandKind::Continuity] { - let seeded = - seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id, &binding)?; - ids.push(seeded.transaction_id); - } - let continuity = ids.pop().context("no continuity")?; - let deploy = ids.pop().context("no deploy")?; + let (world, deploy, continuity) = stop_first_world(workload.clone())?; + // This world serves no host actuator: the target's host workload runs + // through the same port as everything else here. + *world.engine.host_workload.lock().unwrap() = Some(workload); // seam warm_stop_first(&world, &continuity)?; let owner = resident_id(&world, &continuity)?; assert_eq!(owner.value.phase, DeploymentPhase::Warming); @@ -17706,12 +17681,13 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// The seeded transaction `id`, rewritten as the Sealing transaction of a /// stop-then-start target: the host-actuator plan, release and Expected, /// and none of the evidence that follows Sealing. - fn make_stop_first(world: &EngineFixture, id: &str, binding: &str) -> Result<()> { + fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { use crate::deployment_plan::tests::{ RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); + let binding = stop_first_binding(world)?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -17780,12 +17756,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ) -> Result<(EngineFixture, String, String)> { let world = EngineFixture::with_workload(workload)?; let incumbent = admit_incumbent(&world)?; - let binding = stop_first_binding(&world)?; let mut ids = Vec::new(); for kind in [CommandKind::Deploy, CommandKind::Continuity] { let seeded = seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id, &binding)?; + make_stop_first(&world, &seeded.transaction_id)?; ids.push(seeded.transaction_id); } let continuity = ids.pop().context("no continuity")?; From 9844806eb125159bcca9d99fbc2963134c5915cb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:56:28 +0200 Subject: [PATCH 024/122] Cut c2b fix: the waiter's own entry attempt drives the owner harness --- src/control_plane.rs | 62 +++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index dacf5d9..c5a51a8 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15902,21 +15902,20 @@ mod tests { store: Mutex::new(None), armed: std::sync::atomic::AtomicBool::new(false), }); - let world = EngineFixture::with_workload(workload.clone())?; + let (world, _deploy, owner) = stop_first_world(workload.clone())?; + *world.engine.host_workload.lock().unwrap() = Some(workload.clone()); *workload.store.lock().unwrap() = Some(world.state_store.clone()); - let incumbent = admit_incumbent(&world)?; - let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Continuity, Some(&incumbent))?; - let id = live.transaction_id.as_str(); + warm_stop_first(&world, &owner)?; // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. - world.engine.resume_candidate(&resident_id(&world, id)?)?; - let waited = resident_id(&world, id)?.value; - assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?} {:?}", waited.phase, waited.warming); + world.engine.resume_candidate(&resident_id(&world, &owner)?)?; + let waited = resident_id(&world, &owner)?.value; + assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?}", waited.phase); workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); world.engine.resume_backoff.lock().unwrap().clear(); let error = world .engine - .resume_candidate(&resident_id(&world, id)?) + .resume_candidate(&resident_id(&world, &owner)?) .expect_err("a record gone from both stores was read as a finished one"); assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); assert!(read_history_transactions(&world.state_store).is_empty()); @@ -17158,6 +17157,32 @@ mod tests { Ok(()) } + /// What the scheduler does for a deployment waiting at its Sealing tail: it + /// asks to enter Starting, and a refusal ends it before fencing. (A resume + /// pass would stop first at the deployment brake this world never releases.) + fn step_waiter(world: &EngineFixture, id: &str) -> Result<()> { + let waiting = resident_id(world, id)?; + if waiting.value.pre_fencing_abort.is_some() { + return Ok(()); + } + if let Err(error) = world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?) { + world.engine.begin_pre_fencing_abort(&resident_id(world, id)?, error)?; + } + Ok(()) + } + + /// Passes of the scheduler over the owner, each followed by the waiter's + /// own attempt to enter Starting. + fn run_owner_and_waiter(world: &EngineFixture, waiter: &str, passes: usize) -> Result<()> { + for _ in 0..passes { + resume_pass(world)?; + if resident_id(world, waiter).is_ok() { + step_waiter(world, waiter)?; + } + } + Ok(()) + } + fn operator_required(world: &EngineFixture) -> Result> { world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) // head-only } @@ -17315,9 +17340,7 @@ mod tests { ) -> Result<()> { let (world, deploy, continuity) = warm_stop_first_owner(SwitchWorkload::new())?; // The gate-wait is the Warming step itself, not an error of it. - for _ in 0..3 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 3)?; let waiting = resident_id(&world, &continuity)?.value; assert_eq!(waiting.phase, DeploymentPhase::Warming); assert!(waiting.pre_fencing_abort.is_none(), "the Warming step failed: {:?}", waiting.last_error); @@ -17329,7 +17352,7 @@ mod tests { if resident_id(&world, &continuity).is_err() { break; } - resume_pass(&world)?; + run_owner_and_waiter(&world, &deploy, 1)?; } let ended = latest_id(&world, &continuity)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { @@ -17340,12 +17363,7 @@ mod tests { // The pass that retired the owner, or the next, moves the waiter to // Starting; stop there, before its Starting step runs. - for _ in 0..3 { - if resident_id(&world, &deploy)?.value.phase >= DeploymentPhase::Starting { - break; - } - resume_pass(&world)?; - } + step_waiter(&world, &deploy)?; let took = resident_id(&world, &deploy)?.value; assert_eq!(took.phase, DeploymentPhase::Starting); assert!(took.owns_target() && took.pre_fencing_abort.is_none()); @@ -17363,9 +17381,7 @@ mod tests { world .engine .begin_pre_fencing_abort(&resident_id(&world, &owner)?, anyhow!("candidate refused"))?; - for _ in 0..3 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 3)?; // Before the abort's deadline: it waits, unpaged. let aborting = resident_id(&world, &owner)?.value; assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); @@ -17373,9 +17389,7 @@ mod tests { assert!(resident_id(&world, &deploy)?.value.pre_fencing_abort.is_none()); age_deadline_of(&world, &owner)?; - for _ in 0..20 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 20)?; let aborting = resident_id(&world, &owner)?.value; assert!(!aborting.is_terminal() && aborting.owns_target(), "the owner was ended"); assert_eq!(operator_required(&world)?, [owner.as_str()]); // head-only From 8230c544acf3a45f0874c3906e368a94fce63c59 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:58:13 +0200 Subject: [PATCH 025/122] Cut c2b fix: a pre-fencing abort keeps its deadline through its last cleanup write (a Sealing waiter's abort could not finish) --- src/control_plane.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c5a51a8..52842ea 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1808,10 +1808,13 @@ impl DeploymentTransaction { } /// Whether a deadline can describe the record's current activity: a - /// deadline phase, a pending pre-fencing abort, or an admitted Complete. + /// deadline phase, a pre-fencing abort not yet ended, or an admitted Complete. fn carries_a_deadline(&self) -> bool { + // An abort carries its deadline until the record ends, including the + // write that completes its last cleanup: a Sealing or Starting record + // has no phase duration to fall back on, and that write is validated. DEADLINE_PHASES.contains(&self.phase) - || self.pre_fencing_abort_pending() + || (self.completion.is_none() && self.pre_fencing_abort.is_some()) || (self.phase == DeploymentPhase::Complete && matches!(self.completion, Some(TransactionCompletion::Admitted { .. }))) } @@ -15918,7 +15921,7 @@ mod tests { .resume_candidate(&resident_id(&world, &owner)?) .expect_err("a record gone from both stores was read as a finished one"); assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); - assert!(read_history_transactions(&world.state_store).is_empty()); + assert!(read_history_transactions(&world.state_store).iter().all(|archived| archived.transaction_id != owner)); Ok(()) } @@ -17162,7 +17165,7 @@ mod tests { /// pass would stop first at the deployment brake this world never releases.) fn step_waiter(world: &EngineFixture, id: &str) -> Result<()> { let waiting = resident_id(world, id)?; - if waiting.value.pre_fencing_abort.is_some() { + if waiting.value.pre_fencing_abort.is_some() || waiting.value.phase >= DeploymentPhase::Starting { return Ok(()); } if let Err(error) = world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?) { From 58f213fe1525c9f3acacbe40c705e55a03a4d791 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:24:22 +0200 Subject: [PATCH 026/122] Cut c2c: boot lift returns a wedged continuity abort to forward retry A pre-c2c Idunn aborted a continuity at its phase deadline, which stops the only running copy of the admitted body. At boot, a continuity abort that has not stopped its running candidate (candidate_cleanup Pending, lease_withdrawal not Complete, workload observed, issued lease exact on disk) is withdrawn and the record retimed, the exact inverse of begin_post_fencing_abort. Negative tests carry the live shapes of tx-4eaed669 (Routing, adopted lease) and tx-039b91e9 (AwaitingReady, granted lease); the lift's refusals are pinned per predicate input. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 301 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 52842ea..092324d 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5626,6 +5626,79 @@ struct ProjectionReconciliation { } impl Engine { + /// Once at boot: withdraw the abort of a continuity that has not stopped + /// its running candidate, and retime the record so its phase retries + /// forward. A pre-c2c Idunn aborted a continuity at its deadline; that + /// abort stops the only running copy of the admitted body (ruling + /// c2c-boot-returns-wedged-aborts-forward). The result is the exact inverse + /// of `begin_post_fencing_abort`. + /// + /// A continuity abort now begins only over a permanently stopped + /// candidate, so this predicate cannot match a new record: it is a + /// migration with a deletion line (follow-up c2c-delete-boot-lift). A + /// record it cannot decide is reported and left aborting. + fn return_wedged_continuity_aborts_forward(&self) -> Result> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut returned = Vec::new(); + for stored in &snapshot.transactions { + let transaction = &stored.value; + let Some(abort) = &transaction.post_fencing_abort else { + continue; + }; + if transaction.command_kind != CommandKind::Continuity + || transaction.completion.is_some() + || abort.candidate_cleanup != CleanupEvidence::Pending + || abort.lease_withdrawal == CleanupEvidence::Complete + { + continue; + } + let id = &transaction.transaction_id; + let lifted = (|| -> Result { + let plan = required(&transaction.plan, "transaction plan")?; + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(plan)? + .observe(expected, activation, workload)?; + if let Some(lease) = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + { + let path = plan + .parsed_inputs()? + .1 + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + if !CultCacheWriteLeaseDriver::new(&transaction.target, path) + .observe_exact(lease)? + { + return Ok(false); + } + } + let now = now_millis()?; + let mut next = transaction.clone(); + next.post_fencing_abort = None; + next.last_error = Some(abort.error.clone()); + next.updated_at_unix_millis = now; + next.retime(now); + replace_transaction(&self.options.state_store, stored, &next)?; + Ok(true) + })(); + match lifted { + Ok(true) => returned.push(id.clone()), + Ok(false) => {} + Err(error) => eprintln!( + "Idunn could not decide whether continuity {id} may retry forward; its abort is left running: {error:#}" + ), + } + } + Ok(returned) + } + /// Once at boot: demote the activation a failed continuity left standing, /// where the projection still names exactly the activation that failed /// transaction issued. A continuity abort before the single resolution @@ -5739,6 +5812,16 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; + match engine.return_wedged_continuity_aborts_forward() { + Ok(returned) => { + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + } + Err(error) => eprintln!("Idunn boot could not lift wedged continuity aborts: {error:#}"), + } match engine.reconcile_failed_continuity_projections() { Ok(outcome) => { for transaction_id in &outcome.demoted { @@ -22362,6 +22445,18 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// Scheduler passes until `done`, at most twelve. + #[cfg(unix)] + fn drive_until_resuming(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..12 { + if done(&self.transaction()?) { + return Ok(()); + } + self.resume()?; + } + bail!("the transaction never reached the awaited state: {:?}", self.transaction()?.last_error) + } + fn resume_until_terminal(&self) -> Result { for _ in 0..12 { if latest(&self.world)?.completion.is_some() { @@ -22954,6 +23049,212 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// What a pre-c2c Idunn left when a continuity's phase passed its + /// deadline: the abort begun, route restoration done, the lease + /// withdrawal and candidate cleanup still owed, and the deadline stale. + #[cfg(unix)] + fn wedge_abort(routed: &RoutedWorld, error: &str) -> Result<()> { + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("{error}"))?; + routed.resume()?; + let aborting = routed.transaction()?.post_fencing_abort.context("no abort")?; + assert_eq!(aborting.route_restoration, CleanupEvidence::Complete); + assert_eq!(aborting.candidate_cleanup, CleanupEvidence::Pending); + pass_deadline(&routed.world) + } + + #[cfg(unix)] + fn lease_of(routed: &RoutedWorld) -> Result<(IdunnProcessWriteLeaseRecord, String)> { + let leasing = routed.transaction()?.leasing.context("no leasing evidence")?; + Ok(( + leasing.issued_lease().context("no issued lease")?.clone(), + leasing.lease_sha256().context("no granted lease")?.to_owned(), + )) + } + + #[cfg(unix)] + fn lease_driver(routed: &RoutedWorld) -> (CultCacheWriteLeaseDriver, PathBuf) { + let path = routed.world.root.join("lease/process-write-lease.cc"); + (CultCacheWriteLeaseDriver::new("service", &path), path) + } + + /// The live shape of tx-4eaed669 (Odin): Routing, lease granted and + /// adopted, the proof failing, the abort begun at the Routing deadline. + /// A pre-c2c Idunn went on to stop the candidate and the target lost + /// its only running copy. The boot lift returns the record to forward + /// retry; nothing is stopped, and the proof, once it passes, commits. + #[cfg(unix)] + #[test] + fn a_wedged_odin_shaped_continuity_abort_is_returned_to_forward_retry_at_boot() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + assert!(routed.transaction()?.lease_adoption.is_some()); + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + wedge_abort( + &routed, + "the Routing phase passed its deadline at unix ms 1792324133638", + )?; + let stops = routed.workload.stops.load(Ordering::SeqCst); + + let lifted_at = now_millis()?; + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + for _ in 0..10 { + routed.resume()?; + } + assert_eq!( + routed.workload.stops.load(Ordering::SeqCst), + stops, + "the candidate was stopped" + ); + let retrying = routed.transaction()?; + assert_eq!(returned, [retrying.transaction_id.clone()]); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + assert!(retrying.post_fencing_abort.is_none(), "the abort came back"); + assert!(retrying.completion.is_none()); + assert_eq!(retrying.phase, DeploymentPhase::Routing); + let deadline = retrying.phase_deadline.context("no deadline")?; + assert!(deadline.entered_at_unix_millis >= lifted_at, "the stale deadline stayed"); + assert!(routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); + + routed.stub.hang_up.store(false, Ordering::SeqCst); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert!(routed.fragment_path().exists(), "the route membership was not re-installed"); + Ok(()) + } + + /// The live shape of tx-039b91e9 (ghostlight): AwaitingReady with the + /// lease granted and not adopted, aborted at the readiness deadline. + /// Returned to forward retry, the candidate that later answers is + /// adopted and the record enters Routing. + #[cfg(unix)] + #[test] + fn a_wedged_ghostlight_shaped_continuity_abort_is_returned_to_forward_retry_at_boot() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + wedge_abort( + &routed, + "lease-not-adopted: the AwaitingReady phase passed its deadline at unix ms 1792324133638", + )?; + let stops = routed.workload.stops.load(Ordering::SeqCst); + + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + for _ in 0..5 { + routed.resume()?; + } + let waiting = routed.transaction()?; + assert_eq!(returned, [waiting.transaction_id.clone()]); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + assert!(waiting.post_fencing_abort.is_none() && waiting.completion.is_none()); + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.lease_adoption.is_none()); + + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Routing)?; + let adopted = routed.transaction()?; + assert_eq!(adopted.lease_adoption.context("no adoption recorded")?.write_lease_sha256, lease_sha256); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + Ok(()) + } + + /// The lift is a function of its inputs: each case lacks one thing the + /// lift requires, and its abort stays and runs to its end. + #[cfg(unix)] + #[test] + fn the_boot_lift_leaves_every_abort_it_cannot_prove_harmless() -> Result<()> { + #[derive(Clone, Copy, Debug)] + enum Case { + CandidateNotRunning, + LeaseWithdrawn, + CandidateCleaned, + LeaseRevokedUnderIt, + DeployWithRunningCandidate, + CompletedRecord, + } + for case in [ + Case::CandidateNotRunning, + Case::LeaseWithdrawn, + Case::CandidateCleaned, + Case::LeaseRevokedUnderIt, + Case::DeployWithRunningCandidate, + Case::CompletedRecord, + ] { + let deploy = matches!(case, Case::DeployWithRunningCandidate); + // The cleanup comes after the lease withdrawal, so a case with + // cleanup done and the withdrawal not is a stateless target's. + let stateful = !matches!(case, Case::CandidateCleaned); + let routed = build_routed_world( + Odin::Unreachable, + 1, + DeploymentPhase::Fencing, + false, + stateful, + if deploy { CommandKind::Deploy } else { CommandKind::Continuity }, + )?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + routed.world.engine.begin_post_fencing_abort( + &resident(&routed.world)?, + anyhow!("the AwaitingReady phase passed its deadline"), + )?; + // Route restoration; then each case takes the abort one real + // step, or one input, away from the lift's predicate. + routed.resume()?; + match case { + Case::CandidateNotRunning => routed.workload.kill(), + Case::LeaseWithdrawn | Case::CandidateCleaned => routed.resume()?, + Case::LeaseRevokedUnderIt => { + let (lease, _) = lease_of(&routed)?; + lease_driver(&routed).0.revoke_exact(Some(&lease))?; + } + Case::DeployWithRunningCandidate => {} + Case::CompletedRecord => { + routed.resume_until_terminal()?; + } + } + let abort = routed.transaction().ok().and_then(|t| t.post_fencing_abort); + match case { + Case::LeaseWithdrawn => assert_eq!( + abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), + Some((CleanupEvidence::Complete, CleanupEvidence::Pending)) + ), + Case::CandidateCleaned => assert_eq!( + abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), + Some((CleanupEvidence::Skipped, CleanupEvidence::Complete)) + ), + _ => {} + } + let before = ControlSnapshot::read(&routed.world.state_store)?; + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty(), "lifted {returned:?}"); + let after = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!( + before.transactions.iter().map(|t| &t.envelope).collect::>(), + after.transactions.iter().map(|t| &t.envelope).collect::>(), + "the lift wrote a record it left" + ); + let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.completion + ); + } + Ok(()) + } + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word /// carries the granted lease, and that Ready records the adoption, /// sourced from Odin's topology and naming the presence Odin observed. From 063409489e95682117ad5afb29e4441d74bde2e8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:33:29 +0200 Subject: [PATCH 027/122] Cut c2c: a continuity past Fencing retries forward and pages, never aborts PhaseEnd::RetryForward: phase_end answers it for every continuity at or past Fencing, because aborting restores the body its candidate already runs (ruling c2c-retry-decided-by-record). The resolver leaves it to the operator (operator-required incident, report_once) and the phase step keeps running. leaves_to_operator covers it. DeploymentTransaction::expirable is the one spelling the expire CLI, the spent-request retirement and the resolver's requested flag read: a continuity past Fencing cannot be expired. Deploy keeps abort-to-incumbent and the stateful operator-required arm. Tests: the PhaseEnd table, the Routing deadline forward retry and its incident episode, the stateful deadline, expiry refusal and spent-request retirement, and a deployment still aborting to its incumbent. Abort-mechanics tests begin their aborts directly. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 318 ++++++++++++++++++++++++++++++++----------- 1 file changed, 236 insertions(+), 82 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 092324d..3e36787 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -981,6 +981,10 @@ enum PhaseEnd { /// A stateful deployment whose candidate holds an issued lease: the phase /// is reported and not resolved. OperatorRequired, + /// A continuity at or past Fencing: its candidate is the only running copy + /// of the admitted body, so aborting it restores what it already runs. The + /// resolver leaves it to the operator and its phase step keeps running. + RetryForward, } /// How a post-fencing failure recovers. `RestoreIncumbent` is what a @@ -1844,17 +1848,29 @@ impl DeploymentTransaction { && (self.pre_fencing_abort.is_some() || self.post_fencing_abort.is_some() || self.drains_incumbent() - || (self.resolves_by_deadline() && self.phase_end() == PhaseEnd::OperatorRequired)) + || (self.resolves_by_deadline() + && matches!(self.phase_end(), PhaseEnd::OperatorRequired | PhaseEnd::RetryForward))) } - /// What an ended post-fencing phase resolves to (Q3). A candidate that was - /// never issued a write lease cannot have written, so the transaction - /// aborts to the incumbent through the post-fencing path, stateless or not. - /// A continuity always aborts: its candidate is the admitted release, and - /// supervision restarts that release under the continuity backoff. A - /// deployment whose candidate was issued a lease may have written under a - /// new state contract, and restoring the incumbent over that is not - /// Idunn's call: it is left for the operator. + /// Whether an operator's expiry can end this record's phase: it resolves by + /// deadline, and is not a continuity past Fencing, whose expiry abort would + /// stop the only running copy of the admitted body (ruling + /// c2c-expire-refuses-continuity-past-fencing). + fn expirable(&self) -> bool { + self.resolves_by_deadline() + && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) + } + + /// What an ended post-fencing phase resolves to (Q3). A continuity at or + /// past Fencing retries forward: its incumbent is the admitted generation, + /// which supervision mints a continuity only after seeing dead, and from + /// Fencing the continuity owns the target, so aborting restores the body + /// its candidate already runs (ruling c2c-retry-decided-by-record). A + /// deployment whose candidate was never issued a write lease cannot have + /// written, so it aborts to the incumbent through the post-fencing path, + /// stateless or not. One whose candidate was issued a lease may have + /// written under a new state contract, and restoring the incumbent over + /// that is not Idunn's call: it is left for the operator. fn phase_end(&self) -> PhaseEnd { if self.phase < DeploymentPhase::Fencing { return PhaseEnd::AbortBeforeFencing; @@ -1863,8 +1879,9 @@ impl DeploymentTransaction { let granted = leasing.and_then(LeasingEvidence::lease).is_some(); let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); match self.command_kind { + CommandKind::Continuity => PhaseEnd::RetryForward, CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, - _ => PhaseEnd::Abort { + CommandKind::Deploy => PhaseEnd::Abort { lease_not_adopted: granted && self.lease_adoption.is_none(), }, } @@ -3119,8 +3136,13 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> "command has multiple live transactions" ); ensure!( - current.value.resolves_by_deadline(), - "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + current.value.expirable(), + "{}", + if current.value.resolves_by_deadline() { + "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" + } else { + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + } ); let now = now_millis()?; let request = ExpiryRequest { @@ -9460,8 +9482,11 @@ impl Engine { /// Every live record is held to a deadline. Before a transaction runs its /// step, a deadline phase whose deadline has passed, or whose command an - /// operator asked to expire, is resolved as `phase_end` says; an abort or - /// a drain past its deadline is left to the operator and its step retries. + /// operator asked to expire, is resolved as `phase_end` says: aborted before + /// the fence, aborted after it, or left to the operator with its step + /// retrying (a stateful deploy, or a continuity, whose candidate is the only + /// running copy of its body). An abort or a drain past its deadline is left + /// to the operator the same way. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the @@ -9513,10 +9538,11 @@ impl Engine { } return Ok(None); } - let requested = ControlSnapshot::read(&self.options.state_store)? - .expiry_requests - .iter() - .any(|request| request.value.command_id == transaction.command_id); + let requested = transaction.expirable() + && ControlSnapshot::read(&self.options.state_store)? + .expiry_requests + .iter() + .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { format!( "the {:?} phase passed its deadline at unix ms {}", @@ -9548,6 +9574,15 @@ impl Engine { ); Ok(None) } + PhaseEnd::RetryForward => { + self.leave_to_operator( + transaction, + &format!( + "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, stop its unit by hand" + ), + ); + Ok(None) + } } } @@ -9578,7 +9613,7 @@ impl Engine { .filter(|request| { !snapshot.transactions.iter().any(|stored| { stored.value.command_id == request.value.command_id - && stored.value.resolves_by_deadline() + && stored.value.expirable() }) }) .map(|request| request.envelope.clone()) @@ -14061,22 +14096,24 @@ mod tests { (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), - (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), - (Continuity, Some(prepared), None, abort(false)), - (Continuity, Some(granted.clone()), None, abort(true)), - (Continuity, Some(granted), adoption.clone(), abort(false)), + (Continuity, None, None, PhaseEnd::RetryForward), + (Continuity, Some(LeasingEvidence::SkippedStateless), None, PhaseEnd::RetryForward), + (Continuity, Some(prepared), None, PhaseEnd::RetryForward), + (Continuity, Some(granted.clone()), None, PhaseEnd::RetryForward), + (Continuity, Some(granted), adoption.clone(), PhaseEnd::RetryForward), ]; for (kind, leasing, lease_adoption, expected) in cases { transaction.command_kind = kind; transaction.leasing = leasing.clone(); transaction.lease_adoption = lease_adoption.clone(); assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); - // Only a stateful deploy is left to the operator when its phase ends. + // Only a stateful deploy and a continuity are left to the operator + // when their phase ends; the other deploys abort. transaction.retime(1_000); let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; assert_eq!( transaction.leaves_to_operator(due), - expected == PhaseEnd::OperatorRequired, + matches!(expected, PhaseEnd::OperatorRequired | PhaseEnd::RetryForward), "{kind:?} {leasing:?} {lease_adoption:?}" ); assert!(!transaction.leaves_to_operator(due - 1)); @@ -22482,11 +22519,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// Soul's F1 probe, committed. A continuity's route changes are /// Survival, so no ceiling refuses its install and rollback while its - /// route proof keeps failing. The Routing deadline ends it: no attempt - /// runs after the deadline. + /// route proof keeps failing. Its Routing deadline does not end it: the + /// continuity pages the operator and retries forward, never stopping its + /// candidate (ruling post-fence-forward-retry). #[cfg(unix)] #[test] - fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { + fn a_continuity_whose_route_proof_fails_retries_forward_past_its_routing_deadline_and_pages() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; @@ -22504,36 +22542,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(waiting.last_error.is_some()); assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); + let stops = routed.workload.stops.load(Ordering::SeqCst); pass_deadline(&routed.world)?; - routed.resume()?; - let abort = routed - .transaction()? - .post_fencing_abort - .context("the passed deadline did not end the phase")?; - assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); - let finished = routed.resume_until_terminal()?; - assert!( - matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - ), - "{:?}", - finished.completion + for _ in 0..5 { + routed.resume()?; + } + // Past the deadline the continuity is not aborted: aborting it + // would stop the only running copy of the admitted body. It is left + // to the operator, and its Routing step keeps running. + let left = routed.transaction()?; + assert!(left.post_fencing_abort.is_none(), "the passed deadline aborted a continuity"); + assert_eq!(left.phase, DeploymentPhase::Routing); + assert!(left.completion.is_none()); + assert_eq!( + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [left.transaction_id.clone()] ); - // After the deadline no attempt runs; the withdrawal writes and - // reloads once, whatever the rolled-back disk shows. - assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); + assert_eq!(routed.world.engine.incidents().read()?.len(), 1, "one page per episode"); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert!(routed.reloads() > 6, "no attempt ran after the deadline"); + + // The proof passes: the phase moves on, and the first pass after + // closes the incident. + routed.stub.hang_up.store(false, Ordering::SeqCst); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; + routed.resume()?; + assert!( + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty(), + "the incident outlived the episode" + ); + let records = routed.world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); Ok(()) } /// The live shape of F1. A continuity reuses the admitted Expected, so /// its membership is the admitted fragment already on disk: while its /// proof fails, the install and the rollback write nothing and reload - /// nothing, and neither does the withdrawal after its deadline. + /// nothing, and neither does its forward retry after its deadline. #[cfg(unix)] #[test] fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() @@ -22562,8 +22610,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); pass_deadline(&routed.world)?; - routed.resume_until_terminal()?; - assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); + for _ in 0..5 { + routed.resume()?; + } + assert!(routed.transaction()?.post_fencing_abort.is_none(), "a passed deadline aborted it"); + assert_eq!(routed.world.route_stubs.count(""), 0, "its retry ran a host program"); assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); Ok(()) } @@ -22689,10 +22740,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .expect_err("an aborting transaction was expired"); assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let untouched = resident(&routed.world)?.envelope; - expire(&routed.world.state_store, "continuity-service", "operator")?; - assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); + expire(&routed.world.state_store, "up-service", "operator")?; + assert!(expire(&routed.world.state_store, "up-service", "operator").is_err()); let snapshot = ControlSnapshot::read(&routed.world.state_store)?; assert_eq!(snapshot.expiry_requests.len(), 1); assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); @@ -22708,6 +22759,36 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), "the spent request stayed" ); + + // A continuity past Fencing retries forward: expiring it would + // abort the only running copy of its body. The CLI refuses, and a + // request a pre-c2c binary stored for one is retired as spent. + let continuity = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let refused = expire(&continuity.world.state_store, "continuity-service", "operator") + .expect_err("a continuity past Fencing was expired"); + assert!(format!("{refused:#}").contains("retries forward"), "{refused:#}"); + assert!(ControlSnapshot::read(&continuity.world.state_store)?.expiry_requests.is_empty()); + let now = now_millis()?; + let stored = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: "continuity-service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + assert!(SingleFileMessagePackBackingStore::new(&continuity.world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: stored.command_id.clone(), + current: None, + }], + &[expiry_request_envelope(&stored, now)?], + )?); + continuity.resume()?; + assert!( + ControlSnapshot::read(&continuity.world.state_store)?.expiry_requests.is_empty(), + "the request for a continuity past Fencing stayed" + ); + assert!(continuity.transaction()?.post_fencing_abort.is_none(), "the request aborted a continuity"); Ok(()) } @@ -22739,7 +22820,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); // Another target's deployment in flight is not this command's. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let now = now_millis()?; let other = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -22765,10 +22846,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ], &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], )?); - expire(&routed.world.state_store, "continuity-service", "operator")?; + expire(&routed.world.state_store, "up-service", "operator")?; let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; assert_eq!(requests.len(), 1); - assert_eq!(requests[0].value.command_id, "continuity-service"); + assert_eq!(requests[0].value.command_id, "up-service"); Ok(()) } @@ -22776,7 +22857,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// live transactions, a request for the other leaves this one running. #[test] fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let now = now_millis()?; let other = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -22789,8 +22870,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; // The request names a command whose id has this transaction's // length and first bytes, and differs only in its last. - let near = "continuity-servicx"; - assert_eq!(near.len(), "continuity-service".len()); + let near = "up-servicx"; + assert_eq!(near.len(), "up-service".len()); let request = ExpiryRequest { schema_version: EXPIRY_REQUEST_SCHEMA.into(), command_id: near.into(), @@ -22800,11 +22881,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide // And commands whose ids are a strict prefix of this one's, and // extend it: a match is the whole id, in neither direction a part. let prefix = ExpiryRequest { - command_id: "continuity-servic".into(), + command_id: "up-servic".into(), ..request.clone() }; let extension = ExpiryRequest { - command_id: "continuity-service-2".into(), + command_id: "up-service-2".into(), ..request.clone() }; let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); @@ -22835,24 +22916,26 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let mine = snapshot .transactions .iter() - .find(|t| t.value.command_id == "continuity-service") + .find(|t| t.value.command_id == "up-service") .context("no continuity")?; assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); let after = ControlSnapshot::read(&routed.world.state_store)?; let unchanged = after .transactions .iter() - .find(|t| t.value.command_id == "continuity-service") + .find(|t| t.value.command_id == "up-service") .context("no continuity")?; assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); Ok(()) } - /// Q3 at the deadline, through the Engine. A continuity whose candidate - /// was granted a lease and never proved it held it fails as - /// `lease-not-adopted`; one whose Ready proved it restarts the admitted - /// release. A deployment whose candidate holds a lease is left to the - /// operator: nothing is aborted and its step still runs. + /// The deadline of a stateful target, through the Engine. A deployment + /// whose candidate holds a lease is left to the operator: nothing is + /// aborted and its step still runs. So is a continuity, adopted or not, + /// whose candidate is the only running copy of the admitted body. An + /// abort begun on a continuity directly still names its recovery: it + /// restarts the admitted release when its Ready proved the lease, and + /// restores the incumbent when it never did. #[test] fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { let granted = |routed: &RoutedWorld| -> Result { @@ -22866,13 +22949,29 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .to_owned()) }; + // Past its deadline a continuity is not aborted, adopted or not: + // its candidate is the only running copy of the admitted body. It + // is left to the operator and its step runs. + let open = |routed: &RoutedWorld| -> Result> { + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) + }; let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; granted(&unadopted)?; pass_deadline(&unadopted.world)?; - unadopted.resume()?; - let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); - assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); + for _ in 0..3 { + unadopted.resume()?; + } + let held = unadopted.transaction()?; + assert!(held.post_fencing_abort.is_none(), "an unadopted continuity was aborted"); + assert_eq!(held.phase, DeploymentPhase::AwaitingReady); + assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); + assert_eq!(open(&unadopted)?, [held.transaction_id.clone()]); + // An abort begun on it directly (a candidate found dead) still + // restores the incumbent when the lease was never adopted. + unadopted + .world + .engine + .begin_post_fencing_abort(&resident(&unadopted.world)?, anyhow!("test abort"))?; let finished = unadopted.resume_until_terminal()?; assert!(matches!( finished.completion, @@ -22896,10 +22995,19 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); assert_eq!(adoption.source, AdoptionSource::Direct); adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + adopted.stub.hang_up.store(true, Ordering::SeqCst); pass_deadline(&adopted.world)?; - adopted.resume()?; - let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + for _ in 0..3 { + adopted.resume()?; + } + let held = adopted.transaction()?; + assert!(held.post_fencing_abort.is_none(), "an adopted continuity was aborted"); + assert_eq!(held.phase, DeploymentPhase::Routing); + assert_eq!(open(&adopted)?, [held.transaction_id.clone()]); + adopted + .world + .engine + .begin_post_fencing_abort(&resident(&adopted.world)?, anyhow!("test abort"))?; let finished = adopted.resume_until_terminal()?; assert!( matches!( @@ -22991,7 +23099,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } /// B5 Soul A. The physical grant lands before `Granted` is durable. A - /// continuity that dies in that window and passes its deadline is + /// continuity that dies in that window and whose abort begins is /// aborted from `Prepared`; the abort still withdraws the lease on /// disk, so the next fencing of the target is not refused. #[test] @@ -23011,9 +23119,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ); assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); - pass_deadline(&routed.world)?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); routed.resume_until_terminal()?; assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); // What the next fencing does to a candidate's lease path. @@ -23034,9 +23145,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.workload.hold_lease_lock(&lease_path); let driver = CultCacheWriteLeaseDriver::new("service", &lease_path); - pass_deadline(&routed.world)?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); let stops = routed.workload.stops.load(Ordering::SeqCst); let finished = routed.resume_until_terminal()?; assert!( @@ -23049,6 +23163,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// The deadline's abort is a continuity's no longer, and a deployment's + /// still: a deployment's candidate is a new release and its incumbent + /// the admitted body, which an abort restores. + #[cfg(unix)] + #[test] + fn a_deploy_with_a_live_incumbent_still_aborts_to_it_at_its_deadline() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_none(), "aborted before its deadline"); + + pass_deadline(&routed.world)?; + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the passed deadline did not end a deployment")?; + assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!( + routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), + "the boot lift took a deployment's abort" + ); + assert!(routed.transaction()?.post_fencing_abort.is_some()); + let finished = routed.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + ), + "{:?}", + finished.completion + ); + Ok(()) + } + /// What a pre-c2c Idunn left when a continuity's phase passed its /// deadline: the abort begun, route restoration done, the lease /// withdrawal and candidate cleanup still owed, and the deadline stale. From cc4a72ad43bf89f1e34c18be959fd5bbab8b6a57 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:37:03 +0200 Subject: [PATCH 028/122] Cut c2c: a record left to the operator backs off to the continuity cap ResumeBackoff stores the ceiling it was armed with and the skip check reads it. resume_ceiling is the continuity cap (300 s) for a record that leaves_to_operator and the short cap otherwise, for every such record: a forward-retrying continuity, a stateful deploy, a wedged abort, a drain. A step that returns Ok without moving a record left to the operator is backed off like a failed attempt, so a silent candidate is challenged once per backoff, not every poll. Nothing in TargetSupervision is touched. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 136 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 122 insertions(+), 14 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3e36787..f750257 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5112,11 +5112,26 @@ impl RouteActuationGate for EngineRouteGate<'_> { struct ResumeBackoff { failures: u32, not_before_unix_millis: u64, + /// The longest wait this backoff was armed with: the skip check reads it, + /// so the check and the wait cannot disagree. + ceiling_millis: u64, } /// Longest wait between attempts at a failing transaction step. const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; +/// The longest wait between attempts at one transaction's step: the continuity +/// cap for a record left to the operator, which retries on the slow cadence its +/// page promises (ruling c2c-forward-retry-cadence), and the shorter one for +/// every other. +fn resume_ceiling(transaction: &DeploymentTransaction, now: u64) -> u64 { + if transaction.leaves_to_operator(now) { + CONTINUITY_BACKOFF_CEILING_MILLIS + } else { + RESUME_BACKOFF_CEILING_MILLIS + } +} + impl Engine { fn open(options: RuntimeOptions) -> Result { Self::open_with_systemd_workload( @@ -6054,7 +6069,7 @@ impl Engine { .get(id) .is_some_and(|backoff| { now_millis().is_ok_and(|now| { - is_waiting(now, backoff.not_before_unix_millis, RESUME_BACKOFF_CEILING_MILLIS) + is_waiting(now, backoff.not_before_unix_millis, backoff.ceiling_millis) }) }) { @@ -6064,15 +6079,29 @@ impl Engine { self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); return Ok(false); } + let left = now_millis().is_ok_and(|now| current.value.leaves_to_operator(now)); let advanced = self.advance_transaction(current); if advanced.is_ok() { - // The step ran and succeeded: the fault, if any, has recovered, so - // a later failure is news. A skipped or failed attempt clears nothing. - self.resume_backoff - .lock() - .expect("resume backoff mutex") - .remove(id); - self.clear_fault(id); + // A step that returns Ok without moving a record left to the + // operator (a gate-wait on a silent candidate) is an attempt like + // a failed one: it backs off, and the fault it recorded stands. + let unmoved = left + && ControlSnapshot::read(&self.options.state_store)? + .transactions + .iter() + .find(|stored| stored.value.transaction_id == id) + .is_some_and(|live| !state_advanced(¤t.value, &live.value)); + if unmoved { + self.back_off(¤t.value)?; + } else { + // The step ran and succeeded: the fault, if any, has recovered, so + // a later failure is news. A skipped or failed attempt clears nothing. + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + self.clear_fault(id); + } } if let Err(error) = advanced { let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; @@ -6088,7 +6117,7 @@ impl Engine { .remove(id); return Ok(true); } - self.back_off(id)?; + self.back_off(&latest.value)?; if latest.value.phase < DeploymentPhase::Fencing { // Before the fence an abort that cannot finish a step is // resumable, never a post-fence abort: that path refuses a @@ -6142,16 +6171,20 @@ impl Engine { /// Wait twice as long after each consecutive failure of a transaction's /// step, from one poll interval up to a ceiling. - fn back_off(&self, transaction_id: &str) -> Result<()> { + fn back_off(&self, transaction: &DeploymentTransaction) -> Result<()> { let now = now_millis()?; + let ceiling_millis = resume_ceiling(transaction, now); let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); - let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); - let wait = backoff_wait(self.options.poll_millis, failures, RESUME_BACKOFF_CEILING_MILLIS); + let failures = all + .get(&transaction.transaction_id) + .map_or(0, |backoff| backoff.failures); + let wait = backoff_wait(self.options.poll_millis, failures, ceiling_millis); all.insert( - transaction_id.to_string(), + transaction.transaction_id.clone(), ResumeBackoff { failures: failures.saturating_add(1), not_before_unix_millis: now.saturating_add(wait), + ceiling_millis, }, ); Ok(()) @@ -14061,11 +14094,45 @@ mod tests { Ok(()) } + /// The slow cadence belongs to a record that is left to the operator, + /// whichever way it came to be: it waits up to the continuity cap between + /// attempts from its deadline on, and up to the short cap before. + #[test] + fn a_record_left_to_the_operator_backs_off_to_the_continuity_cap() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let lease = write_lease(); + let leased = LeasingEvidence::Granted { + lease_sha256: lease.canonical_sha256()?, + lease, + }; + transaction.retime(1_000); + let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; + let mut aborting = transaction.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "wedged")); + let mut deploy = transaction.clone(); + deploy.command_kind = CommandKind::Deploy; + let mut stateful = deploy.clone(); + stateful.leasing = Some(leased); + let mut continuity = transaction.clone(); + continuity.command_kind = CommandKind::Continuity; + for (what, record) in [ + ("a forward-retrying continuity", &continuity), + ("a stateful deploy", &stateful), + ("a wedged abort", &aborting), + ] { + assert_eq!(resume_ceiling(record, due - 1), RESUME_BACKOFF_CEILING_MILLIS, "{what} before its deadline"); + assert_eq!(resume_ceiling(record, due), CONTINUITY_BACKOFF_CEILING_MILLIS, "{what} at its deadline"); + } + // A deploy that aborts at its deadline is not left to anyone. + assert_eq!(resume_ceiling(&deploy, due), RESUME_BACKOFF_CEILING_MILLIS); + Ok(()) + } + /// Q3 as a table: which ended phases abort, which name the unadopted /// lease, which are left to the operator, and which recovery the terminal /// record names. #[test] - fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { + fn an_ended_phase_is_resolved_by_command_kind_and_lease() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; let lease = write_lease(); let sha256 = lease.canonical_sha256()?; @@ -15794,6 +15861,7 @@ mod tests { ResumeBackoff { failures: 3, not_before_unix_millis, + ceiling_millis: RESUME_BACKOFF_CEILING_MILLIS, }, ); }; @@ -23203,6 +23271,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// A record left to the operator whose step returns Ok without moving it + /// (a gate-wait on a silent candidate) is backed off like a failed + /// attempt, to the continuity cap: its candidate is challenged once per + /// backoff, not once per pass. Before its deadline every pass challenges. + #[test] + fn a_continuity_left_to_the_operator_challenges_a_silent_candidate_once_per_backoff() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let engine = &routed.world.engine; + let id = routed.transaction()?.transaction_id; + let hits = || routed.stub.candidate_hits.load(Ordering::SeqCst); + + let before = hits(); + for _ in 0..3 { + engine.resume_one_transaction()?; + } + assert!(hits() >= before + 3, "a pass inside the deadline did not challenge"); + assert!(engine.resume_backoff.lock().unwrap().get(&id).is_none()); + + pass_deadline(&routed.world)?; + engine.resume_one_transaction()?; + let first = hits(); + assert!(first > before + 3, "the pass past the deadline did not challenge"); + let armed = *engine.resume_backoff.lock().unwrap().get(&id).context("the unmoved step set no backoff")?; + assert_eq!(armed.ceiling_millis, CONTINUITY_BACKOFF_CEILING_MILLIS); + for _ in 0..5 { + engine.resume_one_transaction()?; + } + assert_eq!(hits(), first, "a wedged candidate was challenged inside its backoff"); + assert!(routed.transaction()?.post_fencing_abort.is_none()); + + // The skip check reads the ceiling the backoff was armed with: a + // wait longer than the short cap is still a wait. + engine.resume_backoff.lock().unwrap().get_mut(&id).unwrap().not_before_unix_millis = + now_millis()? + 200_000; + engine.resume_one_transaction()?; + assert_eq!(hits(), first, "the skip check read the short cap"); + Ok(()) + } + /// What a pre-c2c Idunn left when a continuity's phase passed its /// deadline: the abort begun, route restoration done, the lease /// withdrawal and candidate cleanup still owed, and the deadline stale. From bff308aea781817e44a935c38ce2d26e0ddbf47e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:39:32 +0200 Subject: [PATCH 029/122] Cut c2c: expire's refusal names its reason without reading the deadline predicate Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index f750257..ad06ea9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3138,7 +3138,7 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> ensure!( current.value.expirable(), "{}", - if current.value.resolves_by_deadline() { + if current.value.command_kind == CommandKind::Continuity && current.value.phase >= DeploymentPhase::Fencing && current.value.post_fencing_abort.is_none() { "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" } else { "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" From d9846e8847a675008f2877fa26842991fdc45f6d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:39:33 +0200 Subject: [PATCH 030/122] Cut c2c: route-continuity invariant 5 states the continuity forward retry A continuity past its post-fencing deadline is left to the operator and retries forward on a capped backoff; a stateless deployment aborts to its incumbent; a stateful deployment is left to the operator (ruling post-fence-forward-retry). Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index 77c201b..c4d3bc8 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -52,12 +52,13 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: the admitted incarnation. The proof answers an Idunn challenge and is signed with the credential Idunn issued to that launch. Invariant 3 changes where the proof travels, not what it proves. -5. **Every post-fencing phase ends.** Each phase after fencing has an admitted - deadline. When a deadline expires, Idunn resolves it: stateless targets abort - to the incumbent. Stateful targets get an explicit terminal record with a - named recovery. A granted lease that the process does not pick up within its - window is a named failure with an Idunn-owned recovery. It is never a silent - wait. Resumable-forever is gone. +5. **Every post-fencing phase has a deadline and is never a silent wait.** A + deployment's stateless phase that passes its deadline aborts to its + incumbent. A continuity past its deadline is left to the operator + (operator-required incident) and retries its phase forward on a capped + backoff, because aborting it would stop the only running copy of the admitted + body (ruling post-fence-forward-retry). A stateful deployment is left to the + operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never meets drift between the receipt and the projection. Rollback demotes the exact From a7ad3de5593a086e2522eb8bafda1b9f72cd9af4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 19:58:09 +0200 Subject: [PATCH 031/122] c2d negative: a waiting Deploy must land over a forward-retrying continuity past its deadline Red at e529858: the waiter ends FailedBeforeFencing owner-left-to-operator. The yield (ruling continuity-yields-to-deploy) lands in the next commit. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 96 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 11 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index de88449..28ea709 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17643,6 +17643,80 @@ mod tests { } // END c2b-forever-wait harness + // BEGIN c2d continuity-yields + // A deployment waits at Warming beside a continuity that retries forward + // past its deadline (ruling continuity-yields-to-deploy). The pass order is + // the hostile one: the waiter always looks at the owner first. + + /// One pass over the waiter and then the owner, never clearing the resume + /// backoff. A record that left the live set is skipped. + fn waiter_then_owner(world: &EngineFixture, waiter: &str, owner: &str) -> Result<()> { + for id in [waiter, owner] { + if let Ok(stored) = resident_id(world, id) { + world.engine.resume_candidate(&stored)?; + } + } + Ok(()) + } + + /// A continuity at Fencing driven alone to AwaitingReady, where it + /// gate-waits for an Odin word that never comes, beside a Deploy waiting in + /// Warming. Returns the world, the workload, and the two ids. + fn waiter_beside_gate_waiting_continuity() + -> Result<(EngineFixture, Arc, DeploymentTransaction, String, String, AdmittedGeneration)> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, incumbent) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.clone(), continuity.transaction_id.clone()); + drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::AwaitingReady)?; + pass_deadline_of(&world, &owner_id)?; + Ok((world, workload, deploy, deploy_id, owner_id, incumbent)) + } + + /// NEGATIVE. The continuity's deadline has passed and it retries forward; + /// the deployment waiting on it must not end, the continuity must yield. + #[test] + fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { + let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); + for _ in 0..2 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + let waiter = latest_id(&world, &deploy_id)?; + assert!(waiter.completion.is_none(), "the waiter ended: {:?}", waiter.completion); + assert!(waiter.pre_fencing_abort.is_none()); + let owner = resident_id(&world, &owner_id)?.value; + let abort = owner.post_fencing_abort.as_ref().context("the continuity did not yield")?; + assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); + // Nothing stopped the candidate before the yield began. + assert_eq!(workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + + for _ in 0..8 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + world.engine.resume_backoff.lock().unwrap().clear(); + } + // The yield's abort retimed the record: its incident, if one opened, closed. + assert!(!operator_required(&world)?.contains(&owner_id)); + let ended = latest_id(&world, &owner_id)?; + assert!(matches!(ended.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), "{:?}", ended.completion); + assert_eq!(workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops + 1); + + odin_reports_ready(&world, &deploy, seeded_sequence(Some(&incumbent)) + 10)?; + drive_id(&world, &deploy_id, |t| t.is_terminal())?; + let landed = latest_id(&world, &deploy_id)?; + assert!(matches!(landed.completion, Some(TransactionCompletion::Admitted { .. })), "{:?}", landed.completion); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + assert_eq!(admitted.transaction_id, deploy_id); + assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); + Ok(()) + } + // END c2d continuity-yields + #[test] fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { let workload = SwitchWorkload::new(); @@ -22726,7 +22800,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(waiting.last_error.is_some()); assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); pass_deadline(&routed.world)?; for _ in 0..5 { routed.resume()?; @@ -22743,7 +22817,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide [left.transaction_id.clone()] ); assert_eq!(routed.world.engine.incidents().read()?.len(), 1, "one page per episode"); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped"); assert!(routed.reloads() > 6, "no attempt ran after the deadline"); // The proof passes: the phase moves on, and the first pass after @@ -22758,7 +22832,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let records = routed.world.engine.incidents().read()?; assert_eq!(records.len(), 1); assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); Ok(()) } @@ -23335,7 +23409,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let finished = routed.resume_until_terminal()?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), @@ -23343,7 +23417,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide finished.last_error ); assert!(driver.observe_empty()?, "the abort left the lease on disk"); - assert!(routed.workload.stops.load(Ordering::SeqCst) > stops, "the candidate was never stopped"); + assert!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst) > stops, "the candidate was never stopped"); Ok(()) } @@ -23480,7 +23554,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide &routed, "the Routing phase passed its deadline at unix ms 1792324133638", )?; - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let lifted_at = now_millis()?; let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; @@ -23488,7 +23562,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.resume()?; } assert_eq!( - routed.workload.stops.load(Ordering::SeqCst), + routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped" ); @@ -23504,7 +23578,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.stub.hang_up.store(false, Ordering::SeqCst); routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); assert!(routed.fragment_path().exists(), "the route membership was not re-installed"); Ok(()) } @@ -23525,7 +23599,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide &routed, "lease-not-adopted: the AwaitingReady phase passed its deadline at unix ms 1792324133638", )?; - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; for _ in 0..5 { @@ -23533,7 +23607,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } let waiting = routed.transaction()?; assert_eq!(returned, [waiting.transaction_id.clone()]); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped"); assert!(driver.observe_exact(&lease)?, "the lease left the disk"); assert!(waiting.post_fencing_abort.is_none() && waiting.completion.is_none()); assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); @@ -23544,7 +23618,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Routing)?; let adopted = routed.transaction()?; assert_eq!(adopted.lease_adoption.context("no adoption recorded")?.write_lease_sha256, lease_sha256); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); Ok(()) } From 4f68809f0b300c5a8830f51f18c5b2d061e41172 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 19:59:37 +0200 Subject: [PATCH 032/122] Continuity yields to a Deploy awaiting the handover (ruling continuity-yields-to-deploy) A continuity past its deadline at or past Fencing begins its own post-fencing abort when a Deploy on its target awaits the handover; the waiter keeps waiting on an owner that retries forward. Adds retries_forward and awaits_handover, both derived. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 53 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 28ea709..34cf549 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -983,7 +983,9 @@ enum PhaseEnd { OperatorRequired, /// A continuity at or past Fencing: its candidate is the only running copy /// of the admitted body, so aborting it restores what it already runs. The - /// resolver leaves it to the operator and its phase step keeps running. + /// resolver leaves it to the operator and its phase step keeps running, + /// until a deployment awaits the handover, when the continuity yields + /// through its own post-fencing abort (ruling continuity-yields-to-deploy). RetryForward, } @@ -1861,6 +1863,14 @@ impl DeploymentTransaction { && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) } + /// A continuity at or past Fencing that its deadline leaves to the operator + /// while its step runs. It yields to a deployment awaiting the handover + /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. + /// Derived, never stored. + fn retries_forward(&self) -> bool { + self.resolves_by_deadline() && self.phase_end() == PhaseEnd::RetryForward + } + /// What an ended post-fencing phase resolves to (Q3). A continuity at or /// past Fencing retries forward: its incumbent is the admitted generation, /// which supervision mints a continuity only after seeing dead, and from @@ -2064,6 +2074,16 @@ impl DeploymentTransaction { self.phase != DeploymentPhase::Complete } + /// A live deployment whose next move is the handover: not ended, not + /// aborting, one phase short of `takes_target_at`. Derived, never stored. + /// Read by the continuity's yield and the boot lift. + fn awaits_handover(&self) -> bool { + self.command_kind == CommandKind::Deploy + && self.completion.is_none() + && self.pre_fencing_abort.is_none() + && self.phase as u8 + 1 == self.takes_target_at() as u8 + } + /// From the handover until Complete the transaction owns the target's /// incumbent slot: it fences, leases, routes and commits. Derived from the /// phase, never stored. Read by validation (one owner), supervision (which @@ -8535,9 +8555,11 @@ impl Engine { /// A deployment waits on the transaction that owns or drains its target. /// The owner bounds the wait, so the waiter's own Warming deadline is /// disarmed (the resolver re-arms it when the target is free). When the - /// owner's deadline has passed in a state Idunn will not end by itself, - /// the waiter ends through its own pre-fencing abort (the caller's Err - /// path): it never aborts the owner. + /// owner's deadline has passed in a state Idunn will not end by itself and + /// will not yield (a wedged abort or drain), the waiter ends through its own + /// pre-fencing abort (the caller's Err path): it never aborts the owner. A + /// waiter keeps waiting on an owner that retries forward, because that + /// owner yields to it on its next pass. fn wait_for_owner( &self, current: &Stored, @@ -8547,7 +8569,9 @@ impl Engine { let now = now_millis()?; let snapshot = ControlSnapshot::read(&self.options.state_store)?; if snapshot.transactions.iter().any(|stored| { - stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) + stored.value.transaction_id == owner_id + && stored.value.leaves_to_operator(now) + && !stored.value.retries_forward() }) { bail!( "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", @@ -9522,7 +9546,9 @@ impl Engine { /// the fence, aborted after it, or left to the operator with its step /// retrying (a stateful deploy, or a continuity, whose candidate is the only /// running copy of its body). An abort or a drain past its deadline is left - /// to the operator the same way. + /// to the operator the same way. A continuity left to the operator yields, + /// through its own post-fencing abort, to a deployment awaiting the + /// handover; it never writes the deployment. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the @@ -9611,6 +9637,21 @@ impl Engine { Ok(None) } PhaseEnd::RetryForward => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(deploy) = snapshot + .on_target(&transaction.target) + .find(|other| other.value.awaits_handover()) + { + self.begin_post_fencing_abort( + current, + anyhow!( + "continuity-yields: {cause}; deployment {} awaits the handover of {}", + deploy.value.transaction_id, + transaction.target + ), + )?; + return Ok(Some(true)); + } self.leave_to_operator( transaction, &format!( From 196e0dc884f1a573a37c545f80dbb52f14f80127 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:03:48 +0200 Subject: [PATCH 033/122] c2d: a resolver write spends the resume backoff; the boot lift leaves a yield resume_candidate removes the transaction's backoff when the resolver moved its record, so a yield abort is not held behind the forward retry's backoff (ruling c2d-yield-mechanics). return_wedged_continuity_aborts_forward skips an abort whose target holds a Deploy awaiting the handover. Tests: the yield table, the unit rows, the c2b waiter-still-ends row, the backoff spend, and lift case (vii). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 197 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 189 insertions(+), 8 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 34cf549..e2494f9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5690,10 +5690,12 @@ impl Engine { /// c2c-boot-returns-wedged-aborts-forward). The result is the exact inverse /// of `begin_post_fencing_abort`. /// - /// A continuity abort now begins only over a permanently stopped - /// candidate, so this predicate cannot match a new record: it is a - /// migration with a deletion line (follow-up c2c-delete-boot-lift). A - /// record it cannot decide is reported and left aborting. + /// A continuity abort now begins only over a permanently stopped candidate + /// or to yield to a deployment awaiting the handover, so this predicate + /// matches only aborts over a running candidate with nothing awaiting the + /// target, which Idunn no longer creates: it is a migration with a deletion + /// line (follow-up c2c-delete-boot-lift). A record it cannot decide is + /// reported and left aborting. fn return_wedged_continuity_aborts_forward(&self) -> Result> { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut returned = Vec::new(); @@ -5706,6 +5708,9 @@ impl Engine { || transaction.completion.is_some() || abort.candidate_cleanup != CleanupEvidence::Pending || abort.lease_withdrawal == CleanupEvidence::Complete + || snapshot + .on_target(&transaction.target) + .any(|other| other.value.awaits_handover()) { continue; } @@ -6082,6 +6087,11 @@ impl Engine { fn resume_candidate(&self, current: &Stored) -> Result { let id = current.value.transaction_id.as_str(); if let Some(moved) = self.resolve_phase_deadline(current)? { + if moved { + // A phase-ending write is movement: the next step is not held + // behind the backoff a forward retry armed. + self.resume_backoff.lock().expect("resume backoff mutex").remove(id); + } return Ok(moved); } if self @@ -17721,9 +17731,7 @@ mod tests { fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); - for _ in 0..2 { - waiter_then_owner(&world, &deploy_id, &owner_id)?; - } + waiter_then_owner(&world, &deploy_id, &owner_id)?; let waiter = latest_id(&world, &deploy_id)?; assert!(waiter.completion.is_none(), "the waiter ended: {:?}", waiter.completion); assert!(waiter.pre_fencing_abort.is_none()); @@ -17756,6 +17764,167 @@ mod tests { assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); Ok(()) } + + fn stops_of(workload: &SwitchWorkload) -> usize { + workload.stops.load(std::sync::atomic::Ordering::SeqCst) + } + + /// The yield is a function of its inputs: the deadline, the continuity's + /// own record, and a deployment that awaits the handover. + #[test] + fn a_continuity_that_retries_forward_yields_only_past_its_deadline_and_only_to_a_deploy_awaiting_the_handover( + ) -> Result<()> { + // (i) Before the deadline nothing yields and the Deploy waits. + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let stops = stops_of(&workload); + for _ in 0..5 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + let waiting = resident_id(&world, &deploy_id)?.value; + assert!(waiting.pre_fencing_abort.is_none() && waiting.completion.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(&owner_id))); + assert_eq!(stops_of(&workload), stops); + + // (ii) Past the deadline, a Deploy still building awaits nothing. + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let stops = stops_of(&workload); + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + pass_deadline_of(&world, &owner_id)?; + for _ in 0..4 { + waiter_then_owner(&world, "", &owner_id)?; + } + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + + // (iii) A Deploy already ending before the fence awaits nothing. + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + + // (iv) Nor does a Deploy that has ended. + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; + for _ in 0..12 { + if latest_id(&world, &deploy_id)?.completion.is_some() { + break; + } + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, &deploy_id, "")?; + } + assert!(latest_id(&world, &deploy_id)?.completion.is_some()); + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + Ok(()) + } + + /// `awaits_handover` and `retries_forward` row by row. + #[test] + fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let waiting = resident_id(&world, &deploy_id)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.awaits_handover()); + for phase in [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Fencing, + DeploymentPhase::Complete, + ] { + let mut moved = waiting.clone(); + moved.phase = phase; + assert!(!moved.awaits_handover(), "{phase:?}"); + } + let mut ended = waiting.clone(); + ended.completion = Some(TransactionCompletion::FailedBeforeFencing { error: "x".into() }); + assert!(!ended.awaits_handover()); + let mut continuity = resident_id(&world, &owner_id)?.value; + let mut as_continuity = waiting.clone(); + as_continuity.command_kind = CommandKind::Continuity; + assert!(!as_continuity.awaits_handover()); + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("x"))?; + assert!(!resident_id(&world, &deploy_id)?.value.awaits_handover()); + + // The stop-then-start handover is Starting: its Sealing tail awaits it. + let (stop_first, stop_deploy, _) = stop_first_world(Arc::new(StillWorkload))?; + let mut sealing = resident_id(&stop_first, &stop_deploy)?.value; + sealing.phase = DeploymentPhase::Sealing; + assert!(sealing.awaits_handover()); + sealing.phase = DeploymentPhase::Warming; + assert!(!sealing.awaits_handover()); + + // A continuity retries forward from Fencing to Committing, until aborting. + for phase in [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + ] { + continuity.phase = phase; + assert!(continuity.retries_forward(), "{phase:?}"); + } + let mut deploy_record = waiting.clone(); + deploy_record.phase = DeploymentPhase::Fencing; + assert!(!deploy_record.retries_forward()); + continuity.post_fencing_abort = Some(post_fencing_abort_intent(&continuity, "x")); + assert!(!continuity.retries_forward()); + Ok(()) + } + + /// The c2b waiter still ends on an owner that will not yield: the yield's + /// own abort wedged past its deadline. + #[test] + fn the_c2b_waiter_still_ends_on_an_owner_that_will_not_yield() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + workload.fail_stop_of(&runtime_instance_of(&world, &owner_id)?); + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, &deploy_id, &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + pass_deadline_of(&world, &owner_id)?; + for _ in 0..20 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + assert!(!resident_id(&world, &owner_id)?.value.is_terminal()); + assert_ended_for_owner(&world, &deploy_id, &owner_id) + } + + /// The yield's abort is not held behind the backoff the forward retry + /// armed: movement spends it. + #[test] + fn a_yield_abort_is_not_held_behind_the_forward_retry_backoff() -> Result<()> { + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + // The Deploy is still building, so the pass past the deadline retries + // forward and arms the continuity backoff. + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + let armed = *world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&owner_id) + .context("the unmoved step set no backoff")?; + assert_eq!(armed.ceiling_millis, CONTINUITY_BACKOFF_CEILING_MILLIS); + // It reaches the handover: the next pass yields, and the abort runs + // with no clearing and no fake time. + rephase(&world, &deploy_id, DeploymentPhase::Warming)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + for _ in 0..3 { + waiter_then_owner(&world, "", &owner_id)?; + } + let done = latest_id(&world, &owner_id)?; + let cleaned = done.post_fencing_abort.as_ref().map(|abort| abort.candidate_cleanup); + assert_eq!(cleaned, Some(CleanupEvidence::Complete), "{:?}", done.last_error); + Ok(()) + } // END c2d continuity-yields #[test] @@ -23676,6 +23845,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide LeaseRevokedUnderIt, DeployWithRunningCandidate, CompletedRecord, + DeployAwaitsTheTarget, } for case in [ Case::CandidateNotRunning, @@ -23684,6 +23854,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Case::LeaseRevokedUnderIt, Case::DeployWithRunningCandidate, Case::CompletedRecord, + Case::DeployAwaitsTheTarget, ] { let deploy = matches!(case, Case::DeployWithRunningCandidate); // The cleanup comes after the lease withdrawal, so a case with @@ -23713,11 +23884,17 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide lease_driver(&routed).0.revoke_exact(Some(&lease))?; } Case::DeployWithRunningCandidate => {} + // The abort of the Odin-shaped negative, begun to yield: a + // Deploy awaits the handover of its target. + Case::DeployAwaitsTheTarget => { + seeded_transaction(&routed.world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + } Case::CompletedRecord => { routed.resume_until_terminal()?; } } - let abort = routed.transaction().ok().and_then(|t| t.post_fencing_abort); + let owner_id = the_continuity(&routed.world).ok().map(|c| c.value.transaction_id); + let abort = the_continuity(&routed.world).ok().and_then(|c| c.value.post_fencing_abort); match case { Case::LeaseWithdrawn => assert_eq!( abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), @@ -23738,6 +23915,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide after.transactions.iter().map(|t| &t.envelope).collect::>(), "the lift wrote a record it left" ); + if let (Case::DeployAwaitsTheTarget, Some(owner)) = (case, &owner_id) { + assert!(abort.is_some(), "the lift withdrew the yield"); + clear_other_live_transactions(&routed.world, owner)?; + } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), From a8683212598053a2bbce82ca3a454e056de82e6f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:06:00 +0200 Subject: [PATCH 034/122] c2d yield table: name the row an error comes from Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e2494f9..dc57b9d 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17780,7 +17780,7 @@ mod tests { for _ in 0..5 { waiter_then_owner(&world, &deploy_id, &owner_id)?; } - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(i)")?.value.post_fencing_abort.is_none()); let waiting = resident_id(&world, &deploy_id)?.value; assert!(waiting.pre_fencing_abort.is_none() && waiting.completion.is_none()); assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(&owner_id))); @@ -17794,7 +17794,7 @@ mod tests { for _ in 0..4 { waiter_then_owner(&world, "", &owner_id)?; } - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(ii)")?.value.post_fencing_abort.is_none()); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); @@ -17803,7 +17803,7 @@ mod tests { world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; pass_deadline_of(&world, &owner_id)?; waiter_then_owner(&world, "", &owner_id)?; - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(iii)")?.value.post_fencing_abort.is_none()); // (iv) Nor does a Deploy that has ended. let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; @@ -17818,7 +17818,7 @@ mod tests { assert!(latest_id(&world, &deploy_id)?.completion.is_some()); pass_deadline_of(&world, &owner_id)?; waiter_then_owner(&world, "", &owner_id)?; - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(iv)")?.value.post_fencing_abort.is_none()); Ok(()) } From fd6e3e65af2aa781c39b638aeb42ea773a421e62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:06:48 +0200 Subject: [PATCH 035/122] c2d yield table: show how row (i) ends --- src/control_plane.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index dc57b9d..5db88d4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17779,6 +17779,8 @@ mod tests { let stops = stops_of(&workload); for _ in 0..5 { waiter_then_owner(&world, &deploy_id, &owner_id)?; + let owner = latest_id(&world, &owner_id)?; + assert!(owner.completion.is_none(), "(i) {:?} {:?} {:?}", owner.completion, owner.phase, owner.last_error); } assert!(resident_id(&world, &owner_id).with_context(|| "(i)")?.value.post_fencing_abort.is_none()); let waiting = resident_id(&world, &deploy_id)?.value; From f79deecdb70ce6a5b845ca178850e961c01c247a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:07:30 +0200 Subject: [PATCH 036/122] c2d tests: the helper stops short of the deadline; each test passes it --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5db88d4..3b2d51e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17721,7 +17721,6 @@ mod tests { deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; let (deploy_id, owner_id) = (deploy.transaction_id.clone(), continuity.transaction_id.clone()); drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::AwaitingReady)?; - pass_deadline_of(&world, &owner_id)?; Ok((world, workload, deploy, deploy_id, owner_id, incumbent)) } @@ -17730,6 +17729,7 @@ mod tests { #[test] fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + pass_deadline_of(&world, &owner_id)?; let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); waiter_then_owner(&world, &deploy_id, &owner_id)?; let waiter = latest_id(&world, &deploy_id)?; From 09bd12764323b3dc4d023041ca16bd7c845c164f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:09:28 +0200 Subject: [PATCH 037/122] c2d: stateful yield test, an Odin-shaped continuity yields its lease to a waiting Deploy Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3b2d51e..49fe7f9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -23834,6 +23834,76 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// NEGATIVE. The live shape of tx-4eaed669 (Odin) left to the operator + /// at its Routing deadline, with a Deploy now awaiting the handover of + /// its target. The continuity yields through its own abort: the lease + /// leaves the disk only after the stop, the target is free, and the + /// Deploy, unended, fences a path that is already empty. + #[cfg(unix)] + #[test] + fn an_odin_shaped_continuity_left_to_the_operator_yields_its_lease_to_a_waiting_deploy() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + assert!(routed.transaction()?.lease_adoption.is_some()); + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + let owner_id = routed.transaction()?.transaction_id; + pass_deadline(&routed.world)?; + routed.resume()?; + let left = routed.transaction()?; + assert!(left.post_fencing_abort.is_none(), "the continuity aborted with nothing awaiting the target"); + assert_eq!(routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, [owner_id.as_str()]); + assert!(driver.observe_exact(&lease)?); + + let deploy = seeded_transaction(&routed.world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + for _ in 0..12 { + routed.resume()?; + let owner = latest_id(&routed.world, &owner_id)?; + if owner.post_fencing_abort.is_none() { + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(driver.observe_exact(&lease)?, "the lease left before the yield"); + } + if owner.completion.is_some() { + break; + } + } + let ended = latest_id(&routed.world, &owner_id)?; + assert!( + matches!( + ended.completion, + Some(TransactionCompletion::FailedAfterFencing { recovery: TerminalRecovery::RestartAdmitted, .. }) + ), + "{:?}", + ended.completion + ); + assert!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst) > stops); + assert!(!driver.observe_exact(&lease)?, "the lease stayed on disk"); + assert!(ControlSnapshot::read(&routed.world.state_store)?.target_owner("service", None).is_none()); + let waiting = resident_id(&routed.world, &deploy.transaction_id)?.value; + assert!(waiting.completion.is_none() && waiting.pre_fencing_abort.is_none()); + + // Its next move is the fence, over a lease path that is empty. + rephase(&routed.world, &deploy.transaction_id, DeploymentPhase::Fencing)?; + routed.world.engine.advance_transaction(&resident_id(&routed.world, &deploy.transaction_id)?)?; + let fenced = latest_id(&routed.world, &deploy.transaction_id)?; + assert!( + matches!( + fenced.fencing, + Some(FencingEvidence::Revoked { candidate_lease_path_verified_empty: true, .. }) + ), + "{:?} {:?}", + fenced.fencing, + fenced.last_error + ); + Ok(()) + } + /// The lift is a function of its inputs: each case lacks one thing the /// lift requires, and its abort stays and runs to its end. #[cfg(unix)] From 4e66a65758873eab4d3f41ef395a0096dc0ff9bd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:10:18 +0200 Subject: [PATCH 038/122] c2d stateful yield test: drop the Fencing half the seeded Deploy cannot honestly assert Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 49fe7f9..d872c4e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -23838,7 +23838,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// at its Routing deadline, with a Deploy now awaiting the handover of /// its target. The continuity yields through its own abort: the lease /// leaves the disk only after the stop, the target is free, and the - /// Deploy, unended, fences a path that is already empty. + /// Deploy is unended. (The Fencing half is not asserted here: the seeded + /// Deploy plans a stateless target, so its fence skips the lease path.) #[cfg(unix)] #[test] fn an_odin_shaped_continuity_left_to_the_operator_yields_its_lease_to_a_waiting_deploy() -> Result<()> { @@ -23887,20 +23888,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(ControlSnapshot::read(&routed.world.state_store)?.target_owner("service", None).is_none()); let waiting = resident_id(&routed.world, &deploy.transaction_id)?.value; assert!(waiting.completion.is_none() && waiting.pre_fencing_abort.is_none()); - - // Its next move is the fence, over a lease path that is empty. - rephase(&routed.world, &deploy.transaction_id, DeploymentPhase::Fencing)?; - routed.world.engine.advance_transaction(&resident_id(&routed.world, &deploy.transaction_id)?)?; - let fenced = latest_id(&routed.world, &deploy.transaction_id)?; - assert!( - matches!( - fenced.fencing, - Some(FencingEvidence::Revoked { candidate_lease_path_verified_empty: true, .. }) - ), - "{:?} {:?}", - fenced.fencing, - fenced.last_error - ); Ok(()) } From 81bfb037123d97287c024fdab407404253233a1a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:11:10 +0200 Subject: [PATCH 039/122] route-continuity-target invariant 5: the continuity yields to a deployment awaiting the handover Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index c4d3bc8..c014a2a 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -57,7 +57,10 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: incumbent. A continuity past its deadline is left to the operator (operator-required incident) and retries its phase forward on a capped backoff, because aborting it would stop the only running copy of the admitted - body (ruling post-fence-forward-retry). A stateful deployment is left to the + body (ruling post-fence-forward-retry), until a deployment awaits the + handover of its target: then it yields through its own post-fencing abort, + and the deployment fences what is admitted (ruling + continuity-yields-to-deploy). A stateful deployment is left to the operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never From 11f7fdbf09dc4c6447eacb86f83c525f03d828c7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:28:06 +0200 Subject: [PATCH 040/122] c2c fix 1/3: negative test, a lift that cannot observe must not let the abort stop the candidate Fails at 81bfb03: an observe error leaves the abort running and its first step stops the candidate. Also pins the lease_withdrawal==Complete conjunct (case LeaseWithdrawalRecordedComplete) and makes the refusal table's dead-candidate case provably dead (SwitchWorkload::bury). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 106 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d872c4e..e7297b0 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14522,6 +14522,10 @@ mod tests { stops: std::sync::atomic::AtomicUsize, /// Runtime instances whose stop fails, whatever is asked. failing_stops: std::sync::Mutex>, + /// The unit is absent or failed, which the driver reports as + /// permanently stopped: positive evidence of death, unlike an + /// observation that merely errs. + buried: std::sync::atomic::AtomicBool, } impl SwitchWorkload { @@ -14535,9 +14539,16 @@ mod tests { holds: std::sync::Mutex::new(Vec::new()), stops: std::sync::atomic::AtomicUsize::new(0), failing_stops: std::sync::Mutex::new(Vec::new()), + buried: std::sync::atomic::AtomicBool::new(false), }) } + /// The process is dead and the driver can prove it. + fn bury(&self) { + self.kill(); + self.buried.store(true, std::sync::atomic::Ordering::SeqCst); + } + /// From now on a start launches a process distinct from every recorded one. fn launching(&self) { self.launches.store(true, std::sync::atomic::Ordering::SeqCst); @@ -14584,6 +14595,8 @@ mod tests { } fn revive(&self) { + self.buried + .store(false, std::sync::atomic::Ordering::SeqCst); self.alive .store(true, std::sync::atomic::Ordering::SeqCst); self.rebooted @@ -14695,7 +14708,7 @@ mod tests { Ok(()) } fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(false) + Ok(self.buried.load(std::sync::atomic::Ordering::SeqCst)) } fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) @@ -23834,6 +23847,81 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// NEGATIVE. The boot lift runs before anything it needs may answer: a + /// systemctl error, a host-actuator hub no actuator has reached. The + /// abort of a candidate it could not observe is held, not run (its first + /// step stops the candidate), and the lift is retried on later passes. + /// Once observation answers Ok the record is lifted; once it answers + /// dead the abort proceeds. + #[cfg(unix)] + #[test] + fn a_boot_lift_that_cannot_observe_holds_the_abort_and_retries() -> Result<()> { + for hub_unattached in [false, true] { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + wedge_abort(&routed, "the Routing phase passed its deadline at unix ms 1792324133638")?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + let observation_fails = || { + if hub_unattached { + routed.workload.silence(true); + } else { + routed.workload.kill(); + } + }; + + // Boot: observation errs. Nothing is lifted and nothing stops. + observation_fails(); + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty(), "lifted {returned:?} it could not observe"); + for _ in 0..5 { + routed.resume()?; + assert_eq!( + routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), + stops, + "an undecided abort stopped the candidate (hub_unattached {hub_unattached})" + ); + } + let held = routed.transaction()?; + let abort = held.post_fencing_abort.context("the abort was dropped")?; + assert_eq!(abort.lease_withdrawal, CleanupEvidence::Pending); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + + // Observation answers Ok: the next pass lifts the record. + routed.workload.silence(false); + routed.workload.revive(); + routed.resume()?; + let lifted = routed.transaction()?; + assert!(lifted.post_fencing_abort.is_none(), "a later pass did not lift the record"); + assert_eq!(lifted.phase, DeploymentPhase::Routing); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + } + + // Observation answers dead: the abort proceeds to its end. + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + wedge_abort(&routed, "the AwaitingReady phase passed its deadline")?; + routed.workload.silence(true); + assert!(routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty()); + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some_and(|a| a.candidate_cleanup == CleanupEvidence::Pending)); + routed.workload.silence(false); + routed.workload.bury(); + let finished = routed.resume_until_terminal()?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.completion + ); + Ok(()) + } + /// NEGATIVE. The live shape of tx-4eaed669 (Odin) left to the operator /// at its Routing deadline, with a Deploy now awaiting the handover of /// its target. The continuity yields through its own abort: the lease @@ -23899,6 +23987,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[derive(Clone, Copy, Debug)] enum Case { CandidateNotRunning, + LeaseWithdrawalRecordedComplete, LeaseWithdrawn, CandidateCleaned, LeaseRevokedUnderIt, @@ -23908,6 +23997,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } for case in [ Case::CandidateNotRunning, + Case::LeaseWithdrawalRecordedComplete, Case::LeaseWithdrawn, Case::CandidateCleaned, Case::LeaseRevokedUnderIt, @@ -23936,13 +24026,25 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide // step, or one input, away from the lift's predicate. routed.resume()?; match case { - Case::CandidateNotRunning => routed.workload.kill(), + Case::CandidateNotRunning => routed.workload.bury(), Case::LeaseWithdrawn | Case::CandidateCleaned => routed.resume()?, Case::LeaseRevokedUnderIt => { let (lease, _) = lease_of(&routed)?; lease_driver(&routed).0.revoke_exact(Some(&lease))?; } Case::DeployWithRunningCandidate => {} + // The record says the lease was withdrawn while the candidate + // still runs and the lease is still exact on disk: only the + // record's own evidence keeps the lift off it. + Case::LeaseWithdrawalRecordedComplete => { + let stored = resident(&routed.world)?; + let mut next = stored.value.clone(); + next.post_fencing_abort + .as_mut() + .context("no abort")? + .lease_withdrawal = CleanupEvidence::Complete; + replace_transaction(&routed.world.state_store, &stored, &next)?; + } // The abort of the Odin-shaped negative, begun to yield: a // Deploy awaits the handover of its target. Case::DeployAwaitsTheTarget => { From 35d9c7fcf8b3f11be03f0ef0878714e8f70d8807 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:30:18 +0200 Subject: [PATCH 041/122] c2c fix 2/3: the lift defers on an observe error; the retry exit is worded from the driver An abort over a candidate the boot lift could not observe is held in process memory: resume_candidate runs no step of it and retries the lift each pass (retry_held_lift) until observation answers Ok (lifted) or the driver proves the candidate dead (the abort proceeds). The predicate and write move into consider_lifting_continuity_abort, which boot and the retry share. Ruling c2c-lift-defers-on-observe-error. The page and the expire refusal name the manual exit from the plan's workload binding: a unit to stop for systemd, a process to kill for a host actuator. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 233 ++++++++++++++++++++++++++++++++----------- 1 file changed, 173 insertions(+), 60 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e7297b0..786486f 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1863,6 +1863,23 @@ impl DeploymentTransaction { && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) } + /// The operator's only way to restart a forward-retrying continuity, worded + /// from the driver its plan binds: a systemd target has a unit to stop (a + /// stopped transient unit is forgotten, which is permanently stopped); a + /// host-actuator target has no unit, only a process on the managed host + /// that the actuator then reports not running. + fn manual_exit(&self) -> &'static str { + match self + .plan + .as_ref() + .and_then(|plan| plan.parsed_inputs().ok()) + .map(|(_, binding)| binding.workload) + { + Some(WorkloadBinding::HostActuator(_)) => "kill its process on the managed host by hand", + _ => "stop its unit by hand", + } + } + /// A continuity at or past Fencing that its deadline leaves to the operator /// while its step runs. It yields to a deployment awaiting the handover /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. @@ -3159,9 +3176,12 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> current.value.expirable(), "{}", if current.value.command_kind == CommandKind::Continuity && current.value.phase >= DeploymentPhase::Fencing && current.value.post_fencing_abort.is_none() { - "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" + format!( + "a continuity past Fencing retries forward and cannot be expired; to restart it, {} and Idunn aborts it as dead", + current.value.manual_exit() + ) } else { - "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel".to_owned() } ); let now = now_millis()?; @@ -5097,6 +5117,10 @@ struct Engine { /// Process-local on purpose: a restart retries at once, and no schema /// carries a clock that only pacing reads. resume_backoff: Mutex>, + /// Continuity aborts the boot lift could not decide (their candidate could + /// not be observed): no step of them runs until `retry_held_lift` decides. + /// Process-local: a restart asks again at boot. + lift_held: Mutex>, /// When each routed target's next challenge is due, held only when the /// stored `RouteSupervisionState` could not record the last one. The stored /// value owns the schedule and survives restarts; this spaces attempts @@ -5209,6 +5233,7 @@ impl Engine { history_report: ReportOnce::default(), fault_reports: Mutex::default(), resume_backoff: Mutex::default(), + lift_held: Mutex::default(), challenge_wait: Mutex::default(), }) } @@ -5682,6 +5707,17 @@ struct ProjectionReconciliation { unexplained: Vec, } +/// What the lift decided about one continuity abort. +enum LiftDecision { + /// The abort was withdrawn and the record retimed. + Lifted, + /// The abort stands and runs: not a lift candidate, or its candidate is + /// provably dead. + KeepAborting, + /// The candidate could not be observed and is not provably dead. + Undecided(anyhow::Error), +} + impl Engine { /// Once at boot: withdraw the abort of a continuity that has not stopped /// its running candidate, and retime the record so its phase retries @@ -5694,73 +5730,126 @@ impl Engine { /// or to yield to a deployment awaiting the handover, so this predicate /// matches only aborts over a running candidate with nothing awaiting the /// target, which Idunn no longer creates: it is a migration with a deletion - /// line (follow-up c2c-delete-boot-lift). A record it cannot decide is - /// reported and left aborting. + /// line (follow-up c2c-delete-boot-lift). A record whose candidate cannot + /// be observed is held, not aborted: `resume_candidate` runs no step of it + /// and asks `retry_held_lift` each pass, until observation answers (ruling + /// c2c-lift-defers-on-observe-error). fn return_wedged_continuity_aborts_forward(&self) -> Result> { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut returned = Vec::new(); for stored in &snapshot.transactions { - let transaction = &stored.value; - let Some(abort) = &transaction.post_fencing_abort else { - continue; - }; - if transaction.command_kind != CommandKind::Continuity - || transaction.completion.is_some() - || abort.candidate_cleanup != CleanupEvidence::Pending - || abort.lease_withdrawal == CleanupEvidence::Complete - || snapshot - .on_target(&transaction.target) - .any(|other| other.value.awaits_handover()) - { - continue; - } - let id = &transaction.transaction_id; - let lifted = (|| -> Result { - let plan = required(&transaction.plan, "transaction plan")?; - let expected = required(&transaction.expected, "Expected projection")?; - let activation = required(&transaction.activation, "activation")?; - let Some(workload) = &transaction.workload else { - return Ok(false); - }; - self.workload_for(plan)? - .observe(expected, activation, workload)?; - if let Some(lease) = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::issued_lease) - { - let path = plan - .parsed_inputs()? - .1 - .process_write_lease - .context("leased target has no write-lease binding")? - .record_path; - if !CultCacheWriteLeaseDriver::new(&transaction.target, path) - .observe_exact(lease)? - { - return Ok(false); - } + let id = &stored.value.transaction_id; + match self.consider_lifting_continuity_abort(&snapshot, stored) { + LiftDecision::Lifted => returned.push(id.clone()), + LiftDecision::KeepAborting => {} + LiftDecision::Undecided(error) => { + eprintln!( + "Idunn could not decide whether continuity {id} may retry forward; its abort is held, not run: {error:#}" + ); + self.lift_held.lock().expect("lift held mutex").insert(id.clone()); } - let now = now_millis()?; - let mut next = transaction.clone(); - next.post_fencing_abort = None; - next.last_error = Some(abort.error.clone()); - next.updated_at_unix_millis = now; - next.retime(now); - replace_transaction(&self.options.state_store, stored, &next)?; - Ok(true) - })(); - match lifted { - Ok(true) => returned.push(id.clone()), - Ok(false) => {} - Err(error) => eprintln!( - "Idunn could not decide whether continuity {id} may retry forward; its abort is left running: {error:#}" - ), } } Ok(returned) } + /// A held record's later pass: lift it, release it to its abort over + /// positive evidence of death, or keep holding. `Some(moved)` means the + /// pass is spent on the record; `None` lets the pass go on. + fn retry_held_lift(&self, current: &Stored) -> Result> { + let id = current.value.transaction_id.as_str(); + if !self.lift_held.lock().expect("lift held mutex").contains(id) { + return Ok(None); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + match self.consider_lifting_continuity_abort(&snapshot, current) { + LiftDecision::Undecided(error) => { + self.note_fault("holds continuity abort", id, &error); + Ok(Some(false)) + } + LiftDecision::Lifted => { + self.lift_held.lock().expect("lift held mutex").remove(id); + self.clear_fault(id); + println!( + "returned continuity {id} to forward retry: its abort had not stopped its running candidate" + ); + Ok(Some(true)) + } + LiftDecision::KeepAborting => { + self.lift_held.lock().expect("lift held mutex").remove(id); + self.clear_fault(id); + Ok(None) + } + } + } + + /// The lift's predicate and write for one record. An abort proceeds only + /// over positive evidence that its candidate is dead; an observation that + /// merely errs is undecided. + fn consider_lifting_continuity_abort( + &self, + snapshot: &ControlSnapshot, + stored: &Stored, + ) -> LiftDecision { + let transaction = &stored.value; + let Some(abort) = &transaction.post_fencing_abort else { + return LiftDecision::KeepAborting; + }; + if transaction.command_kind != CommandKind::Continuity + || transaction.completion.is_some() + || abort.candidate_cleanup != CleanupEvidence::Pending + || abort.lease_withdrawal == CleanupEvidence::Complete + || snapshot + .on_target(&transaction.target) + .any(|other| other.value.awaits_handover()) + { + return LiftDecision::KeepAborting; + } + let lifted = (|| -> Result { + let plan = required(&transaction.plan, "transaction plan")?; + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(plan)? + .observe(expected, activation, workload)?; + if let Some(lease) = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + { + let path = plan + .parsed_inputs()? + .1 + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + if !CultCacheWriteLeaseDriver::new(&transaction.target, path) + .observe_exact(lease)? + { + return Ok(false); + } + } + let now = now_millis()?; + let mut next = transaction.clone(); + next.post_fencing_abort = None; + next.last_error = Some(abort.error.clone()); + next.updated_at_unix_millis = now; + next.retime(now); + replace_transaction(&self.options.state_store, stored, &next)?; + Ok(true) + })(); + match lifted { + Ok(true) => LiftDecision::Lifted, + Ok(false) => LiftDecision::KeepAborting, + Err(error) => match self.candidate_is_permanently_stopped(transaction) { + Ok(true) => LiftDecision::KeepAborting, + _ => LiftDecision::Undecided(error), + }, + } + } + /// Once at boot: demote the activation a failed continuity left standing, /// where the projection still names exactly the activation that failed /// transaction issued. A continuity abort before the single resolution @@ -6086,6 +6175,9 @@ impl Engine { /// transaction would report progress every tick. fn resume_candidate(&self, current: &Stored) -> Result { let id = current.value.transaction_id.as_str(); + if let Some(moved) = self.retry_held_lift(current)? { + return Ok(moved); + } if let Some(moved) = self.resolve_phase_deadline(current)? { if moved { // A phase-ending write is movement: the next step is not held @@ -9665,7 +9757,8 @@ impl Engine { self.leave_to_operator( transaction, &format!( - "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, stop its unit by hand" + "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, {}", + transaction.manual_exit() ), ); Ok(None) @@ -16173,6 +16266,21 @@ mod tests { } } + /// The operator's exit from a forward-retrying continuity is worded from + /// the driver the plan binds: a host-actuator target has no unit to stop. + #[test] + fn the_manual_exit_is_worded_from_the_targets_driver() -> Result<()> { + let (world, _deploy, host_continuity) = stop_first_world(SwitchWorkload::new())?; + let host = resident_id(&world, &host_continuity)?.value; + assert!(host.manual_exit().contains("managed host"), "{}", host.manual_exit()); + assert!(!host.manual_exit().contains("unit"), "{}", host.manual_exit()); + let systemd = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&systemd)?; + let unit = seeded_transaction(&systemd, DeploymentPhase::Fencing, CommandKind::Continuity, Some(&incumbent))?; + assert!(unit.manual_exit().contains("unit"), "{}", unit.manual_exit()); + Ok(()) + } + /// The step that ends a transaction archives it in its own write, so a /// record gone from the live set is finished only if history holds it. One /// that vanished from both is a fault, never progress. @@ -24080,6 +24188,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(abort.is_some(), "the lift withdrew the yield"); clear_other_live_transactions(&routed.world, owner)?; } + if matches!(case, Case::LeaseWithdrawalRecordedComplete) { + // The record's claim is made true, so the abort can finish. + let (lease, _) = lease_of(&routed)?; + lease_driver(&routed).0.revoke_exact(Some(&lease))?; + } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), From 1beca61ac458081fa171fbc1884c9d5e65ecaa0f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:31:36 +0200 Subject: [PATCH 042/122] c2c fix 3/3: the refusal table's recorded-withdrawal case ends at the lift Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 786486f..3f9ef43 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -24189,9 +24189,9 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide clear_other_live_transactions(&routed.world, owner)?; } if matches!(case, Case::LeaseWithdrawalRecordedComplete) { - // The record's claim is made true, so the abort can finish. - let (lease, _) = lease_of(&routed)?; - lease_driver(&routed).0.revoke_exact(Some(&lease))?; + // An edited record, not a reachable one: the lift's refusal + // is the whole claim, and its abort is not run to the end. + continue; } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( From 04389e6afc2696d895dca271a208e2625484f8ee Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:38:10 +0200 Subject: [PATCH 043/122] c2d fix 1/2: negative tests, no yield to a serving continuity or a braked Deploy Fail at 1beca61 (the c2d logic is 81bfb03's): a Committing continuity past its deadline yields to a waiting Deploy and stops serving; a stop-then-start Deploy at its Sealing tail with the deployment brake unreleased draws the yield. The second test releases the brake and pins that the yield then happens and closes the continuity's operator-required incident Recovered. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 95 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3f9ef43..e34c1c4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17945,7 +17945,7 @@ mod tests { Ok(()) } - /// `awaits_handover` and `retries_forward` row by row. + /// `awaits_handover` and `yields_to_a_deploy` row by row. #[test] fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; @@ -17999,6 +17999,93 @@ mod tests { Ok(()) } + /// NEGATIVE. A continuity past its route proof is serving. Past its deadline + /// it keeps retrying and pages; a waiting Deploy does not stop it. + #[test] + fn a_committing_continuity_past_its_deadline_keeps_serving_and_pages() -> Result<()> { + let (world, workload, _, _, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + rephase(&world, &owner_id, DeploymentPhase::Committing)?; + pass_deadline_of(&world, &owner_id)?; + let stops = stops_of(&workload); + for _ in 0..4 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + let owner = resident_id(&world, &owner_id)?.value; + assert!(owner.post_fencing_abort.is_none(), "a serving continuity yielded"); + assert_eq!(owner.phase, DeploymentPhase::Committing); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + Ok(()) + } + + /// NEGATIVE. A Deploy held at its Sealing tail by the deployment brake + /// cannot take the handover, so a continuity does not stop its candidate + /// for it; once the brake releases, the continuity yields. The yield closes + /// the continuity's operator-required incident Recovered. + #[test] + fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + make_stop_first(&world, &deploy_id)?; + let sealing = resident_id(&world, &deploy_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + assert!(resident_id(&world, &deploy_id)?.value.awaits_handover()); + pass_deadline_of(&world, &owner_id)?; + let stops = stops_of(&workload); + for _ in 0..4 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + let held = resident_id(&world, &owner_id)?.value; + assert!(held.post_fencing_abort.is_none(), "the continuity yielded to a braked Deploy"); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + + // The operator releases the brake for this exact Deploy. + let deploy = resident_id(&world, &deploy_id)?.value; + let authorization = deployment_authorization( + &world, + deploy.expected.as_ref().context("no Expected")?, + &deploy_id, + now_millis()?, + )?; + let store = deploy.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + assert!(SingleFileMessagePackBackingStore::new(&store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + key: record.brake_id.clone(), + current: None, + }], + &[CultCacheEnvelope { + key: record.brake_id.clone(), + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + payload: authorization.canonical_brake_bytes.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), + }], + )?); + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + let yielded = resident_id(&world, &owner_id)?.value; + let abort = yielded.post_fencing_abort.as_ref().context("the continuity did not yield")?; + assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); + // The yield retimed the record: the next pass closes its incident + // Recovered, and it stays closed. + for _ in 0..3 { + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_one_transaction()?; + } + assert!(operator_required(&world)?.is_empty()); + let records = world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].subject, owner_id); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + Ok(()) + } + /// The c2b waiter still ends on an owner that will not yield: the yield's /// own abort wedged past its deadline. #[test] @@ -18311,7 +18398,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide bind_provider( world, &format!("{head}{HOST}[runtime_identity]{identity}[brakes]{brakes}") - .replace("candidate-then-promote", "stop-then-start"), + .replace("candidate-then-promote", "stop-then-start") + .replace( + "/var/lib/gamecult/idunn/service-deployment-brake.cc", + &world.root.join("service-deployment-brake.cc").display().to_string(), + ), ) } From b119e500adb0d318de56a48a1739947c58f34532 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:43:34 +0200 Subject: [PATCH 044/122] c2d fix 2/2: a continuity yields only while unproven, and only to a Deploy that can move serving_proven (Committing, or routing evidence recorded) ends the yield: yields_to_a_deploy replaces retries_forward, and wait_for_owner reads it, so a Deploy no longer waits on an owner that will keep serving. deploy_can_move asks the deployment brake live for a Deploy at its Sealing tail, so a braked Deploy draws no yield and the yield happens once the brake releases. The tests of 04389e6 are repaired to reach their worlds (the failures they showed at 1beca61 were the right ones). Rulings c2d-yield-only-unproven-to-movable. Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 12 ++-- src/control_plane.rs | 109 ++++++++++++++++++++++++-------- 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index c014a2a..7c25d0c 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -57,11 +57,13 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: incumbent. A continuity past its deadline is left to the operator (operator-required incident) and retries its phase forward on a capped backoff, because aborting it would stop the only running copy of the admitted - body (ruling post-fence-forward-retry), until a deployment awaits the - handover of its target: then it yields through its own post-fencing abort, - and the deployment fences what is admitted (ruling - continuity-yields-to-deploy). A stateful deployment is left to the - operator. + body (ruling post-fence-forward-retry), until a deployment that the + deployment brake has released awaits the handover of its target and the + continuity's route proof has not succeeded: then it yields through its own + post-fencing abort, and the deployment fences what is admitted (rulings + continuity-yields-to-deploy, c2d-yield-only-unproven-to-movable). A + continuity that has proven its route is serving and never yields. A + stateful deployment is left to the operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never meets drift between the receipt and the projection. Rollback demotes the exact diff --git a/src/control_plane.rs b/src/control_plane.rs index e34c1c4..5d1389a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1880,12 +1880,23 @@ impl DeploymentTransaction { } } + /// The candidate's route proof has succeeded: it is Committing, or its + /// routing evidence is recorded and only the transition remains. It serves, + /// so no waiting deployment stops it (ruling c2d-yield-only-unproven-to- + /// movable). Derived, never stored. + fn serving_proven(&self) -> bool { + self.phase >= DeploymentPhase::Committing || self.routing.is_some() + } + /// A continuity at or past Fencing that its deadline leaves to the operator - /// while its step runs. It yields to a deployment awaiting the handover - /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. - /// Derived, never stored. - fn retries_forward(&self) -> bool { - self.resolves_by_deadline() && self.phase_end() == PhaseEnd::RetryForward + /// while its step runs, and whose candidate is still unproven: it yields to + /// a deployment that can take the handover (ruling continuity-yields-to- + /// deploy), so a waiter never ends on it. A proven continuity keeps serving + /// and paging and does not yield. Derived, never stored. + fn yields_to_a_deploy(&self) -> bool { + self.resolves_by_deadline() + && self.phase_end() == PhaseEnd::RetryForward + && !self.serving_proven() } /// What an ended post-fencing phase resolves to (Q3). A continuity at or @@ -8546,6 +8557,17 @@ impl Engine { })) } + /// Whether a deployment awaiting the handover is free to take it: the + /// deployment brake has released its exact transaction, so the stop a yield + /// makes is followed by the deployment's own landing. A Deploy at its + /// Sealing tail has not passed the brake yet and is read live; one that is + /// still building, or whose brake cannot be read, is held. An operator's + /// hold is a reason to leave a serving or retrying body alone. + fn deploy_can_move(&self, deploy: &Stored, now: u64) -> bool { + deploy.value.deployment_authorization.is_some() + || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) + } + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { let plan = required(¤t.value.plan, "continuity plan")?; let binding = plan.parsed_inputs()?.1; @@ -8673,7 +8695,7 @@ impl Engine { if snapshot.transactions.iter().any(|stored| { stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) - && !stored.value.retries_forward() + && !stored.value.yields_to_a_deploy() }) { bail!( "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", @@ -9740,10 +9762,12 @@ impl Engine { } PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - if let Some(deploy) = snapshot - .on_target(&transaction.target) - .find(|other| other.value.awaits_handover()) - { + let now = now_millis()?; + if let Some(deploy) = snapshot.on_target(&transaction.target).find(|other| { + transaction.yields_to_a_deploy() + && other.value.awaits_handover() + && self.deploy_can_move(other, now) + }) { self.begin_post_fencing_abort( current, anyhow!( @@ -17947,7 +17971,7 @@ mod tests { /// `awaits_handover` and `yields_to_a_deploy` row by row. #[test] - fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { + fn awaits_handover_and_yields_to_a_deploy_are_exact_functions_of_the_record() -> Result<()> { let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; let waiting = resident_id(&world, &deploy_id)?.value; assert_eq!(waiting.phase, DeploymentPhase::Warming); @@ -17980,22 +18004,29 @@ mod tests { sealing.phase = DeploymentPhase::Warming; assert!(!sealing.awaits_handover()); - // A continuity retries forward from Fencing to Committing, until aborting. + // A continuity yields from Fencing until its route proof succeeds, and + // until aborting; a proven one keeps serving. for phase in [ DeploymentPhase::Fencing, DeploymentPhase::Leasing, DeploymentPhase::AwaitingReady, DeploymentPhase::Routing, - DeploymentPhase::Committing, ] { continuity.phase = phase; - assert!(continuity.retries_forward(), "{phase:?}"); + assert!(continuity.yields_to_a_deploy(), "{phase:?}"); + assert!(!continuity.serving_proven(), "{phase:?}"); } + continuity.phase = DeploymentPhase::Committing; + assert!(continuity.serving_proven() && !continuity.yields_to_a_deploy()); + continuity.phase = DeploymentPhase::Routing; + continuity.routing = Some(RoutingEvidence::SkippedUnrouted); + assert!(continuity.serving_proven() && !continuity.yields_to_a_deploy()); + continuity.routing = None; let mut deploy_record = waiting.clone(); deploy_record.phase = DeploymentPhase::Fencing; - assert!(!deploy_record.retries_forward()); + assert!(!deploy_record.yields_to_a_deploy()); continuity.post_fencing_abort = Some(post_fencing_abort_intent(&continuity, "x")); - assert!(!continuity.retries_forward()); + assert!(!continuity.yields_to_a_deploy()); Ok(()) } @@ -18003,15 +18034,43 @@ mod tests { /// it keeps retrying and pages; a waiting Deploy does not stop it. #[test] fn a_committing_continuity_past_its_deadline_keeps_serving_and_pages() -> Result<()> { - let (world, workload, _, _, owner_id, _) = waiter_beside_gate_waiting_continuity()?; - rephase(&world, &owner_id, DeploymentPhase::Committing)?; + let (world, workload, _, _, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + let owner = resident_id(&world, &owner_id)?.value; + // The continuity's incarnation already has a correlation (the + // incumbent's, same release): Odin re-stamps it Ready for this activation. + let expected = owner.expected.as_ref().context("no Expected")?; + let backing = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); + let key = crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?); + let current = backing + .pull_all_read_only_snapshot()? + .into_iter() + .find(|entry| entry.key == key); + assert!(backing.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + key: key.clone(), + current, + }], + &[CultCacheEnvelope { + key, + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(&world, &owner, seeded_sequence(Some(&incumbent)) + 10, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }], + )?); + let phases = drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::Committing)?; + let at = latest_id(&world, &owner_id)?; + assert_eq!(at.phase, DeploymentPhase::Committing, "{phases:?} {:?} {:?}", at.completion, at.last_error); + // Its Committing step now fails on every attempt: the body is gone. + workload.kill(); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); for _ in 0..4 { world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; } - let owner = resident_id(&world, &owner_id)?.value; + let owner = latest_id(&world, &owner_id)?; assert!(owner.post_fencing_abort.is_none(), "a serving continuity yielded"); assert_eq!(owner.phase, DeploymentPhase::Committing); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); @@ -18026,25 +18085,25 @@ mod tests { #[test] fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; - make_stop_first(&world, &deploy_id)?; - let sealing = resident_id(&world, &deploy_id)?; + make_stop_first(&world, &deploy_id).context("A make_stop_first")?; + let sealing = resident_id(&world, &deploy_id).context("B sealing")?; let mut tail = sealing.value.clone(); tail.deployment_authorization = None; replace_transaction(&world.state_store, &sealing, &tail)?; - assert!(resident_id(&world, &deploy_id)?.value.awaits_handover()); + assert!(resident_id(&world, &deploy_id).context("C awaits")?.value.awaits_handover()); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); for _ in 0..4 { world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; } - let held = resident_id(&world, &owner_id)?.value; + let held = latest_id(&world, &owner_id)?; assert!(held.post_fencing_abort.is_none(), "the continuity yielded to a braked Deploy"); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); // The operator releases the brake for this exact Deploy. - let deploy = resident_id(&world, &deploy_id)?.value; + let deploy = resident_id(&world, &deploy_id).context("E deploy")?.value; let authorization = deployment_authorization( &world, deploy.expected.as_ref().context("no Expected")?, @@ -18069,7 +18128,7 @@ mod tests { )?); world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; - let yielded = resident_id(&world, &owner_id)?.value; + let yielded = resident_id(&world, &owner_id).context("F yielded")?.value; let abort = yielded.post_fencing_abort.as_ref().context("the continuity did not yield")?; assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); // The yield retimed the record: the next pass closes its incident From b993615c69eb34dc8d07e1eb47af8260b100f952 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:47:48 +0200 Subject: [PATCH 045/122] Negative tests: the lift must not yield to a braked waiting Deploy Two tests, from Soul's h2 attacks: a running candidate at boot beside a braked stop-then-start Deploy at its Sealing tail, and a held (unobservable) candidate when such a Deploy appears. Both must leave the candidate unstopped. They fail here: the lift treats any awaits_handover Deploy as a reason to keep aborting. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5d1389a..bf08f52 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17914,6 +17914,74 @@ mod tests { workload.stops.load(std::sync::atomic::Ordering::SeqCst) } + /// A Deploy at its Sealing tail, braked, beside a continuity: it awaits the + /// handover but cannot take it (the brake has not released it). + fn braked_sealing_deploy(world: &EngineFixture, deploy_id: &str) -> Result<()> { + make_stop_first(world, deploy_id)?; + let sealing = resident_id(world, deploy_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + let tail = resident_id(world, deploy_id)?.value; + assert!(tail.awaits_handover()); + assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?, now_millis()?)); + Ok(()) + } + + /// NEGATIVE. The boot lift yields to a Deploy only by the continuity's own + /// yield test: a braked Deploy awaiting the handover is no reason to keep a + /// pre-c2c abort running over the only copy of the body. + #[test] + fn the_boot_lift_returns_a_running_candidate_forward_beside_a_braked_waiting_deploy() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + braked_sealing_deploy(&world, &deploy_id)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + let returned = world.engine.return_wedged_continuity_aborts_forward()?; + assert_eq!(returned, vec![owner_id.clone()]); + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the running candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + Ok(()) + } + + /// NEGATIVE. A held record (its candidate unobservable, not provably dead) + /// is not released to its abort because a braked Deploy starts awaiting the + /// handover. + #[test] + fn a_held_continuity_abort_is_not_released_by_a_braked_waiting_deploy() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + workload.silence(true); + let returned = world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty()); + assert!(world.engine.lift_held.lock().unwrap().contains(&owner_id)); + for _ in 0..3 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + braked_sealing_deploy(&world, &deploy_id)?; + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the unobservable candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + assert!(world.engine.lift_held.lock().unwrap().contains(&owner_id)); + Ok(()) + } + /// The yield is a function of its inputs: the deadline, the continuity's /// own record, and a deployment that awaits the handover. #[test] From c4edc66c48ae80ea8a72ac37a450c16de4bd5ba2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:50:46 +0200 Subject: [PATCH 046/122] The boot lift yields to a Deploy by the continuity's own yield test The lift kept an abort running for any awaits_handover Deploy, so a braked Deploy released a held record to an abort that stopped the only copy of the body. One predicate, deploy_the_continuity_yields_to (candidate unproven, a waiting Deploy that deploy_can_move), is now what both the deadline's yield and the lift ask (rulings c2d-yield-only-unproven-to-movable, c2c-lift-defers-on-observe-error). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 46 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index bf08f52..c681051 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -2104,7 +2104,8 @@ impl DeploymentTransaction { /// A live deployment whose next move is the handover: not ended, not /// aborting, one phase short of `takes_target_at`. Derived, never stored. - /// Read by the continuity's yield and the boot lift. + /// Read by `deploy_the_continuity_yields_to`: the continuity's yield and the + /// boot lift. fn awaits_handover(&self) -> bool { self.command_kind == CommandKind::Deploy && self.completion.is_none() @@ -5810,12 +5811,20 @@ impl Engine { || transaction.completion.is_some() || abort.candidate_cleanup != CleanupEvidence::Pending || abort.lease_withdrawal == CleanupEvidence::Complete - || snapshot - .on_target(&transaction.target) - .any(|other| other.value.awaits_handover()) { return LiftDecision::KeepAborting; } + match now_millis() { + Ok(now) => { + if self + .deploy_the_continuity_yields_to(snapshot, transaction, now) + .is_some() + { + return LiftDecision::KeepAborting; + } + } + Err(error) => return LiftDecision::Undecided(error), + } let lifted = (|| -> Result { let plan = required(&transaction.plan, "transaction plan")?; let expected = required(&transaction.expected, "Expected projection")?; @@ -8568,6 +8577,25 @@ impl Engine { || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) } + /// The deployment a continuity gives way to: one on its target that awaits + /// the handover and can take it, while the continuity's own candidate is + /// unproven (ruling c2d-yield-only-unproven-to-movable). The one test the + /// deadline's yield and the boot lift's abort both ask, so neither stops a + /// serving candidate, or one a braked Deploy cannot replace. + fn deploy_the_continuity_yields_to<'a>( + &self, + snapshot: &'a ControlSnapshot, + continuity: &DeploymentTransaction, + now: u64, + ) -> Option<&'a Stored> { + if continuity.serving_proven() { + return None; + } + snapshot.on_target(&continuity.target).find(|other| { + other.value.awaits_handover() && self.deploy_can_move(other, now) + }) + } + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { let plan = required(¤t.value.plan, "continuity plan")?; let binding = plan.parsed_inputs()?.1; @@ -9763,11 +9791,11 @@ impl Engine { PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let now = now_millis()?; - if let Some(deploy) = snapshot.on_target(&transaction.target).find(|other| { - transaction.yields_to_a_deploy() - && other.value.awaits_handover() - && self.deploy_can_move(other, now) - }) { + if let Some(deploy) = transaction + .yields_to_a_deploy() + .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction, now)) + .flatten() + { self.begin_post_fencing_abort( current, anyhow!( From 8760a4fab377d3153a5621a64336e82a4e509ee7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:51:57 +0200 Subject: [PATCH 047/122] A boot lift that errs fails closed If the boot lift returned Err, boot logged and served with nothing held, so every pending continuity abort ran and stopped a running candidate. Boot now sets lift_unsettled; retry_held_lift runs the lift for each continuity abort until one succeeds, and no abort step runs before. Boot's call moves into Engine::boot_lift so the failure is testable. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 97 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c681051..d2a7635 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5133,6 +5133,10 @@ struct Engine { /// not be observed): no step of them runs until `retry_held_lift` decides. /// Process-local: a restart asks again at boot. lift_held: Mutex>, + /// Set when the boot lift itself failed, so no record's decision exists: + /// every continuity with a post-fencing abort is held until a later lift + /// succeeds. Process-local, like `lift_held`. + lift_unsettled: std::sync::atomic::AtomicBool, /// When each routed target's next challenge is due, held only when the /// stored `RouteSupervisionState` could not record the last one. The stored /// value owns the schedule and survives restarts; this spaces attempts @@ -5246,6 +5250,7 @@ impl Engine { fault_reports: Mutex::default(), resume_backoff: Mutex::default(), lift_held: Mutex::default(), + lift_unsettled: std::sync::atomic::AtomicBool::new(false), challenge_wait: Mutex::default(), }) } @@ -5731,6 +5736,28 @@ enum LiftDecision { } impl Engine { + /// Boot's call of the lift. A lift that errs decided nothing, so boot fails + /// closed: every continuity abort is held, none of its steps run, until + /// `retry_held_lift` completes a lift. + fn boot_lift(&self) { + match self.return_wedged_continuity_aborts_forward() { + Ok(returned) => { + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + } + Err(error) => { + self.lift_unsettled + .store(true, std::sync::atomic::Ordering::SeqCst); + eprintln!( + "Idunn boot could not lift wedged continuity aborts; every continuity abort is held until a lift succeeds: {error:#}" + ); + } + } + } + /// Once at boot: withdraw the abort of a continuity that has not stopped /// its running candidate, and retime the record so its phase retries /// forward. A pre-c2c Idunn aborted a continuity at its deadline; that @@ -5770,6 +5797,29 @@ impl Engine { /// pass is spent on the record; `None` lets the pass go on. fn retry_held_lift(&self, current: &Stored) -> Result> { let id = current.value.transaction_id.as_str(); + if current.value.command_kind == CommandKind::Continuity + && current.value.post_fencing_abort.is_some() + && self.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst) + { + match self.return_wedged_continuity_aborts_forward() { + Err(error) => { + self.note_fault("holds continuity abort", id, &error); + return Ok(Some(false)); + } + Ok(returned) => { + self.lift_unsettled + .store(false, std::sync::atomic::Ordering::SeqCst); + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + if returned.iter().any(|lifted| lifted == id) { + return Ok(Some(true)); + } + } + } + } if !self.lift_held.lock().expect("lift held mutex").contains(id) { return Ok(None); } @@ -5983,16 +6033,7 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; - match engine.return_wedged_continuity_aborts_forward() { - Ok(returned) => { - for transaction_id in &returned { - println!( - "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" - ); - } - } - Err(error) => eprintln!("Idunn boot could not lift wedged continuity aborts: {error:#}"), - } + engine.boot_lift(); match engine.reconcile_failed_continuity_projections() { Ok(outcome) => { for transaction_id in &outcome.demoted { @@ -17979,6 +18020,42 @@ mod tests { Ok(()) } + /// NEGATIVE. A boot lift that errs decided nothing: no continuity abort + /// runs a step until a later lift succeeds, and then the running candidate + /// is returned forward instead of stopped. + #[test] + fn a_boot_lift_that_errs_holds_every_continuity_abort_until_a_lift_succeeds() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + braked_sealing_deploy(&world, &deploy_id)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + // The lift's only fallible read fails: the store is unreadable at boot. + let intact = std::fs::read(&world.state_store)?; + std::fs::write(&world.state_store, b"not a control store")?; + world.engine.boot_lift(); + assert!(world.engine.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)); + std::fs::write(&world.state_store, &intact)?; + let stored = resident_id(&world, &owner_id)?; + assert!(stored.value.post_fencing_abort.is_some()); + // Still failing: the abort is held, no step runs. + std::fs::write(&world.state_store, b"not a control store")?; + assert!(!world.engine.resume_candidate(&stored)?, "a held abort moved"); + std::fs::write(&world.state_store, &intact)?; + assert_eq!(stops_of(&workload), 0); + // A later lift succeeds: the running candidate is returned forward. + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the running candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(!world.engine.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)); + Ok(()) + } + /// NEGATIVE. A held record (its candidate unobservable, not provably dead) /// is not released to its abort because a braked Deploy starts awaiting the /// handover. From 2f8671ac355da1d3e0d404468158f1b57a3b1ca4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 02:36:25 +0200 Subject: [PATCH 048/122] Reincarnation map: cut 2 installed Co-Authored-By: Claude Opus 5.5 --- docs/reincarnation-map.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md index d63a6b3..4f99aed 100644 --- a/docs/reincarnation-map.md +++ b/docs/reincarnation-map.md @@ -575,3 +575,26 @@ its stores and `idunn.running` together, because the cb60965 binary cannot read "migrated 5 control record(s) … idunn.target_supervision.v2". streampixels-service kept running through the restart (unit 73ca40ba, PID 968752, NRestarts 0). The catalog returned 200, and 4443/udp was listening. + +## Installed: cut 2 + +Cuts c2-phase-split, c2b-owner-deadlines, c2c-post-fence-forward and +c2d-continuity-yields (branch tip `8760a4f`) were installed on Yggdrasil on +2026-10-20 at 00:34:06 UTC, with the operator's OK. + +- **The hotfix before it.** `ec45906` (`bounded_detail`) was installed on its + own on 2026-10-19 at 17:32 UTC. It unwedged tx-f978ab86. +- **Build.** `/srv/build/idunn-8760a4f`; `cargo test --lib` passed 471. +- **Backup.** `/root/idunn-store-backup-*-pre-reincarnation-c2` holds the + stores, `incidents.cc` and the binaries. A rollback must restore them + together: older binaries refuse two live transactions on one target, and + they refuse the `operator-required` incident condition. +- **First boot.** The boot lift returned exactly tx-039b91e9 (ghostlight) and + tx-4eaed669 (odin) to forward retry, as the Soul install gate predicted. + Odin's continuity, wedged in Routing since the 2026-10-18 reboot, committed + 4 s later. Generation tx-4eaed669 is admitted, its route is healthy, and the + same PID 2571 still runs. Ghostlight's continuity is retrying forward in + AwaitingReady. Its daemon's readiness is a ghostlight-side fault + (`Permission denied` reading its session-wrapping key). +- **Live processes.** Every live MainPID was the same before and after the + install. The streampixels catalog returned 200, and 4443/udp was listening. From 6e2a31cf31dcc3394bc6cee01f1ca46829f3caf7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 02:40:55 +0200 Subject: [PATCH 049/122] Body map after reincarnation cut 2: two slots, take_target, forward retry, operator-required Co-Authored-By: Claude Opus 5.5 --- docs/authority-map.md | 52 ++++++++++++++++++++----------- docs/deployment-authority.md | 34 +++++++++++++++++--- docs/host-actuator.md | 3 +- docs/watchdog-reporting-map.md | 11 ++++--- docs/watchdog-reporting-target.md | 13 +++++--- 5 files changed, 80 insertions(+), 33 deletions(-) diff --git a/docs/authority-map.md b/docs/authority-map.md index 1cbf3ca..7cf027b 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -173,24 +173,40 @@ health plus Ready correlation. It changes and observes route membership only afterward. Only after the proxy has adopted the candidate may Idunn clear the promotion fence and drain the incumbent. -Ownership of a target is exclusive at each instant. A live -`DeploymentTransaction` owns it from Fencing on (a stop-first rollout, from -Starting); before Fencing a deployment owns only its private candidate and -continuity owns the incumbent slot. Continuity owns the admitted generation's -survival and never stops. There is no stopped state: six charged attempts in -an hour (`restarts_page`) open the ContinuityExhausted incident and gate -nothing, and no deployment path yields to continuity. When continuity next acts -is derived only from its own record in `TargetSupervision` (schema v2, a v1 -record lifts at store open): the attempt streak, the backoff anchor (written -by the dead-and-free pass, the only writer), and the once-per-admission -re-materialization bound. The workload driver alone answers whether a lost unit -was a host reboot (`WorkloadPort::lost_to_host_reboot`). Crash restart, restart -after a sustained run and reboot re-materialization all mint through one CAS; -continuity supervision never aborts a transaction. Operator-facing incidents -and `idunn status` are projections that decide nothing. Cut line: cuts 2 to 4 -of `reincarnation-map.md` still owe continuity restarting alongside a -pre-fencing build, dependency failures not growing the streak, and the -dead-generation route refresh. +A target has two slots: the incumbent (the admitted generation) and a +transaction's candidate. Continuity owns the incumbent slot and keeps the +admitted generation alive; it never stops. A live `DeploymentTransaction` owns +its private candidate before Fencing and takes the target itself at Fencing (a +stop-then-start rollout, at Starting) through one `take_target` CAS, so of two +transactions entering from one view exactly one succeeds. A continuity +restart is itself a transaction and takes the target the same way. + +Phases from Warming through Committing carry deadlines, as do aborts and +drains; an expired deadline never waits silently. A stateless deployment +aborts to its incumbent. A deployment whose candidate was issued a write lease +is left to the operator (`operator-required` incident, the second incident +condition beside `continuity-exhausted`; its subject is the transaction id and +it closes when the record ends or its step goes on). A continuity after +Fencing retries its phase forward on a capped backoff and never stops its live +candidate, since that is the only running copy of the admitted body; the +deadline opens the same incident. The one exception is yielding: a continuity +whose route proof has not succeeded gives way, through its own post-fencing +abort, to a deployment the deployment brake has released and that awaits the +handover; a continuity that has proven its route never yields. The +`ContinuityExhausted` incident (six charged attempts in an hour, +`restarts_page`) pages and gates nothing. When continuity next acts is derived +only from its own record in `TargetSupervision` (schema v2): the attempt +streak, the backoff anchor (written by the dead-and-free pass, the only +writer), and the once-per-admission re-materialization bound. The workload +driver alone answers whether a lost unit was a host reboot +(`WorkloadPort::lost_to_host_reboot`). Operator-facing incidents and +`idunn status` are projections that decide nothing. A boot lift returns a +continuity abort left by an older Idunn, one that had not stopped its running +candidate, to forward retry; if it cannot decide, every continuity abort is +held until a lift succeeds. It is a migration with a deletion line (follow-up +`c2c-delete-boot-lift`). Cut line: cuts 3 and 4 of `reincarnation-map.md` +still owe dependency failures not growing the streak and the dead-generation +route refresh. The deployment brake gates changes to source, release, configuration, schema, unit, or authority binding. The lifecycle brake separately gates restart of an diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index 6d464e5..bdc7efc 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -313,10 +313,36 @@ written when the pass first finds the admitted generation dead and the target free, and a capped doubling wait from that anchor (5 s base, 300 s cap). A run of 600 s or more by the admitted body resets the streak. A generation lost to a host reboot (the workload driver's evidence, `lost_to_host_reboot`) is -re-materialized once per admission without charge or wait. The target is free -for a deployment's candidate between attempts, and a deployment never yields to -continuity before Fencing; from Fencing on (or from Starting for a stop-first -rollout) the live transaction alone owns the target. +re-materialized once per admission without charge or wait. + +Each target has two slots, the incumbent and a transaction's candidate. The +incumbent is continuity's until a transaction takes the target, at Fencing (at +Starting for a stop-then-start rollout), through one `take_target` CAS; of two +transactions entering from one view exactly one succeeds. Before that, a +deployment owns only its private candidate and the target is free for it +between continuity attempts. + +Phases Warming through Committing, post-fencing aborts and drains all carry +deadlines (the plan's frozen `PhaseDeadlines`; `WARMING_DEADLINE_SECONDS`, and +`PRE_FENCING_ABORT_DEADLINE_SECONDS` for a pre-fencing abort). What an expired +post-fencing deadline does depends on the owner. A deployment whose candidate +was never issued a write lease aborts to the incumbent. One whose candidate was +issued a lease may have written under a new state contract and is left to the +operator. A continuity retries its phase forward on a capped backoff and never +stops its live candidate, because that candidate is the only running copy of +the admitted body. Both operator cases open an `operator-required` incident +(subject: the transaction id) that closes when the record ends or its step goes +on. + +The one yield: a continuity whose route proof has not succeeded gives way to a +deployment the deployment brake has released that awaits the handover of the +target. It begins its own post-fencing abort and the deployment fences what is +admitted. A continuity that has proven its route is serving never yields. + +At boot, a lift returns to forward retry any continuity abort an older Idunn +left over a running candidate. If the lift errs or cannot observe the +candidate, the abort is held, not run. The lift is a migration with a deletion +line (follow-up `c2c-delete-boot-lift`). An admitted generation keeps its Ready and latest Odin receipts as durable proof of the exact activation at their recorded admission times. Dependency diff --git a/docs/host-actuator.md b/docs/host-actuator.md index cdb8cde..c595c5d 100644 --- a/docs/host-actuator.md +++ b/docs/host-actuator.md @@ -97,7 +97,8 @@ Each of these was found by running the pipe against Raven, not by reading. bind time. `rollout.strategy = "stop-then-start"` exists for exactly a route-less host-actuator workload: the admitted incumbent is stopped at Starting, before the candidate starts, and supervision leaves the target - alone from Starting on. A systemd-transient workload may not declare it. + alone from Starting on: the transaction takes the target there (one + `take_target` CAS) rather than at Fencing. A systemd-transient workload may not declare it. - **An unobservable host is silence.** The hub returns a typed `HostUnobservable` for not-attached, detached, reconnected and unanswered. Supervision skips that tick and logs it. Before this, an Idunn restart diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index 4c9ada7..4bc8814 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -21,8 +21,8 @@ probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC | I2 | Continuity exhaustion (since `idunn-reincarnation` cut 1: `restarts_page`, a paging threshold only; nothing stops, and the line numbers below are pre-cut) | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | | I3 | Route degradation | `RouteSupervisionState` `:890-942`; `record_failed_challenge` sets `degraded_since_unix_millis` on the **first** failed challenge, `record_proved_challenge` clears it; reported at `:5975` under `route:` | Durable per incarnation (inside `admitted_generation.v4`); a new generation starts from `default()`. Backoff doubles from the topology max age to a cap. | | I4 | Other operator-facing faults | `:5645-5651` (`generation-down:`, "holds a dead admitted generation down"), `:6016` (`route-record:`), `note_fault` per transaction | Also stderr plus `last_error`. Not named by the target; listed so the paging-set question sees them. | -| I5 | `OperatorRequired` on main | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | -| I6 | `OperatorRequired` on B5 | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | +| I5 | `OperatorRequired` on main (pre-cut-2 reading; since `idunn-reincarnation` cut 2 `resolve` reaches it in production through `PhaseEnd::OperatorRequired` and `PhaseEnd::RetryForward`, both calling `leave_to_operator`, which opens the `operator-required` incident `IncidentCondition::OperatorRequired`, subject the transaction id; `close_spent_operator_incidents` closes it when the record no longer `leaves_to_operator`) | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | +| I6 | `OperatorRequired` on B5 (superseded by cut 2: B5's deadlines merged as `PhaseDeadline`, and a continuity past its deadline now also lands here) | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | | I7 | Store kinds | `:55-61`, `:936-966`; B5 adds `idunn.expiry_request.v1` | `control.cc`: `idunn.deployment_command`, `idunn.deployment_transaction`, `idunn.admitted_generation.v4`, `idunn.target_supervision.v1`. `history.cc` beside it holds terminal transactions (`:3653`), retired one per tick (`:5068-5091`). | | I8 | Single-writer precedent inside `control.cc` | B5 `ExpiryRequest` doc comment | "The CLI only writes this record. The daemon's deadline resolver reads it ... No CLI writes a transaction field." One writer per kind. | | I9 | Serve loop | `:5041-5058`, `run_scheduler_tick` `:5098-5110` | 500 ms poll. A tick error is logged, never propagated. | @@ -166,9 +166,10 @@ cut spec fixes them. Empty cells, stated plainly: -- `operator-required` closure: no path on `hands/b5` ends a transaction left to - the operator (I6). The incident can open and never close until B5 names that - action. +- `operator-required` closure: the incident closes when the transaction ends or + its step goes on (`close_spent_operator_incidents`). A stateful deployment + left to the operator still has no Idunn-owned terminal action; the operator's + `idunn expire` re-enters the same arm. - Delivery-failure visibility: no owner. - Discord `enforce_nonce` window: not verified against Discord's current docs in this pass. diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index 5d21c75..509cb6e 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -30,9 +30,12 @@ journald (`report_once`, `control_plane.rs:4599`): - An admitted route is degraded. It fails its stable-route challenge (`supervise_admitted_route`, ~5973). The degraded mark is durable, but no one is told. -- A post-fencing phase deadline expires into `OperatorRequired`. This lives on - B5, which is unmerged on `hands/b5`. The failure record is durable and the - notice is a log line. +- A post-fencing phase deadline leaves a transaction to the operator: a + stateful deployment whose candidate was issued a lease, or a continuity + retrying forward past its deadline. Since `idunn-reincarnation` cut 2 it is + the second incident condition, `operator-required` (subject: the transaction + id), beside `continuity-exhausted`; it closes when the record ends or its + step goes on. Odin's own store condition (`odin.store_condition.v1`, `contested`) is mapped but not built. Idunn reads nothing about Odin's store health today. @@ -85,8 +88,8 @@ the machine has a detector with no record, and a deliverer that nothing feeds. - Odin's store-recovery cuts, O2 included. They produce `odin.store_condition` and are Odin's campaign, still in prose. This campaign consumes the record once it exists. -- Idunn's B5 phase deadlines. B5 is in prose on `hands/b5`. This campaign - reports its `OperatorRequired` outcome once B5 merges. +- Idunn's phase deadlines. They landed in `idunn-reincarnation` cut 2, which + also owns the `operator-required` incident condition. - VoidBot's `notify_owner` MCP tool and its separate DM code. - Bifrost's Persona delivery crossing. - Eve/TUI lowering of incidents. From 491459e6a04fd93036207370da8646647de03123 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 18:47:20 +0200 Subject: [PATCH 050/122] State kind external-directory: process-owned directory, lease-gated like external-file Not built or tested yet; verified at the end of the cut. Co-Authored-By: Claude Sonnet 5.5 --- docs/authority-map.md | 5 ++++- src/deployment.rs | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/authority-map.md b/docs/authority-map.md index 7cf027b..270d592 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -55,7 +55,10 @@ A deployable repository publishes one strict declaration, normally - immutable artifact outputs, optionally pinned to an expected SHA-256; - a constrained launch contract; - one health-contract identity; -- an optional typed state contract with state schema generation and slots; +- an optional typed state contract with state schema generation and slots + (kinds `cultcache-file`, `external-file` and `external-directory`; the + external kinds are process-owned state an external runtime imposes, which + Idunn preserves and fences but never interprets); - provided capabilities, dependency kinds, startup requirements, capacity, and conflicts. diff --git a/src/deployment.rs b/src/deployment.rs index a67b4bc..a81dff4 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -184,6 +184,11 @@ pub enum StateKind { /// Process-owned state imposed by an external runtime contract. Idunn /// preserves and fences the file but never interprets its payload. ExternalFile, + /// A process-owned directory imposed by an external runtime contract (for + /// example Synapse's media store). Idunn preserves and fences it and never + /// interprets its contents. Same writer, recovery and startup rules as + /// `ExternalFile`. + ExternalDirectory, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -2261,6 +2266,17 @@ nodes = ["yggdrasil"] assert!(declaration.write_lease_required()); } + #[test] + fn external_directory_state_is_lease_gated_like_external_file() { + let input = RECIPE.replace("kind = \"cultcache-file\"", "kind = \"external-directory\""); + let declaration = TargetDeclaration::parse(&input).unwrap(); + let slot = &declaration.state.as_ref().unwrap().slots[0]; + assert_eq!(slot.kind, StateKind::ExternalDirectory); + assert_eq!(slot.writer, StateWriter::ProcessBoundSingleWriter); + assert_eq!(slot.startup, StateStartup::CreateOrOpenAfterWriteLease); + assert!(declaration.write_lease_required()); + } + #[test] fn state_slots_cannot_claim_overlapping_paths() { let input = RECIPE.replace( From b9f27ce1e70d6cf01bc02ccddaaee27b78946204 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 18:53:29 +0200 Subject: [PATCH 051/122] idunn-launch: presence launcher, lease hold, closed-set config renderer, route acceptor (unbuilt) Does not build-verified yet; tests follow. drivers.rs: holder-side lease read, one lease-binding rule shared with grant. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.toml | 4 + src/bin/idunn-launch.rs | 86 +++ src/drivers.rs | 105 +++- src/launcher.rs | 1277 +++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + 5 files changed, 1446 insertions(+), 28 deletions(-) create mode 100644 src/bin/idunn-launch.rs create mode 100644 src/launcher.rs diff --git a/Cargo.toml b/Cargo.toml index 713ad33..dcaca2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,10 @@ path = "src/bin/idunn-provision.rs" name = "idunn-host" path = "src/bin/idunn-host.rs" +[[bin]] +name = "idunn-launch" +path = "src/bin/idunn-launch.rs" + [lib] name = "idunn_daemon" path = "src/lib.rs" diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs new file mode 100644 index 0000000..32c6291 --- /dev/null +++ b/src/bin/idunn-launch.rs @@ -0,0 +1,86 @@ +//! The unit's process for an upstream daemon: parse the recipe's argv, read +//! what Idunn set, run the launcher, and map the outcome to an exit code. +#[cfg(unix)] +fn main() -> std::process::ExitCode { + use std::sync::atomic::{AtomicBool, Ordering}; + + use anyhow::Context; + use idunn_daemon::deployment::{ + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + }; + use idunn_daemon::launcher::{self, *}; + + static STOP: AtomicBool = AtomicBool::new(false); + extern "C" fn on_signal(_: libc::c_int) { + STOP.store(true, Ordering::SeqCst); + } + + fn launch() -> anyhow::Result { + let contract = LaunchContract::parse(std::env::args().skip(1))?; + let bundle = std::env::var_os(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT) + .context("the Idunn runtime bundle is not set")?; + let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + .context("the Idunn candidate bind is not set")?; + let (activation, provider) = launcher::inherited_authority_files()?; + let authority = RuntimeAuthority::open(bundle.as_ref(), activation, provider)?; + authority.require_candidate_bind(&bind)?; + let lease_path = std::env::var_os(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT); + anyhow::ensure!( + lease_path.is_some() == authority.expected().write_lease_required, + "the process write lease binding differs from Expected" + ); + let route = std::net::TcpListener::bind(&bind).context("binding the candidate endpoint")?; + let sink = OdinRudpSink::from_expected(authority.expected())?; + let lease = DriverLeaseReader::new( + &authority.expected().target, + lease_path.as_deref().unwrap_or_default().as_ref(), + ); + let health_path = contract.health_path.clone(); + let ports = Ports { + credentials: std::sync::Arc::new(EnvCredentials), + clock: std::sync::Arc::new(SystemClock), + sink: std::sync::Arc::new(sink), + lease: std::sync::Arc::new(lease), + spawner: Box::new(ProcessSpawner), + health: Box::new(move |endpoint| { + std::sync::Arc::new(HttpHealthProbe { + endpoint: endpoint.clone(), + path: health_path.clone(), + }) + }), + }; + // SAFETY: the handler only stores to an atomic. + unsafe { + libc::signal(libc::SIGTERM, on_signal as libc::sighandler_t); + libc::signal(libc::SIGINT, on_signal as libc::sighandler_t); + } + Ok(launcher::run( + &contract, + authority, + route, + &std::env::temp_dir(), + ports, + &STOP, + )) + } + + match launch() { + Ok(outcome) => { + if let Outcome::Failed(code) = &outcome { + eprintln!("idunn-launch ended: {code}"); + } + std::process::ExitCode::from(outcome.exit_code() as u8) + } + Err(error) => { + eprintln!("idunn-launch refused: {error}"); + std::process::ExitCode::from(2) + } + } +} + +#[cfg(not(unix))] +fn main() { + eprintln!("idunn-launch runs only on Unix"); + std::process::exit(2); +} diff --git a/src/drivers.rs b/src/drivers.rs index fdf925c..4bf500a 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -4110,6 +4110,20 @@ impl CultCacheWriteLeaseDriver { fn current(&self) -> Result> { validate_root_authority_path(&self.record_path)?; + self.read_record() + } + + /// The lease as its holder reads it: the same custody, envelope, key and + /// target validation Idunn applies, without requiring the reader to be + /// root. The holder is a service uid by construction; root is who writes. + pub(crate) fn current_for_holder( + &self, + ) -> Result> { + validate_authority_path_custody(&self.record_path)?; + self.read_record() + } + + fn read_record(&self) -> Result> { if !self.record_path.exists() { return Ok(None); } @@ -4156,36 +4170,61 @@ impl CultCacheWriteLeaseDriver { expected.validate()?; activation.validate()?; lease.validate()?; - let expected_sha256 = expected.canonical_sha256()?; - let activation_sha256 = activation.canonical_sha256()?; ensure!( - self.target == expected.target - && lease.target == expected.target - && lease.expected_projection_sha256 == expected_sha256 - && lease.plan_id == expected.plan_id - && lease.incarnation_id == expected.incarnation_id - && lease.sealed_release_id == expected.sealed_release_id - && lease.activation_witness_sha256 == activation_sha256 - && lease.runtime_id == expected.runtime_id - && lease.runtime_instance_id == activation.runtime_instance_id - && lease.state_schema_generation - == expected - .state_schema_generation - .as_deref() - .context("write-lease Expected has no state generation")? - && lease.state_contract_sha256 - == expected - .state_contract_sha256 - .as_deref() - .context("write-lease Expected has no state contract")? - && lease.warming_presence_sha256 == warming.signed_presence_sha256() - && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + self.target == expected.target, "process write lease does not bind the exact warming candidate" ); - Ok(()) + ensure!( + warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + "process write lease does not bind the exact warming candidate" + ); + lease_binds_candidate( + expected, + activation, + lease, + std::slice::from_ref(&warming.signed_presence_sha256().to_owned()), + ) } } +/// The one rule for what makes a process write lease this incarnation's: it +/// binds this Expected, this activation and one of the given provider-signed +/// Warming proofs. Idunn grants with it and the lease holder accepts with it. +pub(crate) fn lease_binds_candidate( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, + warming_presence_sha256es: &[String], +) -> Result<()> { + let expected_sha256 = expected.canonical_sha256()?; + let activation_sha256 = activation.canonical_sha256()?; + ensure!( + lease.target == expected.target + && lease.expected_projection_sha256 == expected_sha256 + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation_sha256 + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id + && lease.state_schema_generation + == expected + .state_schema_generation + .as_deref() + .context("write-lease Expected has no state generation")? + && lease.state_contract_sha256 + == expected + .state_contract_sha256 + .as_deref() + .context("write-lease Expected has no state contract")? + && warming_presence_sha256es + .iter() + .any(|sha256| sha256 == &lease.warming_presence_sha256), + "process write lease does not bind the exact warming candidate" + ); + Ok(()) +} + impl WriteLeasePort for CultCacheWriteLeaseDriver { fn revoke_exact(&self, incumbent: Option<&IdunnProcessWriteLeaseRecord>) -> Result<()> { if let Some(incumbent) = incumbent { @@ -5861,7 +5900,7 @@ impl Answer { } } -fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { +pub(crate) fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) .with_context(|| format!("connecting stable CultNet TCP route {target}")) .map_err(silent)?; @@ -6070,7 +6109,7 @@ fn connect_route_socket(target: SocketAddr) -> std::io::Result { } } -fn exact_route_snapshot_response( +pub(crate) fn exact_route_snapshot_response( response: CultNetMessage, message_id: &str, target: &str, @@ -8335,12 +8374,17 @@ fn authority_lock_path(path: &Path) -> PathBuf { #[cfg(unix)] fn validate_root_authority_path(path: &Path) -> Result<()> { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - ensure!( unsafe { libc::geteuid() } == 0, "process write-lease actuation requires root Idunn" ); + validate_authority_path_custody(path) +} + +#[cfg(unix)] +fn validate_authority_path_custody(path: &Path) -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let parent = path .parent() .context("process write-lease path has no parent")?; @@ -8383,6 +8427,11 @@ fn validate_root_authority_path(_path: &Path) -> Result<()> { bail!("process write-lease actuation requires a Unix authority path") } +#[cfg(not(unix))] +fn validate_authority_path_custody(_path: &Path) -> Result<()> { + bail!("process write-lease actuation requires a Unix authority path") +} + #[cfg(unix)] fn harden_root_authority_file(path: &Path) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; diff --git a/src/launcher.rs b/src/launcher.rs new file mode 100644 index 0000000..f3bee58 --- /dev/null +++ b/src/launcher.rs @@ -0,0 +1,1277 @@ +//! idunn-launch: the unit's process for an upstream daemon that cannot speak +//! Idunn's runtime contract itself (Synapse, MAS). +//! +//! Idunn observes the unit; the launcher, inside it and supervising the child, +//! signs the presence statement with the provider identity and the activation +//! key. The launcher binds the candidate endpoint Idunn challenges, because the +//! child cannot answer a CultNet frame. It starts a stateful child only after +//! it holds the process write lease, renders the child's config from a closed +//! set of placeholders, and ends with the child. +//! +//! Errors, logs and refusals name a field and a code. They never echo an +//! argument, a template line, a path or a credential. + +use std::collections::VecDeque; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::net::UnixStream; +use std::path::{Component, Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultnet_rs::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, + CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, CultNetWireContract, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, + GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, + IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationRecord, + IdunnRuntimeActivationSigner, ServiceIdentitySigner, decode_cultnet_message_from_slice, + encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, +}; + +use crate::deployment::RUNTIME_PRESENCE_IDENTITY_FD_NAME; +use crate::drivers::{CultCacheWriteLeaseDriver, lease_binds_candidate}; + +/// The route challenge is the only thing that begins with this byte: a +/// CultNet frame's 4-byte big-endian length is bounded at 1 MiB, and an HTTP +/// request starts with a method letter. +const CHALLENGE_FIRST_BYTE: u8 = 0x00; +const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; +const CHALLENGE_IO_TIMEOUT: Duration = Duration::from_secs(3); +const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); +const POLL: Duration = Duration::from_millis(100); +const MAX_RECENT_WARMING_PROOFS: usize = 64; +const TERMINATE_GRACE: Duration = Duration::from_secs(20); +const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; +const ODIN_CAPABILITY: &str = "odin.verse-rendezvous"; + +// ---------------------------------------------------------------- contract + +/// Where the child listens. The launcher chooses the address; the child is +/// told it through `{{child.listen}}`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChildListener { + UnixSocket, + LoopbackPort, +} + +/// Everything the recipe declares about the child, parsed from the launcher's +/// argv (`--flag value` pairs; `--child-arg` repeats). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LaunchContract { + pub release_root: PathBuf, + pub child: PathBuf, + pub child_args: Vec, + /// The flag that carries the rendered config path to the child, when the + /// child takes one. + pub config_flag: Option, + pub template: PathBuf, + pub listener: ChildListener, + pub health_path: String, + pub health_contract: String, + pub state_root: Option, + pub lease_wait: Duration, + pub startup: Duration, + pub heartbeat: Duration, +} + +impl LaunchContract { + pub fn parse(args: impl IntoIterator) -> Result { + let mut release_root = None; + let mut child = None; + let mut child_args = Vec::new(); + let mut config_flag = None; + let mut template = None; + let mut listener = None; + let mut health_path = None; + let mut health_contract = None; + let mut state_root = None; + let (mut lease_wait, mut startup, mut heartbeat) = (300_u64, 120_u64, 5_u64); + let mut args = args.into_iter(); + while let Some(flag) = args.next() { + let value = args.next().context("launcher argument has no value")?; + let seconds = |field: &str| -> Result { + value + .parse::() + .ok() + .filter(|seconds| *seconds > 0) + .with_context(|| format!("launcher argument {field} is not a positive integer")) + }; + match flag.as_str() { + "--release-root" => release_root = Some(PathBuf::from(value)), + "--child" => child = Some(release_relative(&value, "--child")?), + "--child-arg" => child_args.push(value), + "--config-flag" => config_flag = Some(value), + "--template" => template = Some(release_relative(&value, "--template")?), + "--listener" => { + listener = Some(match value.as_str() { + "unix" => ChildListener::UnixSocket, + "loopback" => ChildListener::LoopbackPort, + _ => bail!("launcher argument --listener is not unix or loopback"), + }) + } + "--health-path" => { + ensure!( + value.starts_with('/'), + "launcher argument --health-path is not absolute" + ); + health_path = Some(value) + } + "--health-contract" => health_contract = Some(value), + "--state-root" => state_root = Some(PathBuf::from(value)), + "--lease-wait-secs" => lease_wait = seconds("--lease-wait-secs")?, + "--startup-secs" => startup = seconds("--startup-secs")?, + "--heartbeat-secs" => heartbeat = seconds("--heartbeat-secs")?, + _ => bail!("unknown launcher argument"), + } + } + let release_root = release_root.context("launcher argument --release-root is required")?; + ensure!( + release_root.is_absolute(), + "launcher argument --release-root is not absolute" + ); + Ok(Self { + release_root, + child: child.context("launcher argument --child is required")?, + child_args, + config_flag, + template: template.context("launcher argument --template is required")?, + listener: listener.context("launcher argument --listener is required")?, + health_path: health_path.context("launcher argument --health-path is required")?, + health_contract: health_contract + .context("launcher argument --health-contract is required")?, + state_root, + lease_wait: Duration::from_secs(lease_wait), + startup: Duration::from_secs(startup), + heartbeat: Duration::from_secs(heartbeat), + }) + } +} + +fn release_relative(value: &str, field: &str) -> Result { + let path = PathBuf::from(value); + ensure!( + !value.is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))), + "launcher argument {field} is not release-relative" + ); + Ok(path) +} + +// ---------------------------------------------------------------- authority + +/// The bundle, activation credential and provider identity Idunn hands the +/// unit, and the two signatures every presence statement needs. +pub struct RuntimeAuthority { + expected: IdunnExpectedIncarnationRecord, + expected_sha256: String, + activation: IdunnRuntimeActivationRecord, + activation_sha256: String, + activation_signer: IdunnRuntimeActivationSigner, + provider_signer: ServiceIdentitySigner, + sequence: u64, +} + +impl RuntimeAuthority { + pub fn open( + bundle: &Path, + activation_credential: impl Read, + provider_credential: impl Read, + ) -> Result { + ensure!(bundle.is_absolute(), "Idunn runtime bundle is not absolute"); + let (key, payload) = read_single_record( + &bundle.join("expected.cc"), + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + )?; + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&payload)?; + expected.validate()?; + ensure!(key == expected.target, "Expected key is substituted"); + let (key, payload) = read_single_record( + &bundle.join("activation.cc"), + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + )?; + let activation = IdunnRuntimeActivationRecord::decode_canonical(&payload)?; + activation.validate()?; + ensure!(key == expected.target, "activation key is substituted"); + let expected_sha256 = expected.canonical_sha256()?; + ensure!( + activation.expected_projection_sha256 == expected_sha256 + && activation.runtime_id == expected.runtime_id, + "activation does not bind this Expected projection" + ); + let activation_signer = IdunnRuntimeActivationSigner::from_credential_reader( + activation_credential, + )?; + ensure!( + activation_signer.identity_id() == activation.activation_signer_identity_id + && activation_signer.public_key() == activation.activation_signer_public_key, + "activation credential does not belong to this activation" + ); + let provider_signer = + open_service_identity_credential_reader::( + provider_credential, + )?; + ensure!( + provider_signer.entry().identity_id == expected.expected_signer_identity_id, + "provider identity is not the signer selected by Expected" + ); + Ok(Self { + activation_sha256: activation.canonical_sha256()?, + expected, + expected_sha256, + activation, + activation_signer, + provider_signer, + sequence: 0, + }) + } + + pub fn expected(&self) -> &IdunnExpectedIncarnationRecord { + &self.expected + } + + pub fn activation(&self) -> &IdunnRuntimeActivationRecord { + &self.activation + } + + /// The candidate endpoint Idunn will challenge, which the launcher binds. + pub fn require_candidate_bind(&self, bind: &str) -> Result<()> { + let route = self.expected.route.as_ref().context("Expected has no route")?; + ensure!( + route.transport == "tcp" && route.candidate_endpoint == format!("tcp://{bind}"), + "candidate bind differs from the Expected route" + ); + Ok(()) + } + + /// The next presence record, signed by the provider identity and the + /// activation. The sequence is the clock: strictly increasing here and + /// above anything an earlier incarnation of this signer published. + pub fn sign( + &mut self, + state: &str, + detail: &str, + write_lease_sha256: Option, + now_millis: u64, + ) -> Result { + self.sequence = now_millis.max( + self.sequence + .checked_add(1) + .ok_or_else(|| anyhow!("runtime presence sequence exhausted"))?, + ); + let expected = &self.expected; + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.expected_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation_sha256.clone(), + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: state.into(), + detail: detail.into(), + write_lease_sha256, + signer_identity_id: self.provider_signer.entry().identity_id.clone(), + publisher_sequence: self.sequence, + observed_at_unix_millis: now_millis, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + let payload = record.canonical_proof_payload()?; + record.signature = self + .provider_signer + .sign::(&payload) + .signature; + record.activation_signature = self.activation_signer.sign_presence_proof(&record)?; + record.validate()?; + Ok(record) + } +} + +fn read_single_record(path: &Path, record_type: &str, schema: &str) -> Result<(String, Vec)> { + let entries = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading a runtime bundle record")?; + let [envelope] = entries.as_slice() else { + bail!("runtime bundle record does not hold exactly one entry"); + }; + ensure!( + envelope.r#type == record_type && envelope.schema_id.as_deref() == Some(schema), + "runtime bundle record has the wrong type or schema" + ); + Ok((envelope.key.clone(), envelope.payload.clone())) +} + +/// The two authority descriptors systemd opens for the unit, in the order and +/// under the names Idunn's launch contract fixes. +pub fn inherited_authority_files() -> Result<(File, File)> { + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let fds = std::env::var("LISTEN_FDS").context("systemd did not bind authority FDs")?; + let names = std::env::var("LISTEN_FDNAMES").context("systemd did not name authority FDs")?; + ensure!( + fds == "2" + && names + == format!( + "{IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME}:{RUNTIME_PRESENCE_IDENTITY_FD_NAME}" + ), + "systemd authority FDs differ from the launch contract" + ); + for fd in [3, 4] { + // SAFETY: F_GETFD observes one integer descriptor and transfers nothing. + ensure!( + unsafe { libc::fcntl(fd, libc::F_GETFD) } != -1, + "an authority descriptor is not open" + ); + } + // SAFETY: the LISTEN_* contract above assigns these two distinct, open + // descriptors to this process; this is their first and only consumer. + let (activation, provider) = unsafe { (File::from_raw_fd(3), File::from_raw_fd(4)) }; + for (file, exact) in [(&activation, Some(32_u64)), (&provider, None)] { + // OpenFile descriptors arrive without close-on-exec; the child must + // never inherit a signer. + // SAFETY: F_SETFD on a descriptor `file` owns. + ensure!( + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) } != -1, + "an authority descriptor could not be sealed" + ); + let metadata = file.metadata()?; + ensure!( + metadata.is_file() + && metadata.uid() == 0 + && metadata.permissions().mode() & 0o777 == 0o400 + && metadata.nlink() == 1 + && exact.map_or(metadata.len() > 0, |size| metadata.len() == size), + "an authority descriptor is not one root-owned 0400 file" + ); + } + Ok((activation, provider)) +} + +// ---------------------------------------------------------------- ports + +pub trait Clock: Send + Sync { + fn now_millis(&self) -> u64; + fn sleep(&self, duration: Duration); +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_millis(&self) -> u64 { + u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0) + } + + fn sleep(&self, duration: Duration) { + thread::sleep(duration); + } +} + +/// Systemd credentials as Idunn binds them: the environment variable `NAME` +/// holds the path of the credential file. +pub trait CredentialSource: Send + Sync { + fn contents(&self, name: &str) -> Result>; + fn path(&self, name: &str) -> Result; +} + +pub struct EnvCredentials; + +impl CredentialSource for EnvCredentials { + fn contents(&self, name: &str) -> Result> { + fs::read(self.path(name)?).map_err(|_| anyhow!("a credential could not be read")) + } + + fn path(&self, name: &str) -> Result { + std::env::var_os(name) + .map(PathBuf::from) + .ok_or_else(|| anyhow!("a credential is not bound")) + } +} + +/// Where signed presence goes (Odin). +pub trait PresenceSink: Send + Sync { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()>; +} + +/// The lease as the launcher sees it: the current record, and a shared lock on +/// its store that Idunn's revocation cannot take while the child lives. +pub trait LeaseReader: Send + Sync { + fn read(&self) -> Result>; + fn hold_shared(&self) -> Result>; +} + +pub trait HealthProbe: Send + Sync { + fn healthy(&self) -> bool; +} + +pub trait ChildHandle: Send { + /// The exit status once the child has exited. + fn exited(&mut self) -> Option; + /// SIGTERM, then SIGKILL after the grace period; returns once it is gone. + fn terminate(&mut self, grace: Duration); +} + +pub trait ChildSpawner: Send { + fn spawn(&self, program: &Path, args: &[String]) -> Result>; +} + +// ---------------------------------------------------------------- production ports + +pub struct DriverLeaseReader { + driver: CultCacheWriteLeaseDriver, +} + +impl DriverLeaseReader { + pub fn new(target: &str, record_path: &Path) -> Self { + Self { + driver: CultCacheWriteLeaseDriver::new(target, record_path), + } + } +} + +struct SharedLockGuard { + release: Option>, + worker: Option>, +} + +impl Drop for SharedLockGuard { + fn drop(&mut self) { + drop(self.release.take()); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl LeaseReader for DriverLeaseReader { + fn read(&self) -> Result> { + self.driver + .current_for_holder() + .map(|current| current.map(|(_, lease)| lease)) + .map_err(|_| anyhow!("write-lease store refused custody, envelope or target")) + } + + fn hold_shared(&self) -> Result> { + let path = self.driver.record_path.clone(); + let (ready_tx, ready_rx) = mpsc::sync_channel::(1); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let worker = thread::spawn(move || { + let held = SingleFileMessagePackBackingStore::new(&path).with_read_only_shared_snapshot( + |_| { + let _ = ready_tx.send(true); + // Returns when the guard drops its sender. + let _ = release_rx.recv(); + Ok(()) + }, + ); + if held.is_err() { + let _ = ready_tx.send(false); + } + }); + if ready_rx.recv() != Ok(true) { + let _ = worker.join(); + bail!("write-lease lock could not be held"); + } + Ok(Box::new(SharedLockGuard { + release: Some(release_tx), + worker: Some(worker), + })) + } +} + +/// Odin takes presence over RUDP at the endpoint Expected's Odin dependency +/// names. +pub struct OdinRudpSink { + endpoint: SocketAddr, +} + +impl OdinRudpSink { + pub fn from_expected(expected: &IdunnExpectedIncarnationRecord) -> Result { + let dependency = expected + .dependencies + .iter() + .find(|dependency| dependency.capability == ODIN_CAPABILITY) + .context("Expected has no Odin dependency")?; + let endpoint = dependency + .provider_endpoint + .as_deref() + .context("Expected Odin dependency has no endpoint")?; + let endpoint = endpoint.strip_prefix("rudp://").unwrap_or(endpoint); + Ok(Self { + endpoint: endpoint + .parse() + .map_err(|_| anyhow!("Expected Odin endpoint is not a socket address"))?, + }) + } +} + +impl PresenceSink for OdinRudpSink { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { + let message = CultNetMessage::DocumentPutRaw { + message_id: format!( + "runtime-presence:{}:{}:{}", + record.target, record.runtime_instance_id, record.publisher_sequence + ), + document: presence_document( + record, + Some(vec![ + "cultnet.transport.rudp.v0".into(), + "runtime-presence".into(), + ]), + )?, + }; + let bind = if self.endpoint.is_ipv4() { + "0.0.0.0:0" + } else { + "[::]:0" + }; + let socket = UdpSocket::bind(bind)?; + socket.set_read_timeout(Some(Duration::from_millis(100)))?; + let mut transport = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client( + &record.runtime_id, + socket, + self.endpoint, + ODIN_CULTMESH_CATALOG_CONNECTION_ID, + ), + )?; + transport.connect(Vec::new())?; + let deadline = Instant::now() + Duration::from_millis(500); + while !transport.connected() { + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "Odin did not accept the connection"); + } + let receipt = transport.send_reliable( + "schema", + encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?, + )?; + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match transport.reliable_send_status(&receipt) { + CultNetRudpReliableSendStatus::Acknowledged => return Ok(()), + CultNetRudpReliableSendStatus::Invalidated => { + bail!("Odin invalidated the presence acknowledgement") + } + CultNetRudpReliableSendStatus::Pending => {} + } + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "Odin did not acknowledge presence"); + } + } +} + +fn presence_document( + record: &GameCultRuntimePresenceHealthRecord, + tags: Option>, +) -> Result { + let payload = rmp_serde::to_vec(record).context("encoding runtime presence")?; + Ok(CultNetRawDocumentRecord { + schema_id: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: record.target.clone(), + stored_at: chrono::DateTime::from_timestamp_millis( + record.observed_at_unix_millis.try_into()?, + ) + .context("runtime presence timestamp is invalid")? + .to_rfc3339(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: Some(record.runtime_id.clone()), + source_agent_id: Some(record.signer_identity_id.clone()), + source_role: Some("runtime-presence-publisher".into()), + tags, + }) +} + +/// The child's private listener. +#[derive(Clone, Debug)] +pub enum ChildEndpoint { + Unix(PathBuf), + Loopback(SocketAddr), +} + +pub enum ChildStream { + Unix(UnixStream), + Tcp(TcpStream), +} + +impl ChildEndpoint { + pub fn connect(&self) -> io::Result { + match self { + Self::Unix(path) => UnixStream::connect(path).map(ChildStream::Unix), + Self::Loopback(address) => { + TcpStream::connect_timeout(address, HEALTH_PROBE_TIMEOUT).map(ChildStream::Tcp) + } + } + } + + /// What `{{child.listen}}` renders to. + fn listen_value(&self) -> String { + match self { + Self::Unix(path) => path.display().to_string(), + Self::Loopback(address) => address.to_string(), + } + } +} + +impl ChildStream { + fn try_clone(&self) -> io::Result { + match self { + Self::Unix(stream) => stream.try_clone().map(Self::Unix), + Self::Tcp(stream) => stream.try_clone().map(Self::Tcp), + } + } + + fn shutdown_write(&self) { + let _ = match self { + Self::Unix(stream) => stream.shutdown(Shutdown::Write), + Self::Tcp(stream) => stream.shutdown(Shutdown::Write), + }; + } + + fn set_timeout(&self, timeout: Duration) { + let _ = match self { + Self::Unix(stream) => stream + .set_read_timeout(Some(timeout)) + .and_then(|()| stream.set_write_timeout(Some(timeout))), + Self::Tcp(stream) => stream + .set_read_timeout(Some(timeout)) + .and_then(|()| stream.set_write_timeout(Some(timeout))), + }; + } +} + +impl Read for ChildStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self { + Self::Unix(stream) => stream.read(buffer), + Self::Tcp(stream) => stream.read(buffer), + } + } +} + +impl Write for ChildStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + match self { + Self::Unix(stream) => stream.write(buffer), + Self::Tcp(stream) => stream.write(buffer), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Unix(stream) => stream.flush(), + Self::Tcp(stream) => stream.flush(), + } + } +} + +/// `GET ` over the child's private listener; healthy on a 200. +pub struct HttpHealthProbe { + pub endpoint: ChildEndpoint, + pub path: String, +} + +impl HealthProbe for HttpHealthProbe { + fn healthy(&self) -> bool { + let Ok(mut stream) = self.endpoint.connect() else { + return false; + }; + stream.set_timeout(HEALTH_PROBE_TIMEOUT); + let request = format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + self.path + ); + let mut head = [0_u8; 12]; + stream.write_all(request.as_bytes()).is_ok() + && stream.read_exact(&mut head).is_ok() + && head.starts_with(b"HTTP/1.") + && &head[8..12] == b" 200" + } +} + +pub struct ProcessSpawner; + +struct ProcessChild(Child); + +impl ChildSpawner for ProcessSpawner { + fn spawn(&self, program: &Path, args: &[String]) -> Result> { + let mut command = Command::new(program); + // The child gets a bare environment: no Idunn variable, no candidate + // bind, no credential path leaks into it. + command.args(args).env_clear().stdin(Stdio::null()); + for name in ["PATH", "TMPDIR", "LANG", "LC_ALL"] { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } + let child = command.spawn().map_err(|_| anyhow!("the child could not be spawned"))?; + Ok(Box::new(ProcessChild(child))) + } +} + +impl ChildHandle for ProcessChild { + fn exited(&mut self) -> Option { + match self.0.try_wait() { + Ok(Some(status)) => Some(status.code().unwrap_or(1)), + Ok(None) => None, + Err(_) => Some(1), + } + } + + fn terminate(&mut self, grace: Duration) { + if self.exited().is_some() { + return; + } + // SAFETY: the pid is our own live child. + unsafe { libc::kill(self.0.id() as libc::pid_t, libc::SIGTERM) }; + let deadline = Instant::now() + grace; + while Instant::now() < deadline { + if self.exited().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +// ---------------------------------------------------------------- config + +/// The closed placeholder set (decision D6). Any other `{{...}}` refuses. +pub struct ConfigRenderer<'a> { + pub credentials: &'a dyn CredentialSource, + pub listen: &'a str, + pub state_root: Option<&'a Path>, +} + +impl ConfigRenderer<'_> { + pub fn render(&self, template: &str) -> Result { + let mut rendered = String::with_capacity(template.len()); + let mut rest = template; + while let Some(open) = rest.find("{{") { + rendered.push_str(&rest[..open]); + let after = &rest[open + 2..]; + let close = after + .find("}}") + .context("config template has an unterminated placeholder")?; + rendered.push_str(&self.expand(&after[..close])?); + rest = &after[close + 2..]; + } + rendered.push_str(rest); + Ok(rendered) + } + + fn expand(&self, key: &str) -> Result { + if key == "child.listen" { + return Ok(self.listen.to_owned()); + } + if key == "state_root" { + return Ok(self + .state_root + .context("config template needs a state root the launch did not give")? + .display() + .to_string()); + } + if let Some(name) = key.strip_prefix("credential:") { + ensure!(valid_credential_name(name), "credential placeholder name is invalid"); + let bytes = self.credentials.contents(name)?; + let text = String::from_utf8(bytes) + .map_err(|_| anyhow!("a credential is not UTF-8 text"))?; + return Ok(text.strip_suffix('\n').unwrap_or(&text).to_owned()); + } + if let Some(name) = key.strip_prefix("credential_path:") { + ensure!(valid_credential_name(name), "credential placeholder name is invalid"); + return Ok(self.credentials.path(name)?.display().to_string()); + } + bail!("config template uses a placeholder outside the closed set") + } +} + +fn valid_credential_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') +} + +fn write_private_file(directory: &Path, name: &str, contents: &str) -> Result { + let path = directory.join(name); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .map_err(|_| anyhow!("the rendered config could not be created"))?; + file.write_all(contents.as_bytes())?; + Ok(path) +} + +// ---------------------------------------------------------------- lease + +/// A write lease this launcher holds: the record binds this Expected, +/// activation and one of the launcher's own Warming proofs, and its store is +/// share-locked until the hold drops. +pub struct LeaseHold { + reader: Arc, + lease: IdunnProcessWriteLeaseRecord, + sha256: String, + _lock: Box, +} + +impl LeaseHold { + pub fn acquire( + reader: Arc, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + warming_proofs: &[String], + ) -> Result { + let lease = reader.read()?.context("no write lease is granted")?; + lease.validate()?; + lease_binds_candidate(expected, activation, &lease, warming_proofs)?; + let lock = reader.hold_shared()?; + ensure!( + reader.read()?.as_ref() == Some(&lease), + "write lease changed while it was being held" + ); + Ok(Self { + sha256: lease.canonical_sha256()?, + reader, + lease, + _lock: lock, + }) + } + + pub fn sha256(&self) -> &str { + &self.sha256 + } + + pub fn still_current(&self) -> bool { + matches!(self.reader.read(), Ok(Some(current)) if current == self.lease) + } +} + +// ---------------------------------------------------------------- route + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Phase { + Warming, + Active(Option), + Failed, +} + +/// What the launcher currently is, shared with the route acceptor. +struct Shared { + authority: Mutex, + phase: Mutex, + lease: Mutex>, + lease_required: bool, + health: Arc, + clock: Arc, +} + +impl Shared { + fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { + self.authority + .lock() + .map_err(|_| anyhow!("authority lock poisoned"))? + .sign(state, detail, lease, self.clock.now_millis()) + } + + fn lease_current(&self) -> bool { + !self.lease_required + || self + .lease + .lock() + .is_ok_and(|hold| hold.as_ref().is_some_and(LeaseHold::still_current)) + } + + /// The answer to a route observation: active only if the child is healthy + /// and the lease is still ours. + fn observe(&self, request: CultNetMessage) -> Result { + let CultNetMessage::SnapshotRequest { message_id, .. } = request else { + bail!("route challenge is not a snapshot request"); + }; + let detail = format!("route-observation:{message_id}"); + let phase = self.phase.lock().map_err(|_| anyhow!("phase lock poisoned"))?.clone(); + let record = match phase { + Phase::Warming => self.sign("warming", &detail, None)?, + Phase::Failed => self.sign("failed", &detail, None)?, + Phase::Active(lease) => { + let state = if self.health.healthy() && self.lease_current() { + "active" + } else { + "degraded" + }; + self.sign(state, &detail, lease)? + } + }; + Ok(CultNetMessage::SnapshotResponseRaw { + message_id, + documents: vec![presence_document(&record, None)?], + }) + } +} + +type Responder = Arc Result + Send + Sync>; + +/// Owns the candidate endpoint. A leading 0x00 is an Idunn challenge frame and +/// is answered here; every other byte is copied untouched to the child. +pub struct RouteAcceptor { + listener: TcpListener, + child: ChildEndpoint, + responder: Responder, +} + +impl RouteAcceptor { + pub fn new(listener: TcpListener, child: ChildEndpoint, responder: Responder) -> Self { + Self { + listener, + child, + responder, + } + } + + pub fn serve(self, halt: Arc) -> Result<()> { + self.listener.set_nonblocking(true)?; + let acceptor = Arc::new(self); + while !halt.load(Ordering::SeqCst) { + match acceptor.listener.accept() { + Ok((connection, _)) => { + let acceptor = Arc::clone(&acceptor); + thread::spawn(move || { + let _ = connection.set_nonblocking(false); + let _ = acceptor.handle(connection); + }); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => thread::sleep(Duration::from_millis(10)), + } + } + Ok(()) + } + + fn handle(&self, mut connection: TcpStream) -> Result<()> { + let mut first = [0_u8; 1]; + if connection.read(&mut first)? == 0 { + return Ok(()); + } + if first[0] == CHALLENGE_FIRST_BYTE { + return self.answer_challenge(&mut connection); + } + let mut upstream = self.child.connect()?; + upstream.write_all(&first)?; + let mut upstream_reader = upstream.try_clone()?; + let mut client_reader = connection.try_clone()?; + let forward = thread::spawn(move || { + let _ = io::copy(&mut client_reader, &mut upstream); + upstream.shutdown_write(); + }); + let _ = io::copy(&mut upstream_reader, &mut connection); + let _ = connection.shutdown(Shutdown::Write); + let _ = forward.join(); + Ok(()) + } + + fn answer_challenge(&self, connection: &mut TcpStream) -> Result<()> { + connection.set_read_timeout(Some(CHALLENGE_IO_TIMEOUT))?; + connection.set_write_timeout(Some(CHALLENGE_IO_TIMEOUT))?; + let mut rest = [0_u8; 3]; + connection.read_exact(&mut rest)?; + let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; + ensure!( + (1..=CHALLENGE_MAX_BYTES).contains(&length), + "challenge frame is outside its bound" + ); + let mut payload = vec![0_u8; length]; + connection.read_exact(&mut payload)?; + let request = + decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; + let response = (self.responder)(request)?; + let frame = encode_frame(&encode_cultnet_message_to_vec( + &response, + CultNetWireContract::CultNetSchemaV0, + )?)?; + connection.write_all(&frame)?; + connection.flush()?; + Ok(()) + } +} + +// ---------------------------------------------------------------- lifecycle + +pub struct Ports { + pub credentials: Arc, + pub clock: Arc, + pub sink: Arc, + pub lease: Arc, + pub spawner: Box, + /// Built from the endpoint `run` chooses. + pub health: Box Arc + Send>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + /// SIGTERM: the child was stopped and waited for. + Stopped, + /// The unit must die so continuity sees it; the code names why. + Failed(&'static str), +} + +impl Outcome { + pub fn exit_code(&self) -> i32 { + match self { + Self::Stopped => 0, + Self::Failed(_) => 1, + } + } +} + +fn child_endpoint(listener: ChildListener, tmp: &Path) -> Result { + Ok(match listener { + ChildListener::UnixSocket => ChildEndpoint::Unix(tmp.join("child.sock")), + ChildListener::LoopbackPort => { + let probe = TcpListener::bind("127.0.0.1:0")?; + ChildEndpoint::Loopback(probe.local_addr()?) + } + }) +} + +/// The unit's whole life: Warming, lease, child, Active, until the child or the +/// lease ends or SIGTERM arrives. `route` is the already-bound candidate +/// listener; `tmp` is the unit's private temporary directory. +pub fn run( + contract: &LaunchContract, + authority: RuntimeAuthority, + route: TcpListener, + tmp: &Path, + ports: Ports, + stop: &AtomicBool, +) -> Outcome { + let expected = authority.expected().clone(); + let activation = authority.activation().clone(); + if expected.health_contract != contract.health_contract { + return Outcome::Failed("health-contract-mismatch"); + } + let Ok(endpoint) = child_endpoint(contract.listener, tmp) else { + return Outcome::Failed("child-endpoint"); + }; + let health = (ports.health)(&endpoint); + let shared = Arc::new(Shared { + authority: Mutex::new(authority), + phase: Mutex::new(Phase::Warming), + lease: Mutex::new(None), + lease_required: expected.write_lease_required, + health: Arc::clone(&health), + clock: Arc::clone(&ports.clock), + }); + let halt = Arc::new(AtomicBool::new(false)); + let responder: Responder = { + let shared = Arc::clone(&shared); + Arc::new(move |request| shared.observe(request)) + }; + let serving = { + let acceptor = RouteAcceptor::new(route, endpoint.clone(), responder); + let halt = Arc::clone(&halt); + thread::spawn(move || acceptor.serve(halt)) + }; + let outcome = lifecycle(contract, &expected, &activation, &endpoint, tmp, &ports, &shared, stop); + halt.store(true, Ordering::SeqCst); + let _ = serving.join(); + outcome +} + +#[allow(clippy::too_many_arguments)] +fn lifecycle( + contract: &LaunchContract, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + endpoint: &ChildEndpoint, + tmp: &Path, + ports: &Ports, + shared: &Shared, + stop: &AtomicBool, +) -> Outcome { + let clock = &*ports.clock; + let publish = |state: &str, detail: &str, lease: Option| -> Option { + let record = shared.sign(state, detail, lease).ok()?; + let sha = record.canonical_sha256().ok()?; + // A refused or lost publish is retried by the heartbeat; Odin may + // refuse Warming for a while after Idunn starts the unit. + let _ = ports.sink.put(&record); + Some(sha) + }; + let set_phase = |phase: Phase| { + if let Ok(mut current) = shared.phase.lock() { + *current = phase; + } + }; + let fail = |child: Option<&mut Box>, code: &'static str| { + if let Some(child) = child { + child.terminate(TERMINATE_GRACE); + } + set_phase(Phase::Failed); + publish("failed", code, None); + Outcome::Failed(code) + }; + + // 1. Warming reaches Odin before any child exists. + let mut proofs: VecDeque = VecDeque::new(); + let mut remember = |sha: Option, proofs: &mut VecDeque| { + if let Some(sha) = sha { + proofs.push_back(sha); + while proofs.len() > MAX_RECENT_WARMING_PROOFS { + proofs.pop_front(); + } + } + }; + remember(publish("warming", "launcher-started", None), &mut proofs); + let heartbeat = u64::try_from(contract.heartbeat.as_millis()).unwrap_or(u64::MAX); + let mut next_heartbeat = clock.now_millis() + heartbeat; + + // 2. A stateful child waits for the lease; a stateless one does not. + let mut lease_sha = None; + if expected.write_lease_required { + let deadline = clock.now_millis() + contract.lease_wait.as_millis() as u64; + loop { + if stop.load(Ordering::SeqCst) { + return Outcome::Stopped; + } + let held = LeaseHold::acquire( + Arc::clone(&ports.lease), + expected, + activation, + proofs.make_contiguous(), + ); + if let Ok(hold) = held { + lease_sha = Some(hold.sha256().to_owned()); + if let Ok(mut slot) = shared.lease.lock() { + *slot = Some(hold); + } + break; + } + let now = clock.now_millis(); + if now >= deadline { + return fail(None, "lease-timeout"); + } + if now >= next_heartbeat { + remember(publish("warming", "write-lease-pending", None), &mut proofs); + next_heartbeat = now + heartbeat; + } + clock.sleep(POLL); + } + } + + // 3. Render, spawn, and wait for health. + let Some(mut child) = start_child(contract, endpoint, tmp, ports) else { + return fail(None, "child-start-refused"); + }; + let startup_deadline = clock.now_millis() + contract.startup.as_millis() as u64; + loop { + if stop.load(Ordering::SeqCst) { + child.terminate(TERMINATE_GRACE); + return Outcome::Stopped; + } + if child.exited().is_some() { + return fail(None, "child-exited"); + } + if shared.health.healthy() { + break; + } + if clock.now_millis() >= startup_deadline { + return fail(Some(&mut child), "startup-timeout"); + } + clock.sleep(POLL); + } + + // 4. Active, and heartbeats while it holds. + set_phase(Phase::Active(lease_sha.clone())); + publish("active", "child-healthy", lease_sha.clone()); + next_heartbeat = clock.now_millis() + heartbeat; + loop { + if stop.load(Ordering::SeqCst) { + child.terminate(TERMINATE_GRACE); + return Outcome::Stopped; + } + if child.exited().is_some() { + return fail(None, "child-exited"); + } + if !shared.lease_current() { + return fail(Some(&mut child), "lease-lost"); + } + let now = clock.now_millis(); + if now >= next_heartbeat { + let state = if shared.health.healthy() { "active" } else { "degraded" }; + publish(state, "heartbeat", lease_sha.clone()); + next_heartbeat = now + heartbeat; + } + clock.sleep(POLL); + } +} + +fn start_child( + contract: &LaunchContract, + endpoint: &ChildEndpoint, + tmp: &Path, + ports: &Ports, +) -> Option> { + let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; + let listen = endpoint.listen_value(); + let rendered = ConfigRenderer { + credentials: &*ports.credentials, + listen: &listen, + state_root: contract.state_root.as_deref(), + } + .render(&template) + .ok()?; + let config = write_private_file(tmp, "child-config", &rendered).ok()?; + let mut args = contract.child_args.clone(); + if let Some(flag) = &contract.config_flag { + args.push(flag.clone()); + args.push(config.display().to_string()); + } + ports + .spawner + .spawn(&contract.release_root.join(&contract.child), &args) + .ok() +} diff --git a/src/lib.rs b/src/lib.rs index b96d835..e71a17f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,5 +4,7 @@ pub mod deployment_plan; pub mod drivers; pub mod host; pub mod host_actuator; +#[cfg(unix)] +pub mod launcher; mod incident; pub mod provisioning; From 6866d755364b7af08016196df1bcbfe02608bda3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 19:10:58 +0200 Subject: [PATCH 052/122] idunn-launch tests: lease gate, foreign lease, lease loss, stateless, route challenge, verbatim splice, closed placeholders, child exit (unbuilt) Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 713 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 713 insertions(+) diff --git a/src/launcher.rs b/src/launcher.rs index f3bee58..25e38db 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -1275,3 +1275,716 @@ fn start_child( .spawn(&contract.release_root.join(&contract.child), &args) .ok() } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::{AtomicU64, AtomicUsize}; + + use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; + use cultnet_rs::{ + IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, + IdunnServiceIdentity, RuntimePresenceAuthenticationContext, VerifiedRuntimeAuthority, + authenticate_runtime_presence_claim, enroll_service_identity_at, verify_runtime_authority, + }; + + use super::*; + use crate::drivers::{WriteLeasePort, exact_route_snapshot_response, request_tcp_snapshot}; + + fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) + } + + struct Fixture { + dir: tempfile::TempDir, + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + activation_credential: Vec, + provider_credential: Vec, + verified: VerifiedRuntimeAuthority, + } + + fn write_record(path: &Path, key: &str, kind: &str, schema: &str, payload: Vec) { + let envelope = CultCacheEnvelope { + key: key.into(), + r#type: kind.into(), + payload, + stored_at: chrono::Utc::now().to_rfc3339(), + schema_id: Some(schema.into()), + }; + let outcome = SingleFileMessagePackBackingStore::new(path) + .try_compare_exchange_snapshot(&[], &[envelope]) + .unwrap(); + assert!(matches!( + outcome, + TryCompareExchangeSnapshotOutcome::Exchanged + )); + } + + fn fixture(lease_required: bool) -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let provider_path = root.join("provider.cc"); + let provider = + enroll_service_identity_at::(&provider_path).unwrap(); + let idunn = + enroll_service_identity_at::(&root.join("idunn.cc")).unwrap(); + let expected = IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: "synapse".into(), + plan_id: digest('1'), + incarnation_id: "incarnation-1".into(), + sealed_release_id: digest('2'), + source_repository: "github.com/GameCult/gamecult-chat".into(), + source_revision: "3".repeat(40), + recipe_sha256: digest('4'), + runtime_id: "synapse-yggdrasil".into(), + expected_signer_identity_id: provider.entry().identity_id.clone(), + health_contract: "gamecult.chat.synapse.presence-launcher.v1".into(), + artifact_sha256: digest('5'), + state_schema_generation: lease_required.then(|| "state-v1".into()), + state_contract_sha256: lease_required.then(|| digest('6')), + write_lease_required: lease_required, + route: Some(IdunnExpectedRoute { + route_id: "synapse-tcp".into(), + transport: "tcp".into(), + stable_endpoint: "tcp://127.0.0.1:18834".into(), + candidate_endpoint: "tcp://127.0.0.1:18835".into(), + }), + capabilities: vec![IdunnExpectedCapability { + capability: "gamecult.chat.homeserver".into(), + schema: "matrix.client.v3".into(), + compatibility: "v3".into(), + minimum_capacity: 1, + }], + dependencies: Vec::new(), + }; + let launch = + IdunnRuntimeActivationLaunch::issue(&expected, digest('7'), 100, &idunn).unwrap(); + let activation = launch.activation().clone(); + let mut activation_credential = Vec::new(); + launch.write_credential(&mut activation_credential).unwrap(); + let verified = verify_runtime_authority( + &expected, + &activation, + &idunn.trust_anchor().unwrap(), + &provider.entry().public_key, + ) + .unwrap(); + let bundle = root.join("bundle"); + fs::create_dir(&bundle).unwrap(); + write_record( + &bundle.join("expected.cc"), + &expected.target, + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + expected.canonical_bytes().unwrap(), + ); + write_record( + &bundle.join("activation.cc"), + &expected.target, + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + activation.canonical_bytes().unwrap(), + ); + let provider_credential = fs::read(&provider_path).unwrap(); + for directory in ["release/bin", "tmp"] { + fs::create_dir_all(root.join(directory)).unwrap(); + } + Fixture { + dir, + expected, + activation, + activation_credential, + provider_credential, + verified, + } + } + + impl Fixture { + fn authority(&self) -> RuntimeAuthority { + RuntimeAuthority::open( + &self.dir.path().join("bundle"), + &self.activation_credential[..], + &self.provider_credential[..], + ) + .unwrap() + } + + fn lease(&self, warming_sha256: &str) -> IdunnProcessWriteLeaseRecord { + IdunnProcessWriteLeaseRecord { + schema_version: cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: self.expected.target.clone(), + expected_projection_sha256: self.expected.canonical_sha256().unwrap(), + plan_id: self.expected.plan_id.clone(), + incarnation_id: self.expected.incarnation_id.clone(), + sealed_release_id: self.expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256().unwrap(), + state_schema_generation: self.expected.state_schema_generation.clone().unwrap(), + state_contract_sha256: self.expected.state_contract_sha256.clone().unwrap(), + runtime_id: self.expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + warming_presence_sha256: warming_sha256.into(), + lease_epoch: 1, + issued_at_unix_millis: 200, + } + } + + fn contract(&self, template: &str) -> LaunchContract { + fs::write(self.dir.path().join("release/config.tmpl"), template).unwrap(); + LaunchContract { + release_root: self.dir.path().join("release"), + child: PathBuf::from("bin/child"), + child_args: vec!["--serve".into()], + config_flag: Some("--config".into()), + template: PathBuf::from("config.tmpl"), + listener: ChildListener::UnixSocket, + health_path: "/health".into(), + health_contract: self.expected.health_contract.clone(), + state_root: Some(PathBuf::from("/var/lib/synapse")), + lease_wait: Duration::from_secs(3600), + startup: Duration::from_secs(3600), + heartbeat: Duration::from_secs(5), + } + } + } + + /// Virtual time runs a hundred times faster than the wall clock. + struct FastClock(AtomicU64); + + impl Clock for FastClock { + fn now_millis(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } + + fn sleep(&self, duration: Duration) { + self.0 + .fetch_add(duration.as_millis() as u64, Ordering::SeqCst); + thread::sleep(Duration::from_millis(1)); + } + } + + #[derive(Default)] + struct Sink(Mutex>); + + impl PresenceSink for Sink { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { + self.0.lock().unwrap().push(record.clone()); + Ok(()) + } + } + + impl Sink { + fn states(&self) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .map(|record| record.state.clone()) + .collect() + } + + fn first(&self, state: &str) -> Option { + self.0 + .lock() + .unwrap() + .iter() + .find(|record| record.state == state) + .cloned() + } + } + + #[derive(Default)] + struct FakeLease(Mutex>); + + impl LeaseReader for FakeLease { + fn read(&self) -> Result> { + Ok(self.0.lock().unwrap().clone()) + } + + fn hold_shared(&self) -> Result> { + Ok(Box::new(())) + } + } + + struct Health(AtomicBool); + + impl HealthProbe for Health { + fn healthy(&self) -> bool { + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct Child { + spawns: AtomicUsize, + exit: Mutex>, + terminated: AtomicBool, + args: Mutex>>, + } + + struct FakeChild(Arc); + + impl ChildHandle for FakeChild { + fn exited(&mut self) -> Option { + *self.0.exit.lock().unwrap() + } + + fn terminate(&mut self, _grace: Duration) { + self.0.terminated.store(true, Ordering::SeqCst); + *self.0.exit.lock().unwrap() = Some(143); + } + } + + struct FakeSpawner(Arc); + + impl ChildSpawner for FakeSpawner { + fn spawn(&self, _program: &Path, args: &[String]) -> Result> { + self.0.spawns.fetch_add(1, Ordering::SeqCst); + self.0.args.lock().unwrap().push(args.to_vec()); + Ok(Box::new(FakeChild(Arc::clone(&self.0)))) + } + } + + struct Credentials(BTreeMap>); + + impl CredentialSource for Credentials { + fn contents(&self, name: &str) -> Result> { + self.0.get(name).cloned().ok_or_else(|| anyhow!("unbound")) + } + + fn path(&self, name: &str) -> Result { + self.0 + .contains_key(name) + .then(|| PathBuf::from(format!("/run/credentials/unit/{name}"))) + .ok_or_else(|| anyhow!("unbound")) + } + } + + struct Run { + sink: Arc, + lease: Arc, + child: Arc, + stop: Arc, + outcome: Option>, + } + + fn start(fx: &Fixture, contract: LaunchContract, healthy: bool) -> Run { + let sink = Arc::new(Sink::default()); + let lease = Arc::new(FakeLease::default()); + let health = Arc::new(Health(AtomicBool::new(healthy))); + let child = Arc::new(Child::default()); + let stop = Arc::new(AtomicBool::new(false)); + let mut secrets = BTreeMap::new(); + secrets.insert("SYNAPSE_KEY".to_owned(), b"s3cret-value\n".to_vec()); + let ports = Ports { + credentials: Arc::new(Credentials(secrets)), + clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + sink: sink.clone(), + lease: lease.clone(), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(move |_| health.clone()), + }; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let flag = stop.clone(); + let outcome = thread::spawn(move || run(&contract, authority, route, &tmp, ports, &flag)); + Run { + sink, + lease, + child, + stop, + outcome: Some(outcome), + } + } + + impl Run { + fn finish(&mut self) -> Outcome { + self.outcome.take().unwrap().join().unwrap() + } + + fn stop(&mut self) -> Outcome { + self.stop.store(true, Ordering::SeqCst); + self.finish() + } + } + + fn wait_until(what: &str, condition: impl Fn() -> bool) { + for _ in 0..4000 { + if condition() { + return; + } + thread::sleep(Duration::from_millis(5)); + } + panic!("timed out waiting for {what}"); + } + + #[test] + fn stateful_child_never_starts_before_its_lease() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("listen {{child.listen}}"), true); + wait_until("two warming heartbeats", || { + launch.sink.states().iter().filter(|s| *s == "warming").count() >= 2 + }); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); + let warming = launch.sink.first("warming").unwrap(); + assert!(warming.write_lease_sha256.is_none()); + + // The lease binds the launcher's own Warming proof; only then it starts. + let lease = fx.lease(&warming.canonical_sha256().unwrap()); + *launch.lease.0.lock().unwrap() = Some(lease.clone()); + wait_until("child spawn", || { + launch.child.spawns.load(Ordering::SeqCst) == 1 + }); + wait_until("active", || launch.sink.first("active").is_some()); + let active = launch.sink.first("active").unwrap(); + assert_eq!( + active.write_lease_sha256.as_deref(), + Some(lease.canonical_sha256().unwrap().as_str()) + ); + assert_eq!(launch.stop(), Outcome::Stopped); + assert!(launch.child.terminated.load(Ordering::SeqCst)); + } + + #[test] + fn a_lease_for_another_activation_or_expected_is_not_accepted() { + let fx = fixture(true); + let foreign: [(&str, fn(&mut IdunnProcessWriteLeaseRecord)); 3] = [ + ("another runtime instance", |lease| { + lease.runtime_instance_id = "other".into() + }), + ("another Expected", |lease| { + lease.expected_projection_sha256 = digest('9') + }), + ("another warming proof", |lease| { + lease.warming_presence_sha256 = digest('9') + }), + ]; + for (name, edit) in foreign { + let mut contract = fx.contract("x"); + contract.lease_wait = Duration::from_secs(5); + let mut launch = start(&fx, contract, true); + // Installed at once; the launcher's own proofs are never digest('8'). + let mut lease = fx.lease(&digest('8')); + edit(&mut lease); + *launch.lease.0.lock().unwrap() = Some(lease); + assert_eq!(launch.finish(), Outcome::Failed("lease-timeout"), "{name}"); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0, "{name}"); + assert_eq!(launch.sink.states().last().unwrap(), "failed", "{name}"); + } + } + + #[test] + fn lease_loss_stops_the_child() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("warming", || launch.sink.first("warming").is_some()); + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + *launch.lease.0.lock().unwrap() = None; + let outcome = launch.finish(); + assert_eq!(outcome, Outcome::Failed("lease-lost")); + assert_ne!(outcome.exit_code(), 0); + assert!(launch.child.terminated.load(Ordering::SeqCst)); + assert_eq!(launch.sink.states().last().unwrap(), "failed"); + } + + #[test] + fn stateless_child_starts_without_a_lease() { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 1); + assert_eq!(launch.sink.states().first().unwrap(), "warming"); + assert!( + launch + .sink + .first("active") + .unwrap() + .write_lease_sha256 + .is_none() + ); + launch.stop(); + } + + #[test] + fn child_exit_ends_the_launcher() { + for code in [0, 3] { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + *launch.child.exit.lock().unwrap() = Some(code); + let outcome = launch.finish(); + assert_eq!(outcome, Outcome::Failed("child-exited"), "exit {code}"); + assert_eq!(outcome.exit_code(), 1); + assert_eq!(launch.sink.states().last().unwrap(), "failed"); + } + } + + #[test] + fn closed_placeholder_set_renders_and_refuses() { + let fx = fixture(false); + let template = "listen={{child.listen}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; + let mut launch = start(&fx, fx.contract(template), true); + wait_until("spawn", || launch.child.spawns.load(Ordering::SeqCst) == 1); + let args = launch.child.args.lock().unwrap()[0].clone(); + assert_eq!(args[..2], ["--serve", "--config"]); + let config = PathBuf::from(&args[2]); + assert_eq!(config.parent().unwrap(), fx.dir.path().join("tmp")); + assert_eq!( + fs::metadata(&config).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let rendered = fs::read_to_string(&config).unwrap(); + let socket = fx.dir.path().join("tmp/child.sock"); + assert_eq!( + rendered, + format!( + "listen={}\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n", + socket.display() + ) + ); + assert!( + !rendered.contains("18835"), + "the candidate bind never reaches the child" + ); + launch.stop(); + + // Any other placeholder refuses to start, and the refusal names no input. + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("a={{canary_bind}}"), true); + assert_eq!(launch.finish(), Outcome::Failed("child-start-refused")); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); + let credentials = Credentials(BTreeMap::new()); + let renderer = ConfigRenderer { + credentials: &credentials, + listen: "L", + state_root: None, + }; + for template in [ + "{{canary_bind}}", + "{{credential:CANARY-MISSING}}", + "{{credential:canary b}}", + "{{child.listen}} {{canary", + "{{state_root}}", + ] { + let error = format!("{:#}", renderer.render(template).unwrap_err()); + assert!(!error.to_lowercase().contains("canary"), "{error}"); + } + } + + fn shared(fx: &Fixture, health: Arc) -> Arc { + Arc::new(Shared { + authority: Mutex::new(fx.authority()), + phase: Mutex::new(Phase::Active(None)), + lease: Mutex::new(None), + lease_required: false, + health, + clock: Arc::new(SystemClock), + }) + } + + #[test] + fn route_answer_degrades_with_the_child() { + let fx = fixture(false); + let health = Arc::new(Health(AtomicBool::new(true))); + let shared = shared(&fx, health.clone()); + let state_of = |shared: &Shared| { + let request = CultNetMessage::SnapshotRequest { + message_id: "challenge-1".into(), + schema_ids: None, + record_keys: None, + }; + let CultNetMessage::SnapshotResponseRaw { documents, .. } = + shared.observe(request).unwrap() + else { + panic!("not a raw snapshot response") + }; + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&documents[0].payload).unwrap(); + assert_eq!(record.detail, "route-observation:challenge-1"); + record.state + }; + assert_eq!(state_of(&shared), "active"); + health.0.store(false, Ordering::SeqCst); + assert_eq!(state_of(&shared), "degraded"); + } + + #[test] + fn route_challenge_is_answered_with_both_signatures_through_idunns_client() { + let fx = fixture(false); + let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let responder: Responder = Arc::new(move |request| shared.observe(request)); + let halt = Arc::new(AtomicBool::new(false)); + let acceptor = RouteAcceptor::new( + listener, + ChildEndpoint::Unix("/nonexistent".into()), + responder, + ); + let serving = { + let halt = halt.clone(); + thread::spawn(move || acceptor.serve(halt)) + }; + let request = CultNetMessage::SnapshotRequest { + message_id: "challenge-7".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + let response = request_tcp_snapshot(address, &payload).ok().unwrap(); + let message = + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .unwrap(); + let answer = + exact_route_snapshot_response(message, "challenge-7", &fx.expected.target).unwrap(); + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&answer.canonical_presence).unwrap(); + let claim = authenticate_runtime_presence_claim( + &answer.canonical_presence, + &fx.verified, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: record.observed_at_unix_millis, + maximum_age_millis: 10_000, + maximum_future_skew_millis: 10, + }, + ) + .unwrap(); + assert_eq!(claim.record().state, "active"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn member_bytes_reach_the_child_verbatim_and_only_challenge_frames_are_answered() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("child.sock"); + let child = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + let request: Vec = + b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 6\r\n\r\n\x00\x01\x00\x02\x00\x03".to_vec(); + let response: Vec = + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n\x00\xff\x00\x10".to_vec(); + let received = Arc::new(Mutex::new(Vec::new())); + let connections = Arc::new(AtomicUsize::new(0)); + let fake = { + let (received, connections, response, want) = ( + received.clone(), + connections.clone(), + response.clone(), + request.len(), + ); + thread::spawn(move || { + for stream in child.incoming() { + let mut stream = stream.unwrap(); + connections.fetch_add(1, Ordering::SeqCst); + let mut buffer = vec![0_u8; want]; + if stream.read_exact(&mut buffer).is_err() { + continue; + } + *received.lock().unwrap() = buffer; + stream.write_all(&response).unwrap(); + return; + } + }) + }; + let challenges = Arc::new(AtomicUsize::new(0)); + let responder: Responder = { + let challenges = challenges.clone(); + Arc::new(move |_| { + challenges.fetch_add(1, Ordering::SeqCst); + bail!("not under test") + }) + }; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let halt = Arc::new(AtomicBool::new(false)); + let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder); + let serving = { + let halt = halt.clone(); + thread::spawn(move || acceptor.serve(halt)) + }; + + // A leading 0x00 that is not a challenge frame never reaches the child. + let mut probe = TcpStream::connect(address).unwrap(); + probe.write_all(&[0x00, 0xff, 0xff, 0xff, 1, 2, 3]).unwrap(); + let mut answered = Vec::new(); + let _ = probe.read_to_end(&mut answered); + assert!(answered.is_empty()); + assert_eq!(connections.load(Ordering::SeqCst), 0); + + // Everything else is copied untouched, both ways, 0x00 bytes included. + let mut member = TcpStream::connect(address).unwrap(); + member.write_all(&request).unwrap(); + let mut answer = Vec::new(); + member.read_to_end(&mut answer).unwrap(); + assert_eq!(answer, response); + fake.join().unwrap(); + assert_eq!(*received.lock().unwrap(), request); + assert_eq!( + challenges.load(Ordering::SeqCst), + 0, + "a malformed frame fails before the responder; member traffic never reaches it" + ); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn contract_parsing_refuses_escapes_and_echoes_nothing() { + let base = |extra: &[&str]| -> Result { + let mut args: Vec = [ + "--release-root", + "/r", + "--child", + "bin/c", + "--template", + "t", + "--listener", + "unix", + "--health-path", + "/health", + "--health-contract", + "h", + ] + .map(String::from) + .to_vec(); + args.extend(extra.iter().map(|value| value.to_string())); + LaunchContract::parse(args) + }; + assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); + assert!(base(&["--child", "../escape"]).is_err()); + assert!(base(&["--template", "/etc/passwd"]).is_err()); + let error = base(&["--canary-flag", "canary-value"]) + .unwrap_err() + .to_string(); + assert!(!error.contains("canary")); + } + + #[test] + fn real_lease_reader_holds_the_shared_lock_idunn_must_respect() { + let fx = fixture(true); + let path = fx.dir.path().join("lease.cc"); + let warming = digest('a'); + let lease = fx.lease(&warming); + write_record( + &path, + &lease.target, + IdunnProcessWriteLeaseRecord::TYPE, + cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes().unwrap(), + ); + let reader: Arc = Arc::new(DriverLeaseReader::new(&lease.target, &path)); + let hold = LeaseHold::acquire(reader, &fx.expected, &fx.activation, &[warming]).unwrap(); + assert!(hold.still_current()); + let idunn = CultCacheWriteLeaseDriver::new(&lease.target, &path); + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer")); + drop(hold); + idunn.revoke_exact(Some(&lease)).unwrap(); + } +} From d94150d396135759ec1cab13242609e87ee1d892 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 19:58:21 +0200 Subject: [PATCH 053/122] idunn-launch: release root is the launcher's own release; docs for foreign workloads; probe, spawner and authority tests Not built at this commit; verified at the head by the final job. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 25 ++++++++++++++ docs/authority-map.md | 36 ++++++++++++++++++++ src/bin/idunn-launch.rs | 10 ++++-- src/launcher.rs | 74 +++++++++++++++++++++++++++++++++-------- 4 files changed, 129 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 503409e..225c5d8 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,31 @@ omits one. Unknown fields are rejected on both sides. A repository that gets compromised can therefore change what is built, but not what it is allowed to touch on the host. +An upstream daemon that does not speak Idunn's runtime contract is deployed +with `idunn-launch` as its executable artifact and the upstream tree as a +required adjacent artifact; the launcher finds the child inside the release that +holds it. The launcher is the unit's process: it binds the +candidate endpoint, answers Idunn's challenge, renders the child's config, +starts a stateful child only once it holds the write lease, and signs presence. + +```toml +# recipe fragment — the upstream tree is built or fetched as an artifact +[service] +executable_artifact = "idunn-launch" +arguments = [ + { kind = "literal", value = "--child" }, + { kind = "literal", value = "upstream/bin/daemon" }, + { kind = "literal", value = "--template" }, + { kind = "literal", value = "config.tmpl" }, + { kind = "literal", value = "--listener" }, + { kind = "literal", value = "unix" }, + { kind = "literal", value = "--health-path" }, + { kind = "literal", value = "/health" }, + { kind = "literal", value = "--health-contract" }, + { kind = "literal", value = "example.presence-launcher.v1" }, +] +``` + ## Contracts Idunn's typed contracts are published by diff --git a/docs/authority-map.md b/docs/authority-map.md index 270d592..b61a8f2 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -70,6 +70,42 @@ recipe fields. Stateless services declare no state block. Their operator binding therefore has no state root, state-transition policy, or process write-lease record. +## Foreign workloads: idunn-launch + +An upstream daemon that cannot speak Idunn's runtime contract (Synapse, MAS) +runs under `idunn-launch`, the unit's own process. Idunn's two-witness rule is +unchanged: Idunn observes the unit, and the launcher, inside the unit and +supervising the child, signs presence with the provider identity and the +activation key. + +- **Owner:** the launcher owns presence, config rendering and the child's + lifetime inside the unit, and the route answers it signs for that child. + Idunn owns admission, lease, route and continuity, as before. +- **Inputs:** the runtime bundle, the activation credential and provider + identity descriptors, the process write lease, the systemd credentials the + binding names, and the child's health over its private listener. +- **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active + and failed to Odin; route observations on the candidate endpoint), one child + process, and a rendered config in the unit's private tmp that is never read + back. +- **Route:** the launcher, not the child, binds the candidate endpoint. A first + byte of 0x00 is an Idunn challenge frame and is answered by the launcher; + every other byte is copied untouched to the child's private listener. +- **Lease:** with `write_lease_required` the launcher starts the child only + after it holds the lease (bound to this Expected, this activation and one of + its own Warming proofs) and keeps the lease store share-locked while the + child lives. Losing the lease stops the child. +- **Config:** one template from the release is rendered with the closed + placeholder set `{{child.listen}}`, `{{credential:NAME}}`, + `{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder + refuses to start. +- **Forbidden:** the child binding the candidate endpoint; a child of a + lease-requiring target running without a held lease; any presence not signed + by both keys. +- **State kind `external-directory`:** a process-owned directory imposed by an + external runtime contract. Same writer, recovery and startup rules as + `external-file`: it is lease-gated, preserved and fenced, never interpreted. + ## Operator binding The root-admitted operator binding supplies the physical policy a repository diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs index 32c6291..b743f06 100644 --- a/src/bin/idunn-launch.rs +++ b/src/bin/idunn-launch.rs @@ -17,7 +17,11 @@ fn main() -> std::process::ExitCode { } fn launch() -> anyhow::Result { - let contract = LaunchContract::parse(std::env::args().skip(1))?; + let release_root = std::env::current_exe()? + .parent() + .context("the launcher is not inside a release")? + .to_owned(); + let contract = LaunchContract::parse(release_root, std::env::args().skip(1))?; let bundle = std::env::var_os(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT) .context("the Idunn runtime bundle is not set")?; let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) @@ -52,8 +56,8 @@ fn main() -> std::process::ExitCode { }; // SAFETY: the handler only stores to an atomic. unsafe { - libc::signal(libc::SIGTERM, on_signal as libc::sighandler_t); - libc::signal(libc::SIGINT, on_signal as libc::sighandler_t); + libc::signal(libc::SIGTERM, on_signal as *const () as libc::sighandler_t); + libc::signal(libc::SIGINT, on_signal as *const () as libc::sighandler_t); } Ok(launcher::run( &contract, diff --git a/src/launcher.rs b/src/launcher.rs index 25e38db..7748177 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -66,7 +66,8 @@ pub enum ChildListener { } /// Everything the recipe declares about the child, parsed from the launcher's -/// argv (`--flag value` pairs; `--child-arg` repeats). +/// argv (`--flag value` pairs; `--child-arg` repeats). The release root is the +/// sealed release directory that holds the launcher itself. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LaunchContract { pub release_root: PathBuf, @@ -86,8 +87,8 @@ pub struct LaunchContract { } impl LaunchContract { - pub fn parse(args: impl IntoIterator) -> Result { - let mut release_root = None; + pub fn parse(release_root: PathBuf, args: impl IntoIterator) -> Result { + ensure!(release_root.is_absolute(), "the release root is not absolute"); let mut child = None; let mut child_args = Vec::new(); let mut config_flag = None; @@ -108,7 +109,6 @@ impl LaunchContract { .with_context(|| format!("launcher argument {field} is not a positive integer")) }; match flag.as_str() { - "--release-root" => release_root = Some(PathBuf::from(value)), "--child" => child = Some(release_relative(&value, "--child")?), "--child-arg" => child_args.push(value), "--config-flag" => config_flag = Some(value), @@ -135,11 +135,6 @@ impl LaunchContract { _ => bail!("unknown launcher argument"), } } - let release_root = release_root.context("launcher argument --release-root is required")?; - ensure!( - release_root.is_absolute(), - "launcher argument --release-root is not absolute" - ); Ok(Self { release_root, child: child.context("launcher argument --child is required")?, @@ -1157,7 +1152,7 @@ fn lifecycle( // 1. Warming reaches Odin before any child exists. let mut proofs: VecDeque = VecDeque::new(); - let mut remember = |sha: Option, proofs: &mut VecDeque| { + let remember = |sha: Option, proofs: &mut VecDeque| { if let Some(sha) = sha { proofs.push_back(sha); while proofs.len() > MAX_RECENT_WARMING_PROOFS { @@ -1938,8 +1933,6 @@ mod tests { fn contract_parsing_refuses_escapes_and_echoes_nothing() { let base = |extra: &[&str]| -> Result { let mut args: Vec = [ - "--release-root", - "/r", "--child", "bin/c", "--template", @@ -1954,7 +1947,7 @@ mod tests { .map(String::from) .to_vec(); args.extend(extra.iter().map(|value| value.to_string())); - LaunchContract::parse(args) + LaunchContract::parse(PathBuf::from("/r"), args) }; assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); assert!(base(&["--child", "../escape"]).is_err()); @@ -1987,4 +1980,59 @@ mod tests { drop(hold); idunn.revoke_exact(Some(&lease)).unwrap(); } + + #[test] + fn authority_binds_the_candidate_and_signs_strictly_increasing_sequences() { + let fx = fixture(false); + let mut authority = fx.authority(); + assert!(authority.require_candidate_bind("127.0.0.1:18835").is_ok()); + assert!(authority.require_candidate_bind("127.0.0.1:18836").is_err()); + let first = authority.sign("warming", "a", None, 5_000).unwrap(); + let second = authority.sign("warming", "b", None, 5_000).unwrap(); + assert_eq!(first.publisher_sequence, 5_000); + assert_eq!(second.publisher_sequence, 5_001); + assert_eq!( + first.bound_endpoint.as_deref(), + Some("tcp://127.0.0.1:18835") + ); + } + + #[test] + fn health_probe_is_healthy_only_on_a_200_from_the_child() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("child.sock"); + let probe = HttpHealthProbe { + endpoint: ChildEndpoint::Unix(socket.clone()), + path: "/health".into(), + }; + assert!(!probe.healthy(), "no listener is not healthy"); + let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + let server = thread::spawn(move || { + for status in ["200 OK", "503 Service Unavailable"] { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 64]; + let read = stream.read(&mut request).unwrap(); + assert!(request[..read].starts_with(b"GET /health HTTP/1.1\r\n")); + write!(stream, "HTTP/1.1 {status}\r\nContent-Length: 2\r\n\r\nOK").unwrap(); + } + }); + assert!(probe.healthy()); + assert!(!probe.healthy()); + server.join().unwrap(); + } + + #[test] + fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { + let mut gone = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()]) + .unwrap(); + wait_until("child exit", || gone.exited().is_some()); + assert_eq!(gone.exited(), Some(3)); + let mut running = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()]) + .unwrap(); + assert_eq!(running.exited(), None); + running.terminate(Duration::from_secs(5)); + assert!(running.exited().is_some()); + } } From 0e09488c19a9fd564cfd82596fd9f664268e5236 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 20:09:26 +0200 Subject: [PATCH 054/122] idunn-launch tests: wait_until takes FnMut Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/launcher.rs b/src/launcher.rs index 7748177..b6946fa 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -1606,7 +1606,7 @@ mod tests { } } - fn wait_until(what: &str, condition: impl Fn() -> bool) { + fn wait_until(what: &str, mut condition: impl FnMut() -> bool) { for _ in 0..4000 { if condition() { return; From 9c69d006410e4b92bd42c1c606221ed2d4e9a3ad Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 20:48:29 +0200 Subject: [PATCH 055/122] Tests kill survivors: grant binding and holder custody, parse fields, credential names, env credentials, Odin endpoint, half-close splice, heartbeat pacing, SIGTERM escalation Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 34 ++++++++++ src/launcher.rs | 166 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 4bf500a..df361c3 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -13473,6 +13473,40 @@ processes 9 Ok(()) } + #[cfg(unix)] + #[test] + fn write_lease_grant_binds_the_exact_warming_candidate_and_the_holder_read_keeps_custody() + -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let path = temp.path().join("lease.cc"); + let driver = CultCacheWriteLeaseDriver::new("service", &path); + let (expected, activation, warming, _) = authenticated_warming(temp.path())?; + let good = lease(&expected, &activation, &warming); + let mut foreign = good.clone(); + foreign.warming_presence_sha256 = digest('8'); + assert!( + driver + .grant(&expected, &activation, &warming, &foreign) + .is_err() + ); + let other = CultCacheWriteLeaseDriver::new("other", temp.path().join("other.cc")); + assert!(other.grant(&expected, &activation, &warming, &good).is_err()); + assert!(driver.current_for_holder()?.is_none()); + driver.grant(&expected, &activation, &warming, &good)?; + assert_eq!( + driver.current_for_holder()?.map(|(_, held)| held), + Some(good) + ); + fs::set_permissions(&path, fs::Permissions::from_mode(0o666))?; + assert!( + driver.current_for_holder().is_err(), + "a service-writable lease store is not authority" + ); + Ok(()) + } + #[cfg(unix)] #[test] fn installed_release_is_root_owned_and_nonwritable() -> Result<()> { diff --git a/src/launcher.rs b/src/launcher.rs index b6946fa..b1eeb69 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2021,6 +2021,172 @@ mod tests { server.join().unwrap(); } + #[test] + fn contract_parsing_reads_every_declared_field() { + let args: Vec = [ + "--child", "bin/c", "--child-arg", "-x", "--child-arg", "y", "--config-flag", "-c", + "--template", "t", "--listener", "loopback", "--health-path", "/h", + "--health-contract", "hc", "--state-root", "/s", "--lease-wait-secs", "11", + "--startup-secs", "22", "--heartbeat-secs", "33", + ] + .map(String::from) + .to_vec(); + let contract = LaunchContract::parse(PathBuf::from("/r"), args.clone()).unwrap(); + assert_eq!(contract.child_args, ["-x", "y"]); + assert_eq!(contract.config_flag.as_deref(), Some("-c")); + assert_eq!(contract.listener, ChildListener::LoopbackPort); + assert_eq!(contract.state_root, Some(PathBuf::from("/s"))); + assert_eq!( + (contract.lease_wait, contract.startup, contract.heartbeat), + (Duration::from_secs(11), Duration::from_secs(22), Duration::from_secs(33)) + ); + for zero in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { + let mut bad = args.clone(); + bad.extend([zero.to_owned(), "0".to_owned()]); + assert!(LaunchContract::parse(PathBuf::from("/r"), bad).is_err(), "{zero}"); + } + } + + #[test] + fn credential_names_cannot_escape_or_be_empty() { + struct Any; + impl CredentialSource for Any { + fn contents(&self, _: &str) -> Result> { + Ok(b"v".to_vec()) + } + fn path(&self, name: &str) -> Result { + Ok(PathBuf::from(name)) + } + } + let renderer = ConfigRenderer { credentials: &Any, listen: "L", state_root: None }; + for ok in ["A", "a-b_9", "Z0"] { + assert_eq!(renderer.render(&format!("{{{{credential:{ok}}}}}")).unwrap(), "v"); + } + for bad in ["", "../x", "a/b", "a b", "a.b", "a:b", "é"] { + assert!(renderer.render(&format!("{{{{credential:{bad}}}}}")).is_err(), "{bad}"); + assert!(renderer.render(&format!("{{{{credential_path:{bad}}}}}")).is_err(), "{bad}"); + } + assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "aLbLc"); + } + + #[test] + fn env_credentials_resolve_through_the_variable_idunn_sets() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("cred"); + fs::write(&file, b"abc\n").unwrap(); + // SAFETY: a variable no other test reads. + unsafe { std::env::set_var("IDUNN_LAUNCH_TEST_CREDENTIAL", &file) }; + assert_eq!(EnvCredentials.path("IDUNN_LAUNCH_TEST_CREDENTIAL").unwrap(), file); + assert_eq!(EnvCredentials.contents("IDUNN_LAUNCH_TEST_CREDENTIAL").unwrap(), b"abc\n"); + assert!(EnvCredentials.path("IDUNN_LAUNCH_TEST_UNBOUND").is_err()); + } + + #[test] + fn odin_sink_takes_its_endpoint_only_from_the_odin_dependency() { + let fx = fixture(false); + let dependency = |capability: &str, endpoint: &str| cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: capability.into(), + schema: "s".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: None, + provider_authority: None, + provider_expected_projection_sha256: None, + provider_endpoint: Some(endpoint.into()), + }; + let mut expected = fx.expected.clone(); + assert!(OdinRudpSink::from_expected(&expected).is_err()); + expected.dependencies = vec![dependency("other.thing", "rudp://127.0.0.1:1")]; + assert!(OdinRudpSink::from_expected(&expected).is_err()); + expected.dependencies = vec![ + dependency("other.thing", "rudp://127.0.0.1:1"), + dependency(ODIN_CAPABILITY, "rudp://127.0.0.1:4100"), + ]; + assert_eq!( + OdinRudpSink::from_expected(&expected).unwrap().endpoint, + "127.0.0.1:4100".parse().unwrap() + ); + expected.dependencies = vec![dependency(ODIN_CAPABILITY, "rudp://not-an-address")]; + assert!(OdinRudpSink::from_expected(&expected).is_err()); + } + + #[test] + fn half_closed_member_requests_still_reach_a_child_that_waits_for_eof() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("child.sock"); + let child = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + let fake = thread::spawn(move || { + let (mut stream, _) = child.accept().unwrap(); + let mut received = Vec::new(); + stream.read_to_end(&mut received).unwrap(); + stream.write_all(&received.iter().rev().copied().collect::>()).unwrap(); + }); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let halt = Arc::new(AtomicBool::new(false)); + let responder: Responder = Arc::new(|_| bail!("never")); + let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder); + let serving = { + let halt = halt.clone(); + thread::spawn(move || acceptor.serve(halt)) + }; + let mut member = TcpStream::connect(address).unwrap(); + member.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + member.write_all(b"GET /abc").unwrap(); + member.shutdown(Shutdown::Write).unwrap(); + let mut answer = Vec::new(); + member.read_to_end(&mut answer).unwrap(); + assert_eq!(answer, b"cba/ TEG"); + fake.join().unwrap(); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn heartbeats_are_paced_and_a_stop_exits_zero() { + let fx = fixture(true); + let launch = start(&fx, fx.contract("x"), true); + thread::sleep(Duration::from_millis(250)); + let waiting = launch + .sink + .0 + .lock() + .unwrap() + .iter() + .filter(|record| record.detail == "write-lease-pending") + .count(); + assert!((1..=12).contains(&waiting), "{waiting} lease-wait heartbeats"); + let mut launch = launch; + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + thread::sleep(Duration::from_millis(250)); + let beats = launch + .sink + .0 + .lock() + .unwrap() + .iter() + .filter(|record| record.detail == "heartbeat") + .count(); + assert!((1..=12).contains(&beats), "{beats} active heartbeats"); + assert_eq!(launch.stop().exit_code(), 0); + } + + #[test] + fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { + let mut stubborn = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()]) + .unwrap(); + thread::sleep(Duration::from_millis(200)); + let started = Instant::now(); + stubborn.terminate(Duration::from_millis(400)); + assert!(started.elapsed() >= Duration::from_millis(400), "SIGTERM gets its grace period"); + assert!(stubborn.exited().is_some(), "then SIGKILL ends it"); + } + #[test] fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { let mut gone = ProcessSpawner From d36786f76c61cae332f80cef5a171463d6f613ca Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 23:16:42 +0200 Subject: [PATCH 056/122] idunn-launch: credentials only under CREDENTIALS_DIRECTORY, bounded route acceptor, lease lock inherited by the child, startup probe rules pinned Soul s1 fixes. Credential placeholders resolve files under $CREDENTIALS_DIRECTORY only, never an environment variable. The route acceptor has a first-byte deadline, a total challenge deadline and a connection cap with unspawned refusal; a dead acceptor degrades presence. The shared lease lock is a plain descriptor the child inherits, so the lock lives as long as any writer. Tests pin a late-healthy child and a hung probe. One commit: the spawner and lease trait signatures change together with their fakes. Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/bin/idunn-launch.rs | 3 +- src/launcher.rs | 715 ++++++++++++++++++++++++++++++++++------ 2 files changed, 624 insertions(+), 94 deletions(-) diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs index b743f06..41b6c76 100644 --- a/src/bin/idunn-launch.rs +++ b/src/bin/idunn-launch.rs @@ -42,11 +42,12 @@ fn main() -> std::process::ExitCode { ); let health_path = contract.health_path.clone(); let ports = Ports { - credentials: std::sync::Arc::new(EnvCredentials), + credentials: std::sync::Arc::new(CredentialsDirectory::from_environment()), clock: std::sync::Arc::new(SystemClock), sink: std::sync::Arc::new(sink), lease: std::sync::Arc::new(lease), spawner: Box::new(ProcessSpawner), + route_bounds: RouteBounds::default(), health: Box::new(move |endpoint| { std::sync::Arc::new(HttpHealthProbe { endpoint: endpoint.clone(), diff --git a/src/launcher.rs b/src/launcher.rs index b1eeb69..cf34c9d 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -15,12 +15,14 @@ use std::collections::VecDeque; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::fd::{AsRawFd, RawFd}; use std::os::unix::fs::OpenOptionsExt; use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt; use std::path::{Component, Path, PathBuf}; use std::process::{Child, Command, Stdio}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, mpsc}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -48,6 +50,12 @@ use crate::drivers::{CultCacheWriteLeaseDriver, lease_binds_candidate}; const CHALLENGE_FIRST_BYTE: u8 = 0x00; const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; const CHALLENGE_IO_TIMEOUT: Duration = Duration::from_secs(3); +/// How long a public connection may stay silent before its first byte. +const FIRST_BYTE_DEADLINE: Duration = Duration::from_secs(5); +/// A whole challenge, from its first byte to its answer, however it trickles. +const CHALLENGE_DEADLINE: Duration = Duration::from_secs(5); +/// Concurrent connections the acceptor serves; the rest are refused unspawned. +const MAX_CONNECTIONS: usize = 256; const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); const POLL: Duration = Duration::from_millis(100); const MAX_RECENT_WARMING_PROOFS: usize = 64; @@ -398,24 +406,44 @@ impl Clock for SystemClock { } } -/// Systemd credentials as Idunn binds them: the environment variable `NAME` -/// holds the path of the credential file. +/// Systemd credentials as Idunn binds them: the files `NAME` under the unit's +/// credentials directory. pub trait CredentialSource: Send + Sync { fn contents(&self, name: &str) -> Result>; fn path(&self, name: &str) -> Result; } -pub struct EnvCredentials; +/// Resolves a credential only as a file directly under the credentials +/// directory systemd gives the unit (`$CREDENTIALS_DIRECTORY`, which Idunn +/// pins to `/run/credentials/`). The process environment is never a +/// credential namespace: a name is a plain file name, never a variable. +pub struct CredentialsDirectory(Option); -impl CredentialSource for EnvCredentials { +impl CredentialsDirectory { + pub fn new(directory: PathBuf) -> Self { + Self(Some(directory)) + } + + /// The directory systemd exported; no directory means no credential is bound. + pub fn from_environment() -> Self { + Self(std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from)) + } +} + +impl CredentialSource for CredentialsDirectory { fn contents(&self, name: &str) -> Result> { fs::read(self.path(name)?).map_err(|_| anyhow!("a credential could not be read")) } fn path(&self, name: &str) -> Result { - std::env::var_os(name) - .map(PathBuf::from) - .ok_or_else(|| anyhow!("a credential is not bound")) + ensure!(valid_credential_name(name), "credential name is invalid"); + let path = self + .0 + .as_ref() + .ok_or_else(|| anyhow!("a credential is not bound"))? + .join(name); + ensure!(fs::symlink_metadata(&path).is_ok(), "a credential is not bound"); + Ok(path) } } @@ -428,7 +456,14 @@ pub trait PresenceSink: Send + Sync { /// its store that Idunn's revocation cannot take while the child lives. pub trait LeaseReader: Send + Sync { fn read(&self) -> Result>; - fn hold_shared(&self) -> Result>; + fn hold_shared(&self) -> Result>; +} + +/// The shared lock on the lease store, held until dropped. Its descriptor is +/// handed to the child (`ChildSpawner::spawn`), so the lock lives as long as +/// the launcher or any process that inherited it. +pub trait LeaseLock: Send { + fn descriptor(&self) -> Option; } pub trait HealthProbe: Send + Sync { @@ -443,7 +478,14 @@ pub trait ChildHandle: Send { } pub trait ChildSpawner: Send { - fn spawn(&self, program: &Path, args: &[String]) -> Result>; + /// `inherit` is a descriptor the child keeps open across exec (the lease + /// lock); every other descriptor of the launcher is close-on-exec. + fn spawn( + &self, + program: &Path, + args: &[String], + inherit: Option, + ) -> Result>; } // ---------------------------------------------------------------- production ports @@ -460,17 +502,11 @@ impl DriverLeaseReader { } } -struct SharedLockGuard { - release: Option>, - worker: Option>, -} +struct SharedLockFile(File); -impl Drop for SharedLockGuard { - fn drop(&mut self) { - drop(self.release.take()); - if let Some(worker) = self.worker.take() { - let _ = worker.join(); - } +impl LeaseLock for SharedLockFile { + fn descriptor(&self) -> Option { + Some(self.0.as_raw_fd()) } } @@ -482,31 +518,20 @@ impl LeaseReader for DriverLeaseReader { .map_err(|_| anyhow!("write-lease store refused custody, envelope or target")) } - fn hold_shared(&self) -> Result> { - let path = self.driver.record_path.clone(); - let (ready_tx, ready_rx) = mpsc::sync_channel::(1); - let (release_tx, release_rx) = mpsc::channel::<()>(); - let worker = thread::spawn(move || { - let held = SingleFileMessagePackBackingStore::new(&path).with_read_only_shared_snapshot( - |_| { - let _ = ready_tx.send(true); - // Returns when the guard drops its sender. - let _ = release_rx.recv(); - Ok(()) - }, - ); - if held.is_err() { - let _ = ready_tx.send(false); - } - }); - if ready_rx.recv() != Ok(true) { - let _ = worker.join(); - bail!("write-lease lock could not be held"); - } - Ok(Box::new(SharedLockGuard { - release: Some(release_tx), - worker: Some(worker), - })) + /// LOCK_SH on `.lock`, the lock Idunn's revocation respects. The + /// descriptor is a plain file, so flock's lock belongs to its open file + /// description: a child that inherits it keeps the lock after the launcher + /// dies. + fn hold_shared(&self) -> Result> { + let mut lock = self.driver.record_path.clone().into_os_string(); + lock.push(".lock"); + let file = File::open(&lock).map_err(|_| anyhow!("write-lease lock could not be held"))?; + // SAFETY: flock on a descriptor this function owns. + ensure!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0, + "write-lease lock could not be held" + ); + Ok(Box::new(SharedLockFile(file))) } } @@ -727,7 +752,12 @@ pub struct ProcessSpawner; struct ProcessChild(Child); impl ChildSpawner for ProcessSpawner { - fn spawn(&self, program: &Path, args: &[String]) -> Result> { + fn spawn( + &self, + program: &Path, + args: &[String], + inherit: Option, + ) -> Result> { let mut command = Command::new(program); // The child gets a bare environment: no Idunn variable, no candidate // bind, no credential path leaks into it. @@ -737,6 +767,18 @@ impl ChildSpawner for ProcessSpawner { command.env(name, value); } } + if let Some(descriptor) = inherit { + // SAFETY: runs in the forked child before exec; fcntl is + // async-signal-safe and the parent's descriptor stays close-on-exec. + unsafe { + command.pre_exec(move || { + if libc::fcntl(descriptor, libc::F_SETFD, 0) == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + } let child = command.spawn().map_err(|_| anyhow!("the child could not be spawned"))?; Ok(Box::new(ProcessChild(child))) } @@ -849,7 +891,7 @@ pub struct LeaseHold { reader: Arc, lease: IdunnProcessWriteLeaseRecord, sha256: String, - _lock: Box, + lock: Box, } impl LeaseHold { @@ -871,7 +913,7 @@ impl LeaseHold { sha256: lease.canonical_sha256()?, reader, lease, - _lock: lock, + lock, }) } @@ -879,6 +921,11 @@ impl LeaseHold { &self.sha256 } + /// The locked descriptor, for the child to inherit. + pub fn lock_descriptor(&self) -> Option { + self.lock.descriptor() + } + pub fn still_current(&self) -> bool { matches!(self.reader.read(), Ok(Some(current)) if current == self.lease) } @@ -901,9 +948,21 @@ struct Shared { lease_required: bool, health: Arc, clock: Arc, + /// False once the route acceptor has stopped for any reason but a halt: + /// the launcher cannot answer Idunn's challenge, so it must not say active. + acceptor_alive: AtomicBool, } impl Shared { + /// Active only while the child is healthy and the route is being served. + fn serving_state(&self) -> &'static str { + if self.health.healthy() && self.acceptor_alive.load(Ordering::SeqCst) { + "active" + } else { + "degraded" + } + } + fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { self.authority .lock() @@ -931,8 +990,8 @@ impl Shared { Phase::Warming => self.sign("warming", &detail, None)?, Phase::Failed => self.sign("failed", &detail, None)?, Phase::Active(lease) => { - let state = if self.health.healthy() && self.lease_current() { - "active" + let state = if self.lease_current() { + self.serving_state() } else { "degraded" }; @@ -948,39 +1007,90 @@ impl Shared { type Responder = Arc Result + Send + Sync>; +/// Bounds on what the public route endpoint spends on one peer. +#[derive(Clone, Copy, Debug)] +pub struct RouteBounds { + pub first_byte: Duration, + pub challenge: Duration, + pub max_connections: usize, +} + +impl Default for RouteBounds { + fn default() -> Self { + Self { + first_byte: FIRST_BYTE_DEADLINE, + challenge: CHALLENGE_DEADLINE, + max_connections: MAX_CONNECTIONS, + } + } +} + +/// Counts a live connection; dropping it frees the slot. +struct Slot(Arc); + +impl Drop for Slot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + /// Owns the candidate endpoint. A leading 0x00 is an Idunn challenge frame and -/// is answered here; every other byte is copied untouched to the child. +/// is answered here; every other byte is copied untouched to the child. The +/// endpoint is public, so it spends bounded resources on a peer: a first-byte +/// deadline, a total challenge deadline, and a cap on live connections whose +/// overflow is closed without a thread. pub struct RouteAcceptor { listener: TcpListener, child: ChildEndpoint, responder: Responder, + bounds: RouteBounds, + live: Arc, } impl RouteAcceptor { - pub fn new(listener: TcpListener, child: ChildEndpoint, responder: Responder) -> Self { + pub fn new(listener: TcpListener, child: ChildEndpoint, responder: Responder, bounds: RouteBounds) -> Self { Self { listener, child, responder, + bounds, + live: Arc::new(AtomicUsize::new(0)), } } + /// Serves until `halt`. Any other end is an error: a failed accept or a + /// thread that cannot be created means the route is no longer served. pub fn serve(self, halt: Arc) -> Result<()> { self.listener.set_nonblocking(true)?; let acceptor = Arc::new(self); while !halt.load(Ordering::SeqCst) { match acceptor.listener.accept() { Ok((connection, _)) => { - let acceptor = Arc::clone(&acceptor); - thread::spawn(move || { - let _ = connection.set_nonblocking(false); - let _ = acceptor.handle(connection); - }); + if acceptor.live.fetch_add(1, Ordering::SeqCst) >= acceptor.bounds.max_connections { + acceptor.live.fetch_sub(1, Ordering::SeqCst); + continue; + } + let slot = Slot(Arc::clone(&acceptor.live)); + let worker = Arc::clone(&acceptor); + thread::Builder::new() + .spawn(move || { + let _slot = slot; + let _ = connection.set_nonblocking(false); + let _ = worker.handle(connection); + }) + .map_err(|_| anyhow!("a route connection thread could not be created"))?; } - Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock + | io::ErrorKind::Interrupted + | io::ErrorKind::ConnectionAborted + ) => + { thread::sleep(Duration::from_millis(10)); } - Err(_) => thread::sleep(Duration::from_millis(10)), + Err(_) => bail!("the route listener could not accept"), } } Ok(()) @@ -988,9 +1098,7 @@ impl RouteAcceptor { fn handle(&self, mut connection: TcpStream) -> Result<()> { let mut first = [0_u8; 1]; - if connection.read(&mut first)? == 0 { - return Ok(()); - } + read_by(&mut connection, &mut first, Instant::now() + self.bounds.first_byte)?; if first[0] == CHALLENGE_FIRST_BYTE { return self.answer_challenge(&mut connection); } @@ -998,10 +1106,15 @@ impl RouteAcceptor { upstream.write_all(&first)?; let mut upstream_reader = upstream.try_clone()?; let mut client_reader = connection.try_clone()?; - let forward = thread::spawn(move || { - let _ = io::copy(&mut client_reader, &mut upstream); - upstream.shutdown_write(); - }); + let forward = thread::Builder::new() + .spawn(move || { + let _ = io::copy(&mut client_reader, &mut upstream); + upstream.shutdown_write(); + }) + .map_err(|_| anyhow!("a route splice thread could not be created"))?; + // The first-byte deadline bounded the silent peer; a spliced session + // runs as long as the member and the child keep it. + connection.set_read_timeout(None)?; let _ = io::copy(&mut upstream_reader, &mut connection); let _ = connection.shutdown(Shutdown::Write); let _ = forward.join(); @@ -1009,17 +1122,19 @@ impl RouteAcceptor { } fn answer_challenge(&self, connection: &mut TcpStream) -> Result<()> { - connection.set_read_timeout(Some(CHALLENGE_IO_TIMEOUT))?; + // The challenge began with its first byte; the rest must arrive within + // the total deadline, however slowly the peer sends it. + let deadline = Instant::now() + self.bounds.challenge; connection.set_write_timeout(Some(CHALLENGE_IO_TIMEOUT))?; let mut rest = [0_u8; 3]; - connection.read_exact(&mut rest)?; + read_by(connection, &mut rest, deadline)?; let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; ensure!( (1..=CHALLENGE_MAX_BYTES).contains(&length), "challenge frame is outside its bound" ); let mut payload = vec![0_u8; length]; - connection.read_exact(&mut payload)?; + read_by(connection, &mut payload, deadline)?; let request = decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; let response = (self.responder)(request)?; @@ -1033,6 +1148,25 @@ impl RouteAcceptor { } } +/// Fills `buffer` or fails when `deadline` passes, however the peer paces it. +fn read_by(connection: &mut TcpStream, buffer: &mut [u8], deadline: Instant) -> Result<()> { + let mut filled = 0; + while filled < buffer.len() { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .context("the peer did not send in time")?; + connection.set_read_timeout(Some(remaining))?; + match connection.read(&mut buffer[filled..]) { + Ok(0) => bail!("the peer closed early"), + Ok(count) => filled += count, + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(_) => bail!("the peer did not send in time"), + } + } + Ok(()) +} + // ---------------------------------------------------------------- lifecycle pub struct Ports { @@ -1041,6 +1175,7 @@ pub struct Ports { pub sink: Arc, pub lease: Arc, pub spawner: Box, + pub route_bounds: RouteBounds, /// Built from the endpoint `run` chooses. pub health: Box Arc + Send>, } @@ -1099,6 +1234,7 @@ pub fn run( lease_required: expected.write_lease_required, health: Arc::clone(&health), clock: Arc::clone(&ports.clock), + acceptor_alive: AtomicBool::new(true), }); let halt = Arc::new(AtomicBool::new(false)); let responder: Responder = { @@ -1106,9 +1242,16 @@ pub fn run( Arc::new(move |request| shared.observe(request)) }; let serving = { - let acceptor = RouteAcceptor::new(route, endpoint.clone(), responder); + let acceptor = RouteAcceptor::new(route, endpoint.clone(), responder, ports.route_bounds); let halt = Arc::clone(&halt); - thread::spawn(move || acceptor.serve(halt)) + let shared = Arc::clone(&shared); + thread::spawn(move || { + let served = acceptor.serve(halt); + if served.is_err() { + shared.acceptor_alive.store(false, Ordering::SeqCst); + } + served + }) }; let outcome = lifecycle(contract, &expected, &activation, &endpoint, tmp, &ports, &shared, stop); halt.store(true, Ordering::SeqCst); @@ -1166,6 +1309,7 @@ fn lifecycle( // 2. A stateful child waits for the lease; a stateless one does not. let mut lease_sha = None; + let mut lease_fd = None; if expected.write_lease_required { let deadline = clock.now_millis() + contract.lease_wait.as_millis() as u64; loop { @@ -1180,6 +1324,7 @@ fn lifecycle( ); if let Ok(hold) = held { lease_sha = Some(hold.sha256().to_owned()); + lease_fd = hold.lock_descriptor(); if let Ok(mut slot) = shared.lease.lock() { *slot = Some(hold); } @@ -1198,7 +1343,7 @@ fn lifecycle( } // 3. Render, spawn, and wait for health. - let Some(mut child) = start_child(contract, endpoint, tmp, ports) else { + let Some(mut child) = start_child(contract, endpoint, tmp, ports, lease_fd) else { return fail(None, "child-start-refused"); }; let startup_deadline = clock.now_millis() + contract.startup.as_millis() as u64; @@ -1221,7 +1366,7 @@ fn lifecycle( // 4. Active, and heartbeats while it holds. set_phase(Phase::Active(lease_sha.clone())); - publish("active", "child-healthy", lease_sha.clone()); + publish(shared.serving_state(), "child-healthy", lease_sha.clone()); next_heartbeat = clock.now_millis() + heartbeat; loop { if stop.load(Ordering::SeqCst) { @@ -1236,8 +1381,7 @@ fn lifecycle( } let now = clock.now_millis(); if now >= next_heartbeat { - let state = if shared.health.healthy() { "active" } else { "degraded" }; - publish(state, "heartbeat", lease_sha.clone()); + publish(shared.serving_state(), "heartbeat", lease_sha.clone()); next_heartbeat = now + heartbeat; } clock.sleep(POLL); @@ -1249,6 +1393,7 @@ fn start_child( endpoint: &ChildEndpoint, tmp: &Path, ports: &Ports, + lease_fd: Option, ) -> Option> { let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; let listen = endpoint.listen_value(); @@ -1267,7 +1412,7 @@ fn start_child( } ports .spawner - .spawn(&contract.release_root.join(&contract.child), &args) + .spawn(&contract.release_root.join(&contract.child), &args, lease_fd) .ok() } @@ -1276,6 +1421,7 @@ mod tests { use std::collections::BTreeMap; use std::os::unix::fs::PermissionsExt; use std::sync::atomic::{AtomicU64, AtomicUsize}; + use std::sync::mpsc; use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; use cultnet_rs::{ @@ -1498,8 +1644,19 @@ mod tests { Ok(self.0.lock().unwrap().clone()) } - fn hold_shared(&self) -> Result> { - Ok(Box::new(())) + fn hold_shared(&self) -> Result> { + Ok(Box::new(FakeLock)) + } + } + + /// The descriptor number a fake lock claims; the launcher must hand it on. + const FAKE_LOCK_FD: RawFd = 1234; + + struct FakeLock; + + impl LeaseLock for FakeLock { + fn descriptor(&self) -> Option { + Some(FAKE_LOCK_FD) } } @@ -1517,6 +1674,7 @@ mod tests { exit: Mutex>, terminated: AtomicBool, args: Mutex>>, + inherited: Mutex>>, } struct FakeChild(Arc); @@ -1535,7 +1693,13 @@ mod tests { struct FakeSpawner(Arc); impl ChildSpawner for FakeSpawner { - fn spawn(&self, _program: &Path, args: &[String]) -> Result> { + fn spawn( + &self, + _program: &Path, + args: &[String], + inherit: Option, + ) -> Result> { + self.0.inherited.lock().unwrap().push(inherit); self.0.spawns.fetch_add(1, Ordering::SeqCst); self.0.args.lock().unwrap().push(args.to_vec()); Ok(Box::new(FakeChild(Arc::clone(&self.0)))) @@ -1579,6 +1743,7 @@ mod tests { sink: sink.clone(), lease: lease.clone(), spawner: Box::new(FakeSpawner(child.clone())), + route_bounds: RouteBounds::default(), health: Box::new(move |_| health.clone()), }; let authority = fx.authority(); @@ -1779,6 +1944,7 @@ mod tests { lease_required: false, health, clock: Arc::new(SystemClock), + acceptor_alive: AtomicBool::new(true), }) } @@ -1820,6 +1986,7 @@ mod tests { listener, ChildEndpoint::Unix("/nonexistent".into()), responder, + RouteBounds::default(), ); let serving = { let halt = halt.clone(); @@ -1898,7 +2065,7 @@ mod tests { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let halt = Arc::new(AtomicBool::new(false)); - let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder); + let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder, RouteBounds::default()); let serving = { let halt = halt.clone(); thread::spawn(move || acceptor.serve(halt)) @@ -2070,15 +2237,51 @@ mod tests { } #[test] - fn env_credentials_resolve_through_the_variable_idunn_sets() { + fn credentials_resolve_only_as_files_under_the_credentials_directory() { let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("cred"); - fs::write(&file, b"abc\n").unwrap(); - // SAFETY: a variable no other test reads. - unsafe { std::env::set_var("IDUNN_LAUNCH_TEST_CREDENTIAL", &file) }; - assert_eq!(EnvCredentials.path("IDUNN_LAUNCH_TEST_CREDENTIAL").unwrap(), file); - assert_eq!(EnvCredentials.contents("IDUNN_LAUNCH_TEST_CREDENTIAL").unwrap(), b"abc\n"); - assert!(EnvCredentials.path("IDUNN_LAUNCH_TEST_UNBOUND").is_err()); + fs::write(dir.path().join("cred"), b"abc\n").unwrap(); + let credentials = CredentialsDirectory::new(dir.path().to_owned()); + assert_eq!(credentials.path("cred").unwrap(), dir.path().join("cred")); + assert_eq!(credentials.contents("cred").unwrap(), b"abc\n"); + assert!(credentials.path("unbound").is_err()); + for bad in ["", "..", "../cred", "a/b", ".", "a.b"] { + assert!(credentials.path(bad).is_err(), "{bad}"); + assert!(credentials.contents(bad).is_err(), "{bad}"); + } + assert!(CredentialsDirectory(None).path("cred").is_err()); + } + + #[test] + fn credential_placeholders_never_read_the_process_environment() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("elsewhere"); + fs::write(&file, b"canary-file\n").unwrap(); + // SAFETY: variables no other test reads. + unsafe { + std::env::set_var("GAMECULT_IDUNN_CANDIDATE_BIND", "127.0.0.1:18835"); + std::env::set_var("IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", &file); + } + let credentials = CredentialsDirectory::new(dir.path().to_owned()); + let renderer = ConfigRenderer { credentials: &credentials, listen: "L", state_root: None }; + for name in [ + "GAMECULT_IDUNN_CANDIDATE_BIND", + "IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", + "../elsewhere", + "a/b", + "..", + ] { + for kind in ["credential", "credential_path"] { + let error = renderer.render(&format!("x={{{{{kind}:{name}}}}}")); + assert!(error.is_err(), "{kind}:{name}"); + assert!(!format!("{:#}", error.unwrap_err()).contains("18835")); + } + } + // The same name resolves when it is a credential file, never otherwise. + fs::write(dir.path().join("GAMECULT_IDUNN_CANDIDATE_BIND"), b"file-value\n").unwrap(); + assert_eq!( + renderer.render("{{credential:GAMECULT_IDUNN_CANDIDATE_BIND}}").unwrap(), + "file-value" + ); } #[test] @@ -2127,7 +2330,7 @@ mod tests { let address = listener.local_addr().unwrap(); let halt = Arc::new(AtomicBool::new(false)); let responder: Responder = Arc::new(|_| bail!("never")); - let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder); + let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder, RouteBounds::default()); let serving = { let halt = halt.clone(); thread::spawn(move || acceptor.serve(halt)) @@ -2178,7 +2381,7 @@ mod tests { #[test] fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { let mut stubborn = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()]) + .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()], None) .unwrap(); thread::sleep(Duration::from_millis(200)); let started = Instant::now(); @@ -2190,15 +2393,341 @@ mod tests { #[test] fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { let mut gone = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()]) + .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()], None) .unwrap(); wait_until("child exit", || gone.exited().is_some()); assert_eq!(gone.exited(), Some(3)); let mut running = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()]) + .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()], None) .unwrap(); assert_eq!(running.exited(), None); running.terminate(Duration::from_secs(5)); assert!(running.exited().is_some()); } + + // ------------------------------------------------------------ Soul s1 fixes + + fn serve_with( + fx: &Fixture, + bounds: RouteBounds, + ) -> (SocketAddr, Arc, thread::JoinHandle>) { + let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let responder: Responder = Arc::new(move |request| shared.observe(request)); + let acceptor = + RouteAcceptor::new(listener, ChildEndpoint::Unix("/nonexistent".into()), responder, bounds); + let halt = Arc::new(AtomicBool::new(false)); + let flag = halt.clone(); + (address, halt, thread::spawn(move || acceptor.serve(flag))) + } + + fn quick(first_byte_ms: u64, challenge_ms: u64, max_connections: usize) -> RouteBounds { + RouteBounds { + first_byte: Duration::from_millis(first_byte_ms), + challenge: Duration::from_millis(challenge_ms), + max_connections, + } + } + + /// True when the peer closed (EOF or reset) within `within`. + fn closed_within(stream: &mut TcpStream, within: Duration) -> bool { + stream.set_read_timeout(Some(within)).unwrap(); + let mut byte = [0_u8; 64]; + match stream.read(&mut byte) { + Ok(0) => true, + Ok(_) => false, + Err(error) => !matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut), + } + } + + fn challenge_answered(address: SocketAddr, fx: &Fixture) -> bool { + let request = CultNetMessage::SnapshotRequest { + message_id: "bounded".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + request_tcp_snapshot(address, &payload).is_ok() + } + + #[test] + fn an_idle_connection_is_closed_at_the_first_byte_deadline() { + let fx = fixture(false); + let (address, halt, serving) = serve_with(&fx, quick(300, 5000, 64)); + let mut idle = TcpStream::connect(address).unwrap(); + let started = Instant::now(); + assert!(closed_within(&mut idle, Duration::from_secs(4)), "never closed"); + assert!(started.elapsed() >= Duration::from_millis(250), "closed before its deadline"); + assert!(challenge_answered(address, &fx)); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_its_total_deadline() { + let fx = fixture(false); + // Every pause is far inside the first-byte deadline: only the total + // challenge deadline can end this. + let (address, halt, serving) = serve_with(&fx, quick(5000, 600, 64)); + let mut slow = TcpStream::connect(address).unwrap(); + slow.write_all(&[0x00, 0x00, 0x01, 0x00]).unwrap(); + let started = Instant::now(); + let mut closed = false; + while started.elapsed() < Duration::from_secs(4) { + if closed_within(&mut slow, Duration::from_millis(150)) || slow.write_all(&[0x01]).is_err() { + closed = true; + break; + } + } + assert!(closed, "the trickle was never cut off"); + assert!(started.elapsed() >= Duration::from_millis(500), "cut before its deadline"); + assert!(challenge_answered(address, &fx)); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + #[test] + fn live_connections_are_capped_and_the_overflow_is_refused_unspawned() { + let fx = fixture(false); + let (address, halt, serving) = serve_with(&fx, quick(30_000, 5000, 4)); + let mut held: Vec = (0..4).map(|_| TcpStream::connect(address).unwrap()).collect(); + thread::sleep(Duration::from_millis(300)); + let threads_before = thread_count(); + let mut overflow: Vec = (0..40).map(|_| TcpStream::connect(address).unwrap()).collect(); + for stream in &mut overflow { + assert!(closed_within(stream, Duration::from_secs(3)), "an overflow connection was served"); + } + assert!(thread_count() < threads_before + 8, "overflow connections cost threads"); + for stream in &mut held { + assert!(!closed_within(stream, Duration::from_millis(100)), "a held connection was cut"); + } + // A freed slot serves the next peer. + held.pop(); + let mut answered = false; + for _ in 0..40 { + if challenge_answered(address, &fx) { + answered = true; + break; + } + thread::sleep(Duration::from_millis(50)); + } + assert!(answered, "a freed slot was never reused"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap().unwrap(); + } + + fn thread_count() -> usize { + fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("Threads:")) + .unwrap() + .trim() + .parse() + .unwrap() + } + + /// A listener whose accept always fails: a datagram socket's descriptor. + fn failing_listener() -> TcpListener { + use std::os::fd::{FromRawFd, IntoRawFd}; + let datagram = UdpSocket::bind("127.0.0.1:0").unwrap(); + // SAFETY: the descriptor is owned by the new listener alone. + unsafe { TcpListener::from_raw_fd(datagram.into_raw_fd()) } + } + + #[test] + fn an_acceptor_that_cannot_accept_ends_with_an_error() { + let fx = fixture(false); + let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); + let responder: Responder = Arc::new(move |request| shared.observe(request)); + let acceptor = RouteAcceptor::new( + failing_listener(), + ChildEndpoint::Unix("/nonexistent".into()), + responder, + RouteBounds::default(), + ); + assert!(acceptor.serve(Arc::new(AtomicBool::new(false))).is_err()); + } + + #[test] + fn presence_stops_claiming_active_when_the_route_acceptor_dies() { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let health = Arc::new(Health(AtomicBool::new(true))); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(child)), + route_bounds: RouteBounds::default(), + health: Box::new(move |_| health.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = thread::spawn(move || run(&contract, authority, failing_listener(), &tmp, ports, &flag)); + wait_until("two degraded heartbeats", || { + sink.states().iter().filter(|state| *state == "degraded").count() >= 2 + }); + // The child is healthy throughout; only the dead acceptor degrades it, + // and nothing says active after that. + let states = sink.states(); + let first_degraded = states.iter().position(|state| state == "degraded").unwrap(); + assert!(states[first_degraded..].iter().all(|state| state != "active"), "{states:?}"); + stop.store(true, Ordering::SeqCst); + assert_eq!(handle.join().unwrap(), Outcome::Stopped); + } + + #[test] + fn a_live_acceptor_keeps_presence_active() { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("two active heartbeats", || { + launch.sink.states().iter().filter(|state| *state == "active").count() >= 2 + }); + assert!(launch.sink.first("degraded").is_none()); + launch.stop(); + } + + /// A store with a lease for `fx`, and Idunn's own driver over it. + fn lease_store( + fx: &Fixture, + name: &str, + ) -> (DriverLeaseReader, CultCacheWriteLeaseDriver, IdunnProcessWriteLeaseRecord) { + let dir = fx.dir.path().join(name); + fs::create_dir(&dir).unwrap(); + let path = dir.join("lease.cc"); + let lease = fx.lease(&digest('a')); + write_record( + &path, + &lease.target, + IdunnProcessWriteLeaseRecord::TYPE, + cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes().unwrap(), + ); + fs::write(dir.join("lease.cc.lock"), b"").unwrap(); + ( + DriverLeaseReader::new(&lease.target, &path), + CultCacheWriteLeaseDriver::new(&lease.target, &path), + lease, + ) + } + + #[test] + fn a_child_that_inherits_the_lease_lock_holds_it_after_the_launcher_is_gone() { + let fx = fixture(true); + let (reader, idunn, lease) = lease_store(&fx, "inherited"); + let guard = reader.hold_shared().unwrap(); + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], guard.descriptor()) + .unwrap(); + // The launcher dies: its own descriptor goes with it. + drop(guard); + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); + child.terminate(Duration::from_secs(5)); + wait_until("child gone", || child.exited().is_some()); + idunn.revoke_exact(Some(&lease)).unwrap(); + } + + #[test] + fn a_child_spawned_without_the_descriptor_leaves_the_lock_with_the_launcher() { + let fx = fixture(true); + let (reader, idunn, lease) = lease_store(&fx, "uninherited"); + let guard = reader.hold_shared().unwrap(); + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], None) + .unwrap(); + drop(guard); + idunn.revoke_exact(Some(&lease)).unwrap(); + child.terminate(Duration::from_secs(5)); + } + + #[test] + fn the_launcher_hands_its_lease_descriptor_to_the_child_it_spawns() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("warming", || launch.sink.first("warming").is_some()); + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(*launch.child.inherited.lock().unwrap(), [Some(FAKE_LOCK_FD)]); + launch.stop(); + + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(*launch.child.inherited.lock().unwrap(), [None]); + launch.stop(); + } + + /// Unhealthy for its first `probes` calls, healthy from then on. + struct LateHealth { + calls: AtomicUsize, + probes: usize, + } + + impl HealthProbe for LateHealth { + fn healthy(&self) -> bool { + self.calls.fetch_add(1, Ordering::SeqCst) >= self.probes + } + } + + #[test] + fn a_child_that_becomes_healthy_late_is_not_killed_at_its_first_probe() { + let fx = fixture(false); + let mut contract = fx.contract("x"); + contract.startup = Duration::from_secs(60); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let health = Arc::new(LateHealth { calls: AtomicUsize::new(0), probes: 5 }); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(child.clone())), + route_bounds: RouteBounds::default(), + health: Box::new(move |_| health.clone()), + }; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = thread::spawn(move || run(&contract, authority, route, &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert!(!child.terminated.load(Ordering::SeqCst), "killed while it was still starting"); + assert_eq!(child.spawns.load(Ordering::SeqCst), 1); + assert!(sink.first("failed").is_none()); + stop.store(true, Ordering::SeqCst); + assert_eq!(handle.join().unwrap(), Outcome::Stopped); + } + + #[test] + fn a_hung_health_probe_times_out_instead_of_blocking() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = ChildEndpoint::Loopback(listener.local_addr().unwrap()); + // Accepts and never answers. + let held = thread::spawn(move || { + let _connection = listener.accept().unwrap(); + thread::sleep(Duration::from_secs(8)); + }); + let probe = HttpHealthProbe { endpoint, path: "/health".into() }; + let (done, verdict) = mpsc::channel(); + thread::spawn(move || { + let _ = done.send(probe.healthy()); + }); + let healthy = verdict + .recv_timeout(Duration::from_secs(5)) + .expect("the probe outlived its timeout against a hung child"); + assert!(!healthy); + drop(held); + } } From 8246de7c3737aa7ff95d0378863a0977dcfc4400 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 23:26:35 +0200 Subject: [PATCH 057/122] idunn-launch tests: lock-release wait tolerates a concurrent fork; health probe test reads the request line, not a 64-byte read Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index cf34c9d..6c72c0b 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2177,9 +2177,10 @@ mod tests { let server = thread::spawn(move || { for status in ["200 OK", "503 Service Unavailable"] { let (mut stream, _) = listener.accept().unwrap(); - let mut request = [0_u8; 64]; - let read = stream.read(&mut request).unwrap(); - assert!(request[..read].starts_with(b"GET /health HTTP/1.1\r\n")); + let mut line = String::new(); + io::BufRead::read_line(&mut io::BufReader::new(stream.try_clone().unwrap()), &mut line) + .unwrap(); + assert_eq!(line, "GET /health HTTP/1.1\r\n"); write!(stream, "HTTP/1.1 {status}\r\nContent-Length: 2\r\n\r\nOK").unwrap(); } }); @@ -2645,7 +2646,9 @@ mod tests { .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], None) .unwrap(); drop(guard); - idunn.revoke_exact(Some(&lease)).unwrap(); + // Another test's fork may hold a copy for the instant before its exec; + // the child under test would hold it for thirty seconds. + wait_until("lock released", || idunn.revoke_exact(Some(&lease)).is_ok()); child.terminate(Duration::from_secs(5)); } From e2dee7c16a96e179116e6429cb43f081eed3b401 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 00:15:33 +0200 Subject: [PATCH 058/122] idunn-launch tests: the inherited lease lock must stay held for a second, so a concurrent fork cannot mask a missing descriptor Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 6c72c0b..689c35a 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2630,8 +2630,13 @@ mod tests { .unwrap(); // The launcher dies: its own descriptor goes with it. drop(guard); - let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); - assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); + // Held for a full second: a fork of some other test could pin the lock + // for an instant, only the child under test keeps it that long. + for _ in 0..10 { + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); + thread::sleep(Duration::from_millis(100)); + } child.terminate(Duration::from_secs(5)); wait_until("child gone", || child.exited().is_some()); idunn.revoke_exact(Some(&lease)).unwrap(); From 7805c0aa9ac40dcc14821141fb765b2b6fd020ac Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 01:18:26 +0200 Subject: [PATCH 059/122] Idunn routes: an optional [route.member] renders a second nginx server block and sets GAMECULT_IDUNN_MEMBER_BIND; Idunn-owned launch names live in one list The member candidate port is derived from the candidate port; both stable sockets get firewall rules; native targets render the same bytes as before. Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 1 + src/deployment.rs | 337 +++++++++++++++++++++++++++++++++++++------ src/drivers.rs | 256 ++++++++++++++++++++++++++------ 3 files changed, 512 insertions(+), 82 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d2a7635..9597d43 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -10723,6 +10723,7 @@ mod tests { private_port_end, config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), reload_unit: "nginx.service".into(), + member: None, } } diff --git a/src/deployment.rs b/src/deployment.rs index a81dff4..b01deed 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -9,12 +9,27 @@ pub const TARGET_DECLARATION_SCHEMA: &str = "gamecult.idunn.target_declaration.v pub const OPERATOR_BINDING_SCHEMA: &str = "gamecult.idunn.operator_binding.v2"; pub const IDUNN_RUNTIME_BUNDLE_ENVIRONMENT: &str = "GAMECULT_IDUNN_RUNTIME_BUNDLE"; pub const IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_CANDIDATE_BIND"; +pub const IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_MEMBER_BIND"; pub const IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT: &str = "GAMECULT_IDUNN_PROCESS_WRITE_LEASE"; pub const RUNTIME_PRESENCE_IDENTITY_BINDING: &str = "GAMECULT_RUNTIME_PRESENCE_IDENTITY"; /// A host actuator hands the workload its activation credential as a file /// named by this variable; on the Idunn host it arrives as a parent-only /// descriptor instead and this name is never set. pub const IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT: &str = "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL"; +/// systemd hands a unit its credential directory under this name; Idunn owns +/// the directory, so no binding may set it. +pub const SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT: &str = "CREDENTIALS_DIRECTORY"; +/// Every launch-environment name Idunn sets or reserves. A binding may name +/// none of them; this is the one list both the binding check and the +/// launch-input check read. +pub(crate) const IDUNN_OWNED_LAUNCH_ENVIRONMENT: [&str; 6] = [ + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT, +]; pub const RUNTIME_PRESENCE_IDENTITY_FD_NAME: &str = "gamecult-runtime-presence-identity"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -509,8 +524,7 @@ impl WorkloadBinding { { require_environment_name(name)?; ensure!( - name != IDUNN_RUNTIME_BUNDLE_ENVIRONMENT - && name != IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + !IDUNN_OWNED_LAUNCH_ENVIRONMENT.contains(&name.as_str()), "operator binding attempts to replace Idunn-owned workload environment" ); } @@ -667,40 +681,87 @@ pub struct RouteBinding { pub private_port_end: u16, pub config_path: PathBuf, pub reload_unit: String, + /// A second listener for the route's members. When set, members reach the + /// workload straight through a second nginx server block, and the + /// workload's launcher answers presence only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub member: Option, +} + +/// The member listener of a route: where members connect, and the first of the +/// private ports the workload binds for them. The private range is as wide as +/// the route's own. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemberListenerBinding { + pub stable_endpoint: String, + pub private_port_start: u16, } impl RouteBinding { pub(crate) fn stable_socket(&self) -> Result<(IpAddr, u16)> { - let authority = match self.driver { - RouteDriver::NginxStreamTcp => self - .stable_endpoint - .strip_prefix("http://") - .or_else(|| self.stable_endpoint.strip_prefix("tcp://")), - RouteDriver::NginxStreamUdp => self.stable_endpoint.strip_prefix("rudp://"), - } - .context("stable endpoint transport does not match its route driver")?; - ensure!( - !authority - .chars() - .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), - "stable endpoint is not a plain IP address and port" - ); - let (host, port) = authority - .rsplit_once(':') - .context("stable endpoint has no port")?; - let host = host - .parse::() - .context("stable endpoint host is not an IP address")?; - ensure!( - matches!(host, IpAddr::V4(_)), - "route binding v2 stable endpoints use explicit IPv4 addresses" - ); - let port = port - .parse::() - .context("stable endpoint port is not a u16")?; - ensure!(port > 0, "stable endpoint port is zero"); - Ok((host, port)) + parse_stable_socket(self.driver, &self.stable_endpoint, "stable endpoint") } + + /// The member listener's stable socket, when the route has one. + pub(crate) fn member_stable_socket(&self) -> Result> { + self.member + .as_ref() + .map(|member| { + parse_stable_socket(self.driver, &member.stable_endpoint, "member endpoint") + }) + .transpose() + } + + /// The private port the workload binds for members when its presence + /// candidate is `candidate_port`. Derived, never stored: the member range + /// is the private range shifted to the member start. + pub(crate) fn member_candidate_port(&self, candidate_port: u16) -> Result> { + let Some(member) = self.member.as_ref() else { + return Ok(None); + }; + ensure!( + (self.private_port_start..=self.private_port_end).contains(&candidate_port), + "candidate port is outside the route binding" + ); + let offset = candidate_port - self.private_port_start; + let port = member + .private_port_start + .checked_add(offset) + .context("member private port overflows")?; + Ok(Some(port)) + } +} + +fn parse_stable_socket(driver: RouteDriver, endpoint: &str, what: &str) -> Result<(IpAddr, u16)> { + let authority = match driver { + RouteDriver::NginxStreamTcp => endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("tcp://")), + RouteDriver::NginxStreamUdp => endpoint.strip_prefix("rudp://"), + } + .with_context(|| format!("{what} transport does not match its route driver"))?; + ensure!( + !authority + .chars() + .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), + "{what} is not a plain IP address and port" + ); + let (host, port) = authority + .rsplit_once(':') + .with_context(|| format!("{what} has no port"))?; + let host = host + .parse::() + .with_context(|| format!("{what} host is not an IP address"))?; + ensure!( + matches!(host, IpAddr::V4(_)), + "route binding v2 endpoints use explicit IPv4 addresses" + ); + let port = port + .parse::() + .with_context(|| format!("{what} port is not a u16"))?; + ensure!(port > 0, "{what} port is zero"); + Ok((host, port)) } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -924,6 +985,14 @@ impl TargetDeclaration { == self.service.route_required, "routed services must require exactly one Idunn-owned candidate bind" ); + ensure!( + !self + .service + .required_environment + .contains(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + || self.service.route_required, + "only routed services may require the Idunn-owned member bind" + ); ensure!( self.service .required_environment @@ -1381,7 +1450,35 @@ impl OperatorBinding { route.private_port_start < route.private_port_end, "candidate rollout requires at least two private ports" ); - route.stable_socket()?; + let stable = route.stable_socket()?; + if let Some(member) = route.member.as_ref() { + ensure!( + route.driver == RouteDriver::NginxStreamTcp, + "only a TCP route may carry a member listener" + ); + require_value(&member.stable_endpoint, "member endpoint")?; + let member_stable = route + .member_stable_socket()? + .context("route has no member listener")?; + ensure!( + member_stable != stable, + "member endpoint equals the route's stable endpoint" + ); + ensure!( + member.private_port_start > 0, + "member private port range starts at zero" + ); + let width = route.private_port_end - route.private_port_start; + let member_end = member + .private_port_start + .checked_add(width) + .context("member private port range overflows")?; + ensure!( + member_end < route.private_port_start + || member.private_port_start > route.private_port_end, + "member private port range overlaps the route's private range" + ); + } require_absolute_path(&route.config_path, "route config")?; require_unit(&route.reload_unit, "route reload unit")?; } @@ -1559,12 +1656,7 @@ impl OperatorBinding { .keys() .chain(self.workload.secret_environment_names()) .collect(); - for owned in [ - IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, - IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, - IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, - ] { + for owned in IDUNN_OWNED_LAUNCH_ENVIRONMENT { ensure!( !workload_names.contains(&owned.to_string()), "operator binding cannot replace Idunn-owned launch environment {owned}" @@ -1586,6 +1678,9 @@ impl OperatorBinding { } if declaration.service.route_required { available_environment.insert(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT.into()); + if self.route.as_ref().is_some_and(|route| route.member.is_some()) { + available_environment.insert(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into()); + } } if declaration.write_lease_required() { available_environment.insert(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT.into()); @@ -1933,10 +2028,10 @@ fn require_host(value: &str) -> Result<()> { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - const RECIPE: &str = r#" + pub(crate) const RECIPE: &str = r#" schema = "gamecult.idunn.target_declaration.v1" target = "test-service" source_stamp_environment = "TEST_SERVICE_BUILD_COMMIT" @@ -2008,7 +2103,7 @@ compatibility = "v1" startup = "before-promotion" "#; - const BINDING: &str = r#" + pub(crate) const BINDING: &str = r#" schema = "gamecult.idunn.operator_binding.v2" target = "test-service" profiles = ["aetheria", "full-gamecult"] @@ -2235,6 +2330,166 @@ nodes = ["yggdrasil"] } } + /// The recipe and binding of a routed service whose members connect to a + /// listener of their own. + pub(crate) fn member_listener_pair() -> (String, String) { + let recipe = RECIPE.replace( + "required_environment = [", + "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", + ); + let binding = BINDING.replace( + "reload_unit = \"nginx.service\"\n", + "reload_unit = \"nginx.service\"\n\n[route.member]\nstable_endpoint = \"http://127.0.0.1:18840\"\nprivate_port_start = 18841\n", + ); + (recipe, binding) + } + + fn member_binding_with(replacement: (&str, &str)) -> Result { + let (recipe, binding) = member_listener_pair(); + let recipe = TargetDeclaration::parse(&recipe)?; + let binding = OperatorBinding::parse(&binding.replace(replacement.0, replacement.1))?; + binding.admit(&recipe)?; + Ok(binding) + } + + #[test] + fn a_member_listener_admits_with_a_recipe_that_requires_the_member_bind() { + let binding = member_binding_with(("never-matches", "")).unwrap(); + let route = binding.route.unwrap(); + assert_eq!(route.member_stable_socket().unwrap().unwrap().1, 18840); + assert_eq!(route.member_candidate_port(18831).unwrap(), Some(18841)); + assert_eq!(route.member_candidate_port(18839).unwrap(), Some(18849)); + assert!(route.member_candidate_port(18840).is_err()); + } + + #[test] + fn a_binding_without_a_member_listener_has_no_member_ports() { + let binding = OperatorBinding::parse(BINDING).unwrap(); + let route = binding.route.unwrap(); + assert_eq!(route.member_stable_socket().unwrap(), None); + assert_eq!(route.member_candidate_port(18831).unwrap(), None); + } + + #[test] + fn a_member_listener_needs_a_recipe_that_requires_the_member_bind() { + let (recipe, binding) = member_listener_pair(); + // Bound but not required: the recipe never asked for the variable. + let plain_recipe = TargetDeclaration::parse(RECIPE).unwrap(); + assert!( + OperatorBinding::parse(&binding) + .unwrap() + .admit(&plain_recipe) + .is_err() + ); + // Required but not bound: the variable would never be set. + let recipe = TargetDeclaration::parse(&recipe).unwrap(); + assert!( + OperatorBinding::parse(BINDING) + .unwrap() + .admit(&recipe) + .is_err() + ); + } + + #[test] + fn only_a_routed_recipe_may_require_the_member_bind() { + let recipe = RECIPE + .replace("route_required = true", "route_required = false") + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", ", + "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", + ); + assert!(recipe.contains("GAMECULT_IDUNN_MEMBER_BIND")); + assert!(TargetDeclaration::parse(&recipe).is_err()); + } + + #[test] + fn member_listener_bindings_are_validated() { + let start = |value: &str| { + member_binding_with(( + "private_port_start = 18841", + &format!("private_port_start = {value}"), + )) + }; + // Overlapping the route's own private range, in either direction. + assert!(start("18835").is_err()); + assert!(start("18825").is_err()); + // Adjacent but disjoint is fine. + assert!(start("18840").is_ok()); + // Zero, and a width that overflows u16. + assert!(start("0").is_err()); + assert!(start("65530").is_err()); + // The member endpoint may not be the route's stable endpoint. + assert!(member_binding_with(("http://127.0.0.1:18840", "http://127.0.0.1:18830")).is_err()); + // The same transport and address rules as the stable endpoint. + assert!(member_binding_with(("http://127.0.0.1:18840", "rudp://127.0.0.1:18840")).is_err()); + assert!(member_binding_with(("http://127.0.0.1:18840", "http://example.test:18840")).is_err()); + } + + #[test] + fn a_udp_route_cannot_carry_a_member_listener() { + let (recipe, binding) = member_listener_pair(); + let binding = binding + .replace("nginx-stream-tcp", "nginx-stream-udp") + .replace("http://127.0.0.1:18830", "rudp://127.0.0.1:18830") + .replace("http://127.0.0.1:18840", "rudp://127.0.0.1:18840"); + let recipe = TargetDeclaration::parse(&recipe).unwrap(); + assert!( + OperatorBinding::parse(&binding) + .and_then(|binding| binding.admit(&recipe)) + .is_err() + ); + } + + #[test] + fn idunn_owned_names_are_reserved_in_one_list() { + let recipe = TargetDeclaration::parse(RECIPE).unwrap(); + let owned = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", + "GAMECULT_IDUNN_PROCESS_WRITE_LEASE", + "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", + "CREDENTIALS_DIRECTORY", + ]; + let mut listed = IDUNN_OWNED_LAUNCH_ENVIRONMENT.to_vec(); + listed.sort_unstable(); + let mut expected = owned.to_vec(); + expected.sort_unstable(); + assert_eq!(listed, expected); + + let as_environment = |name: &str| { + BINDING.replace( + "[workload.argument_bindings]", + &format!("[workload.environment]\n{name} = \"value\"\n\n[workload.argument_bindings]"), + ) + }; + let as_secret = |name: &str| { + BINDING.replace( + "[workload.secret_files]", + &format!("[workload.secret_files]\n{name} = \"/etc/gamecult/test-service/other.cc\""), + ) + }; + // The fixtures are well formed: an unowned name parses in both + // shapes, so the refusals below are about the name. + assert!(OperatorBinding::parse(&as_environment("UNOWNED_NAME")).is_ok()); + assert!(OperatorBinding::parse(&as_secret("UNOWNED_NAME")).is_ok()); + for name in owned { + assert!( + OperatorBinding::parse(&as_environment(name)) + .and_then(|binding| binding.admit(&recipe)) + .is_err(), + "{name} accepted as environment" + ); + assert!( + OperatorBinding::parse(&as_secret(name)) + .and_then(|binding| binding.admit(&recipe)) + .is_err(), + "{name} accepted as a secret file" + ); + } + } + #[test] fn state_slot_path_cannot_escape_the_operator_owned_root() { let input = RECIPE.replace( diff --git a/src/drivers.rs b/src/drivers.rs index df361c3..fb4ff1b 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -22,7 +22,8 @@ use crate::control_plane::SequenceAdmittedWarming; use crate::deployment::{ ArtifactOutput, ArtifactSource, DockerRunnerBinding, GitlinkBinding, IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, LaunchArgument, OperatorBinding, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + LaunchArgument, OperatorBinding, RUNTIME_PRESENCE_IDENTITY_BINDING, RUNTIME_PRESENCE_IDENTITY_FD_NAME, RouteBinding, RouteDriver, SourceSelectionPolicy, TargetDeclaration, WorkloadNetwork, }; @@ -3377,10 +3378,32 @@ impl SystemdTransientWorkloadDriver { .is_none(), "operator binding attempts to replace the Idunn candidate bind" ); + let member_port = binding + .route + .as_ref() + .map(|binding_route| binding_route.member_candidate_port(port)) + .transpose()? + .flatten(); + match member_port { + Some(member_port) => ensure!( + environment + .insert( + IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into(), + format!("{host}:{member_port}"), + ) + .is_none(), + "operator binding attempts to replace the Idunn member bind" + ), + None => ensure!( + !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), + "workload without a member listener carries an Idunn member bind" + ), + } } None => ensure!( - !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT), - "unrouted workload carries an Idunn candidate bind" + !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + && !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), + "unrouted workload carries an Idunn route bind" ), } match &binding.process_write_lease { @@ -5241,50 +5264,67 @@ impl NginxRouteDriver { } } - /// The one firewall rule this route owns: inbound to the stable endpoint's - /// address and port, its transport's protocol, tagged with the route id. - fn endpoint_rule(&self) -> Result> { - let (host, port) = self.binding.stable_socket()?; + /// The firewall rules this route owns, one per stable socket (the + /// presence endpoint, then the member endpoint when the binding has one): + /// inbound to the socket's address and port, its transport's protocol. + fn endpoint_rules(&self) -> Result>> { let protocol = match self.binding.driver { RouteDriver::NginxStreamTcp => "tcp", RouteDriver::NginxStreamUdp => "udp", }; - Ok(vec![ - OsString::from("allow"), - OsString::from("in"), - OsString::from("to"), - OsString::from(host.to_string()), - OsString::from("port"), - OsString::from(port.to_string()), - OsString::from("proto"), - OsString::from(protocol), - ]) + let mut sockets = vec![self.binding.stable_socket()?]; + sockets.extend(self.binding.member_stable_socket()?); + Ok(sockets + .into_iter() + .map(|(host, port)| { + vec![ + OsString::from("allow"), + OsString::from("in"), + OsString::from("to"), + OsString::from(host.to_string()), + OsString::from("port"), + OsString::from(port.to_string()), + OsString::from("proto"), + OsString::from(protocol), + ] + }) + .collect()) } fn admit_endpoint(&self) -> Result<()> { - let mut args = self.endpoint_rule()?; - args.push(OsString::from("comment")); - args.push(OsString::from(format!( - "Idunn route {}", - self.binding.route_id - ))); - self.command(&self.ufw_program, args) - .context("admitting the stable endpoint on the host firewall")?; + for mut args in self.endpoint_rules()? { + args.push(OsString::from("comment")); + args.push(OsString::from(format!( + "Idunn route {}", + self.binding.route_id + ))); + self.command(&self.ufw_program, args) + .context("admitting a route endpoint on the host firewall")?; + } Ok(()) } fn withdraw_endpoint(&self) -> Result<()> { - let mut args = vec![OsString::from("delete")]; - args.extend(self.endpoint_rule()?); - match self.command(&self.ufw_program, args) { - Ok(_) => Ok(()), - // ufw reports a rule that is already gone as a failure; for a - // withdrawal that is the state being asked for. - Err(error) if format!("{error:#}").contains("non-existent") => Ok(()), - Err(error) => { - Err(error).context("withdrawing the stable endpoint from the host firewall") + let mut first_error = None; + for rule in self.endpoint_rules()? { + let mut args = vec![OsString::from("delete")]; + args.extend(rule); + match self.command(&self.ufw_program, args) { + Ok(_) => {} + // ufw reports a rule that is already gone as a failure; for a + // withdrawal that is the state being asked for. + Err(error) if format!("{error:#}").contains("non-existent") => {} + Err(error) => { + first_error.get_or_insert( + error.context("withdrawing a route endpoint from the host firewall"), + ); + } } } + match first_error { + Some(error) => Err(error), + None => Ok(()), + } } fn command(&self, program: &Path, args: I) -> Result @@ -5345,8 +5385,20 @@ impl NginxRouteDriver { RouteDriver::NginxStreamTcp => { let (stable_host, stable_port) = endpoint_host_port(&route.stable_endpoint, endpoint_prefix)?; + let member_blocks = match ( + self.binding.member_stable_socket()?, + self.binding.member_candidate_port(candidate_port)?, + ) { + (Some((member_host, member_stable_port)), Some(member_port)) => { + let member_upstream = format!("{upstream}_member"); + format!( + "upstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n proxy_pass {member_upstream};\n}}\n" + ) + } + _ => String::new(), + }; format!( - "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n{member_blocks}" ) } RouteDriver::NginxStreamUdp => { @@ -8366,7 +8418,7 @@ fn ensure_source_directory(path: &Path, identity: Option) -> Re } } -fn authority_lock_path(path: &Path) -> PathBuf { +pub(crate) fn authority_lock_path(path: &Path) -> PathBuf { let mut lock = path.as_os_str().to_os_string(); lock.push(".lock"); PathBuf::from(lock) @@ -8698,6 +8750,7 @@ mod tests { private_port_end: port, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }) .request_candidate_runtime_presence(&candidate, "candidate-probe") } @@ -9521,6 +9574,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 17998, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -12253,6 +12307,7 @@ processes 9 private_port_end: 17979, config_path: temp.path().join("odin.conf"), reload_unit: "nginx.service".into(), + member: None, }, nginx_program: program("nginx", "#!/bin/sh\nexit 0\n")?, systemd_run_program: program("systemd-run", "#!/bin/sh\nexit 64\n")?, @@ -12262,12 +12317,15 @@ processes 9 }; assert_eq!( driver - .endpoint_rule()? + .endpoint_rules()? .iter() - .map(|value| value.to_string_lossy().into_owned()) - .collect::>() - .join(" "), - "allow in to 10.77.0.1 port 17971 proto udp" + .map(|rule| rule + .iter() + .map(|value| value.to_string_lossy().into_owned()) + .collect::>() + .join(" ")) + .collect::>(), + vec!["allow in to 10.77.0.1 port 17971 proto udp"] ); driver.restore(None, &Unmetered)?; @@ -12327,6 +12385,7 @@ processes 9 private_port_end: 4109, config_path: temp.join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }, &RouteActuators { nginx: program("nginx")?, @@ -12361,6 +12420,117 @@ processes 9 }) } + fn with_member_listener(driver: &mut NginxRouteDriver) { + driver.binding.member = Some(crate::deployment::MemberListenerBinding { + stable_endpoint: "http://127.0.0.1:4120".into(), + private_port_start: 4121, + }); + } + + /// The fragment as the base commit rendered it, computed from its format + /// string and not from the code under test. + #[cfg(unix)] + fn base_fragment(sha: &str, candidate_port: u16) -> String { + format!( + "# Idunn Expected {sha}\nupstream idunn_service_route {{\n server 127.0.0.1:{candidate_port};\n}}\nserver {{\n listen 127.0.0.1:4103;\n proxy_pass idunn_service_route;\n}}\n" + ) + } + + /// A route without a member listener renders the bytes it always did; a + /// member listener adds a second upstream and server block to the one + /// fragment, under the one Expected comment. + #[cfg(unix)] + #[test] + fn a_member_listener_renders_a_second_block_only_when_bound() -> Result<()> { + let temp = tempfile::tempdir()?; + let (mut driver, mut candidate, _calls) = stubbed_route_driver(temp.path())?; + let sha = candidate.canonical_sha256()?; + assert_eq!( + String::from_utf8(driver.render(&candidate)?)?, + base_fragment(&sha, 4104) + ); + + with_member_listener(&mut driver); + let member_blocks = |member_port: u16| { + format!( + "upstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n proxy_pass idunn_service_route_member;\n}}\n" + ) + }; + let rendered = String::from_utf8(driver.render(&candidate)?)?; + assert_eq!(rendered, base_fragment(&sha, 4104) + &member_blocks(4121)); + assert_eq!(rendered.matches("# Idunn Expected").count(), 1); + assert_eq!(rendered.matches("server {").count(), 2); + + candidate.route.as_mut().unwrap().candidate_endpoint = "http://127.0.0.1:4106".into(); + let sha = candidate.canonical_sha256()?; + assert_eq!( + String::from_utf8(driver.render(&candidate)?)?, + base_fragment(&sha, 4106) + &member_blocks(4123) + ); + Ok(()) + } + + /// Both stable sockets are admitted on the firewall, and both withdrawn. + #[cfg(unix)] + #[test] + fn both_route_endpoints_are_admitted_and_withdrawn() -> Result<()> { + let temp = tempfile::tempdir()?; + let (mut driver, _candidate, calls) = stubbed_route_driver(temp.path())?; + with_member_listener(&mut driver); + driver.admit_endpoint()?; + driver.withdraw_endpoint()?; + let log = fs::read_to_string(&calls)?; + let ufw: Vec<&str> = log.lines().filter(|line| line.starts_with("ufw ")).collect(); + assert_eq!( + ufw, + vec![ + "ufw allow in to 127.0.0.1 port 4103 proto tcp comment Idunn route service-route", + "ufw allow in to 127.0.0.1 port 4120 proto tcp comment Idunn route service-route", + "ufw delete allow in to 127.0.0.1 port 4103 proto tcp", + "ufw delete allow in to 127.0.0.1 port 4120 proto tcp", + ] + ); + Ok(()) + } + + /// Idunn sets the member bind for a bound member listener, derived from the + /// candidate, and for no other workload. + #[test] + fn the_member_bind_is_set_only_for_a_bound_member_listener() -> Result<()> { + use crate::deployment::tests::{BINDING, RECIPE, member_listener_pair}; + + let driver = SystemdTransientWorkloadDriver::default(); + let mut routed = expected(); + routed.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "test-service-public".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:18830".into(), + candidate_endpoint: "http://127.0.0.1:18833".into(), + }); + let bundle = Path::new("/etc/gamecult/test-service/runtime/bundle"); + let unit = "idunn-test-service-abc"; + + let (recipe, binding) = member_listener_pair(); + let binding = OperatorBinding::parse(&binding)?; + binding.admit(&TargetDeclaration::parse(&recipe)?)?; + let environment = driver.launch_environment(&binding, bundle, &routed, unit)?; + assert_eq!( + environment.get(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT).map(String::as_str), + Some("127.0.0.1:18843") + ); + assert_eq!( + environment.get(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT).map(String::as_str), + Some("127.0.0.1:18833") + ); + + let plain = OperatorBinding::parse(BINDING)?; + plain.admit(&TargetDeclaration::parse(RECIPE)?)?; + let environment = driver.launch_environment(&plain, bundle, &routed, unit)?; + assert!(!environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT)); + assert!(environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT)); + Ok(()) + } + /// A refused forward change stops before any host mutation: no program /// runs, no fragment is written, and no private-mount unit starts. #[cfg(unix)] @@ -12680,6 +12850,7 @@ processes 9 private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), + member: None, }, nginx_program: program("nginx")?, systemd_run_program: program("systemd-run")?, @@ -12740,6 +12911,7 @@ processes 9 private_port_end: 27879, config_path: config_path.clone(), reload_unit: "nginx.service".into(), + member: None, }); assert_eq!(driver.current_configuration()?, None); // What systemd leaves behind when it materializes the preflight's bind @@ -12766,6 +12938,7 @@ processes 9 private_port_end: 27879, config_path: "/etc/nginx/idunn-stream-routes/odin-rudp.conf".into(), reload_unit: "nginx.service".into(), + member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -12827,6 +13000,7 @@ processes 9 private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), + member: None, }; let driver = NginxRouteDriver { binding, From 9173a0e8e65a204c8111b1310a658b4e2292d590 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 01:28:30 +0200 Subject: [PATCH 060/122] idunn-launch answers presence only: the splice, the unix child endpoint and --listener are deleted; the child binds the member bind and the probe reaches it over TCP PresenceAcceptor serves one connection at a time within a 1 s deadline and never ends on an accept error; a dead presence thread fails the unit. Builds. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 29 +- docs/authority-map.md | 20 +- src/bin/idunn-launch.rs | 13 +- src/deployment.rs | 4 +- src/launcher.rs | 865 ++++++++++++++++------------------------ 5 files changed, 396 insertions(+), 535 deletions(-) diff --git a/README.md b/README.md index 225c5d8..3714ea9 100644 --- a/README.md +++ b/README.md @@ -154,8 +154,11 @@ An upstream daemon that does not speak Idunn's runtime contract is deployed with `idunn-launch` as its executable artifact and the upstream tree as a required adjacent artifact; the launcher finds the child inside the release that holds it. The launcher is the unit's process: it binds the -candidate endpoint, answers Idunn's challenge, renders the child's config, -starts a stateful child only once it holds the write lease, and signs presence. +candidate endpoint and answers Idunn's challenge there, renders the child's +config, starts a stateful child only once it holds the write lease, and signs +presence. Members never pass through it: the binding's `[route.member]` makes +Idunn route a second listener straight to the child, which binds the address +Idunn gives it in `GAMECULT_IDUNN_MEMBER_BIND`. ```toml # recipe fragment — the upstream tree is built or fetched as an artifact @@ -166,13 +169,31 @@ arguments = [ { kind = "literal", value = "upstream/bin/daemon" }, { kind = "literal", value = "--template" }, { kind = "literal", value = "config.tmpl" }, - { kind = "literal", value = "--listener" }, - { kind = "literal", value = "unix" }, { kind = "literal", value = "--health-path" }, { kind = "literal", value = "/health" }, { kind = "literal", value = "--health-contract" }, { kind = "literal", value = "example.presence-launcher.v1" }, ] +required_environment = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", +] +``` + +```toml +# binding fragment — members connect to the member endpoint, which Idunn +# forwards to the private port derived from the candidate: member +# private_port_start + (candidate port - route private_port_start) +[route] +stable_endpoint = "tcp://10.77.0.1:8008" +private_port_start = 18831 +private_port_end = 18839 +# ... + +[route.member] +stable_endpoint = "tcp://10.77.0.1:8448" +private_port_start = 18841 ``` ## Contracts diff --git a/docs/authority-map.md b/docs/authority-map.md index b61a8f2..2db896f 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -83,24 +83,30 @@ activation key. Idunn owns admission, lease, route and continuity, as before. - **Inputs:** the runtime bundle, the activation credential and provider identity descriptors, the process write lease, the systemd credentials the - binding names, and the child's health over its private listener. + binding names, and the child's health over TCP at the member bind. - **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active and failed to Odin; route observations on the candidate endpoint), one child process, and a rendered config in the unit's private tmp that is never read back. -- **Route:** the launcher, not the child, binds the candidate endpoint. A first - byte of 0x00 is an Idunn challenge frame and is answered by the launcher; - every other byte is copied untouched to the child's private listener. +- **Route:** Idunn owns both listeners of the route: the Expected route's + stable endpoint (presence) and, when the binding has `[route.member]`, the + member endpoint, rendered as a second server block in the one nginx fragment. + The launcher binds the candidate endpoint and answers Idunn's challenge there, + one connection at a time within a one-second deadline; any other first byte + closes the connection. The child binds the member bind Idunn sets in + `GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes + never touch the launcher. - **Lease:** with `write_lease_required` the launcher starts the child only after it holds the lease (bound to this Expected, this activation and one of its own Warming proofs) and keeps the lease store share-locked while the child lives. Losing the lease stops the child. - **Config:** one template from the release is rendered with the closed - placeholder set `{{child.listen}}`, `{{credential:NAME}}`, + placeholder set `{{child.listen}}`, `{{child.listen_host}}`, + `{{child.listen_port}}` (all three the member bind), `{{credential:NAME}}`, `{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder refuses to start. -- **Forbidden:** the child binding the candidate endpoint; a child of a - lease-requiring target running without a held lease; any presence not signed +- **Forbidden:** member traffic through the launcher; the child binding the + candidate endpoint; a child of a lease-requiring target running without a held lease; any presence not signed by both keys. - **State kind `external-directory`:** a process-owned directory imposed by an external runtime contract. Same writer, recovery and startup rules as diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs index 41b6c76..3cf71e6 100644 --- a/src/bin/idunn-launch.rs +++ b/src/bin/idunn-launch.rs @@ -7,7 +7,7 @@ fn main() -> std::process::ExitCode { use anyhow::Context; use idunn_daemon::deployment::{ IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, }; use idunn_daemon::launcher::{self, *}; @@ -26,6 +26,11 @@ fn main() -> std::process::ExitCode { .context("the Idunn runtime bundle is not set")?; let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) .context("the Idunn candidate bind is not set")?; + let member = launcher::parse_member_bind( + &std::env::var(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + .context("the Idunn member bind is not set")?, + &bind, + )?; let (activation, provider) = launcher::inherited_authority_files()?; let authority = RuntimeAuthority::open(bundle.as_ref(), activation, provider)?; authority.require_candidate_bind(&bind)?; @@ -47,10 +52,9 @@ fn main() -> std::process::ExitCode { sink: std::sync::Arc::new(sink), lease: std::sync::Arc::new(lease), spawner: Box::new(ProcessSpawner), - route_bounds: RouteBounds::default(), - health: Box::new(move |endpoint| { + health: Box::new(move |member| { std::sync::Arc::new(HttpHealthProbe { - endpoint: endpoint.clone(), + member, path: health_path.clone(), }) }), @@ -64,6 +68,7 @@ fn main() -> std::process::ExitCode { &contract, authority, route, + member, &std::env::temp_dir(), ports, &STOP, diff --git a/src/deployment.rs b/src/deployment.rs index b01deed..067f3b8 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -657,9 +657,7 @@ pub struct ProcessWriteLeaseBinding { impl ProcessWriteLeaseBinding { pub fn lock_path(&self) -> PathBuf { - let mut lock_path = self.record_path.as_os_str().to_os_string(); - lock_path.push(".lock"); - PathBuf::from(lock_path) + crate::drivers::authority_lock_path(&self.record_path) } } diff --git a/src/launcher.rs b/src/launcher.rs index 689c35a..391ee9e 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -4,9 +4,11 @@ //! Idunn observes the unit; the launcher, inside it and supervising the child, //! signs the presence statement with the provider identity and the activation //! key. The launcher binds the candidate endpoint Idunn challenges, because the -//! child cannot answer a CultNet frame. It starts a stateful child only after -//! it holds the process write lease, renders the child's config from a closed -//! set of placeholders, and ends with the child. +//! child cannot answer a CultNet frame, and answers presence there and nowhere +//! else: members reach the child on the member bind Idunn routes them to. It +//! starts a stateful child only after it holds the process write lease, renders +//! the child's config from a closed set of placeholders, and ends with the +//! child. //! //! Errors, logs and refusals name a field and a code. They never echo an //! argument, a template line, a path or a credential. @@ -14,14 +16,13 @@ use std::collections::VecDeque; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; -use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::net::{SocketAddr, TcpListener, TcpStream, UdpSocket}; use std::os::fd::{AsRawFd, RawFd}; use std::os::unix::fs::OpenOptionsExt; -use std::os::unix::net::UnixStream; use std::os::unix::process::CommandExt; use std::path::{Component, Path, PathBuf}; use std::process::{Child, Command, Stdio}; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -41,21 +42,22 @@ use cultnet_rs::{ encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, }; -use crate::deployment::RUNTIME_PRESENCE_IDENTITY_FD_NAME; -use crate::drivers::{CultCacheWriteLeaseDriver, lease_binds_candidate}; +use crate::deployment::{RUNTIME_PRESENCE_IDENTITY_FD_NAME, SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT}; +use crate::drivers::{CultCacheWriteLeaseDriver, authority_lock_path, lease_binds_candidate}; /// The route challenge is the only thing that begins with this byte: a /// CultNet frame's 4-byte big-endian length is bounded at 1 MiB, and an HTTP /// request starts with a method letter. const CHALLENGE_FIRST_BYTE: u8 = 0x00; const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; -const CHALLENGE_IO_TIMEOUT: Duration = Duration::from_secs(3); -/// How long a public connection may stay silent before its first byte. -const FIRST_BYTE_DEADLINE: Duration = Duration::from_secs(5); -/// A whole challenge, from its first byte to its answer, however it trickles. -const CHALLENGE_DEADLINE: Duration = Duration::from_secs(5); -/// Concurrent connections the acceptor serves; the rest are refused unspawned. -const MAX_CONNECTIONS: usize = 256; +/// One whole presence exchange on the candidate endpoint, from accept to +/// answer, however the peer paces it. It is shorter than the 3 s Idunn waits +/// for a challenge answer, so one stalled peer cannot cost the next challenge +/// its reply. +const PRESENCE_DEADLINE: Duration = Duration::from_secs(1); +/// How long the presence acceptor waits after an accept error before trying +/// again. No accept error ends the acceptor. +const ACCEPT_BACKOFF: Duration = Duration::from_millis(50); const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); const POLL: Duration = Duration::from_millis(100); const MAX_RECENT_WARMING_PROOFS: usize = 64; @@ -65,14 +67,6 @@ const ODIN_CAPABILITY: &str = "odin.verse-rendezvous"; // ---------------------------------------------------------------- contract -/// Where the child listens. The launcher chooses the address; the child is -/// told it through `{{child.listen}}`. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ChildListener { - UnixSocket, - LoopbackPort, -} - /// Everything the recipe declares about the child, parsed from the launcher's /// argv (`--flag value` pairs; `--child-arg` repeats). The release root is the /// sealed release directory that holds the launcher itself. @@ -85,7 +79,6 @@ pub struct LaunchContract { /// child takes one. pub config_flag: Option, pub template: PathBuf, - pub listener: ChildListener, pub health_path: String, pub health_contract: String, pub state_root: Option, @@ -101,7 +94,6 @@ impl LaunchContract { let mut child_args = Vec::new(); let mut config_flag = None; let mut template = None; - let mut listener = None; let mut health_path = None; let mut health_contract = None; let mut state_root = None; @@ -121,13 +113,6 @@ impl LaunchContract { "--child-arg" => child_args.push(value), "--config-flag" => config_flag = Some(value), "--template" => template = Some(release_relative(&value, "--template")?), - "--listener" => { - listener = Some(match value.as_str() { - "unix" => ChildListener::UnixSocket, - "loopback" => ChildListener::LoopbackPort, - _ => bail!("launcher argument --listener is not unix or loopback"), - }) - } "--health-path" => { ensure!( value.starts_with('/'), @@ -149,7 +134,6 @@ impl LaunchContract { child_args, config_flag, template: template.context("launcher argument --template is required")?, - listener: listener.context("launcher argument --listener is required")?, health_path: health_path.context("launcher argument --health-path is required")?, health_contract: health_contract .context("launcher argument --health-contract is required")?, @@ -426,7 +410,7 @@ impl CredentialsDirectory { /// The directory systemd exported; no directory means no credential is bound. pub fn from_environment() -> Self { - Self(std::env::var_os("CREDENTIALS_DIRECTORY").map(PathBuf::from)) + Self(std::env::var_os(SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT).map(PathBuf::from)) } } @@ -523,9 +507,7 @@ impl LeaseReader for DriverLeaseReader { /// description: a child that inherits it keeps the lock after the launcher /// dies. fn hold_shared(&self) -> Result> { - let mut lock = self.driver.record_path.clone().into_os_string(); - lock.push(".lock"); - let file = File::open(&lock).map_err(|_| anyhow!("write-lease lock could not be held"))?; + let file = File::open(authority_lock_path(&self.driver.record_path)).map_err(|_| anyhow!("write-lease lock could not be held"))?; // SAFETY: flock on a descriptor this function owns. ensure!( unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0, @@ -640,108 +622,34 @@ fn presence_document( }) } -/// The child's private listener. -#[derive(Clone, Debug)] -pub enum ChildEndpoint { - Unix(PathBuf), - Loopback(SocketAddr), -} - -pub enum ChildStream { - Unix(UnixStream), - Tcp(TcpStream), -} - -impl ChildEndpoint { - pub fn connect(&self) -> io::Result { - match self { - Self::Unix(path) => UnixStream::connect(path).map(ChildStream::Unix), - Self::Loopback(address) => { - TcpStream::connect_timeout(address, HEALTH_PROBE_TIMEOUT).map(ChildStream::Tcp) - } - } - } - - /// What `{{child.listen}}` renders to. - fn listen_value(&self) -> String { - match self { - Self::Unix(path) => path.display().to_string(), - Self::Loopback(address) => address.to_string(), - } - } -} - -impl ChildStream { - fn try_clone(&self) -> io::Result { - match self { - Self::Unix(stream) => stream.try_clone().map(Self::Unix), - Self::Tcp(stream) => stream.try_clone().map(Self::Tcp), - } - } - - fn shutdown_write(&self) { - let _ = match self { - Self::Unix(stream) => stream.shutdown(Shutdown::Write), - Self::Tcp(stream) => stream.shutdown(Shutdown::Write), - }; - } - - fn set_timeout(&self, timeout: Duration) { - let _ = match self { - Self::Unix(stream) => stream - .set_read_timeout(Some(timeout)) - .and_then(|()| stream.set_write_timeout(Some(timeout))), - Self::Tcp(stream) => stream - .set_read_timeout(Some(timeout)) - .and_then(|()| stream.set_write_timeout(Some(timeout))), - }; - } -} - -impl Read for ChildStream { - fn read(&mut self, buffer: &mut [u8]) -> io::Result { - match self { - Self::Unix(stream) => stream.read(buffer), - Self::Tcp(stream) => stream.read(buffer), - } - } -} - -impl Write for ChildStream { - fn write(&mut self, buffer: &[u8]) -> io::Result { - match self { - Self::Unix(stream) => stream.write(buffer), - Self::Tcp(stream) => stream.write(buffer), - } - } - - fn flush(&mut self) -> io::Result<()> { - match self { - Self::Unix(stream) => stream.flush(), - Self::Tcp(stream) => stream.flush(), - } - } -} - -/// `GET ` over the child's private listener; healthy on a 200. +/// `GET ` over TCP at the child's member bind; healthy on a 200. One +/// deadline covers connect, write and read, so a full backlog or a silent child +/// answers false within `HEALTH_PROBE_TIMEOUT`. pub struct HttpHealthProbe { - pub endpoint: ChildEndpoint, + pub member: SocketAddr, pub path: String, } impl HealthProbe for HttpHealthProbe { fn healthy(&self) -> bool { - let Ok(mut stream) = self.endpoint.connect() else { + let deadline = Instant::now() + HEALTH_PROBE_TIMEOUT; + let Ok(mut stream) = TcpStream::connect_timeout(&self.member, HEALTH_PROBE_TIMEOUT) else { + return false; + }; + let Some(remaining) = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + else { return false; }; - stream.set_timeout(HEALTH_PROBE_TIMEOUT); let request = format!( "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", self.path ); let mut head = [0_u8; 12]; - stream.write_all(request.as_bytes()).is_ok() - && stream.read_exact(&mut head).is_ok() + stream.set_write_timeout(Some(remaining)).is_ok() + && stream.write_all(request.as_bytes()).is_ok() + && read_by(&mut stream, &mut head, deadline).is_ok() && head.starts_with(b"HTTP/1.") && &head[8..12] == b" 200" } @@ -816,7 +724,8 @@ impl ChildHandle for ProcessChild { /// The closed placeholder set (decision D6). Any other `{{...}}` refuses. pub struct ConfigRenderer<'a> { pub credentials: &'a dyn CredentialSource, - pub listen: &'a str, + /// The member bind the child serves on. + pub listen: SocketAddr, pub state_root: Option<&'a Path>, } @@ -838,8 +747,11 @@ impl ConfigRenderer<'_> { } fn expand(&self, key: &str) -> Result { - if key == "child.listen" { - return Ok(self.listen.to_owned()); + match key { + "child.listen" => return Ok(self.listen.to_string()), + "child.listen_host" => return Ok(self.listen.ip().to_string()), + "child.listen_port" => return Ok(self.listen.port().to_string()), + _ => {} } if key == "state_root" { return Ok(self @@ -940,7 +852,7 @@ enum Phase { Failed, } -/// What the launcher currently is, shared with the route acceptor. +/// What the launcher currently is, shared with the presence acceptor. struct Shared { authority: Mutex, phase: Mutex, @@ -948,15 +860,13 @@ struct Shared { lease_required: bool, health: Arc, clock: Arc, - /// False once the route acceptor has stopped for any reason but a halt: - /// the launcher cannot answer Idunn's challenge, so it must not say active. - acceptor_alive: AtomicBool, } impl Shared { - /// Active only while the child is healthy and the route is being served. + /// Active only while the child is healthy. A dead presence acceptor does + /// not degrade presence: it fails the unit. fn serving_state(&self) -> &'static str { - if self.health.healthy() && self.acceptor_alive.load(Ordering::SeqCst) { + if self.health.healthy() { "active" } else { "degraded" @@ -1007,134 +917,66 @@ impl Shared { type Responder = Arc Result + Send + Sync>; -/// Bounds on what the public route endpoint spends on one peer. -#[derive(Clone, Copy, Debug)] -pub struct RouteBounds { - pub first_byte: Duration, - pub challenge: Duration, - pub max_connections: usize, -} - -impl Default for RouteBounds { - fn default() -> Self { - Self { - first_byte: FIRST_BYTE_DEADLINE, - challenge: CHALLENGE_DEADLINE, - max_connections: MAX_CONNECTIONS, - } - } -} - -/// Counts a live connection; dropping it frees the slot. -struct Slot(Arc); - -impl Drop for Slot { - fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::SeqCst); - } -} - -/// Owns the candidate endpoint. A leading 0x00 is an Idunn challenge frame and -/// is answered here; every other byte is copied untouched to the child. The -/// endpoint is public, so it spends bounded resources on a peer: a first-byte -/// deadline, a total challenge deadline, and a cap on live connections whose -/// overflow is closed without a thread. -pub struct RouteAcceptor { +/// Owns the candidate endpoint and answers presence there and nowhere else. +/// Members never reach it: Idunn routes them to the child's member bind. +/// +/// The endpoint is public, so it spends bounded resources on a peer: one +/// connection at a time, so a connection holds at most one descriptor; a total +/// deadline for the whole exchange; and any first byte but a challenge's closes +/// the connection unanswered. No accept error ends it: it backs off and accepts +/// again, and only `halt` stops it. +pub struct PresenceAcceptor { listener: TcpListener, - child: ChildEndpoint, responder: Responder, - bounds: RouteBounds, - live: Arc, } -impl RouteAcceptor { - pub fn new(listener: TcpListener, child: ChildEndpoint, responder: Responder, bounds: RouteBounds) -> Self { - Self { - listener, - child, - responder, - bounds, - live: Arc::new(AtomicUsize::new(0)), - } +impl PresenceAcceptor { + pub fn new(listener: TcpListener, responder: Responder) -> Self { + Self { listener, responder } } - /// Serves until `halt`. Any other end is an error: a failed accept or a - /// thread that cannot be created means the route is no longer served. - pub fn serve(self, halt: Arc) -> Result<()> { - self.listener.set_nonblocking(true)?; - let acceptor = Arc::new(self); + pub fn serve(self, halt: Arc) { + self.serve_accepting(halt, |listener| listener.accept()); + } + + fn serve_accepting( + self, + halt: Arc, + mut accept: impl FnMut(&TcpListener) -> io::Result<(TcpStream, SocketAddr)>, + ) { + // A listener that cannot be made non-blocking could never observe + // `halt`; ending here is a dead acceptor, which fails the unit. + if self.listener.set_nonblocking(true).is_err() { + return; + } while !halt.load(Ordering::SeqCst) { - match acceptor.listener.accept() { + match accept(&self.listener) { Ok((connection, _)) => { - if acceptor.live.fetch_add(1, Ordering::SeqCst) >= acceptor.bounds.max_connections { - acceptor.live.fetch_sub(1, Ordering::SeqCst); - continue; - } - let slot = Slot(Arc::clone(&acceptor.live)); - let worker = Arc::clone(&acceptor); - thread::Builder::new() - .spawn(move || { - let _slot = slot; - let _ = connection.set_nonblocking(false); - let _ = worker.handle(connection); - }) - .map_err(|_| anyhow!("a route connection thread could not be created"))?; + let _ = connection.set_nonblocking(false); + let _ = self.answer(connection); } - Err(error) - if matches!( - error.kind(), - io::ErrorKind::WouldBlock - | io::ErrorKind::Interrupted - | io::ErrorKind::ConnectionAborted - ) => - { + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { thread::sleep(Duration::from_millis(10)); } - Err(_) => bail!("the route listener could not accept"), + Err(_) => thread::sleep(ACCEPT_BACKOFF), } } - Ok(()) } - fn handle(&self, mut connection: TcpStream) -> Result<()> { + fn answer(&self, mut connection: TcpStream) -> Result<()> { + let deadline = Instant::now() + PRESENCE_DEADLINE; let mut first = [0_u8; 1]; - read_by(&mut connection, &mut first, Instant::now() + self.bounds.first_byte)?; - if first[0] == CHALLENGE_FIRST_BYTE { - return self.answer_challenge(&mut connection); - } - let mut upstream = self.child.connect()?; - upstream.write_all(&first)?; - let mut upstream_reader = upstream.try_clone()?; - let mut client_reader = connection.try_clone()?; - let forward = thread::Builder::new() - .spawn(move || { - let _ = io::copy(&mut client_reader, &mut upstream); - upstream.shutdown_write(); - }) - .map_err(|_| anyhow!("a route splice thread could not be created"))?; - // The first-byte deadline bounded the silent peer; a spliced session - // runs as long as the member and the child keep it. - connection.set_read_timeout(None)?; - let _ = io::copy(&mut upstream_reader, &mut connection); - let _ = connection.shutdown(Shutdown::Write); - let _ = forward.join(); - Ok(()) - } - - fn answer_challenge(&self, connection: &mut TcpStream) -> Result<()> { - // The challenge began with its first byte; the rest must arrive within - // the total deadline, however slowly the peer sends it. - let deadline = Instant::now() + self.bounds.challenge; - connection.set_write_timeout(Some(CHALLENGE_IO_TIMEOUT))?; + read_by(&mut connection, &mut first, deadline)?; + ensure!(first[0] == CHALLENGE_FIRST_BYTE, "not a challenge frame"); let mut rest = [0_u8; 3]; - read_by(connection, &mut rest, deadline)?; + read_by(&mut connection, &mut rest, deadline)?; let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; ensure!( (1..=CHALLENGE_MAX_BYTES).contains(&length), "challenge frame is outside its bound" ); let mut payload = vec![0_u8; length]; - read_by(connection, &mut payload, deadline)?; + read_by(&mut connection, &mut payload, deadline)?; let request = decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; let response = (self.responder)(request)?; @@ -1142,6 +984,11 @@ impl RouteAcceptor { &response, CultNetWireContract::CultNetSchemaV0, )?)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .context("the answer was not ready in time")?; + connection.set_write_timeout(Some(remaining))?; connection.write_all(&frame)?; connection.flush()?; Ok(()) @@ -1175,9 +1022,8 @@ pub struct Ports { pub sink: Arc, pub lease: Arc, pub spawner: Box, - pub route_bounds: RouteBounds, - /// Built from the endpoint `run` chooses. - pub health: Box Arc + Send>, + /// Built from the member bind the child serves on. + pub health: Box Arc + Send>, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -1197,23 +1043,32 @@ impl Outcome { } } -fn child_endpoint(listener: ChildListener, tmp: &Path) -> Result { - Ok(match listener { - ChildListener::UnixSocket => ChildEndpoint::Unix(tmp.join("child.sock")), - ChildListener::LoopbackPort => { - let probe = TcpListener::bind("127.0.0.1:0")?; - ChildEndpoint::Loopback(probe.local_addr()?) - } - }) +/// The member bind Idunn hands the unit: a socket address on a loopback IP, +/// other than the candidate bind. Errors name the variable, never its value. +pub fn parse_member_bind(raw: &str, candidate_bind: &str) -> Result { + let member: SocketAddr = raw + .parse() + .map_err(|_| anyhow!("the Idunn member bind is not a socket address"))?; + ensure!( + member.ip().is_loopback(), + "the Idunn member bind is not on a loopback address" + ); + let candidate: SocketAddr = candidate_bind + .parse() + .map_err(|_| anyhow!("the Idunn candidate bind is not a socket address"))?; + ensure!(member != candidate, "the Idunn member bind equals the candidate bind"); + Ok(member) } /// The unit's whole life: Warming, lease, child, Active, until the child or the /// lease ends or SIGTERM arrives. `route` is the already-bound candidate -/// listener; `tmp` is the unit's private temporary directory. +/// listener, answered for presence only; `member` is the address the child +/// serves members on; `tmp` is the unit's private temporary directory. pub fn run( contract: &LaunchContract, authority: RuntimeAuthority, route: TcpListener, + member: SocketAddr, tmp: &Path, ports: Ports, stop: &AtomicBool, @@ -1223,10 +1078,7 @@ pub fn run( if expected.health_contract != contract.health_contract { return Outcome::Failed("health-contract-mismatch"); } - let Ok(endpoint) = child_endpoint(contract.listener, tmp) else { - return Outcome::Failed("child-endpoint"); - }; - let health = (ports.health)(&endpoint); + let health = (ports.health)(member); let shared = Arc::new(Shared { authority: Mutex::new(authority), phase: Mutex::new(Phase::Warming), @@ -1234,26 +1086,23 @@ pub fn run( lease_required: expected.write_lease_required, health: Arc::clone(&health), clock: Arc::clone(&ports.clock), - acceptor_alive: AtomicBool::new(true), }); let halt = Arc::new(AtomicBool::new(false)); let responder: Responder = { let shared = Arc::clone(&shared); Arc::new(move |request| shared.observe(request)) }; + let acceptor = PresenceAcceptor::new(route, responder); let serving = { - let acceptor = RouteAcceptor::new(route, endpoint.clone(), responder, ports.route_bounds); let halt = Arc::clone(&halt); - let shared = Arc::clone(&shared); - thread::spawn(move || { - let served = acceptor.serve(halt); - if served.is_err() { - shared.acceptor_alive.store(false, Ordering::SeqCst); - } - served - }) + thread::Builder::new() + .name("presence-acceptor".into()) + .spawn(move || acceptor.serve(halt)) }; - let outcome = lifecycle(contract, &expected, &activation, &endpoint, tmp, &ports, &shared, stop); + let Ok(serving) = serving else { + return Outcome::Failed("presence-acceptor"); + }; + let outcome = lifecycle(contract, &expected, &activation, member, tmp, &ports, &shared, &serving, stop); halt.store(true, Ordering::SeqCst); let _ = serving.join(); outcome @@ -1264,10 +1113,11 @@ fn lifecycle( contract: &LaunchContract, expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, - endpoint: &ChildEndpoint, + member: SocketAddr, tmp: &Path, ports: &Ports, shared: &Shared, + presence: &thread::JoinHandle<()>, stop: &AtomicBool, ) -> Outcome { let clock = &*ports.clock; @@ -1316,6 +1166,9 @@ fn lifecycle( if stop.load(Ordering::SeqCst) { return Outcome::Stopped; } + if presence.is_finished() { + return fail(None, "presence-acceptor"); + } let held = LeaseHold::acquire( Arc::clone(&ports.lease), expected, @@ -1343,7 +1196,7 @@ fn lifecycle( } // 3. Render, spawn, and wait for health. - let Some(mut child) = start_child(contract, endpoint, tmp, ports, lease_fd) else { + let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { return fail(None, "child-start-refused"); }; let startup_deadline = clock.now_millis() + contract.startup.as_millis() as u64; @@ -1355,6 +1208,9 @@ fn lifecycle( if child.exited().is_some() { return fail(None, "child-exited"); } + if presence.is_finished() { + return fail(Some(&mut child), "presence-acceptor"); + } if shared.health.healthy() { break; } @@ -1376,6 +1232,9 @@ fn lifecycle( if child.exited().is_some() { return fail(None, "child-exited"); } + if presence.is_finished() { + return fail(Some(&mut child), "presence-acceptor"); + } if !shared.lease_current() { return fail(Some(&mut child), "lease-lost"); } @@ -1390,16 +1249,15 @@ fn lifecycle( fn start_child( contract: &LaunchContract, - endpoint: &ChildEndpoint, + member: SocketAddr, tmp: &Path, ports: &Ports, lease_fd: Option, ) -> Option> { let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; - let listen = endpoint.listen_value(); let rendered = ConfigRenderer { credentials: &*ports.credentials, - listen: &listen, + listen: member, state_root: contract.state_root.as_deref(), } .render(&template) @@ -1421,7 +1279,6 @@ mod tests { use std::collections::BTreeMap; use std::os::unix::fs::PermissionsExt; use std::sync::atomic::{AtomicU64, AtomicUsize}; - use std::sync::mpsc; use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; use cultnet_rs::{ @@ -1433,6 +1290,11 @@ mod tests { use super::*; use crate::drivers::{WriteLeasePort, exact_route_snapshot_response, request_tcp_snapshot}; + /// The member bind Idunn would hand these fixtures' unit. + fn member() -> SocketAddr { + "127.0.0.1:18845".parse().unwrap() + } + fn digest(byte: char) -> String { format!("sha256-{}", byte.to_string().repeat(64)) } @@ -1580,7 +1442,6 @@ mod tests { child_args: vec!["--serve".into()], config_flag: Some("--config".into()), template: PathBuf::from("config.tmpl"), - listener: ChildListener::UnixSocket, health_path: "/health".into(), health_contract: self.expected.health_contract.clone(), state_root: Some(PathBuf::from("/var/lib/synapse")), @@ -1743,14 +1604,14 @@ mod tests { sink: sink.clone(), lease: lease.clone(), spawner: Box::new(FakeSpawner(child.clone())), - route_bounds: RouteBounds::default(), health: Box::new(move |_| health.clone()), }; let authority = fx.authority(); let route = TcpListener::bind("127.0.0.1:0").unwrap(); let tmp = fx.dir.path().join("tmp"); let flag = stop.clone(); - let outcome = thread::spawn(move || run(&contract, authority, route, &tmp, ports, &flag)); + let outcome = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); Run { sink, lease, @@ -1887,7 +1748,7 @@ mod tests { #[test] fn closed_placeholder_set_renders_and_refuses() { let fx = fixture(false); - let template = "listen={{child.listen}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; + let template = "listen={{child.listen}}\nhost={{child.listen_host}}\nport={{child.listen_port}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; let mut launch = start(&fx, fx.contract(template), true); wait_until("spawn", || launch.child.spawns.load(Ordering::SeqCst) == 1); let args = launch.child.args.lock().unwrap()[0].clone(); @@ -1899,13 +1760,9 @@ mod tests { 0o600 ); let rendered = fs::read_to_string(&config).unwrap(); - let socket = fx.dir.path().join("tmp/child.sock"); assert_eq!( rendered, - format!( - "listen={}\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n", - socket.display() - ) + "listen=127.0.0.1:18845\nhost=127.0.0.1\nport=18845\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n" ); assert!( !rendered.contains("18835"), @@ -1921,7 +1778,7 @@ mod tests { let credentials = Credentials(BTreeMap::new()); let renderer = ConfigRenderer { credentials: &credentials, - listen: "L", + listen: member(), state_root: None, }; for template in [ @@ -1944,7 +1801,6 @@ mod tests { lease_required: false, health, clock: Arc::new(SystemClock), - acceptor_alive: AtomicBool::new(true), }) } @@ -1982,12 +1838,7 @@ mod tests { let address = listener.local_addr().unwrap(); let responder: Responder = Arc::new(move |request| shared.observe(request)); let halt = Arc::new(AtomicBool::new(false)); - let acceptor = RouteAcceptor::new( - listener, - ChildEndpoint::Unix("/nonexistent".into()), - responder, - RouteBounds::default(), - ); + let acceptor = PresenceAcceptor::new(listener, responder); let serving = { let halt = halt.clone(); thread::spawn(move || acceptor.serve(halt)) @@ -2019,81 +1870,7 @@ mod tests { .unwrap(); assert_eq!(claim.record().state, "active"); halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); - } - - #[test] - fn member_bytes_reach_the_child_verbatim_and_only_challenge_frames_are_answered() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("child.sock"); - let child = std::os::unix::net::UnixListener::bind(&socket).unwrap(); - let request: Vec = - b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 6\r\n\r\n\x00\x01\x00\x02\x00\x03".to_vec(); - let response: Vec = - b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n\x00\xff\x00\x10".to_vec(); - let received = Arc::new(Mutex::new(Vec::new())); - let connections = Arc::new(AtomicUsize::new(0)); - let fake = { - let (received, connections, response, want) = ( - received.clone(), - connections.clone(), - response.clone(), - request.len(), - ); - thread::spawn(move || { - for stream in child.incoming() { - let mut stream = stream.unwrap(); - connections.fetch_add(1, Ordering::SeqCst); - let mut buffer = vec![0_u8; want]; - if stream.read_exact(&mut buffer).is_err() { - continue; - } - *received.lock().unwrap() = buffer; - stream.write_all(&response).unwrap(); - return; - } - }) - }; - let challenges = Arc::new(AtomicUsize::new(0)); - let responder: Responder = { - let challenges = challenges.clone(); - Arc::new(move |_| { - challenges.fetch_add(1, Ordering::SeqCst); - bail!("not under test") - }) - }; - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let halt = Arc::new(AtomicBool::new(false)); - let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder, RouteBounds::default()); - let serving = { - let halt = halt.clone(); - thread::spawn(move || acceptor.serve(halt)) - }; - - // A leading 0x00 that is not a challenge frame never reaches the child. - let mut probe = TcpStream::connect(address).unwrap(); - probe.write_all(&[0x00, 0xff, 0xff, 0xff, 1, 2, 3]).unwrap(); - let mut answered = Vec::new(); - let _ = probe.read_to_end(&mut answered); - assert!(answered.is_empty()); - assert_eq!(connections.load(Ordering::SeqCst), 0); - - // Everything else is copied untouched, both ways, 0x00 bytes included. - let mut member = TcpStream::connect(address).unwrap(); - member.write_all(&request).unwrap(); - let mut answer = Vec::new(); - member.read_to_end(&mut answer).unwrap(); - assert_eq!(answer, response); - fake.join().unwrap(); - assert_eq!(*received.lock().unwrap(), request); - assert_eq!( - challenges.load(Ordering::SeqCst), - 0, - "a malformed frame fails before the responder; member traffic never reaches it" - ); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); + serving.join().unwrap(); } #[test] @@ -2104,8 +1881,6 @@ mod tests { "bin/c", "--template", "t", - "--listener", - "unix", "--health-path", "/health", "--health-contract", @@ -2118,6 +1893,9 @@ mod tests { }; assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); assert!(base(&["--child", "../escape"]).is_err()); + // The child serves members on Idunn's member bind: there is no + // listener to choose. + assert!(base(&["--listener", "loopback"]).is_err()); assert!(base(&["--template", "/etc/passwd"]).is_err()); let error = base(&["--canary-flag", "canary-value"]) .unwrap_err() @@ -2166,14 +1944,9 @@ mod tests { #[test] fn health_probe_is_healthy_only_on_a_200_from_the_child() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("child.sock"); - let probe = HttpHealthProbe { - endpoint: ChildEndpoint::Unix(socket.clone()), - path: "/health".into(), - }; - assert!(!probe.healthy(), "no listener is not healthy"); - let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + let probe = HttpHealthProbe { member, path: "/health".into() }; let server = thread::spawn(move || { for status in ["200 OK", "503 Service Unavailable"] { let (mut stream, _) = listener.accept().unwrap(); @@ -2187,13 +1960,14 @@ mod tests { assert!(probe.healthy()); assert!(!probe.healthy()); server.join().unwrap(); + assert!(!probe.healthy(), "no listener is not healthy"); } #[test] fn contract_parsing_reads_every_declared_field() { let args: Vec = [ "--child", "bin/c", "--child-arg", "-x", "--child-arg", "y", "--config-flag", "-c", - "--template", "t", "--listener", "loopback", "--health-path", "/h", + "--template", "t", "--health-path", "/h", "--health-contract", "hc", "--state-root", "/s", "--lease-wait-secs", "11", "--startup-secs", "22", "--heartbeat-secs", "33", ] @@ -2202,7 +1976,6 @@ mod tests { let contract = LaunchContract::parse(PathBuf::from("/r"), args.clone()).unwrap(); assert_eq!(contract.child_args, ["-x", "y"]); assert_eq!(contract.config_flag.as_deref(), Some("-c")); - assert_eq!(contract.listener, ChildListener::LoopbackPort); assert_eq!(contract.state_root, Some(PathBuf::from("/s"))); assert_eq!( (contract.lease_wait, contract.startup, contract.heartbeat), @@ -2226,7 +1999,7 @@ mod tests { Ok(PathBuf::from(name)) } } - let renderer = ConfigRenderer { credentials: &Any, listen: "L", state_root: None }; + let renderer = ConfigRenderer { credentials: &Any, listen: member(), state_root: None }; for ok in ["A", "a-b_9", "Z0"] { assert_eq!(renderer.render(&format!("{{{{credential:{ok}}}}}")).unwrap(), "v"); } @@ -2234,7 +2007,7 @@ mod tests { assert!(renderer.render(&format!("{{{{credential:{bad}}}}}")).is_err(), "{bad}"); assert!(renderer.render(&format!("{{{{credential_path:{bad}}}}}")).is_err(), "{bad}"); } - assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "aLbLc"); + assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "a127.0.0.1:18845b127.0.0.1:18845c"); } #[test] @@ -2263,7 +2036,7 @@ mod tests { std::env::set_var("IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", &file); } let credentials = CredentialsDirectory::new(dir.path().to_owned()); - let renderer = ConfigRenderer { credentials: &credentials, listen: "L", state_root: None }; + let renderer = ConfigRenderer { credentials: &credentials, listen: member(), state_root: None }; for name in [ "GAMECULT_IDUNN_CANDIDATE_BIND", "IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", @@ -2316,38 +2089,6 @@ mod tests { assert!(OdinRudpSink::from_expected(&expected).is_err()); } - #[test] - fn half_closed_member_requests_still_reach_a_child_that_waits_for_eof() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("child.sock"); - let child = std::os::unix::net::UnixListener::bind(&socket).unwrap(); - let fake = thread::spawn(move || { - let (mut stream, _) = child.accept().unwrap(); - let mut received = Vec::new(); - stream.read_to_end(&mut received).unwrap(); - stream.write_all(&received.iter().rev().copied().collect::>()).unwrap(); - }); - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let halt = Arc::new(AtomicBool::new(false)); - let responder: Responder = Arc::new(|_| bail!("never")); - let acceptor = RouteAcceptor::new(listener, ChildEndpoint::Unix(socket), responder, RouteBounds::default()); - let serving = { - let halt = halt.clone(); - thread::spawn(move || acceptor.serve(halt)) - }; - let mut member = TcpStream::connect(address).unwrap(); - member.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); - member.write_all(b"GET /abc").unwrap(); - member.shutdown(Shutdown::Write).unwrap(); - let mut answer = Vec::new(); - member.read_to_end(&mut answer).unwrap(); - assert_eq!(answer, b"cba/ TEG"); - fake.join().unwrap(); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); - } - #[test] fn heartbeats_are_paced_and_a_stop_exits_zero() { let fx = fixture(true); @@ -2406,32 +2147,35 @@ mod tests { assert!(running.exited().is_some()); } - // ------------------------------------------------------------ Soul s1 fixes + // ------------------------------------------------------------ presence acceptor - fn serve_with( + type Accepted = io::Result<(TcpStream, SocketAddr)>; + + fn serve_presence_accepting( fx: &Fixture, - bounds: RouteBounds, - ) -> (SocketAddr, Arc, thread::JoinHandle>) { - let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + responder: Responder, + accept: impl FnMut(&TcpListener) -> Accepted + Send + 'static, + ) -> (SocketAddr, Arc, thread::JoinHandle<()>) { + let _ = fx; let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); - let responder: Responder = Arc::new(move |request| shared.observe(request)); - let acceptor = - RouteAcceptor::new(listener, ChildEndpoint::Unix("/nonexistent".into()), responder, bounds); + let acceptor = PresenceAcceptor::new(listener, responder); let halt = Arc::new(AtomicBool::new(false)); let flag = halt.clone(); - (address, halt, thread::spawn(move || acceptor.serve(flag))) + (address, halt, thread::spawn(move || acceptor.serve_accepting(flag, accept))) } - fn quick(first_byte_ms: u64, challenge_ms: u64, max_connections: usize) -> RouteBounds { - RouteBounds { - first_byte: Duration::from_millis(first_byte_ms), - challenge: Duration::from_millis(challenge_ms), - max_connections, - } + fn observing(fx: &Fixture) -> Responder { + let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + Arc::new(move |request| shared.observe(request)) } - /// True when the peer closed (EOF or reset) within `within`. + fn serve_presence(fx: &Fixture) -> (SocketAddr, Arc, thread::JoinHandle<()>) { + serve_presence_accepting(fx, observing(fx), |listener| listener.accept()) + } + + /// True when the peer closed (EOF or reset) within `within`, having sent + /// nothing. fn closed_within(stream: &mut TcpStream, within: Duration) -> bool { stream.set_read_timeout(Some(within)).unwrap(); let mut byte = [0_u8; 64]; @@ -2454,24 +2198,58 @@ mod tests { } #[test] - fn an_idle_connection_is_closed_at_the_first_byte_deadline() { + fn only_challenge_frames_are_answered_and_anything_else_is_closed_unanswered() { let fx = fixture(false); - let (address, halt, serving) = serve_with(&fx, quick(300, 5000, 64)); - let mut idle = TcpStream::connect(address).unwrap(); - let started = Instant::now(); - assert!(closed_within(&mut idle, Duration::from_secs(4)), "never closed"); - assert!(started.elapsed() >= Duration::from_millis(250), "closed before its deadline"); + let reached = Arc::new(AtomicUsize::new(0)); + let responder: Responder = { + let (inner, reached) = (observing(&fx), reached.clone()); + Arc::new(move |request| { + reached.fetch_add(1, Ordering::SeqCst); + inner(request) + }) + }; + let (address, halt, serving) = + serve_presence_accepting(&fx, responder, |listener| listener.accept()); + // Member traffic is not presence: an HTTP request gets no byte back. + let mut member = TcpStream::connect(address).unwrap(); + member + .write_all(b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 2\r\n\r\n\x00\x01") + .unwrap(); + assert!(closed_within(&mut member, Duration::from_secs(3)), "member bytes were answered"); + // A leading 0x00 with a frame length outside its bound never reaches + // the responder. + let mut malformed = TcpStream::connect(address).unwrap(); + malformed.write_all(&[0x00, 0xff, 0xff, 0xff]).unwrap(); + assert!(closed_within(&mut malformed, Duration::from_secs(3)), "a malformed frame was answered"); + assert_eq!(reached.load(Ordering::SeqCst), 0); + // A real challenge is answered, and is the only thing that reaches it. assert!(challenge_answered(address, &fx)); + assert_eq!(reached.load(Ordering::SeqCst), 1); halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); + serving.join().unwrap(); } #[test] - fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_its_total_deadline() { + fn an_idle_peer_holds_the_acceptor_for_the_presence_deadline_and_no_longer() { let fx = fixture(false); - // Every pause is far inside the first-byte deadline: only the total - // challenge deadline can end this. - let (address, halt, serving) = serve_with(&fx, quick(5000, 600, 64)); + let (address, halt, serving) = serve_presence(&fx); + let mut idle = TcpStream::connect(address).unwrap(); + let started = Instant::now(); + // Idunn waits 3 s for a challenge answer; one stalled peer ahead of it + // must not cost it that. + assert!(challenge_answered(address, &fx)); + let waited = started.elapsed(); + assert!(waited >= Duration::from_millis(800), "the idle peer was cut early: {waited:?}"); + assert!(waited < Duration::from_millis(2200), "the idle peer held the acceptor: {waited:?}"); + assert!(closed_within(&mut idle, Duration::from_secs(1)), "the idle peer stayed connected"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_the_presence_deadline() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); let mut slow = TcpStream::connect(address).unwrap(); slow.write_all(&[0x00, 0x00, 0x01, 0x00]).unwrap(); let started = Instant::now(); @@ -2483,51 +2261,34 @@ mod tests { } } assert!(closed, "the trickle was never cut off"); - assert!(started.elapsed() >= Duration::from_millis(500), "cut before its deadline"); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(800), "cut before its deadline: {elapsed:?}"); + assert!(elapsed < Duration::from_millis(2200), "cut after its deadline: {elapsed:?}"); assert!(challenge_answered(address, &fx)); halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); + serving.join().unwrap(); } #[test] - fn live_connections_are_capped_and_the_overflow_is_refused_unspawned() { + fn accept_errors_never_end_the_presence_acceptor() { let fx = fixture(false); - let (address, halt, serving) = serve_with(&fx, quick(30_000, 5000, 4)); - let mut held: Vec = (0..4).map(|_| TcpStream::connect(address).unwrap()).collect(); - thread::sleep(Duration::from_millis(300)); - let threads_before = thread_count(); - let mut overflow: Vec = (0..40).map(|_| TcpStream::connect(address).unwrap()).collect(); - for stream in &mut overflow { - assert!(closed_within(stream, Duration::from_secs(3)), "an overflow connection was served"); - } - assert!(thread_count() < threads_before + 8, "overflow connections cost threads"); - for stream in &mut held { - assert!(!closed_within(stream, Duration::from_millis(100)), "a held connection was cut"); - } - // A freed slot serves the next peer. - held.pop(); - let mut answered = false; - for _ in 0..40 { - if challenge_answered(address, &fx) { - answered = true; - break; - } - thread::sleep(Duration::from_millis(50)); - } - assert!(answered, "a freed slot was never reused"); + // Descriptor exhaustion: the first accepts fail, then descriptors free. + let failures = Arc::new(AtomicUsize::new(0)); + let (address, halt, serving) = { + let failures = failures.clone(); + serve_presence_accepting(&fx, observing(&fx), move |listener| { + if failures.fetch_add(1, Ordering::SeqCst) < 3 { + Err(io::Error::from_raw_os_error(libc::EMFILE)) + } else { + listener.accept() + } + }) + }; + assert!(challenge_answered(address, &fx), "the acceptor did not recover"); + assert!(failures.load(Ordering::SeqCst) >= 4); + assert!(!serving.is_finished()); halt.store(true, Ordering::SeqCst); - serving.join().unwrap().unwrap(); - } - - fn thread_count() -> usize { - fs::read_to_string("/proc/self/status") - .unwrap() - .lines() - .find_map(|line| line.strip_prefix("Threads:")) - .unwrap() - .trim() - .parse() - .unwrap() + serving.join().unwrap(); } /// A listener whose accept always fails: a datagram socket's descriptor. @@ -2539,50 +2300,57 @@ mod tests { } #[test] - fn an_acceptor_that_cannot_accept_ends_with_an_error() { + fn an_acceptor_whose_every_accept_fails_runs_until_halted() { let fx = fixture(false); - let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); - let responder: Responder = Arc::new(move |request| shared.observe(request)); - let acceptor = RouteAcceptor::new( - failing_listener(), - ChildEndpoint::Unix("/nonexistent".into()), - responder, - RouteBounds::default(), - ); - assert!(acceptor.serve(Arc::new(AtomicBool::new(false))).is_err()); + let acceptor = PresenceAcceptor::new(failing_listener(), observing(&fx)); + let halt = Arc::new(AtomicBool::new(false)); + let flag = halt.clone(); + let serving = thread::spawn(move || acceptor.serve(flag)); + thread::sleep(Duration::from_millis(400)); + assert!(!serving.is_finished(), "the acceptor ended on an accept error"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + /// Healthy for everyone but the presence thread, which it kills: the + /// stand-in for a presence acceptor that dies while the child is fine. + struct PresenceTripwire; + + impl HealthProbe for PresenceTripwire { + fn healthy(&self) -> bool { + assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); + true + } } #[test] - fn presence_stops_claiming_active_when_the_route_acceptor_dies() { + fn a_dead_presence_acceptor_fails_the_unit() { let fx = fixture(false); let sink = Arc::new(Sink::default()); let child = Arc::new(Child::default()); - let health = Arc::new(Health(AtomicBool::new(true))); let ports = Ports { credentials: Arc::new(Credentials(BTreeMap::new())), clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), sink: sink.clone(), lease: Arc::new(FakeLease::default()), - spawner: Box::new(FakeSpawner(child)), - route_bounds: RouteBounds::default(), - health: Box::new(move |_| health.clone()), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(|_| Arc::new(PresenceTripwire)), }; let contract = fx.contract("x"); let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); let tmp = fx.dir.path().join("tmp"); let stop = Arc::new(AtomicBool::new(false)); let flag = stop.clone(); - let handle = thread::spawn(move || run(&contract, authority, failing_listener(), &tmp, ports, &flag)); - wait_until("two degraded heartbeats", || { - sink.states().iter().filter(|state| *state == "degraded").count() >= 2 - }); - // The child is healthy throughout; only the dead acceptor degrades it, - // and nothing says active after that. - let states = sink.states(); - let first_degraded = states.iter().position(|state| state == "degraded").unwrap(); - assert!(states[first_degraded..].iter().all(|state| state != "active"), "{states:?}"); - stop.store(true, Ordering::SeqCst); - assert_eq!(handle.join().unwrap(), Outcome::Stopped); + let handle = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + // The first challenge reaches the tripwire on the presence thread. + assert!(!challenge_answered(address, &fx)); + assert_eq!(handle.join().unwrap(), Outcome::Failed("presence-acceptor")); + assert_eq!(sink.first("failed").unwrap().detail, "presence-acceptor"); + assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); } #[test] @@ -2701,7 +2469,6 @@ mod tests { sink: sink.clone(), lease: Arc::new(FakeLease::default()), spawner: Box::new(FakeSpawner(child.clone())), - route_bounds: RouteBounds::default(), health: Box::new(move |_| health.clone()), }; let authority = fx.authority(); @@ -2709,7 +2476,8 @@ mod tests { let tmp = fx.dir.path().join("tmp"); let stop = Arc::new(AtomicBool::new(false)); let flag = stop.clone(); - let handle = thread::spawn(move || run(&contract, authority, route, &tmp, ports, &flag)); + let handle = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); wait_until("active", || sink.first("active").is_some()); assert!(!child.terminated.load(Ordering::SeqCst), "killed while it was still starting"); assert_eq!(child.spawns.load(Ordering::SeqCst), 1); @@ -2719,23 +2487,86 @@ mod tests { } #[test] - fn a_hung_health_probe_times_out_instead_of_blocking() { + fn a_silent_child_fails_the_probe_within_its_timeout() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let endpoint = ChildEndpoint::Loopback(listener.local_addr().unwrap()); + let member = listener.local_addr().unwrap(); // Accepts and never answers. let held = thread::spawn(move || { let _connection = listener.accept().unwrap(); - thread::sleep(Duration::from_secs(8)); + thread::sleep(Duration::from_secs(4)); }); - let probe = HttpHealthProbe { endpoint, path: "/health".into() }; - let (done, verdict) = mpsc::channel(); - thread::spawn(move || { - let _ = done.send(probe.healthy()); - }); - let healthy = verdict - .recv_timeout(Duration::from_secs(5)) - .expect("the probe outlived its timeout against a hung child"); - assert!(!healthy); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let started = Instant::now(); + assert!(!probe.healthy()); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(900), "gave up before its timeout: {elapsed:?}"); + assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its timeout: {elapsed:?}"); drop(held); } + + #[test] + fn the_probe_deadline_covers_the_whole_response() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + // Every byte arrives inside any per-read timeout, but the head takes + // three seconds in all. + let dribbler = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + for byte in b"HTTP/1.1 200 OK" { + if stream.write_all(&[*byte]).is_err() { + return; + } + thread::sleep(Duration::from_millis(250)); + } + }); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let started = Instant::now(); + assert!(!probe.healthy()); + let elapsed = started.elapsed(); + assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its deadline: {elapsed:?}"); + let _ = dribbler.join(); + } + + #[test] + fn a_process_spawner_refuses_an_invalid_lease_descriptor_and_runs_no_child() { + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("ran"); + let script = format!("touch '{}'", marker.display()); + for invalid in [-1, 9999] { + let refused = ProcessSpawner.spawn( + Path::new("/bin/sh"), + &["-c".to_owned(), script.clone()], + Some(invalid), + ); + assert!(refused.is_err(), "descriptor {invalid} was accepted"); + } + thread::sleep(Duration::from_millis(300)); + assert!(!marker.exists(), "a child ran past an invalid descriptor"); + // Control: the same spawn without a descriptor runs the child. + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".to_owned(), script], None) + .unwrap(); + wait_until("the control child to exit", || child.exited().is_some()); + assert!(marker.exists()); + } + + #[test] + fn the_member_bind_is_a_loopback_address_other_than_the_candidate() { + let candidate = "127.0.0.1:18835"; + assert_eq!( + parse_member_bind("127.0.0.1:18845", candidate).unwrap(), + "127.0.0.1:18845".parse().unwrap() + ); + for bad in [ + "127.0.0.1:18835", + "10.0.0.5:18845", + "0.0.0.0:18845", + "canary-not-an-address", + "", + ] { + let error = parse_member_bind(bad, candidate).unwrap_err().to_string(); + assert!(!error.contains("canary") && !error.contains("10.0.0.5"), "{error}"); + } + assert!(parse_member_bind("127.0.0.1:18845", "not-an-address").is_err()); + } } From b379fbd70035d55b745442a9419a64da19268f62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 01:45:19 +0200 Subject: [PATCH 061/122] Tests kill mutation survivors: each owned-name site is pinned alone, the member range below the route's is accepted, a challenge near the frame bound is answered; accept errors share one backoff Builds. Co-Authored-By: Claude Sonnet 5.5 --- src/deployment.rs | 39 ++++++++++++++++++++++++++++++--------- src/launcher.rs | 29 ++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/deployment.rs b/src/deployment.rs index 067f3b8..6fdf339 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -2412,8 +2412,10 @@ nodes = ["yggdrasil"] // Overlapping the route's own private range, in either direction. assert!(start("18835").is_err()); assert!(start("18825").is_err()); - // Adjacent but disjoint is fine. + // Adjacent but disjoint is fine, above the route's range or below it. assert!(start("18840").is_ok()); + assert!(start("18822").is_ok()); + assert!(start("18823").is_err()); // Zero, and a width that overflows u16. assert!(start("0").is_err()); assert!(start("65530").is_err()); @@ -2473,18 +2475,37 @@ nodes = ["yggdrasil"] assert!(OperatorBinding::parse(&as_environment("UNOWNED_NAME")).is_ok()); assert!(OperatorBinding::parse(&as_secret("UNOWNED_NAME")).is_ok()); for name in owned { + // Refused when the binding is parsed... assert!( - OperatorBinding::parse(&as_environment(name)) - .and_then(|binding| binding.admit(&recipe)) - .is_err(), - "{name} accepted as environment" + OperatorBinding::parse(&as_environment(name)).is_err(), + "{name} accepted as environment at parse" ); assert!( - OperatorBinding::parse(&as_secret(name)) - .and_then(|binding| binding.admit(&recipe)) - .is_err(), - "{name} accepted as a secret file" + OperatorBinding::parse(&as_secret(name)).is_err(), + "{name} accepted as a secret file at parse" ); + // ...and again at admission, for a binding that did not come + // through parse. Each site reads the one list. + for secret in [false, true] { + let mut binding = OperatorBinding::parse(BINDING).unwrap(); + let WorkloadBinding::SystemdTransient(workload) = &mut binding.workload else { + panic!("the fixture is a systemd workload"); + }; + if secret { + workload.secret_files.insert(name.into(), "/etc/gamecult/test-service/other.cc".into()); + } else { + workload.environment.insert(name.into(), "forged".into()); + } + assert!( + binding.workload.validate_launch_environment().is_err(), + "{name} (secret: {secret}) accepted by the binding check" + ); + let error = binding.admit(&recipe).unwrap_err().to_string(); + assert!( + error.contains("Idunn-owned launch environment"), + "{name} (secret: {secret}) accepted by the launch-input check: {error}" + ); + } } } diff --git a/src/launcher.rs b/src/launcher.rs index 391ee9e..539c396 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -955,9 +955,8 @@ impl PresenceAcceptor { let _ = connection.set_nonblocking(false); let _ = self.answer(connection); } - Err(error) if error.kind() == io::ErrorKind::WouldBlock => { - thread::sleep(Duration::from_millis(10)); - } + // An idle non-blocking accept is an error too; every one waits + // and tries again. Err(_) => thread::sleep(ACCEPT_BACKOFF), } } @@ -2229,6 +2228,30 @@ mod tests { serving.join().unwrap(); } + #[test] + fn a_challenge_up_to_the_frame_bound_is_answered_and_one_past_it_is_not() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); + // A request far larger than a few KiB, well inside the 1 MiB bound. + let request = CultNetMessage::SnapshotRequest { + message_id: "large".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some((0..300).map(|index| format!("{index:0>40}")).collect()), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert!(payload.len() > 10_000 && payload.len() < CHALLENGE_MAX_BYTES); + assert!(request_tcp_snapshot(address, &payload).is_ok()); + // One byte past the bound is closed unanswered. + let past = (CHALLENGE_MAX_BYTES as u32 + 1).to_be_bytes(); + assert_eq!(past[0], CHALLENGE_FIRST_BYTE); + let mut oversized = TcpStream::connect(address).unwrap(); + oversized.write_all(&past).unwrap(); + assert!(closed_within(&mut oversized, Duration::from_secs(3)), "an oversized frame was read"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + #[test] fn an_idle_peer_holds_the_acceptor_for_the_presence_deadline_and_no_longer() { let fx = fixture(false); From 2218ffb8a82cb037a0d78b3184ea1aa4f4335e19 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:10:12 +0200 Subject: [PATCH 062/122] Idunn routes: the member server block renders a limit_conn on a per-route zone; [route.member].max_connections defaults to 128 Co-Authored-By: Claude Sonnet 5.5 --- src/deployment.rs | 30 +++++++++ src/drivers.rs | 151 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/src/deployment.rs b/src/deployment.rs index 6fdf339..75a30ab 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -694,6 +694,18 @@ pub struct RouteBinding { pub struct MemberListenerBinding { pub stable_endpoint: String, pub private_port_start: u16, + /// Most member connections nginx holds open at once for this route. Each + /// costs nginx two connections (client and upstream) from the pool the + /// route's presence block shares, so this limit is what keeps members from + /// exhausting it. + #[serde(default = "default_member_max_connections")] + pub max_connections: u32, +} + +const DEFAULT_MEMBER_MAX_CONNECTIONS: u32 = 128; + +fn default_member_max_connections() -> u32 { + DEFAULT_MEMBER_MAX_CONNECTIONS } impl RouteBinding { @@ -1466,6 +1478,10 @@ impl OperatorBinding { member.private_port_start > 0, "member private port range starts at zero" ); + ensure!( + member.max_connections > 0, + "member connection limit is zero" + ); let width = route.private_port_end - route.private_port_start; let member_end = member .private_port_start @@ -2416,6 +2432,20 @@ nodes = ["yggdrasil"] assert!(start("18840").is_ok()); assert!(start("18822").is_ok()); assert!(start("18823").is_err()); + // The connection limit defaults, and zero is refused. + let limit = |value: &str| { + member_binding_with(( + "private_port_start = 18841", + &format!("private_port_start = 18841 +max_connections = {value}"), + )) + }; + assert_eq!( + member_binding_with(("never-matches", "")).unwrap().route.unwrap().member.unwrap().max_connections, + 128 + ); + assert_eq!(limit("7").unwrap().route.unwrap().member.unwrap().max_connections, 7); + assert!(limit("0").is_err()); // Zero, and a width that overflows u16. assert!(start("0").is_err()); assert!(start("65530").is_err()); diff --git a/src/drivers.rs b/src/drivers.rs index fb4ff1b..d3f6abf 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5391,8 +5391,16 @@ impl NginxRouteDriver { ) { (Some((member_host, member_stable_port)), Some(member_port)) => { let member_upstream = format!("{upstream}_member"); + let max_connections = self + .binding + .member + .as_ref() + .map_or(0, |member| member.max_connections); + // One constant key: the limit is on the route's members + // together, so a flood of them can never take the + // connections the presence block needs. format!( - "upstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n proxy_pass {member_upstream};\n}}\n" + "limit_conn_zone \"{member_upstream}\" zone={member_upstream}:64k;\nupstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n limit_conn {member_upstream} {max_connections};\n proxy_pass {member_upstream};\n}}\n" ) } _ => String::new(), @@ -12424,6 +12432,7 @@ processes 9 driver.binding.member = Some(crate::deployment::MemberListenerBinding { stable_endpoint: "http://127.0.0.1:4120".into(), private_port_start: 4121, + max_connections: 200, }); } @@ -12493,6 +12502,146 @@ processes 9 Ok(()) } + /// Real nginx over the rendered two-block fragment, a real presence + /// acceptor behind the presence block and a flood of idle members on the + /// member block. Needs an `nginx` on PATH and the fixed loopback ports + /// 4103, 4104, 4120 and 4121, so it runs on demand in a container: + /// `cargo test --lib real_nginx -- --ignored`. + #[cfg(unix)] + fn challenges_answered_during_a_member_flood(max_connections: u32) -> Result { + use std::net::{TcpListener, TcpStream}; + use std::process::{Child, Command}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + struct Nginx(Child); + impl Drop for Nginx { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + let temp = tempfile::tempdir()?; + let (mut driver, candidate, _calls) = stubbed_route_driver(temp.path())?; + with_member_listener(&mut driver); + driver.binding.member.as_mut().unwrap().max_connections = max_connections; + let fragment = temp.path().join("route.conf"); + fs::write(&fragment, driver.render(&candidate)?)?; + // Debian builds the stream module dynamic. + let module = Path::new("/usr/lib/nginx/modules/ngx_stream_module.so"); + let load = if module.exists() { + format!("load_module {}; +", module.display()) + } else { + String::new() + }; + let conf = temp.path().join("nginx.conf"); + fs::write( + &conf, + format!( + "{load}worker_processes 1;\npid {dir}/nginx.pid;\nerror_log {dir}/error.log;\nevents {{ worker_connections 512; }}\nstream {{ include {fragment}; }}\n", + dir = temp.path().display(), + fragment = fragment.display() + ), + )?; + let halt = Arc::new(AtomicBool::new(false)); + let presence = crate::launcher::PresenceAcceptor::new( + TcpListener::bind("127.0.0.1:4104")?, + Arc::new(|request| match request { + CultNetMessage::SnapshotRequest { message_id, .. } => { + Ok(CultNetMessage::SnapshotResponseRaw { message_id, documents: vec![] }) + } + _ => bail!("not a snapshot request"), + }), + ); + let presence_halt = halt.clone(); + let presence = thread::spawn(move || presence.serve(presence_halt)); + // The member backend accepts and holds every connection. + let members = TcpListener::bind("127.0.0.1:4121")?; + members.set_nonblocking(true)?; + let member_halt = halt.clone(); + let members = thread::spawn(move || { + let mut held = Vec::new(); + while !member_halt.load(Ordering::SeqCst) { + match members.accept() { + Ok((stream, _)) => held.push(stream), + Err(_) => thread::sleep(Duration::from_millis(5)), + } + } + }); + let _nginx = Nginx( + Command::new("nginx") + .args(["-p"]) + .arg(temp.path()) + .arg("-c") + .arg(&conf) + .args(["-g", "daemon off;"]) + .spawn()?, + ); + let stable: SocketAddr = "127.0.0.1:4103".parse()?; + let ready = Instant::now(); + while TcpStream::connect(stable).is_err() { + ensure!(ready.elapsed() < Duration::from_secs(10), "nginx never listened"); + thread::sleep(Duration::from_millis(50)); + } + // The probe connection above reached the presence acceptor as an idle + // peer; let its deadline lapse so only the flood is measured. + thread::sleep(Duration::from_millis(1200)); + + let mut flood = Vec::new(); + for _ in 0..300 { + if let Ok(mut stream) = TcpStream::connect("127.0.0.1:4120") { + let _ = stream.write_all(b"x"); + flood.push(stream); + } + } + thread::sleep(Duration::from_millis(500)); + + let payload = encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: "flood".into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + )?; + let mut answered = 0; + for _ in 0..20 { + let started = Instant::now(); + if request_tcp_snapshot(stable, &payload).is_ok() + && started.elapsed() < Duration::from_secs(3) + { + answered += 1; + } + } + drop(flood); + halt.store(true, Ordering::SeqCst); + presence.join().unwrap(); + members.join().unwrap(); + Ok(answered) + } + + #[cfg(unix)] + #[test] + #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] + fn real_nginx_keeps_presence_answerable_under_a_member_flood() -> Result<()> { + // The rendered default limit keeps the route's members out of the + // presence block's share of a 512-connection pool. + assert_eq!(challenges_answered_during_a_member_flood(128)?, 20); + Ok(()) + } + + #[cfg(unix)] + #[test] + #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] + fn real_nginx_flood_starves_presence_without_a_member_limit() -> Result<()> { + // The control: the same flood against a limit too high to bind starves + // presence, so the test above measures the limit and not the load. + assert!(challenges_answered_during_a_member_flood(100_000)? < 20); + Ok(()) + } + /// Idunn sets the member bind for a bound member listener, derived from the /// candidate, and for no other workload. #[test] From 36114b8b7baa6dcfb656428c14b0250ba72aa228 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:12:56 +0200 Subject: [PATCH 063/122] Idunn routes tests: the rendered member block pins its limit zone; real-nginx flood tests prove presence stays answerable Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers.rs b/src/drivers.rs index d3f6abf..060917d 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -12462,7 +12462,7 @@ processes 9 with_member_listener(&mut driver); let member_blocks = |member_port: u16| { format!( - "upstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n proxy_pass idunn_service_route_member;\n}}\n" + "limit_conn_zone \"idunn_service_route_member\" zone=idunn_service_route_member:64k;\nupstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n limit_conn idunn_service_route_member 200;\n proxy_pass idunn_service_route_member;\n}}\n" ) }; let rendered = String::from_utf8(driver.render(&candidate)?)?; From e5294db40cce39f0b9d013503f5b29bf077c5630 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:12:57 +0200 Subject: [PATCH 064/122] idunn-launch: a challenge answers from the lifecycle's last probe, stale or failed reads degraded; tests bound their waits and pin startup timeout and heartbeat cadence Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 331 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 294 insertions(+), 37 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 539c396..f15882e 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -59,6 +59,11 @@ const PRESENCE_DEADLINE: Duration = Duration::from_secs(1); /// again. No accept error ends the acceptor. const ACCEPT_BACKOFF: Duration = Duration::from_millis(50); const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); +/// How often the lifecycle probes the child once it is Active. +const HEALTH_REFRESH: Duration = Duration::from_millis(500); +/// A probe result older than this is not evidence of health: the lifecycle +/// that refreshes it is itself stuck. +const HEALTH_STALE: Duration = Duration::from_secs(5); const POLL: Duration = Duration::from_millis(100); const MAX_RECENT_WARMING_PROOFS: usize = 64; const TERMINATE_GRACE: Duration = Duration::from_secs(20); @@ -859,18 +864,36 @@ struct Shared { lease: Mutex>, lease_required: bool, health: Arc, + /// The last probe result and the clock time it started. Only the lifecycle + /// probes; a challenge reads this and never waits on the child. + probed: Mutex>, clock: Arc, } impl Shared { - /// Active only while the child is healthy. A dead presence acceptor does - /// not degrade presence: it fails the unit. - fn serving_state(&self) -> &'static str { - if self.health.healthy() { - "active" - } else { - "degraded" + /// Probes the child now and keeps the result for presence to answer from. + fn refresh_health(&self) -> bool { + let started = self.clock.now_millis(); + let healthy = self.health.healthy(); + if let Ok(mut probed) = self.probed.lock() { + *probed = Some((healthy, started)); } + healthy + } + + /// Active only while the last probe found the child healthy and is recent. + /// A child never probed, wedged, or probed too long ago reads degraded. A + /// dead presence acceptor does not degrade presence: it fails the unit. + fn serving_state(&self) -> &'static str { + let now = self.clock.now_millis(); + let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); + let healthy = self + .probed + .lock() + .ok() + .and_then(|probed| *probed) + .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); + if healthy { "active" } else { "degraded" } } fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { @@ -1084,6 +1107,7 @@ pub fn run( lease: Mutex::new(None), lease_required: expected.write_lease_required, health: Arc::clone(&health), + probed: Mutex::new(None), clock: Arc::clone(&ports.clock), }); let halt = Arc::new(AtomicBool::new(false)); @@ -1210,7 +1234,7 @@ fn lifecycle( if presence.is_finished() { return fail(Some(&mut child), "presence-acceptor"); } - if shared.health.healthy() { + if shared.refresh_health() { break; } if clock.now_millis() >= startup_deadline { @@ -1222,6 +1246,8 @@ fn lifecycle( // 4. Active, and heartbeats while it holds. set_phase(Phase::Active(lease_sha.clone())); publish(shared.serving_state(), "child-healthy", lease_sha.clone()); + let refresh = u64::try_from(HEALTH_REFRESH.as_millis()).unwrap_or(u64::MAX); + let mut next_probe = clock.now_millis() + refresh; next_heartbeat = clock.now_millis() + heartbeat; loop { if stop.load(Ordering::SeqCst) { @@ -1238,6 +1264,10 @@ fn lifecycle( return fail(Some(&mut child), "lease-lost"); } let now = clock.now_millis(); + if now >= next_probe { + shared.refresh_health(); + next_probe = now + refresh; + } if now >= next_heartbeat { publish(shared.serving_state(), "heartbeat", lease_sha.clone()); next_heartbeat = now + heartbeat; @@ -1793,13 +1823,18 @@ mod tests { } fn shared(fx: &Fixture, health: Arc) -> Arc { + shared_on(fx, health, Arc::new(SystemClock)) + } + + fn shared_on(fx: &Fixture, health: Arc, clock: Arc) -> Arc { Arc::new(Shared { authority: Mutex::new(fx.authority()), phase: Mutex::new(Phase::Active(None)), lease: Mutex::new(None), lease_required: false, health, - clock: Arc::new(SystemClock), + probed: Mutex::new(None), + clock, }) } @@ -1824,8 +1859,14 @@ mod tests { assert_eq!(record.detail, "route-observation:challenge-1"); record.state }; + // Never probed: nothing says the child is healthy. + assert_eq!(state_of(&shared), "degraded"); + shared.refresh_health(); assert_eq!(state_of(&shared), "active"); + // A challenge answers from the last probe and never runs one. health.0.store(false, Ordering::SeqCst); + assert_eq!(state_of(&shared), "active"); + shared.refresh_health(); assert_eq!(state_of(&shared), "degraded"); } @@ -1833,6 +1874,7 @@ mod tests { fn route_challenge_is_answered_with_both_signatures_through_idunns_client() { let fx = fixture(false); let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); + shared.refresh_health(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let responder: Responder = Arc::new(move |request| shared.observe(request)); @@ -2088,34 +2130,248 @@ mod tests { assert!(OdinRudpSink::from_expected(&expected).is_err()); } + #[test] - fn heartbeats_are_paced_and_a_stop_exits_zero() { + fn a_probe_result_is_evidence_only_while_it_is_recent() { + let fx = fixture(false); + let clock = Arc::new(FastClock(AtomicU64::new(1_000_000))); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + let stale = HEALTH_STALE.as_millis() as u64; + clock.0.fetch_add(stale, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "active", "a result exactly at the bound is still recent"); + clock.0.fetch_add(1, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "degraded", "a result past the bound is not evidence"); + shared.refresh_health(); + assert_eq!(shared.serving_state(), "active"); + } + + /// Runs `work` and returns its result, failing the test when it has not + /// finished within `limit`, so a mutant that makes it wait forever fails + /// instead of hanging the suite. + fn within(limit: Duration, work: impl FnOnce() -> T + Send + 'static) -> T { + join_within(limit, thread::spawn(work)) + } + + fn join_within(limit: Duration, handle: thread::JoinHandle) -> T { + let started = Instant::now(); + while !handle.is_finished() { + assert!(started.elapsed() < limit, "still running after {limit:?}"); + thread::sleep(Duration::from_millis(10)); + } + handle.join().unwrap() + } + + /// The state the presence endpoint signs in answer to a challenge, or None + /// when it is not answered promptly. + fn challenge_state(address: SocketAddr, fx: &Fixture) -> Option { + let request = CultNetMessage::SnapshotRequest { + message_id: "wedge".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + let started = Instant::now(); + let answer = request_tcp_snapshot(address, &payload).ok()?; + if started.elapsed() > Duration::from_millis(750) { + return None; + } + let CultNetMessage::SnapshotResponseRaw { documents, .. } = + decode_cultnet_message_from_slice(&answer, CultNetWireContract::CultNetSchemaV0).ok()? + else { + return None; + }; + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&documents.first()?.payload).ok()?; + Some(record.state) + } + + #[derive(Clone, Copy)] + enum Wedge { + /// Accepts and never answers. + Silent, + /// Never accepts; the kernel backlog fills. + FullBacklog, + /// Accepts and answers 200 after 1.1 s, past the probe's deadline. + Slow, + } + + type WedgeSwitch = Arc>>; + + /// A child that answers 200 until wedged. Returns the member address, the + /// switch, and the thread to join after `halt`. + fn serving_child(halt: Arc) -> (SocketAddr, WedgeSwitch, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let wedge: WedgeSwitch = Arc::default(); + let mode = wedge.clone(); + let serving = thread::spawn(move || { + while !halt.load(Ordering::SeqCst) { + let current = *mode.lock().unwrap(); + if matches!(current, Some(Wedge::FullBacklog)) { + thread::sleep(Duration::from_millis(5)); + continue; + } + let Ok((mut stream, _)) = listener.accept() else { + thread::sleep(Duration::from_millis(5)); + continue; + }; + stream.set_nonblocking(false).unwrap(); + match current { + None => { + let mut request = [0_u8; 256]; + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + let _ = stream.read(&mut request); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + } + Some(Wedge::Slow) => { + thread::spawn(move || { + thread::sleep(Duration::from_millis(1100)); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + }); + } + _ => { + thread::spawn(move || { + thread::sleep(Duration::from_secs(8)); + drop(stream); + }); + } + } + } + }); + (address, wedge, serving) + } + + /// Whatever wedges the child, every challenge is still answered promptly, + /// and the answer turns to a signed degraded within a few probes. + fn a_wedged_child_is_answered_degraded(wedge: Wedge) { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let halt = Arc::new(AtomicBool::new(false)); + let (member, mode, child) = serving_child(halt.clone()); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(SystemClock), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(|member| Arc::new(HttpHealthProbe { member, path: "/health".into() })), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member, &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); + + *mode.lock().unwrap() = Some(wedge); + // A backlog fills only once clients have connected without being + // accepted. + let mut queued = Vec::new(); + if matches!(wedge, Wedge::FullBacklog) { + for _ in 0..4096 { + match TcpStream::connect_timeout(&member, Duration::from_millis(20)) { + Ok(stream) => queued.push(stream), + Err(_) => break, + } + } + } + let started = Instant::now(); + let mut degraded = false; + while started.elapsed() < Duration::from_secs(6) { + let state = challenge_state(address, &fx); + assert!(state.is_some(), "a challenge went unanswered after {:?}", started.elapsed()); + if state.as_deref() == Some("degraded") { + degraded = true; + break; + } + thread::sleep(Duration::from_millis(100)); + } + assert!(degraded, "the wedged child was never reported degraded"); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + halt.store(true, Ordering::SeqCst); + join_within(Duration::from_secs(10), child); + drop(queued); + } + + #[test] + fn a_silent_child_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::Silent); + } + + #[test] + fn a_child_with_a_full_backlog_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::FullBacklog); + } + + #[test] + fn a_child_answering_after_the_probe_deadline_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::Slow); + } + + /// The virtual time (milliseconds) of each record that has the given + /// detail. + fn times_of(sink: &Sink, detail: &str) -> Vec { + sink.0 + .lock() + .unwrap() + .iter() + .filter(|record| record.detail == detail) + .map(|record| record.observed_at_unix_millis) + .collect() + } + + #[test] + fn a_child_that_never_becomes_healthy_fails_the_unit_at_the_startup_timeout() { + let fx = fixture(false); + let mut contract = fx.contract("x"); + contract.startup = Duration::from_secs(5); + let mut launch = start(&fx, contract, false); + let sink = launch.sink.clone(); + let child = launch.child.clone(); + let outcome = within(Duration::from_secs(20), move || launch.finish()); + assert_eq!(outcome, Outcome::Failed("startup-timeout")); + assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); + let started = times_of(&sink, "launcher-started")[0]; + let failed = times_of(&sink, "startup-timeout")[0]; + // Virtual time: the deadline, found within one poll of it. + let waited = failed - started; + assert!( + (5_000..=5_000 + POLL.as_millis() as u64).contains(&waited), + "failed after {waited} ms of virtual time" + ); + } + + #[test] + fn heartbeats_follow_the_contracts_cadence_while_waiting_for_the_lease_and_while_active() { let fx = fixture(true); - let launch = start(&fx, fx.contract("x"), true); - thread::sleep(Duration::from_millis(250)); - let waiting = launch - .sink - .0 - .lock() - .unwrap() - .iter() - .filter(|record| record.detail == "write-lease-pending") - .count(); - assert!((1..=12).contains(&waiting), "{waiting} lease-wait heartbeats"); - let mut launch = launch; + let mut contract = fx.contract("x"); + contract.heartbeat = Duration::from_secs(1); + let mut launch = start(&fx, contract, true); + let cadence = 1_000..=1_000 + POLL.as_millis() as u64; + wait_until("lease-wait heartbeats", || { + times_of(&launch.sink, "write-lease-pending").len() >= 4 + }); let warming = launch.sink.first("warming").unwrap(); + let mut beats = vec![warming.observed_at_unix_millis]; + beats.extend(times_of(&launch.sink, "write-lease-pending").into_iter().take(4)); + for pair in beats.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "lease-wait beats {beats:?}"); + } *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); - wait_until("active", || launch.sink.first("active").is_some()); - thread::sleep(Duration::from_millis(250)); - let beats = launch - .sink - .0 - .lock() - .unwrap() - .iter() - .filter(|record| record.detail == "heartbeat") - .count(); - assert!((1..=12).contains(&beats), "{beats} active heartbeats"); + wait_until("active heartbeats", || times_of(&launch.sink, "heartbeat").len() >= 4); + let mut beats = vec![times_of(&launch.sink, "child-healthy")[0]]; + beats.extend(times_of(&launch.sink, "heartbeat").into_iter().take(4)); + for pair in beats.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "active beats {beats:?}"); + } assert_eq!(launch.stop().exit_code(), 0); } @@ -2166,6 +2422,7 @@ mod tests { fn observing(fx: &Fixture) -> Responder { let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + shared.refresh_health(); Arc::new(move |request| shared.observe(request)) } @@ -2371,7 +2628,7 @@ mod tests { wait_until("active", || sink.first("active").is_some()); // The first challenge reaches the tripwire on the presence thread. assert!(!challenge_answered(address, &fx)); - assert_eq!(handle.join().unwrap(), Outcome::Failed("presence-acceptor")); + assert_eq!(join_within(Duration::from_secs(20), handle), Outcome::Failed("presence-acceptor")); assert_eq!(sink.first("failed").unwrap().detail, "presence-acceptor"); assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); } @@ -2520,7 +2777,7 @@ mod tests { }); let probe = HttpHealthProbe { member, path: "/health".into() }; let started = Instant::now(); - assert!(!probe.healthy()); + assert!(!within(Duration::from_secs(5), move || probe.healthy())); let elapsed = started.elapsed(); assert!(elapsed >= Duration::from_millis(900), "gave up before its timeout: {elapsed:?}"); assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its timeout: {elapsed:?}"); @@ -2544,10 +2801,10 @@ mod tests { }); let probe = HttpHealthProbe { member, path: "/health".into() }; let started = Instant::now(); - assert!(!probe.healthy()); + assert!(!within(Duration::from_secs(5), move || probe.healthy())); let elapsed = started.elapsed(); assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its deadline: {elapsed:?}"); - let _ = dribbler.join(); + join_within(Duration::from_secs(10), dribbler); } #[test] From c38bb365601cdf5d9a9a01e86cf007de15f5f3fe Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:15:50 +0200 Subject: [PATCH 065/122] Fix route limit bound (nginx allows below 65536); the dead-acceptor test kills the presence thread through the clock Co-Authored-By: Claude Sonnet 5.5 --- src/deployment.rs | 8 +++++--- src/drivers.rs | 6 +++--- src/launcher.rs | 17 +++++++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/deployment.rs b/src/deployment.rs index 75a30ab..56cd8d8 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -699,12 +699,12 @@ pub struct MemberListenerBinding { /// route's presence block shares, so this limit is what keeps members from /// exhausting it. #[serde(default = "default_member_max_connections")] - pub max_connections: u32, + pub max_connections: u16, } -const DEFAULT_MEMBER_MAX_CONNECTIONS: u32 = 128; +const DEFAULT_MEMBER_MAX_CONNECTIONS: u16 = 128; -fn default_member_max_connections() -> u32 { +fn default_member_max_connections() -> u16 { DEFAULT_MEMBER_MAX_CONNECTIONS } @@ -2446,6 +2446,8 @@ max_connections = {value}"), ); assert_eq!(limit("7").unwrap().route.unwrap().member.unwrap().max_connections, 7); assert!(limit("0").is_err()); + assert!(limit("65535").is_ok()); + assert!(limit("65536").is_err()); // Zero, and a width that overflows u16. assert!(start("0").is_err()); assert!(start("65530").is_err()); diff --git a/src/drivers.rs b/src/drivers.rs index 060917d..73feb5f 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -12508,7 +12508,7 @@ processes 9 /// 4103, 4104, 4120 and 4121, so it runs on demand in a container: /// `cargo test --lib real_nginx -- --ignored`. #[cfg(unix)] - fn challenges_answered_during_a_member_flood(max_connections: u32) -> Result { + fn challenges_answered_during_a_member_flood(max_connections: u16) -> Result { use std::net::{TcpListener, TcpStream}; use std::process::{Child, Command}; use std::sync::Arc; @@ -12636,9 +12636,9 @@ processes 9 #[test] #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] fn real_nginx_flood_starves_presence_without_a_member_limit() -> Result<()> { - // The control: the same flood against a limit too high to bind starves + // The control: the same flood against a limit too high to bind (nginx allows up to 65535) starves // presence, so the test above measures the limit and not the load. - assert!(challenges_answered_during_a_member_flood(100_000)? < 20); + assert!(challenges_answered_during_a_member_flood(60_000)? < 20); Ok(()) } diff --git a/src/launcher.rs b/src/launcher.rs index f15882e..b787386 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -28,7 +28,7 @@ use std::thread; use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultcacheclock: Arc::new(PresenceTripwire(FastClock(AtomicU64::new(1_000_000)))),health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))),rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, @@ -2592,14 +2592,19 @@ mod tests { serving.join().unwrap(); } - /// Healthy for everyone but the presence thread, which it kills: the + /// A clock that kills the presence thread when it reads the time: the /// stand-in for a presence acceptor that dies while the child is fine. - struct PresenceTripwire; + /// Answering a challenge signs, and signing reads the clock. + struct PresenceTripwire(FastClock); - impl HealthProbe for PresenceTripwire { - fn healthy(&self) -> bool { + impl Clock for PresenceTripwire { + fn now_millis(&self) -> u64 { assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); - true + self.0.now_millis() + } + + fn sleep(&self, duration: Duration) { + self.0.sleep(duration); } } From d6ebd328e156b1538e35851e9fc5f94a32c5f2ab Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:16:44 +0200 Subject: [PATCH 066/122] Fix route limit bound (nginx allows below 65536); the dead-acceptor test kills the presence thread through the clock Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index b787386..85ed592 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -28,7 +28,7 @@ use std::thread; use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcacheclock: Arc::new(PresenceTripwire(FastClock(AtomicU64::new(1_000_000)))),health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))),rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, @@ -2615,11 +2615,11 @@ mod tests { let child = Arc::new(Child::default()); let ports = Ports { credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + clock: Arc::new(PresenceTripwire(FastClock(AtomicU64::new(1_000_000)))), sink: sink.clone(), lease: Arc::new(FakeLease::default()), spawner: Box::new(FakeSpawner(child.clone())), - health: Box::new(|_| Arc::new(PresenceTripwire)), + health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), }; let contract = fx.contract("x"); let authority = fx.authority(); From 60b64e283d577785f7803ee2754f0c6bea1bc3f6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:33:23 +0200 Subject: [PATCH 067/122] Real-nginx tests stop nginx through nginx so its workers do not hold the test's pipe Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 73feb5f..4fc1577 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -12514,10 +12514,18 @@ processes 9 use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; - struct Nginx(Child); + /// Stopped through nginx itself: killing the master would orphan its + /// workers, which keep the test's output pipe open. + struct Nginx(Child, PathBuf, PathBuf); impl Drop for Nginx { fn drop(&mut self) { - let _ = self.0.kill(); + let _ = Command::new("nginx") + .arg("-p") + .arg(&self.1) + .arg("-c") + .arg(&self.2) + .args(["-s", "stop"]) + .status(); let _ = self.0.wait(); } } @@ -12577,7 +12585,11 @@ processes 9 .arg("-c") .arg(&conf) .args(["-g", "daemon off;"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) .spawn()?, + temp.path().to_path_buf(), + conf.clone(), ); let stable: SocketAddr = "127.0.0.1:4103".parse()?; let ready = Instant::now(); From 1207f219487383653d5e0b7459c67a880e555608 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:34:14 +0200 Subject: [PATCH 068/122] Real-nginx tests report how many challenges were answered Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/drivers.rs b/src/drivers.rs index 4fc1577..44d9976 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -12627,6 +12627,7 @@ processes 9 answered += 1; } } + eprintln!("answered {answered} of 20 challenges with a member limit of {max_connections}"); drop(flood); halt.store(true, Ordering::SeqCst); presence.join().unwrap(); From 2ea0572a4cdd80f1f9fe43f507ab573a2bc422e9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:34:28 +0200 Subject: [PATCH 069/122] Docs: member connection limit and the cached presence answer Co-Authored-By: Claude Sonnet 5.5 --- README.md | 1 + docs/authority-map.md | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3714ea9..db728a9 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ private_port_end = 18839 [route.member] stable_endpoint = "tcp://10.77.0.1:8448" private_port_start = 18841 +# max_connections = 128 # nginx limit_conn on the member listener (1-65535) ``` ## Contracts diff --git a/docs/authority-map.md b/docs/authority-map.md index 2db896f..c8a8546 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -90,10 +90,14 @@ activation key. back. - **Route:** Idunn owns both listeners of the route: the Expected route's stable endpoint (presence) and, when the binding has `[route.member]`, the - member endpoint, rendered as a second server block in the one nginx fragment. + member endpoint, rendered as a second server block in the one nginx fragment + with a `limit_conn` on one per-route zone (`max_connections`, default 128), so + members cannot exhaust the connections the presence block shares with them. The launcher binds the candidate endpoint and answers Idunn's challenge there, one connection at a time within a one-second deadline; any other first byte - closes the connection. The child binds the member bind Idunn sets in + closes the connection. The answer never waits on the child: it reports the + lifecycle's last probe, and a failed, missing or stale probe reads `degraded`. + The child binds the member bind Idunn sets in `GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes never touch the launcher. - **Lease:** with `write_lease_required` the launcher starts the child only From 6b1d14d1596b3acdb0e83b0bb18a3649b65fb985 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:43:04 +0200 Subject: [PATCH 070/122] idunn-launch tests: an Active child is probed at the refresh cadence Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/launcher.rs b/src/launcher.rs index 85ed592..62a04b1 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2375,6 +2375,53 @@ mod tests { assert_eq!(launch.stop().exit_code(), 0); } + + /// Healthy, and notes the virtual time of every probe. + struct RecordingProbe { + clock: Arc, + at: Mutex>, + } + + impl HealthProbe for RecordingProbe { + fn healthy(&self) -> bool { + self.at.lock().unwrap().push(self.clock.now_millis()); + true + } + } + + #[test] + fn an_active_child_is_probed_at_the_refresh_cadence() { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let clock = Arc::new(FastClock(AtomicU64::new(1_000_000))); + let probe = Arc::new(RecordingProbe { clock: clock.clone(), at: Mutex::new(Vec::new()) }); + let seen = probe.clone(); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock, + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(move |_| probe.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("five probes", || seen.at.lock().unwrap().len() >= 5); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + let at = seen.at.lock().unwrap().clone(); + // The first probe ends startup; the rest follow the refresh cadence. + let cadence = HEALTH_REFRESH.as_millis() as u64..=HEALTH_REFRESH.as_millis() as u64 + POLL.as_millis() as u64; + for pair in at[1..].windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "probes at {at:?}"); + } + } + #[test] fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { let mut stubborn = ProcessSpawner From b5d13652e10cf5eda6a324043a91d51d8481a799 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 04:50:09 +0200 Subject: [PATCH 071/122] idunn-launch tests: the first Active probe is also held to the refresh cadence Co-Authored-By: Claude Sonnet 5.5 --- src/launcher.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 62a04b1..b2a17be 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2415,9 +2415,9 @@ mod tests { stop.store(true, Ordering::SeqCst); assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); let at = seen.at.lock().unwrap().clone(); - // The first probe ends startup; the rest follow the refresh cadence. + // The first probe ends startup; every later one follows the cadence. let cadence = HEALTH_REFRESH.as_millis() as u64..=HEALTH_REFRESH.as_millis() as u64 + POLL.as_millis() as u64; - for pair in at[1..].windows(2) { + for pair in at.windows(2) { assert!(cadence.contains(&(pair[1] - pair[0])), "probes at {at:?}"); } } From 5d8f7e18c538cb0b098116c2d6c2680b9b050aae Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 12:26:22 +0200 Subject: [PATCH 072/122] idunn-launch: freshness, deadlines and the probe schedule run on a monotonic clock Clock gains monotonic_millis; the cached health result's age, the freshness decision, the probe schedule, heartbeat and lease/startup deadlines all read it. Wall time stays on the signed record only. A result stamped after now is not evidence. Tests step the wall clock both ways, pin the warming-proof cap, and bound the lease-timeout join. Co-Authored-By: Claude Opus 5.5 --- src/launcher.rs | 288 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 261 insertions(+), 27 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index b2a17be..4b775f0 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -379,7 +379,12 @@ pub fn inherited_authority_files() -> Result<(File, File)> { // ---------------------------------------------------------------- ports pub trait Clock: Send + Sync { + /// Wall-clock time, for values signed or published to another party. No + /// decision about freshness, deadlines or schedules reads it. fn now_millis(&self) -> u64; + /// Monotonic time: never steps back, unaffected by wall-clock changes. Every + /// age, deadline and schedule in the launcher reads this one source. + fn monotonic_millis(&self) -> u64; fn sleep(&self, duration: Duration); } @@ -390,6 +395,12 @@ impl Clock for SystemClock { u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0) } + fn monotonic_millis(&self) -> u64 { + static START: std::sync::OnceLock = std::sync::OnceLock::new(); + u64::try_from(START.get_or_init(std::time::Instant::now).elapsed().as_millis()) + .unwrap_or(u64::MAX) + } + fn sleep(&self, duration: Duration) { thread::sleep(duration); } @@ -864,7 +875,7 @@ struct Shared { lease: Mutex>, lease_required: bool, health: Arc, - /// The last probe result and the clock time it started. Only the lifecycle + /// The last probe result and the monotonic time it started. Only the lifecycle /// probes; a challenge reads this and never waits on the child. probed: Mutex>, clock: Arc, @@ -873,7 +884,7 @@ struct Shared { impl Shared { /// Probes the child now and keeps the result for presence to answer from. fn refresh_health(&self) -> bool { - let started = self.clock.now_millis(); + let started = self.clock.monotonic_millis(); let healthy = self.health.healthy(); if let Ok(mut probed) = self.probed.lock() { *probed = Some((healthy, started)); @@ -885,14 +896,14 @@ impl Shared { /// A child never probed, wedged, or probed too long ago reads degraded. A /// dead presence acceptor does not degrade presence: it fails the unit. fn serving_state(&self) -> &'static str { - let now = self.clock.now_millis(); + let now = self.clock.monotonic_millis(); let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); let healthy = self .probed .lock() .ok() .and_then(|probed| *probed) - .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); + .is_some_and(|(healthy, at)| healthy && now.checked_sub(at).is_some_and(|age| age <= stale)); if healthy { "active" } else { "degraded" } } @@ -1178,13 +1189,13 @@ fn lifecycle( }; remember(publish("warming", "launcher-started", None), &mut proofs); let heartbeat = u64::try_from(contract.heartbeat.as_millis()).unwrap_or(u64::MAX); - let mut next_heartbeat = clock.now_millis() + heartbeat; + let mut next_heartbeat = clock.monotonic_millis() + heartbeat; // 2. A stateful child waits for the lease; a stateless one does not. let mut lease_sha = None; let mut lease_fd = None; if expected.write_lease_required { - let deadline = clock.now_millis() + contract.lease_wait.as_millis() as u64; + let deadline = clock.monotonic_millis() + contract.lease_wait.as_millis() as u64; loop { if stop.load(Ordering::SeqCst) { return Outcome::Stopped; @@ -1206,7 +1217,7 @@ fn lifecycle( } break; } - let now = clock.now_millis(); + let now = clock.monotonic_millis(); if now >= deadline { return fail(None, "lease-timeout"); } @@ -1222,7 +1233,7 @@ fn lifecycle( let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { return fail(None, "child-start-refused"); }; - let startup_deadline = clock.now_millis() + contract.startup.as_millis() as u64; + let startup_deadline = clock.monotonic_millis() + contract.startup.as_millis() as u64; loop { if stop.load(Ordering::SeqCst) { child.terminate(TERMINATE_GRACE); @@ -1237,7 +1248,7 @@ fn lifecycle( if shared.refresh_health() { break; } - if clock.now_millis() >= startup_deadline { + if clock.monotonic_millis() >= startup_deadline { return fail(Some(&mut child), "startup-timeout"); } clock.sleep(POLL); @@ -1247,8 +1258,8 @@ fn lifecycle( set_phase(Phase::Active(lease_sha.clone())); publish(shared.serving_state(), "child-healthy", lease_sha.clone()); let refresh = u64::try_from(HEALTH_REFRESH.as_millis()).unwrap_or(u64::MAX); - let mut next_probe = clock.now_millis() + refresh; - next_heartbeat = clock.now_millis() + heartbeat; + let mut next_probe = clock.monotonic_millis() + refresh; + next_heartbeat = clock.monotonic_millis() + heartbeat; loop { if stop.load(Ordering::SeqCst) { child.terminate(TERMINATE_GRACE); @@ -1263,7 +1274,7 @@ fn lifecycle( if !shared.lease_current() { return fail(Some(&mut child), "lease-lost"); } - let now = clock.now_millis(); + let now = clock.monotonic_millis(); if now >= next_probe { shared.refresh_health(); next_probe = now + refresh; @@ -1307,7 +1318,7 @@ fn start_child( mod tests { use std::collections::BTreeMap; use std::os::unix::fs::PermissionsExt; - use std::sync::atomic::{AtomicU64, AtomicUsize}; + use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize}; use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; use cultnet_rs::{ @@ -1482,15 +1493,34 @@ mod tests { } /// Virtual time runs a hundred times faster than the wall clock. - struct FastClock(AtomicU64); + struct FastClock { + monotonic: AtomicU64, + wall_step: AtomicI64, + } + + impl FastClock { + fn new(start: u64) -> Self { + Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(0) } + } + + /// Steps the wall clock alone, as timesyncd or a VM restore would. + fn step_wall(&self, millis: i64) { + self.wall_step.fetch_add(millis, Ordering::SeqCst); + } + } impl Clock for FastClock { fn now_millis(&self) -> u64 { - self.0.load(Ordering::SeqCst) + let wall = self.monotonic.load(Ordering::SeqCst) as i64 + self.wall_step.load(Ordering::SeqCst); + u64::try_from(wall).unwrap_or(0) + } + + fn monotonic_millis(&self) -> u64 { + self.monotonic.load(Ordering::SeqCst) } fn sleep(&self, duration: Duration) { - self.0 + self.monotonic .fetch_add(duration.as_millis() as u64, Ordering::SeqCst); thread::sleep(Duration::from_millis(1)); } @@ -1629,7 +1659,7 @@ mod tests { secrets.insert("SYNAPSE_KEY".to_owned(), b"s3cret-value\n".to_vec()); let ports = Ports { credentials: Arc::new(Credentials(secrets)), - clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + clock: Arc::new(FastClock::new(1_000_000)), sink: sink.clone(), lease: lease.clone(), spawner: Box::new(FakeSpawner(child.clone())), @@ -1720,9 +1750,13 @@ mod tests { let mut lease = fx.lease(&digest('8')); edit(&mut lease); *launch.lease.0.lock().unwrap() = Some(lease); - assert_eq!(launch.finish(), Outcome::Failed("lease-timeout"), "{name}"); - assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0, "{name}"); - assert_eq!(launch.sink.states().last().unwrap(), "failed", "{name}"); + let (child, sink) = (launch.child.clone(), launch.sink.clone()); + // Bounded: a lease deadline that never arrives fails here, not at the + // suite's outer timeout. + let outcome = within(Duration::from_secs(20), move || launch.finish()); + assert_eq!(outcome, Outcome::Failed("lease-timeout"), "{name}"); + assert_eq!(child.spawns.load(Ordering::SeqCst), 0, "{name}"); + assert_eq!(sink.states().last().unwrap(), "failed", "{name}"); } } @@ -2134,18 +2168,214 @@ mod tests { #[test] fn a_probe_result_is_evidence_only_while_it_is_recent() { let fx = fixture(false); - let clock = Arc::new(FastClock(AtomicU64::new(1_000_000))); + let clock = Arc::new(FastClock::new(1_000_000)); let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); shared.refresh_health(); let stale = HEALTH_STALE.as_millis() as u64; - clock.0.fetch_add(stale, Ordering::SeqCst); + clock.monotonic.fetch_add(stale, Ordering::SeqCst); assert_eq!(shared.serving_state(), "active", "a result exactly at the bound is still recent"); - clock.0.fetch_add(1, Ordering::SeqCst); + clock.monotonic.fetch_add(1, Ordering::SeqCst); assert_eq!(shared.serving_state(), "degraded", "a result past the bound is not evidence"); shared.refresh_health(); assert_eq!(shared.serving_state(), "active"); } + /// A lease reader that grants, at the first read once `after` Warming + /// records exist, a lease bound to the Warming record `back` places before + /// the newest. The launcher reads it from its own thread between publishes, + /// so the record it names is at a known place in the retained window. + struct WindowLease { + sink: Arc, + template: IdunnProcessWriteLeaseRecord, + after: usize, + back: usize, + granted: Mutex>, + } + + impl LeaseReader for WindowLease { + fn read(&self) -> Result> { + let mut granted = self.granted.lock().unwrap(); + if granted.is_none() { + let records = self.sink.0.lock().unwrap(); + let warming: Vec<_> = records.iter().filter(|r| r.state == "warming").collect(); + if warming.len() >= self.after { + let mut lease = self.template.clone(); + lease.warming_presence_sha256 = + warming[warming.len() - self.back].canonical_sha256().unwrap(); + *granted = Some(lease); + } + } + Ok(granted.clone()) + } + + fn hold_shared(&self) -> Result> { + Ok(Box::new(FakeLock)) + } + } + + /// Starts a stateful unit waiting on a `WindowLease` and reports whether it + /// accepted the lease (spawned the child) or retried well past the grant. + fn warming_window_accepts(back: usize) -> bool { + let fx = fixture(true); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let lease = Arc::new(WindowLease { + sink: sink.clone(), + template: fx.lease(""), + after: MAX_RECENT_WARMING_PROOFS + 6, + back, + granted: Mutex::new(None), + }); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock::new(1_000_000)), + sink: sink.clone(), + lease: lease.clone(), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), + }; + let mut contract = fx.contract("x"); + contract.heartbeat = POLL; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("the lease is granted", || lease.granted.lock().unwrap().is_some()); + let warming = || sink.states().iter().filter(|s| *s == "warming").count(); + let granted_at = warming(); + let mut spawned = false; + for _ in 0..4000 { + spawned = child.spawns.load(Ordering::SeqCst) == 1; + if spawned || warming() >= granted_at + 10 { + break; + } + thread::sleep(Duration::from_millis(5)); + } + stop.store(true, Ordering::SeqCst); + // A mutant may wedge the unit; only a unit that took the lease is joined. + if spawned { + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + } + spawned + } + + #[test] + fn the_warming_proof_window_keeps_exactly_the_cap() { + assert!( + warming_window_accepts(MAX_RECENT_WARMING_PROOFS), + "the oldest of the retained Warming proofs is accepted" + ); + assert!( + !warming_window_accepts(MAX_RECENT_WARMING_PROOFS + 1), + "a Warming proof past the cap is no longer accepted" + ); + } + + /// The child is healthy, then the wall clock steps by `step_millis`, then + /// the child wedges. A challenge must keep reading the truth throughout. + fn a_wall_clock_step_never_decides_freshness(step_millis: i64) { + struct Toggle { + healthy: AtomicBool, + probes: AtomicUsize, + } + impl HealthProbe for Toggle { + fn healthy(&self) -> bool { + self.probes.fetch_add(1, Ordering::SeqCst); + self.healthy.load(Ordering::SeqCst) + } + } + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let clock = Arc::new(FastClock::new(100_000_000)); + let probe = Arc::new(Toggle { healthy: AtomicBool::new(true), probes: AtomicUsize::new(0) }); + let given = probe.clone(); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: clock.clone(), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(move |_| given.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); + + clock.step_wall(step_millis); + for _ in 0..10 { + assert_eq!( + challenge_state(address, &fx).as_deref(), + Some("active"), + "a healthy child reads active across a wall-clock step of {step_millis} ms" + ); + thread::sleep(Duration::from_millis(20)); + } + + probe.healthy.store(false, Ordering::SeqCst); + let probed = probe.probes.load(Ordering::SeqCst); + let published = sink.0.lock().unwrap().len(); + let started = Instant::now(); + let mut state = None; + while started.elapsed() < Duration::from_secs(5) { + state = challenge_state(address, &fx); + if state.as_deref() == Some("degraded") { + break; + } + thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + state.as_deref(), + Some("degraded"), + "a wedged child reads degraded within the freshness window after a wall-clock step of {step_millis} ms" + ); + assert!(probe.probes.load(Ordering::SeqCst) > probed, "probing continued"); + wait_until("a heartbeat after the step", || sink.0.lock().unwrap().len() > published); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + } + + #[test] + fn a_backward_wall_clock_step_does_not_keep_a_wedged_child_active() { + a_wall_clock_step_never_decides_freshness(-600_000); + } + + #[test] + fn a_forward_wall_clock_step_does_not_make_a_healthy_child_degraded() { + a_wall_clock_step_never_decides_freshness(600_000); + } + + #[test] + fn the_cached_result_ages_on_the_monotonic_clock_alone() { + let fx = fixture(false); + let clock = Arc::new(FastClock::new(100_000_000)); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + for step in [-600_000, 1_200_000] { + clock.step_wall(step); + assert_eq!(shared.serving_state(), "active", "wall step {step}"); + } + clock.monotonic.fetch_add(HEALTH_STALE.as_millis() as u64 + 1, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "degraded", "monotonic age decides, whatever the wall says"); + } + + #[test] + fn a_result_stamped_after_now_is_not_evidence() { + let fx = fixture(false); + let clock = Arc::new(FastClock::new(100_000_000)); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + *shared.probed.lock().unwrap() = Some((true, 100_000_001)); + assert_eq!(shared.serving_state(), "degraded"); + } + /// Runs `work` and returns its result, failing the test when it has not /// finished within `limit`, so a mutant that makes it wait forever fails /// instead of hanging the suite. @@ -2384,7 +2614,7 @@ mod tests { impl HealthProbe for RecordingProbe { fn healthy(&self) -> bool { - self.at.lock().unwrap().push(self.clock.now_millis()); + self.at.lock().unwrap().push(self.clock.monotonic_millis()); true } } @@ -2393,7 +2623,7 @@ mod tests { fn an_active_child_is_probed_at_the_refresh_cadence() { let fx = fixture(false); let sink = Arc::new(Sink::default()); - let clock = Arc::new(FastClock(AtomicU64::new(1_000_000))); + let clock = Arc::new(FastClock::new(1_000_000)); let probe = Arc::new(RecordingProbe { clock: clock.clone(), at: Mutex::new(Vec::new()) }); let seen = probe.clone(); let ports = Ports { @@ -2650,6 +2880,10 @@ mod tests { self.0.now_millis() } + fn monotonic_millis(&self) -> u64 { + self.0.monotonic_millis() + } + fn sleep(&self, duration: Duration) { self.0.sleep(duration); } @@ -2662,7 +2896,7 @@ mod tests { let child = Arc::new(Child::default()); let ports = Ports { credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(PresenceTripwire(FastClock(AtomicU64::new(1_000_000)))), + clock: Arc::new(PresenceTripwire(FastClock::new(1_000_000))), sink: sink.clone(), lease: Arc::new(FakeLease::default()), spawner: Box::new(FakeSpawner(child.clone())), @@ -2797,7 +3031,7 @@ mod tests { let health = Arc::new(LateHealth { calls: AtomicUsize::new(0), probes: 5 }); let ports = Ports { credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(FastClock(AtomicU64::new(1_000_000))), + clock: Arc::new(FastClock::new(1_000_000)), sink: sink.clone(), lease: Arc::new(FakeLease::default()), spawner: Box::new(FakeSpawner(child.clone())), From 06782d38f20387ad47e247ca0f137190377d16b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 12:27:38 +0200 Subject: [PATCH 073/122] idunn-launch tests: the presence tripwire trips on the monotonic read, before any lock is held Co-Authored-By: Claude Opus 5.5 --- src/launcher.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 4b775f0..d1cc49f 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -2871,16 +2871,17 @@ mod tests { /// A clock that kills the presence thread when it reads the time: the /// stand-in for a presence acceptor that dies while the child is fine. - /// Answering a challenge signs, and signing reads the clock. + /// Answering a challenge reads the cached result's age, which reads the + /// monotonic clock before any lock is held. struct PresenceTripwire(FastClock); impl Clock for PresenceTripwire { fn now_millis(&self) -> u64 { - assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); self.0.now_millis() } fn monotonic_millis(&self) -> u64 { + assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); self.0.monotonic_millis() } From d111986093ab168ef8360b80981be003819297ce Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 14:23:28 +0200 Subject: [PATCH 074/122] Launcher: pin deadline clock sources, read presence freshness in a race-free order, bound declared durations The test clock's wall now starts a Unix-epoch away from the monotonic clock, so a deadline or schedule that reads the wrong source fails. serving_state reads the cached probe before the clock (a stamp is taken before it is stored, so the later reading is never behind it) and ages with saturating_sub; the test that pinned the old wrong answer is replaced by a deterministic descheduled-reader test. The lease-wait, startup and heartbeat seconds are bounded to one day at parse with a refusal that names the flag and not the value, and every deadline is computed by one saturating helper. Co-Authored-By: Claude Opus 5.5 --- src/launcher.rs | 144 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 119 insertions(+), 25 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index d1cc49f..13937f6 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -65,6 +65,9 @@ const HEALTH_REFRESH: Duration = Duration::from_millis(500); /// that refreshes it is itself stuck. const HEALTH_STALE: Duration = Duration::from_secs(5); const POLL: Duration = Duration::from_millis(100); +/// The largest duration a recipe may declare for the lease wait, the startup +/// wait or the heartbeat: one day. Past it a deadline is not a deadline. +const MAX_DECLARED_SECONDS: u64 = 86_400; const MAX_RECENT_WARMING_PROOFS: usize = 64; const TERMINATE_GRACE: Duration = Duration::from_secs(20); const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; @@ -110,8 +113,10 @@ impl LaunchContract { value .parse::() .ok() - .filter(|seconds| *seconds > 0) - .with_context(|| format!("launcher argument {field} is not a positive integer")) + .filter(|seconds| (1..=MAX_DECLARED_SECONDS).contains(seconds)) + .with_context(|| { + format!("launcher argument {field} is outside 1 to {MAX_DECLARED_SECONDS} seconds") + }) }; match flag.as_str() { "--child" => child = Some(release_relative(&value, "--child")?), @@ -896,14 +901,13 @@ impl Shared { /// A child never probed, wedged, or probed too long ago reads degraded. A /// dead presence acceptor does not degrade presence: it fails the unit. fn serving_state(&self) -> &'static str { - let now = self.clock.monotonic_millis(); let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); - let healthy = self - .probed - .lock() - .ok() - .and_then(|probed| *probed) - .is_some_and(|(healthy, at)| healthy && now.checked_sub(at).is_some_and(|age| age <= stale)); + // The result is read before the clock: a stamp is taken before it is + // stored, so a later reading of the monotonic clock is never behind it, + // whatever the presence thread does between the two. + let probed = self.probed.lock().ok().and_then(|probed| *probed); + let now = self.clock.monotonic_millis(); + let healthy = probed.is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); if healthy { "active" } else { "degraded" } } @@ -1093,6 +1097,12 @@ pub fn parse_member_bind(raw: &str, candidate_bind: &str) -> Result Ok(member) } +/// The monotonic reading `span` after `now`; a span too long to represent is a +/// deadline that never arrives. +fn after(now: u64, span: Duration) -> u64 { + now.saturating_add(u64::try_from(span.as_millis()).unwrap_or(u64::MAX)) +} + /// The unit's whole life: Warming, lease, child, Active, until the child or the /// lease ends or SIGTERM arrives. `route` is the already-bound candidate /// listener, answered for presence only; `member` is the address the child @@ -1188,14 +1198,14 @@ fn lifecycle( } }; remember(publish("warming", "launcher-started", None), &mut proofs); - let heartbeat = u64::try_from(contract.heartbeat.as_millis()).unwrap_or(u64::MAX); - let mut next_heartbeat = clock.monotonic_millis() + heartbeat; + let heartbeat = contract.heartbeat; + let mut next_heartbeat = after(clock.monotonic_millis(), heartbeat); // 2. A stateful child waits for the lease; a stateless one does not. let mut lease_sha = None; let mut lease_fd = None; if expected.write_lease_required { - let deadline = clock.monotonic_millis() + contract.lease_wait.as_millis() as u64; + let deadline = after(clock.monotonic_millis(), contract.lease_wait); loop { if stop.load(Ordering::SeqCst) { return Outcome::Stopped; @@ -1223,7 +1233,7 @@ fn lifecycle( } if now >= next_heartbeat { remember(publish("warming", "write-lease-pending", None), &mut proofs); - next_heartbeat = now + heartbeat; + next_heartbeat = after(now, heartbeat); } clock.sleep(POLL); } @@ -1233,7 +1243,7 @@ fn lifecycle( let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { return fail(None, "child-start-refused"); }; - let startup_deadline = clock.monotonic_millis() + contract.startup.as_millis() as u64; + let startup_deadline = after(clock.monotonic_millis(), contract.startup); loop { if stop.load(Ordering::SeqCst) { child.terminate(TERMINATE_GRACE); @@ -1257,9 +1267,8 @@ fn lifecycle( // 4. Active, and heartbeats while it holds. set_phase(Phase::Active(lease_sha.clone())); publish(shared.serving_state(), "child-healthy", lease_sha.clone()); - let refresh = u64::try_from(HEALTH_REFRESH.as_millis()).unwrap_or(u64::MAX); - let mut next_probe = clock.monotonic_millis() + refresh; - next_heartbeat = clock.monotonic_millis() + heartbeat; + let mut next_probe = after(clock.monotonic_millis(), HEALTH_REFRESH); + next_heartbeat = after(clock.monotonic_millis(), heartbeat); loop { if stop.load(Ordering::SeqCst) { child.terminate(TERMINATE_GRACE); @@ -1277,11 +1286,11 @@ fn lifecycle( let now = clock.monotonic_millis(); if now >= next_probe { shared.refresh_health(); - next_probe = now + refresh; + next_probe = after(now, HEALTH_REFRESH); } if now >= next_heartbeat { publish(shared.serving_state(), "heartbeat", lease_sha.clone()); - next_heartbeat = now + heartbeat; + next_heartbeat = after(now, heartbeat); } clock.sleep(POLL); } @@ -1492,6 +1501,8 @@ mod tests { } } + const WALL_EPOCH_MILLIS: i64 = 1_760_000_000_000; + /// Virtual time runs a hundred times faster than the wall clock. struct FastClock { monotonic: AtomicU64, @@ -1499,8 +1510,11 @@ mod tests { } impl FastClock { + /// The wall clock starts a Unix-epoch-sized distance from the monotonic + /// one, as the real pair does, so a deadline or schedule that reads the + /// wrong source is off by that distance and no test can pass by it. fn new(start: u64) -> Self { - Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(0) } + Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(WALL_EPOCH_MILLIS) } } /// Steps the wall clock alone, as timesyncd or a VM restore would. @@ -2367,13 +2381,93 @@ mod tests { assert_eq!(shared.serving_state(), "degraded", "monotonic age decides, whatever the wall says"); } + /// A monotonic clock whose read, on a thread that marks itself, returns the + /// current value and then waits to be released: the presence thread + /// descheduled right after it reads the time. + struct HeldClock { + monotonic: AtomicU64, + held: Mutex>>, + release: Mutex>>, + } + + thread_local! { + static HOLD_THIS_READ: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + + impl Clock for HeldClock { + fn now_millis(&self) -> u64 { + 0 + } + + fn monotonic_millis(&self) -> u64 { + let value = self.monotonic.load(Ordering::SeqCst); + if HOLD_THIS_READ.with(std::cell::Cell::get) { + self.held.lock().unwrap().take().unwrap().send(()).unwrap(); + self.release.lock().unwrap().take().unwrap().recv().unwrap(); + } + value + } + + fn sleep(&self, _: Duration) {} + } + #[test] - fn a_result_stamped_after_now_is_not_evidence() { + fn a_presence_read_descheduled_across_a_probe_still_sees_a_healthy_child() { let fx = fixture(false); - let clock = Arc::new(FastClock::new(100_000_000)); + let (held_tx, held_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let clock = Arc::new(HeldClock { + monotonic: AtomicU64::new(1_000), + held: Mutex::new(Some(held_tx)), + release: Mutex::new(Some(release_rx)), + }); let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); - *shared.probed.lock().unwrap() = Some((true, 100_000_001)); - assert_eq!(shared.serving_state(), "degraded"); + shared.refresh_health(); + let reader = { + let shared = shared.clone(); + thread::spawn(move || { + HOLD_THIS_READ.with(|hold| hold.set(true)); + shared.serving_state() + }) + }; + // The reader is descheduled in its clock read. The lifecycle probes a + // millisecond later and stores a result stamped after the reader's time. + held_rx.recv().unwrap(); + clock.monotonic.store(1_001, Ordering::SeqCst); + assert!(shared.refresh_health()); + release_tx.send(()).unwrap(); + assert_eq!(reader.join().unwrap(), "active"); + } + + #[test] + fn the_declared_durations_are_bounded_at_parse() { + let base: Vec = [ + "--child", "bin/c", "--template", "t", "--health-path", "/h", "--health-contract", "hc", + ] + .map(String::from) + .to_vec(); + for flag in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { + for (value, accepted) in [ + (MAX_DECLARED_SECONDS.to_string(), true), + ((MAX_DECLARED_SECONDS + 1).to_string(), false), + (u64::MAX.to_string(), false), + ] { + let mut args = base.clone(); + args.extend([flag.to_owned(), value.clone()]); + let parsed = LaunchContract::parse(PathBuf::from("/r"), args); + assert_eq!(parsed.is_ok(), accepted, "{flag} {value}"); + if let Err(error) = parsed { + assert!(!format!("{error:#}").contains(&value), "the refusal echoes its input"); + } + } + } + } + + #[test] + fn a_span_too_long_for_the_clock_is_a_deadline_that_never_arrives() { + assert_eq!(after(10, Duration::from_millis(5)), 15); + assert_eq!(after(u64::MAX - 1, Duration::from_secs(1)), u64::MAX); + assert_eq!(after(10, Duration::MAX), u64::MAX); } /// Runs `work` and returns its result, failing the test when it has not @@ -2872,7 +2966,7 @@ mod tests { /// A clock that kills the presence thread when it reads the time: the /// stand-in for a presence acceptor that dies while the child is fine. /// Answering a challenge reads the cached result's age, which reads the - /// monotonic clock before any lock is held. + /// monotonic clock. struct PresenceTripwire(FastClock); impl Clock for PresenceTripwire { From d74ac8da5c6ba9a2f81ff61599d16a013c3f8303 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 15:01:56 +0200 Subject: [PATCH 075/122] Launcher: presence freshness treats a stamp after the reading as age zero; pin the bound literally Reading the clock after the cached result still left the answer to ordering; the one place freshness is decided is the age subtraction, and monotonic time cannot run backwards, so a result stamped after the reading is fresher than it and ages zero. The bound test names 86400 and 86401 so raising the constant is caught. Co-Authored-By: Claude Opus 5.5 --- src/launcher.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/launcher.rs b/src/launcher.rs index 13937f6..435f8b1 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -902,12 +902,17 @@ impl Shared { /// dead presence acceptor does not degrade presence: it fails the unit. fn serving_state(&self) -> &'static str { let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); - // The result is read before the clock: a stamp is taken before it is - // stored, so a later reading of the monotonic clock is never behind it, - // whatever the presence thread does between the two. - let probed = self.probed.lock().ok().and_then(|probed| *probed); let now = self.clock.monotonic_millis(); - let healthy = probed.is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); + // A probe can finish between this thread reading the clock and locking + // the result, so a stamp later than `now` is a result fresher than the + // reading, age zero: monotonic time does not run backwards, so the stamp + // is not wrong. + let healthy = self + .probed + .lock() + .ok() + .and_then(|probed| *probed) + .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); if healthy { "active" } else { "degraded" } } @@ -2448,8 +2453,8 @@ mod tests { .to_vec(); for flag in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { for (value, accepted) in [ - (MAX_DECLARED_SECONDS.to_string(), true), - ((MAX_DECLARED_SECONDS + 1).to_string(), false), + ("86400".to_owned(), true), + ("86401".to_owned(), false), (u64::MAX.to_string(), false), ] { let mut args = base.clone(); From 332dbd01494909034f8c824a4317c59bf24fe5c6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 19:42:34 +0200 Subject: [PATCH 076/122] deploy-provenance map: tree executable members (body facts, rationale) Imagination pass for cut tree-executable-members (session self-2026-10-23-eureka-b): seal, digest and receipt facts, symlink rule, live binary and install path, disk sizing for the acceptance job. Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-map.md | 113 ++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 docs/deploy-provenance-map.md diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md new file mode 100644 index 0000000..cfbd11b --- /dev/null +++ b/docs/deploy-provenance-map.md @@ -0,0 +1,113 @@ +# Deployment provenance: map + +Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, +session `self-2026-10-23-eureka-b`). The ends are in +[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, +rulings, cut specs and follow-ups live in Eureka's mind; this page holds what +has no kind: body facts, rationale and sequencing. Line numbers are at +`d74ac8d` (origin/main on 2026-10-23). + +## Cut `tree-executable-members` + +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r1`. It +serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's +admission waits on it) and will serve gamecult-chat C3 too: the presence +launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact +(README, launcher fragment), and Synapse needs a Python interpreter there. + +### Body facts + +- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable` + bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and + `validate` (`:740-760`) requires it to equal the declaration. +- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its + executable set from `root.join(receipt.destination)` only. `harden_root_tree` + (`:6858-6892`) seals every directory 0555 and every file 0444 unless that + exact path is in the set. A directory artifact with `executable = true` is + therefore silently meaningless today: its files are all 0444. Nothing refuses + it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin, + StreamPixels service and web surveyed): `executable = true` is on files only. +- T3. A tree output is copied by `copy_artifact`/`copy_tree` + (`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest + (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not + mode bits. So runner or git exec bits never reach the release: harden is the + only mode authority. +- T4. `validate_release_symlink` (`:6895-6914`) keeps a symlink only if its + target is relative with no `..` and lexically inside the release. That rule is + sound without physical resolution (no `..` means a target can only descend). + Absolute links and `..` links refuse the release. The frozen-source path has + a physical resolver (`resolve_frozen_source_symlink`, tested at `:11826`) that + admits in-tree `..`; reuse it if a real tree needs `..` links, not a new rule. +- T5. The sealed release id hashes `(schema, plan_id, artifacts, + external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes + the whole plan including `recipe_blob`. A changed member list is therefore + already a new plan and a new release. The receipt must still carry the list, + because `harden_installed_release` reads receipts only; its two callers + (`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden + every installed release on each observation. +- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with + `#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every + existing receipt's bytes and release id: the plan's `deadlines` field + (`:574-577`) is the precedent. +- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file + whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB) + is held in Idunn's memory three times per install (seal, install, every + validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB + chunks (S9). The cut collapses both onto one streaming helper. +- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the + artifact destination a single filename. Members are paths inside that + destination, so it does not change. `require_relative_path` (`:1936-1949`) + with `allow_current = false` already refuses absolute, `..`, root and prefix + components. +- T9. The host actuator builds receipts separately (`src/host.rs:1043-1081`) + and must carry the same field and the same shape check. +- T10. The transient unit runs with `ProtectSystem=strict` and the release in + `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no + `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. + Shared libraries need no exec bit. +- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`, + installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is + identical between `8760a4f` and `6e2a31c`, so the live code is the + reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in + `src/`: the presence launcher and member route). Installing this cut ships all + of them. Idunn's own body changes only through gamecult-ops + `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, + bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three + copies (runner workspace, staging, installed). Nothing collects old releases or + staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every + Vili deploy leaves about 7 GiB behind. + +### Contract, corrected against the body + +- Any directory artifact may declare `executable_members` (a set of relative + paths). The follow-up limited it to runner-output; that limit protects no + invariant, and the launcher's worktree or fetched trees need the same feature. +- `executable = true` names a file artifact. On a directory output the release + is refused, and `executable_members` on a file output is refused (T2). +- A member is refused at recipe parse if it is empty, absolute, or holds `.`, + `..`, a root or a prefix. At seal it is refused if it is missing, is a + symlink, passes through a symlink or non-directory, or is not a regular file. + Refusal happens before `SealedRelease::new`, so no sealed release can name a + member it cannot seal. +- Harden seals each member 0555 and fails if any declared executable path was + not visited as a regular file. "Every declared executable is sealed 0555, or + the install fails" holds structurally, not by a lookup that can miss. +- Symlinks: relative, no `..`, in-tree links survive (T4). Vili's trees must + not come from `python -m venv`: its `bin/python` is an absolute link and its + `pyvenv.cfg` names the build workspace. Install packages into the relocatable + interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). + +### Authority + +Owner: the seal (`collect_artifact`, and `host.rs` for host targets) decides +membership, from the recipe, and writes it into the receipt. Harden enforces +the receipt and decides nothing. Runner exec bits and git `100755` modes are +dead inputs for release modes. + +### Acceptance sizing + +The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch +on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It +removes it at the end. If the trees hold any absolute or `..` symlink, the job +stops and the T4 resolver reuse becomes a revision of this spec. From 022dbece5becdb2f30af0aed19c28362c42ecc00 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 19:53:16 +0200 Subject: [PATCH 077/122] Directory artifacts declare executable_members; the seal checks them, the receipt carries them, harden seals exactly those 0555 ArtifactOutput and ArtifactReceipt gain executable_members (the receipt field is skipped when empty, so existing sealed release ids are unchanged). check_artifact_modes is the one seal-time owner of mode declarations and runs from both collect_artifact and the host materializer. harden_installed_release drains the declared set and fails when a declared path was never sealed. Digests stream through one hash_file_streamed helper; the encoding is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- src/deployment.rs | 82 ++++++ src/deployment_plan.rs | 117 ++++++++- src/drivers.rs | 581 ++++++++++++++++++++++++++++++++++++++--- src/host.rs | 7 +- 4 files changed, 753 insertions(+), 34 deletions(-) diff --git a/src/deployment.rs b/src/deployment.rs index 56cd8d8..b7cbdc8 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -113,6 +113,10 @@ pub struct ArtifactOutput { pub expected_sha256: Option, #[serde(default)] pub executable: bool, + /// Regular files inside a directory artifact that are sealed executable + /// (0555); every other file in the tree is sealed 0444. + #[serde(default)] + pub executable_members: BTreeSet, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -912,6 +916,14 @@ impl TargetDeclaration { if let Some(expected_sha256) = &artifact.expected_sha256 { require_sha256(expected_sha256, "expected artifact sha256")?; } + ensure!( + !(artifact.executable && !artifact.executable_members.is_empty()), + "artifact {} declares both executable and executable_members", + artifact.id + ); + for member in &artifact.executable_members { + require_relative_path(member, "executable member", false)?; + } ensure!( destinations.insert(artifact.destination.as_path()), "artifact destination {} is declared twice", @@ -2211,6 +2223,76 @@ nodes = ["yggdrasil"] assert!(TargetDeclaration::parse(&input).is_err()); } + const MEMBER_CANARY: &str = "zzcanary"; + + fn recipe_with_members(members: &str) -> String { + let input = RECIPE.replace("executable = true", &format!("executable_members = [{members}]")); + assert_ne!(input, RECIPE, "the fixture lost its executable line"); + input + } + + /// A refused member list is refused by name of the field, never by echoing + /// the member. + fn assert_members_refused(members: &[&str]) { + let control = TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).unwrap(); + assert!(control.artifacts[0].executable_members.contains(Path::new("bin/daemon"))); + for member in members { + let error = TargetDeclaration::parse(&recipe_with_members(&format!("\"{member}\""))) + .expect_err(member); + assert!( + !format!("{error:#}").contains(MEMBER_CANARY), + "the refusal echoed a member path" + ); + } + } + + #[test] + fn recipe_executable_member_refuses_parent_traversal() { + assert_members_refused(&[ + "zzcanary/../b", + "../zzcanary", + "bin/zzcanary/..", + "bin/../../zzcanary", + ]); + } + + #[test] + fn recipe_executable_member_refuses_absolute_path() { + assert_members_refused(&["/zzcanary/sh", "/zzcanary"]); + } + + #[test] + fn recipe_executable_member_refuses_empty_and_current_dir() { + assert_members_refused(&["", ".", "./", "zzcanary/.."]); + } + + #[test] + fn recipe_refuses_executable_flag_with_members() { + let input = RECIPE.replace( + "executable = true", + "executable = true\nexecutable_members = [\"bin/daemon\"]", + ); + let error = TargetDeclaration::parse(&input).unwrap_err(); + assert!(format!("{error:#}").contains("executable_members")); + assert!(!format!("{error:#}").contains("bin/daemon")); + // executable = true alone, and members alone, both stay admitted. + assert!(TargetDeclaration::parse(RECIPE).is_ok()); + assert!(TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).is_ok()); + } + + #[test] + fn recipe_admits_members_on_both_source_kinds() { + let runner_output = TargetDeclaration::parse(&recipe_with_members("\"a/b\", \"c\"")).unwrap(); + assert_eq!(runner_output.artifacts[0].source_kind, ArtifactSource::RunnerOutput); + assert_eq!(runner_output.artifacts[0].executable_members.len(), 2); + let worktree = recipe_with_members("\"a/b\", \"c\"") + .replace("source_kind = \"runner-output\"\nrunner = \"rust\"\n", "source_kind = \"worktree-tree\"\n") + .replace("expected_sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"\n", ""); + let worktree = TargetDeclaration::parse(&worktree).unwrap(); + assert_eq!(worktree.artifacts[0].source_kind, ArtifactSource::WorktreeTree); + assert_eq!(worktree.artifacts[0].executable_members.len(), 2); + } + #[test] fn operator_cannot_omit_an_affordance_derived_from_the_recipe() { let recipe = TargetDeclaration::parse(RECIPE).unwrap(); diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 1363c48..eceb513 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -734,6 +734,11 @@ pub struct ArtifactReceipt { pub sha256: String, pub size_bytes: u64, pub executable: bool, + /// Files sealed executable inside a directory artifact. Skipped when empty + /// so a receipt with no members keeps its original encoding, and so its + /// sealed release id. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub executable_members: BTreeSet, } impl ArtifactReceipt { @@ -747,7 +752,9 @@ impl ArtifactReceipt { .find(|artifact| artifact.id == self.artifact_id) .with_context(|| format!("plan declares no artifact {}", self.artifact_id))?; ensure!( - self.destination == declared.destination && self.executable == declared.executable, + self.destination == declared.destination + && self.executable == declared.executable + && self.executable_members == declared.executable_members, "artifact receipt differs from declared output" ); if let Some(expected_sha256) = &declared.expected_sha256 { @@ -1425,6 +1432,7 @@ nodes = ["yggdrasil"] sha256: digest(b'c'), size_bytes: 42, executable: true, + executable_members: BTreeSet::new(), } } @@ -1791,6 +1799,113 @@ nodes = ["yggdrasil"] assert!(changed.validate_against(&plan).is_err()); } + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + } + + #[test] + fn receipt_without_members_encodes_as_before() { + // Both values were computed outside this code, from the five-field + // msgpack array the receipt encoded to at base d74ac8d. + const BASE_RECEIPT: &str = "95a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac3"; + const BASE_RELEASE_ID: &str = + "sha256-b9143d8f546ad71c493c70037813547aaec10fd87994a6f3bdc77ac3a8b3495e"; + let receipt = artifact_receipt(); + let encoded = rmp_serde::to_vec(&receipt).unwrap(); + assert_eq!(hex(&encoded), BASE_RECEIPT); + let bytes = (0..BASE_RECEIPT.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&BASE_RECEIPT[at..at + 2], 16).unwrap()) + .collect::>(); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), receipt); + let release = SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![receipt], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + assert_eq!(release.recomputed_release_id().unwrap(), BASE_RELEASE_ID); + } + + #[test] + fn sealed_release_address_covers_executable_members() { + let release_with = |members: &[&str]| SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![ArtifactReceipt { + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + ..artifact_receipt() + }], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + let none = release_with(&[]).recomputed_release_id().unwrap(); + let one = release_with(&["bin/daemon"]).recomputed_release_id().unwrap(); + let other = release_with(&["bin/other"]).recomputed_release_id().unwrap(); + let two = release_with(&["bin/daemon", "bin/other"]) + .recomputed_release_id() + .unwrap(); + let all = [&none, &one, &other, &two]; + for (index, left) in all.iter().enumerate() { + for right in &all[index + 1..] { + assert_ne!(left, right); + } + } + + // The declaration is the other half: a receipt must carry exactly the + // declared list, in a plan that declares one and in one that does not. + let plain = plan(); + let mut with_members = artifact_receipt(); + with_members.executable_members = BTreeSet::from([PathBuf::from("bin/daemon")]); + assert!( + SealedRelease::new(&plain, vec![with_members], vec![external_input_receipt()], 120) + .is_err() + ); + let recipe = RECIPE.replace("executable = true", "executable_members = [\"bin/daemon\", \"lib/tool\"]"); + let declaring = compile_deployment_plan( + recipe.as_bytes(), + BINDING.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(18001), + 110, + &[ready_odin_provider("odin", "odin-yggdrasil", 1)], + ) + .unwrap(); + let honest = ArtifactReceipt { + executable: false, + executable_members: BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]), + ..artifact_receipt() + }; + let release = SealedRelease::new( + &declaring, + vec![honest.clone()], + vec![external_input_receipt()], + 120, + ) + .unwrap(); + release.validate_against(&declaring).unwrap(); + for drifted in [ + BTreeSet::from([PathBuf::from("bin/daemon")]), + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/other")]), + BTreeSet::new(), + ] { + let mut receipt = honest.clone(); + receipt.executable_members = drifted; + assert!( + SealedRelease::new(&declaring, vec![receipt], vec![external_input_receipt()], 120) + .is_err() + ); + } + let mut changed = release.clone(); + changed.artifacts[0].executable_members.insert(PathBuf::from("lib/extra")); + assert!(changed.validate_against(&declaring).is_err()); + } + fn binding_with_deadlines(table: &str) -> String { BINDING.replace("[placement]", &format!("{table}\n[placement]")) } diff --git a/src/drivers.rs b/src/drivers.rs index 44d9976..165dfb0 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; @@ -2309,6 +2309,7 @@ impl DockerRunnerDriver { "artifact destination escaped its release staging root" ); copy_artifact(&containment_root, &source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; let (sha256, size_bytes) = digest_artifact(&destination)?; if let Some(expected) = &artifact.expected_sha256 { ensure!( @@ -2322,6 +2323,7 @@ impl DockerRunnerDriver { sha256: format!("sha256-{sha256}"), size_bytes, executable: artifact.executable, + executable_members: artifact.executable_members.clone(), }) } } @@ -6810,21 +6812,10 @@ fn hash_frozen_source_tree(root: &Path, current: &Path, hasher: &mut Sha256) -> ); hasher.update(b"\0"); hasher.update(metadata.len().to_le_bytes()); - // S9: streamed in fixed-size chunks rather than `fs::read`ing the - // whole file, so peak RSS is bounded by the buffer, not by the - // largest file in the tree. - let mut file = fs::File::open(&path) - .with_context(|| format!("opening {} to hash it", path.display()))?; - let mut buffer = [0u8; 1 << 16]; - loop { - let read = file - .read(&mut buffer) - .with_context(|| format!("reading {} to hash it", path.display()))?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - } + ensure!( + hash_file_streamed(&path, hasher)? == metadata.len(), + "frozen source file changed while hashing" + ); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -6846,19 +6837,31 @@ fn frozen_source_sha256(_root: &Path) -> Result { #[cfg(unix)] fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Result<()> { - let executable_paths = artifacts - .iter() - .filter(|artifact| artifact.executable) - .map(|artifact| root.join(&artifact.destination)) - .collect::>(); - harden_root_tree(root, root, &executable_paths) + let mut executable_paths = BTreeSet::new(); + for artifact in artifacts { + let destination = root.join(&artifact.destination); + for member in &artifact.executable_members { + executable_paths.insert(destination.join(member)); + } + if artifact.executable { + executable_paths.insert(destination); + } + } + harden_root_tree(root, root, &mut executable_paths)?; + ensure!( + executable_paths.is_empty(), + "declared executable is missing or not a regular file" + ); + Ok(()) } +/// Each regular file the walk seals 0555 is removed from `executable_paths`; +/// whatever remains was declared executable and never sealed. #[cfg(unix)] fn harden_root_tree( root: &Path, current: &Path, - executable_paths: &std::collections::BTreeSet, + executable_paths: &mut BTreeSet, ) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -6871,13 +6874,13 @@ fn harden_root_tree( let mut entries = fs::read_dir(current)?.collect::>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { - harden_root_tree(root, &entry.path(), executable_paths)?; + harden_root_tree(root, &entry.path(), &mut *executable_paths)?; } fs::set_permissions(current, fs::Permissions::from_mode(0o555))?; } else if metadata.is_file() { fs::set_permissions( current, - fs::Permissions::from_mode(if executable_paths.contains(current) { + fs::Permissions::from_mode(if executable_paths.remove(current) { 0o555 } else { 0o444 @@ -8125,11 +8128,81 @@ pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> R } } +/// The one seal-time owner of mode declarations for a copied artifact: a +/// directory output is never `executable` (it declares `executable_members`), +/// a file output never declares members, and every member is a regular file +/// reached through real directories, never through a symlink. Errors name the +/// rule, never the member path. +pub(crate) fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { + let is_dir = fs::symlink_metadata(destination) + .context("inspecting a copied artifact")? + .is_dir(); + ensure!( + !(is_dir && artifact.executable), + "directory artifact {} cannot be executable; declare executable_members", + artifact.id + ); + if !is_dir { + ensure!( + artifact.executable_members.is_empty(), + "file artifact {} cannot declare executable_members", + artifact.id + ); + return Ok(()); + } + for member in &artifact.executable_members { + ensure!(!member.as_os_str().is_empty(), "executable member is empty"); + let mut current = destination.to_path_buf(); + let mut components = member.components().peekable(); + while let Some(component) = components.next() { + let std::path::Component::Normal(name) = component else { + bail!("executable member has a forbidden path component"); + }; + current.push(name); + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if error.kind() == ErrorKind::NotFound => { + bail!("executable member is missing") + } + Err(error) => return Err(error).context("inspecting an executable member"), + }; + if components.peek().is_some() { + ensure!( + metadata.is_dir(), + "executable member passes through a symlink or non-directory" + ); + } else { + ensure!(metadata.is_file(), "executable member is not a regular file"); + } + } + } + Ok(()) +} + +/// Streams `path` into `hasher` in 64 KiB chunks, so peak memory is the buffer +/// and not the largest file in the tree. Returns the bytes read; a caller that +/// hashed a length first must check the two agree. +pub(crate) fn hash_file_streamed(path: &Path, hasher: &mut Sha256) -> Result { + let mut file = fs::File::open(path).context("opening a file to hash it")?; + let mut buffer = [0u8; 1 << 16]; + let mut total = 0_u64; + loop { + let read = file.read(&mut buffer).context("reading a file to hash it")?; + if read == 0 { + return Ok(total); + } + hasher.update(&buffer[..read]); + total += read as u64; + } +} + pub(crate) fn digest_artifact(path: &Path) -> Result<(String, u64)> { let metadata = fs::symlink_metadata(path)?; if metadata.is_file() { - let bytes = fs::read(path)?; - return Ok((raw_sha256(&bytes), bytes.len().try_into()?)); + let mut hasher = Sha256::new(); + let streamed = hash_file_streamed(path, &mut hasher)?; + ensure!(streamed == metadata.len(), "artifact file changed while hashing"); + return Ok((format!("{:x}", hasher.finalize()), streamed)); } ensure!(metadata.is_dir(), "artifact is not a file or directory"); let mut hasher = Sha256::new(); @@ -8153,13 +8226,16 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) hasher.update(b"\0"); digest_tree(root, &path, hasher, size)?; } else if metadata.is_file() { - let bytes = fs::read(&path)?; + let length = metadata.len(); hasher.update(b"file\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); - hasher.update((bytes.len() as u64).to_le_bytes()); - hasher.update(&bytes); - *size = size.saturating_add(bytes.len().try_into()?); + hasher.update(length.to_le_bytes()); + ensure!( + hash_file_streamed(&path, hasher)? == length, + "artifact file changed while hashing" + ); + *size = size.saturating_add(length); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -13862,6 +13938,7 @@ processes 9 sha256: sha256_id(b"sealed"), size_bytes: 6, executable: true, + executable_members: BTreeSet::new(), }], )?; let root_metadata = fs::symlink_metadata(&root)?; @@ -13873,6 +13950,448 @@ processes 9 Ok(()) } + #[cfg(unix)] + const MEMBER_CANARY: &str = "zzcanary"; + + #[cfg(unix)] + fn tree_output(executable: bool, members: &[&str]) -> ArtifactOutput { + ArtifactOutput { + id: "upstream".into(), + source_kind: ArtifactSource::WorktreeTree, + runner: None, + source: PathBuf::from("upstream"), + destination: PathBuf::from("upstream"), + expected_sha256: None, + executable, + executable_members: members.iter().map(PathBuf::from).collect(), + } + } + + #[cfg(unix)] + fn write_file(path: &Path, bytes: &[u8]) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::create_dir_all(path.parent().context("file has no parent")?)?; + fs::write(path, bytes)?; + // Builds and git leave arbitrary exec bits; the seal must not inherit them. + fs::set_permissions(path, fs::Permissions::from_mode(0o777))?; + Ok(()) + } + + #[cfg(unix)] + fn mode_of(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + + fs::symlink_metadata(path).unwrap().permissions().mode() & 0o777 + } + + /// `/src/upstream` is the worktree tree; `/staging` the release. + #[cfg(unix)] + fn sample_tree(root: &Path) -> Result<(PathBuf, PathBuf)> { + let source = root.join("src"); + let staging = root.join("staging"); + fs::create_dir_all(&staging)?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/daemon"), b"daemon")?; + write_file(&tree.join("bin/helper"), b"helper")?; + write_file(&tree.join("lib/tool"), b"tool")?; + write_file(&tree.join("lib/data.txt"), b"data")?; + Ok((source, staging)) + } + + #[cfg(unix)] + fn collect_tree( + source: &Path, + staging: &Path, + output: ArtifactOutput, + ) -> Result { + let mut declaration = + TargetDeclaration::parse(crate::deployment::tests::RECIPE)?; + let id = output.id.clone(); + declaration.artifacts = vec![output]; + let frozen = FrozenSource { + receipt: FrozenSourceReceipt { + transaction_id: "tx".into(), + plan_id: "plan".into(), + snapshot_sha256: "snapshot".into(), + }, + facts: crate::deployment_plan::tests::source(crate::deployment::tests::RECIPE), + recipe_bytes: Vec::new(), + root: source.to_path_buf(), + }; + DockerRunnerDriver::default().collect_artifact( + &declaration, + &frozen, + &BTreeMap::new(), + staging, + &id, + ) + } + + #[cfg(unix)] + fn refusal_text(result: Result) -> String { + let Err(error) = result else { + panic!("the release was not refused"); + }; + let text = format!("{error:#}"); + assert!( + !text.contains(MEMBER_CANARY), + "the refusal echoed a member path: {text}" + ); + text + } + + #[cfg(unix)] + #[test] + fn tree_member_sealed_0555_and_siblings_0444() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let receipt = collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib/tool"]), + )?; + assert_eq!( + receipt.executable_members, + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]) + ); + harden_installed_release(&staging, &[receipt])?; + let tree = staging.join("upstream"); + assert_eq!(mode_of(&tree.join("bin/daemon")), 0o555); + assert_eq!(mode_of(&tree.join("lib/tool")), 0o555); + assert_eq!(mode_of(&tree.join("bin/helper")), 0o444); + assert_eq!(mode_of(&tree.join("lib/data.txt")), 0o444); + for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { + assert_eq!(mode_of(directory), 0o555); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn missing_member_refuses_the_release() -> Result<()> { + // The bad member sorts after the good one in one case and before it in + // the other, so a check that stops at the first (or last) member fails. + for members in [ + &["bin/daemon", "lib/zzcanary-missing"][..], + &["aaa-zzcanary-missing", "bin/daemon"][..], + &["zzcanary-dir/tool"][..], + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, members))); + assert!(text.contains("executable member is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn symlink_member_refuses_the_release() -> Result<()> { + for link_target in ["daemon", "../zzcanary-nowhere"] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink( + link_target, + source.join("upstream/bin/zzcanary-link"), + )?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "bin/zzcanary-link"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn member_through_symlinked_directory_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("real-bin/zzcanary-tool"), b"tool")?; + std::os::unix::fs::symlink("real-bin", tree.join("linked-bin"))?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["linked-bin/zzcanary-tool"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + // A member below a regular file is refused the same way. + let second_staging = temp.path().join("second-staging"); + fs::create_dir_all(&second_staging)?; + let text = refusal_text(collect_tree( + &source, + &second_staging, + tree_output(false, &["bin/daemon/zzcanary-below-file"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn directory_member_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn executable_flag_on_tree_output_refuses() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(true, &[]))); + assert!(text.contains("cannot be executable"), "{text}"); + // The same flag on a file output stays admitted. + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + let mut output = tree_output(true, &[]); + output.id = "file".into(); + check_artifact_modes(&output, &file)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn members_on_file_output_refuse() -> Result<()> { + let temp = tempfile::tempdir()?; + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + check_artifact_modes(&tree_output(false, &[]), &file)?; + let text = refusal_text(check_artifact_modes( + &tree_output(false, &["zzcanary"]), + &file, + )); + assert!(text.contains("cannot declare executable_members"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn check_artifact_modes_refuses_a_member_that_is_not_a_clean_relative_path() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&temp.path().join("zzcanary-outside"), b"x")?; + for member in ["../../zzcanary-outside", "bin/../bin/daemon", "/zzcanary", ""] { + refusal_text(check_artifact_modes(&tree_output(false, &[member]), &tree)); + } + check_artifact_modes(&tree_output(false, &["bin/daemon"]), &tree)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn harden_fails_when_a_declared_member_is_absent() -> Result<()> { + let receipt = |members: &[&str]| ArtifactReceipt { + artifact_id: "upstream".into(), + destination: PathBuf::from("upstream"), + sha256: sha256_id(b"x"), + size_bytes: 1, + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + }; + let installed = |root: &Path| -> Result<()> { + write_file(&root.join("upstream/bin/daemon"), b"daemon")?; + write_file(&root.join("upstream/lib/tool"), b"tool")?; + fs::create_dir_all(root.join("upstream/zzcanary-dir"))?; + std::os::unix::fs::symlink("daemon", root.join("upstream/bin/zzcanary-link"))?; + Ok(()) + }; + let temp = tempfile::tempdir()?; + installed(temp.path())?; + harden_installed_release(temp.path(), &[receipt(&["bin/daemon", "lib/tool"])])?; + for members in [ + &["bin/daemon", "lib/zzcanary-gone"][..], + &["zzcanary-gone", "bin/daemon"][..], + &["zzcanary-dir"][..], + &["bin/zzcanary-link"][..], + &["bin/daemon/zzcanary-below-file"][..], + ] { + let temp = tempfile::tempdir()?; + installed(temp.path())?; + let text = refusal_text(harden_installed_release(temp.path(), &[receipt(members)])); + assert!(text.contains("declared executable is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn relative_in_tree_symlink_survives_the_seal() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/python3.12"), b"python")?; + std::os::unix::fs::symlink("python3.12", tree.join("bin/python3"))?; + let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/python3.12"]))?; + let (before, _) = digest_artifact(&staging.join("upstream"))?; + harden_installed_release(&staging, &[receipt])?; + let link = staging.join("upstream/bin/python3"); + assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); + assert_eq!(mode_of(&staging.join("upstream/bin/python3.12")), 0o555); + assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); + + // A link out of the release, by `..` or by absolute target, still refuses. + for (target, rule) in [ + ("../../zzcanary-up", "parent traversal"), + ("/zzcanary/absolute", "absolute"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink(target, source.join("upstream/lib/zzcanary-link"))?; + let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))?; + let text = refusal_text(harden_installed_release(&staging, &[receipt])); + assert!(text.contains(rule), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn streamed_digest_is_byte_compatible() -> Result<()> { + // Not a multiple of the 64 KiB read buffer, so a short last chunk + // matters; the aligned and empty files are the other boundaries. + let big = (0..3 * 65536 + 17).map(|i| (i % 251) as u8).collect::>(); + let aligned = vec![7_u8; 65536]; + let temp = tempfile::tempdir()?; + let tree = temp.path().join("tree"); + write_file(&tree.join("a.txt"), b"alpha")?; + write_file(&tree.join("empty"), b"")?; + write_file(&tree.join("sub/aligned.bin"), &aligned)?; + write_file(&tree.join("sub/big.bin"), &big)?; + std::os::unix::fs::symlink("a.txt", tree.join("zlink"))?; + + // The encoding digest_tree had before it streamed, written out by hand. + fn add_file(hasher: &mut Sha256, name: &str, bytes: &[u8]) { + hasher.update(b"file\0"); + hasher.update(name.as_bytes()); + hasher.update(b"\0"); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + let mut expected = Sha256::new(); + add_file(&mut expected, "a.txt", b"alpha"); + add_file(&mut expected, "empty", b""); + let mut expected_size = 5_u64; + expected.update(b"dir\0sub\0"); + add_file(&mut expected, "sub/aligned.bin", &aligned); + add_file(&mut expected, "sub/big.bin", &big); + expected_size += (aligned.len() + big.len()) as u64; + expected.update(b"link\0zlink\0a.txt"); + expected_size += "a.txt".len() as u64; + assert_eq!( + digest_artifact(&tree)?, + (format!("{:x}", expected.finalize()), expected_size) + ); + + let single = tree.join("sub/big.bin"); + assert_eq!( + digest_artifact(&single)?, + (raw_sha256(&big), big.len() as u64) + ); + let mut hasher = Sha256::new(); + assert_eq!(hash_file_streamed(&single, &mut hasher)?, big.len() as u64); + assert_eq!(format!("{:x}", hasher.finalize()), raw_sha256(&big)); + Ok(()) + } + + /// Acceptance on the real trees: `IDUNN_BIG_TREE` holds `python/` and + /// `blender/`. Run on Yggdrasil as its own job (see the cut spec). + #[cfg(unix)] + #[test] + #[ignore = "needs IDUNN_BIG_TREE: a CPython+torch tree and a Blender tree"] + fn big_tree_seals_from_env() -> Result<()> { + use std::time::Instant; + + fn symlinks(path: &Path) -> Result { + let mut count = 0; + for entry in fs::read_dir(path)? { + let entry = entry?; + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.is_dir() { + count += symlinks(&entry.path())?; + } else if metadata.file_type().is_symlink() { + count += 1; + } + } + Ok(count) + } + fn peak_rss() -> String { + fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| { + status + .lines() + .find(|line| line.starts_with("VmHWM")) + .map(str::to_owned) + }) + .unwrap_or_default() + } + + let tree = PathBuf::from( + std::env::var_os("IDUNN_BIG_TREE").context("IDUNN_BIG_TREE is not set")?, + ); + let members = ["python/bin/python3.12", "blender/blender"]; + let temp = tempfile::tempdir()?; + let staging = temp.path().join("staging"); + fs::create_dir_all(&staging)?; + let destination = staging.join("tree"); + let output = ArtifactOutput { + destination: PathBuf::from("tree"), + ..tree_output(false, &members) + }; + + let started = Instant::now(); + copy_artifact(&tree, &tree, &destination)?; + println!("copy {:?} {}", started.elapsed(), peak_rss()); + let started = Instant::now(); + let first = digest_artifact(&destination)?; + let second = digest_artifact(&destination)?; + assert_eq!(first, second); + println!("digest x2 {:?} {}", started.elapsed(), peak_rss()); + check_artifact_modes(&output, &destination)?; + let links = symlinks(&destination)?; + let started = Instant::now(); + harden_installed_release( + &staging, + &[ArtifactReceipt { + artifact_id: "tree".into(), + destination: PathBuf::from("tree"), + sha256: format!("sha256-{}", first.0), + size_bytes: first.1, + executable: false, + executable_members: output.executable_members.clone(), + }], + )?; + println!("harden {:?} {}", started.elapsed(), peak_rss()); + for member in members { + assert_eq!(mode_of(&destination.join(member)), 0o555); + } + let sample = fs::read_dir(&destination)? + .map(|entry| entry.map(|entry| entry.path())) + .collect::>>()?; + for path in sample { + if fs::symlink_metadata(&path)?.is_file() { + assert_eq!(mode_of(&path), 0o444); + } + } + assert_eq!(symlinks(&destination)?, links); + println!("bytes {} symlinks {links}", first.1); + Ok(()) + } + #[cfg(unix)] #[test] fn runner_cache_is_created_once_for_one_exact_identity() -> Result<()> { diff --git a/src/host.rs b/src/host.rs index 5221f12..6d05478 100644 --- a/src/host.rs +++ b/src/host.rs @@ -35,8 +35,9 @@ use crate::deployment::{ }; use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease}; use crate::drivers::{ - HostWorkloadObservation, InstalledReleaseObservation, copy_artifact, copy_tree, - digest_artifact, release_artifact, remove_tree_inside, sha256_id, write_runtime_bundle_records, + HostWorkloadObservation, InstalledReleaseObservation, check_artifact_modes, copy_artifact, + copy_tree, digest_artifact, release_artifact, remove_tree_inside, sha256_id, + write_runtime_bundle_records, }; use crate::host_actuator::{ HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS, @@ -1064,6 +1065,7 @@ fn materialize( "artifact destination escaped its staging root" ); copy_artifact(&source, &source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; let (sha256, size_bytes) = digest_artifact(&destination)?; if let Some(expected) = &artifact.expected_sha256 { ensure!( @@ -1078,6 +1080,7 @@ fn materialize( sha256: format!("sha256-{sha256}"), size_bytes, executable: artifact.executable, + executable_members: artifact.executable_members.clone(), }); } let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?; From 04ff2c8ce3e90d0e8a2a722389d50ef0f5fe4fff Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 19:53:16 +0200 Subject: [PATCH 078/122] README: a directory artifact is sealed 0444 except its executable_members Co-Authored-By: Claude Sonnet 5.5 --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index db728a9..4936cbe 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,26 @@ required_environment = [ ] ``` +A directory artifact is sealed read-only: its files are 0444 and its +directories 0555, whatever mode the build or git gave them. A file the tree +must run, such as the launcher's `upstream/bin/daemon` child, is declared in +`executable_members` and sealed 0555. The members are part of the sealed release +address, so changing the list makes a new release. Idunn refuses the release if a +member is absolute, climbs with `..`, is missing, is a symlink or sits behind +one, or is not a regular file; `executable = true` on a directory output and +`executable_members` on a file output are refused too. + +```toml +# recipe fragment — the upstream artifact declares the child it launches +[[artifacts]] +id = "upstream" +source_kind = "runner-output" +runner = "build" +source = "dist/upstream" +destination = "upstream" +executable_members = ["bin/daemon"] +``` + ```toml # binding fragment — members connect to the member endpoint, which Idunn # forwards to the private port derived from the candidate: member From 76cec0b5403c0bd54d0aa560e68d29e667eec8f4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 19:54:35 +0200 Subject: [PATCH 079/122] Tests: the sealed-mode helper no longer redefines the module's existing mode_of Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 165dfb0..347dd27 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -13979,7 +13979,7 @@ processes 9 } #[cfg(unix)] - fn mode_of(path: &Path) -> u32 { + fn sealed_mode(path: &Path) -> u32 { use std::os::unix::fs::PermissionsExt; fs::symlink_metadata(path).unwrap().permissions().mode() & 0o777 @@ -14057,12 +14057,12 @@ processes 9 ); harden_installed_release(&staging, &[receipt])?; let tree = staging.join("upstream"); - assert_eq!(mode_of(&tree.join("bin/daemon")), 0o555); - assert_eq!(mode_of(&tree.join("lib/tool")), 0o555); - assert_eq!(mode_of(&tree.join("bin/helper")), 0o444); - assert_eq!(mode_of(&tree.join("lib/data.txt")), 0o444); + assert_eq!(sealed_mode(&tree.join("bin/daemon")), 0o555); + assert_eq!(sealed_mode(&tree.join("lib/tool")), 0o555); + assert_eq!(sealed_mode(&tree.join("bin/helper")), 0o444); + assert_eq!(sealed_mode(&tree.join("lib/data.txt")), 0o444); for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { - assert_eq!(mode_of(directory), 0o555); + assert_eq!(sealed_mode(directory), 0o555); } Ok(()) } @@ -14240,7 +14240,7 @@ processes 9 let link = staging.join("upstream/bin/python3"); assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); - assert_eq!(mode_of(&staging.join("upstream/bin/python3.12")), 0o555); + assert_eq!(sealed_mode(&staging.join("upstream/bin/python3.12")), 0o555); assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); // A link out of the release, by `..` or by absolute target, still refuses. @@ -14377,14 +14377,14 @@ processes 9 )?; println!("harden {:?} {}", started.elapsed(), peak_rss()); for member in members { - assert_eq!(mode_of(&destination.join(member)), 0o555); + assert_eq!(sealed_mode(&destination.join(member)), 0o555); } let sample = fs::read_dir(&destination)? .map(|entry| entry.map(|entry| entry.path())) .collect::>>()?; for path in sample { if fs::symlink_metadata(&path)?.is_file() { - assert_eq!(mode_of(&path), 0o444); + assert_eq!(sealed_mode(&path), 0o444); } } assert_eq!(symlinks(&destination)?, links); From fd040f09c162033bd126e54bbdb286b9b574a424 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 20:28:15 +0200 Subject: [PATCH 080/122] Tests: an io error other than NotFound is not reported as a missing member Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 347dd27..2dfeda6 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -14190,6 +14190,22 @@ processes 9 Ok(()) } + #[cfg(unix)] + #[test] + fn an_unreadable_member_is_not_reported_as_missing() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + // A component past NAME_MAX fails with an io error that is not NotFound. + let long = "z".repeat(300); + let text = refusal_text(check_artifact_modes( + &tree_output(false, &[long.as_str()]), + &source.join("upstream"), + )); + assert!(!text.contains("missing"), "{text}"); + assert!(text.contains("inspecting an executable member"), "{text}"); + Ok(()) + } + #[cfg(unix)] #[test] fn harden_fails_when_a_declared_member_is_absent() -> Result<()> { From 07616697aff1098cfda7f23d7162611630024922 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 21:54:00 +0200 Subject: [PATCH 081/122] deploy-provenance map: link containment rule, host targets, shared seal path (r2) Imagination r2 for cut tree-executable-members after Soul s1: link census of the real PBS and Blender trees, why lexical normalisation does not prove containment, the leading-dotdot-within-depth rule and its prior art, members on host targets carried not applied, one seal step for collect and host. Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-map.md | 124 +++++++++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 18 deletions(-) diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md index cfbd11b..195867d 100644 --- a/docs/deploy-provenance-map.md +++ b/docs/deploy-provenance-map.md @@ -4,12 +4,14 @@ Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, session `self-2026-10-23-eureka-b`). The ends are in [`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, rulings, cut specs and follow-ups live in Eureka's mind; this page holds what -has no kind: body facts, rationale and sequencing. Line numbers are at -`d74ac8d` (origin/main on 2026-10-23). +has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are +at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`, +the head of Hands attempt 1, which revision 2 of the spec builds on. ## Cut `tree-executable-members` -Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r1`. It +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2` +(revision 2, after Soul pass 1 found the real interpreter tree refused). It serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's admission waits on it) and will serve gamecult-chat C3 too: the presence launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact @@ -32,12 +34,16 @@ launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not mode bits. So runner or git exec bits never reach the release: harden is the only mode authority. -- T4. `validate_release_symlink` (`:6895-6914`) keeps a symlink only if its - target is relative with no `..` and lexically inside the release. That rule is - sound without physical resolution (no `..` means a target can only descend). - Absolute links and `..` links refuse the release. The frozen-source path has - a physical resolver (`resolve_frozen_source_symlink`, tested at `:11826`) that - admits in-tree `..`; reuse it if a real tree needs `..` links, not a new rule. +- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink + only if its target is relative with no `..`. It runs only inside + `harden_root_tree`, at install, so a tree with a `..` or absolute link copies, + digests and seals, then fails at `install_release` (Soul s1: copying and + digesting the real tree twice took 160 s before harden refused it). The rule + is sound but too narrow: it refuses the real interpreter tree (T13). The + frozen-source path has a physical resolver (`resolve_frozen_source_symlink`, + `:6617`) that walks the filesystem with a 40-link budget and a memo. Its + first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs + the tree on disk to answer. Revision 2 does not reuse it (T14). - T5. The sealed release id hashes `(schema, plan_id, artifacts, external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes the whole plan including `recipe_blob`. A changed member list is therefore @@ -59,8 +65,11 @@ launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact destination, so it does not change. `require_relative_path` (`:1936-1949`) with `allow_current = false` already refuses absolute, `..`, root and prefix components. -- T9. The host actuator builds receipts separately (`src/host.rs:1043-1081`) - and must carry the same field and the same shape check. +- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy, + mode check, digest, pinned-digest check and receipt, the same steps as + `collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode + check survives every test. Revision 2 collapses both onto one `seal_artifact` + in `drivers.rs`. Each caller keeps only how it finds the source path. - T10. The transient unit runs with `ProtectSystem=strict` and the release in `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. @@ -73,6 +82,68 @@ launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact of them. Idunn's own body changes only through gamecult-ops `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T13. Link census of the real interpreter tree (Yggdrasil probe of + `cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`, + sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049 + after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all + their `..` components before the first name (every one is + `share/terminfo// -> ..//`, one `..` each). 0 have a `..` + after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it. + Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1). +- T14. Lexically normalising `/` does not check + containment. The kernel resolves a target one component at a time, so a `..` + after a symlinked name climbs from where that link points, not from where its + name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x` + normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x` + beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same + way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`, + `L -> D/D/../../../etc/passwd` passed a lexical check). + The sound lexical rule constrains the target's shape: all its `..` components + come before its first name, and they number no more than the link's depth + below the artifact's top. The climb then starts from the link's own + directory, which the walk reached through real directories only (copy and + digest never descend through a link), so it lands on a real ancestor inside + the artifact. After that the target only descends, and any link it passes + through obeys the same rule. By induction on the kernel's hops (capped at 40, + then ELOOP), every hop stays inside. The check is one pass over the target's + components with no filesystem reads, so its cost is linear in the tree and no + resolver is needed. The shapes it refuses (`s/../x`) occur in neither real + tree (T13). + Prior art, from documentation as known to Imagination (high confidence for Go + and securejoin, medium for the tar details): + - Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account + for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug. + - Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc, + containerd and umoci) resolve each component inside the root when a path is + opened, refusing or clamping an escape. They guard one later access, need + the tree on disk, and check again on every open. Idunn instead certifies a + sealed tree once, so that the kernel's own resolution stays inside on every + later access. + - Zip-slip defences and archive extractors check entry names lexically and + protect against link targets by refusing to write through links + (libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for + names; GNU tar creates such links last). They protect the extraction, not + what a kept link points at afterwards. + - OCI layer extraction keeps link targets verbatim, absolute ones included, + because the runtime resolves them inside the container's root. Idunn runs a + release in the host's mount namespace, so absolute targets stay refused. + - Nix keeps link targets verbatim and tracks dependencies by scanning for + references. The whole store is its root, so it has no per-artifact + containment to compare with. + None of these is a static per-link rule for a tree that runs unconfined. The + T14 shape rule fits that case. +- T15. Host targets: `idunn-host` observes, starts and stops processes on + Windows only (the non-Windows `platform` module in `src/host.rs` bails), and + `host.rs` sets no file modes. On Windows a file runs by its type, not by a + mode bit, so `executable` and `executable_members` have nothing to set there. + The file `executable` flag has always been carried and never applied on host + targets. A recipe does not know which workload kind its binding will choose. +- T16. The release root holds only artifact destinations. Runner workspaces, + where external inputs land, are removed before `SealedRelease::new` + (`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and + `validate_installed_release` (`:2815-2822`) digest every receipt's + destination before calling harden. So a rule in the digest walk runs at seal, + at install and at every validate, over every link harden would see. - T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three copies (runner workspace, staging, installed). Nothing collects old releases or staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every @@ -93,21 +164,38 @@ launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact - Harden seals each member 0555 and fails if any declared executable path was not visited as a regular file. "Every declared executable is sealed 0555, or the install fails" holds structurally, not by a lookup that can miss. -- Symlinks: relative, no `..`, in-tree links survive (T4). Vili's trees must +- Symlinks: a link in a tree artifact is kept when its target is relative, its + `..` components all come before its first name, and they number no more than + the link's depth below the artifact's top (T14). One function, + `check_release_link`, decides this. Only the digest walk calls it, so a bad + link refuses the release at seal, before `SealedRelease::new`, and again at + install and validate (T16). Harden no longer judges links. The climb is + bounded by the artifact's top, not the release root: no live release has a + `..` link (the old rule refused them all), so nothing installed is newly + refused, and each artifact's digest describes a closed tree. +- Host targets carry `executable` and `executable_members` in the receipt, + shape-checked at seal by the shared `seal_artifact`, and do not apply them + (T15). Refusing members there would make a recipe's validity depend on its + binding. +- Vili's trees must not come from `python -m venv`: its `bin/python` is an absolute link and its `pyvenv.cfg` names the build workspace. Install packages into the relocatable interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). ### Authority -Owner: the seal (`collect_artifact`, and `host.rs` for host targets) decides -membership, from the recipe, and writes it into the receipt. Harden enforces -the receipt and decides nothing. Runner exec bits and git `100755` modes are -dead inputs for release modes. +Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd +targets and by `materialize` in `host.rs` for host targets) decides +membership from the recipe and writes it into the receipt. Harden enforces the +receipt and decides nothing. Runner exec bits and git `100755` modes are dead +inputs for release modes. Link containment belongs to `check_release_link`, +which only `digest_tree` calls. `validate_release_symlink` is deleted, and +harden's symlink arm leaves links as they are. ### Acceptance sizing The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It -removes it at the end. If the trees hold any absolute or `..` symlink, the job -stops and the T4 resolver reuse becomes a revision of this spec. +removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 +terminfo links (T13). If a tree holds a link the T14 rule refuses, the job +prints it and stops, and the link is reported to Self, not deleted. From 68d6f96808c40c04637624c9077d42bcfc02c6dd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 21:57:07 +0200 Subject: [PATCH 082/122] Seal: one seal_artifact for both targets; link containment decided at seal by a shape rule collect_artifact and host.rs materialize both call seal_artifact. digest_tree judges every link with check_release_link (relative, `..` only first, climbs no higher than the link's depth); validate_release_symlink and harden's link judgement are gone. hash_file_exact owns the streamed-length check. A file directly under a tree's top is pinned 0444. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 6 + src/drivers.rs | 297 ++++++++++++++++++++++++++++++------------------- src/host.rs | 48 ++------ 3 files changed, 201 insertions(+), 150 deletions(-) diff --git a/README.md b/README.md index 4936cbe..a5c48bc 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,12 @@ member is absolute, climbs with `..`, is missing, is a symlink or sits behind one, or is not a regular file; `executable = true` on a directory output and `executable_members` on a file output are refused too. +A tree may hold relative symlinks whose `..` steps all come first and climb no +higher than the artifact's top (`share/terminfo/4/4025ex -> ../t/tek4025ex` is +kept). An absolute link, a `..` after a name, or a climb out of the artifact +refuses the release when it is sealed. Host targets (Windows) record +`executable` and `executable_members` in the receipt and do not apply them. + ```toml # recipe fragment — the upstream artifact declares the child it launches [[artifacts]] diff --git a/src/drivers.rs b/src/drivers.rs index 2dfeda6..d82e5ba 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -2302,29 +2302,7 @@ impl DockerRunnerDriver { } ArtifactSource::WorktreeTree => (source.root.clone(), source.root.join(&artifact.source)), }; - ensure!(source_path.exists(), "declared artifact output is absent"); - let destination = staging_root.join(&artifact.destination); - ensure!( - destination.starts_with(staging_root), - "artifact destination escaped its release staging root" - ); - copy_artifact(&containment_root, &source_path, &destination)?; - check_artifact_modes(artifact, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact output differs from its recipe-pinned digest" - ); - } - Ok(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - executable_members: artifact.executable_members.clone(), - }) + seal_artifact(artifact, &containment_root, &source_path, staging_root) } } @@ -6812,10 +6790,7 @@ fn hash_frozen_source_tree(root: &Path, current: &Path, hasher: &mut Sha256) -> ); hasher.update(b"\0"); hasher.update(metadata.len().to_le_bytes()); - ensure!( - hash_file_streamed(&path, hasher)? == metadata.len(), - "frozen source file changed while hashing" - ); + hash_file_exact(&path, metadata.len(), hasher)?; } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -6847,7 +6822,7 @@ fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Resul executable_paths.insert(destination); } } - harden_root_tree(root, root, &mut executable_paths)?; + harden_root_tree(root, &mut executable_paths)?; ensure!( executable_paths.is_empty(), "declared executable is missing or not a regular file" @@ -6858,11 +6833,7 @@ fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Resul /// Each regular file the walk seals 0555 is removed from `executable_paths`; /// whatever remains was declared executable and never sealed. #[cfg(unix)] -fn harden_root_tree( - root: &Path, - current: &Path, - executable_paths: &mut BTreeSet, -) -> Result<()> { +fn harden_root_tree(current: &Path, executable_paths: &mut BTreeSet) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let metadata = fs::symlink_metadata(current)?; @@ -6874,7 +6845,7 @@ fn harden_root_tree( let mut entries = fs::read_dir(current)?.collect::>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { - harden_root_tree(root, &entry.path(), &mut *executable_paths)?; + harden_root_tree(&entry.path(), &mut *executable_paths)?; } fs::set_permissions(current, fs::Permissions::from_mode(0o555))?; } else if metadata.is_file() { @@ -6887,35 +6858,15 @@ fn harden_root_tree( }), )?; } else if metadata.file_type().is_symlink() { - validate_release_symlink(root, current)?; + // Left untouched: containment was decided by check_release_link in + // digest_tree, which install_release and validate_installed_release + // run over every receipt before this walk (map T16). } else { bail!("sealed release contains a special filesystem entry") } Ok(()) } -#[cfg(unix)] -fn validate_release_symlink(root: &Path, path: &Path) -> Result<()> { - let target = fs::read_link(path)?; - ensure!(target.is_relative(), "sealed release symlink is absolute"); - ensure!( - target.components().all(|component| matches!( - component, - std::path::Component::CurDir | std::path::Component::Normal(_) - )), - "sealed release symlink has a parent traversal" - ); - let destination = path - .parent() - .context("sealed release symlink has no parent")? - .join(target); - ensure!( - destination.starts_with(root), - "sealed release symlink escaped its release" - ); - Ok(()) -} - #[cfg(not(unix))] fn harden_installed_release(_root: &Path, _artifacts: &[ArtifactReceipt]) -> Result<()> { bail!("systemd workload installation requires Unix permissions") @@ -8105,7 +8056,7 @@ fn copy_symlink(source: &Path, destination: &Path) -> Result<()> { /// checked to contain the fully resolved `source` with the same /// whole-chain-aware resolver `validate_frozen_source_symlink` uses, before /// anything is read from it. -pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { +fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { let canonical_root = root.canonicalize().context("resolving artifact root")?; let canonical_source = source .canonicalize() @@ -8133,7 +8084,7 @@ pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> R /// a file output never declares members, and every member is a regular file /// reached through real directories, never through a symlink. Errors name the /// rule, never the member path. -pub(crate) fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { +fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { let is_dir = fs::symlink_metadata(destination) .context("inspecting a copied artifact")? .is_dir(); @@ -8179,30 +8130,73 @@ pub(crate) fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path Ok(()) } +/// The one seal step for a declared artifact, for systemd and host targets +/// alike: copy it into the staging root, check its mode declarations, digest it +/// and check the recipe pin. Modes are applied by harden on systemd targets; +/// host targets (Windows, map T15) carry `executable` and `executable_members` +/// in the receipt and do not apply them. +pub(crate) fn seal_artifact( + artifact: &ArtifactOutput, + containment_root: &Path, + source_path: &Path, + staging_root: &Path, +) -> Result { + ensure!( + source_path.exists(), + "declared artifact output {} is absent", + artifact.id + ); + let destination = staging_root.join(&artifact.destination); + ensure!( + destination.starts_with(staging_root), + "artifact {} destination escaped its staging root", + artifact.id + ); + copy_artifact(containment_root, source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; + let (sha256, size_bytes) = digest_artifact(&destination)?; + if let Some(expected) = &artifact.expected_sha256 { + ensure!( + &sha256 == expected, + "artifact {} differs from its recipe-pinned digest", + artifact.id + ); + } + Ok(ArtifactReceipt { + artifact_id: artifact.id.clone(), + destination: artifact.destination.clone(), + sha256: format!("sha256-{sha256}"), + size_bytes, + executable: artifact.executable, + executable_members: artifact.executable_members.clone(), + }) +} + /// Streams `path` into `hasher` in 64 KiB chunks, so peak memory is the buffer -/// and not the largest file in the tree. Returns the bytes read; a caller that -/// hashed a length first must check the two agree. -pub(crate) fn hash_file_streamed(path: &Path, hasher: &mut Sha256) -> Result { +/// and not the largest file in the tree, and refuses if the bytes read differ +/// from `expected_len`. The only owner of the length check. +pub(crate) fn hash_file_exact(path: &Path, expected_len: u64, hasher: &mut Sha256) -> Result<()> { let mut file = fs::File::open(path).context("opening a file to hash it")?; let mut buffer = [0u8; 1 << 16]; let mut total = 0_u64; loop { let read = file.read(&mut buffer).context("reading a file to hash it")?; if read == 0 { - return Ok(total); + break; } hasher.update(&buffer[..read]); total += read as u64; } + ensure!(total == expected_len, "file changed while hashing"); + Ok(()) } pub(crate) fn digest_artifact(path: &Path) -> Result<(String, u64)> { let metadata = fs::symlink_metadata(path)?; if metadata.is_file() { let mut hasher = Sha256::new(); - let streamed = hash_file_streamed(path, &mut hasher)?; - ensure!(streamed == metadata.len(), "artifact file changed while hashing"); - return Ok((format!("{:x}", hasher.finalize()), streamed)); + hash_file_exact(path, metadata.len(), &mut hasher)?; + return Ok((format!("{:x}", hasher.finalize()), metadata.len())); } ensure!(metadata.is_dir(), "artifact is not a file or directory"); let mut hasher = Sha256::new(); @@ -8231,13 +8225,11 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) hasher.update(relative.as_bytes()); hasher.update(b"\0"); hasher.update(length.to_le_bytes()); - ensure!( - hash_file_streamed(&path, hasher)? == length, - "artifact file changed while hashing" - ); + hash_file_exact(&path, length, hasher)?; *size = size.saturating_add(length); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; + check_release_link(path.strip_prefix(root)?, &target)?; hasher.update(b"link\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); @@ -8250,6 +8242,35 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) Ok(()) } +/// The one judge of link containment in a release tree, pure and static: +/// `link` is the link's path relative to its artifact's top and `target` what +/// it reads. A target is relative, its `..` steps all come first, and they +/// climb no higher than the link's own directory depth, so the link can only +/// land inside the artifact. Lexical normalisation of parent and target is not +/// enough: with `s -> ..` it admits `s/../x` and escapes (map T14). +fn check_release_link(link: &Path, target: &Path) -> Result<()> { + use std::path::Component; + + let depth = link.components().count().saturating_sub(1); + let mut climbs = 0_usize; + let mut named = false; + for component in target.components() { + match component { + Component::CurDir => {} + Component::RootDir | Component::Prefix(_) => { + bail!("sealed release symlink is absolute") + } + Component::ParentDir => { + ensure!(!named, "sealed release symlink climbs after a name"); + climbs += 1; + } + Component::Normal(_) => named = true, + } + } + ensure!(climbs <= depth, "sealed release symlink escapes its artifact"); + Ok(()) +} + pub(crate) fn raw_sha256(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } @@ -13996,6 +14017,7 @@ processes 9 write_file(&tree.join("bin/helper"), b"helper")?; write_file(&tree.join("lib/tool"), b"tool")?; write_file(&tree.join("lib/data.txt"), b"data")?; + write_file(&tree.join("NOTICE"), b"notice")?; Ok((source, staging)) } @@ -14061,12 +14083,44 @@ processes 9 assert_eq!(sealed_mode(&tree.join("lib/tool")), 0o555); assert_eq!(sealed_mode(&tree.join("bin/helper")), 0o444); assert_eq!(sealed_mode(&tree.join("lib/data.txt")), 0o444); + assert_eq!(sealed_mode(&tree.join("NOTICE")), 0o444); for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { assert_eq!(sealed_mode(directory), 0o555); } Ok(()) } + #[test] + fn release_link_rule_is_containment() { + for (link, target) in [ + ("a/b/l", "../t/x"), + ("a/l", ".."), + ("a/l", "./../b"), + ("a/l", "b/./c"), + ("l", "python3.12"), + ("a/b/c/l", "../../x"), + ] { + check_release_link(Path::new(link), Path::new(target)) + .unwrap_or_else(|error| panic!("({link}, {target}) refused: {error:#}")); + } + for (link, target, rule) in [ + ("l", "/x", "absolute"), + ("a/l", "s/../x", "climbs after a name"), + ("a/l", "D/D/../../../etc/passwd", "climbs after a name"), + ("a/l", "../../x", "escapes its artifact"), + ("l", "..", "escapes its artifact"), + ("a/b/l", "../../../x", "escapes its artifact"), + ("a/l", "./s/../x", "climbs after a name"), + ] { + let text = format!( + "{:#}", + check_release_link(Path::new(link), Path::new(target)) + .expect_err(&format!("({link}, {target}) was admitted")) + ); + assert!(text.contains(rule), "({link}, {target}): {text}"); + } + } + #[cfg(unix)] #[test] fn missing_member_refuses_the_release() -> Result<()> { @@ -14250,6 +14304,9 @@ processes 9 let tree = source.join("upstream"); write_file(&tree.join("bin/python3.12"), b"python")?; std::os::unix::fs::symlink("python3.12", tree.join("bin/python3"))?; + write_file(&tree.join("lib/terminfo/b/real"), b"terminfo")?; + fs::create_dir_all(tree.join("lib/terminfo/a"))?; + std::os::unix::fs::symlink("../b/real", tree.join("lib/terminfo/a/alias"))?; let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/python3.12"]))?; let (before, _) = digest_artifact(&staging.join("upstream"))?; harden_installed_release(&staging, &[receipt])?; @@ -14257,18 +14314,22 @@ processes 9 assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); assert_eq!(sealed_mode(&staging.join("upstream/bin/python3.12")), 0o555); + let alias = staging.join("upstream/lib/terminfo/a/alias"); + assert!(fs::symlink_metadata(&alias)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&alias)?, PathBuf::from("../b/real")); assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); - // A link out of the release, by `..` or by absolute target, still refuses. + // A link that escapes, is absolute or climbs after a name is refused at + // the seal, before any receipt exists. for (target, rule) in [ - ("../../zzcanary-up", "parent traversal"), + ("../../zzcanary-up", "escapes its artifact"), ("/zzcanary/absolute", "absolute"), + ("zzcanary-s/../x", "climbs after a name"), ] { let temp = tempfile::tempdir()?; let (source, staging) = sample_tree(temp.path())?; std::os::unix::fs::symlink(target, source.join("upstream/lib/zzcanary-link"))?; - let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))?; - let text = refusal_text(harden_installed_release(&staging, &[receipt])); + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); assert!(text.contains(rule), "{text}"); } Ok(()) @@ -14318,8 +14379,12 @@ processes 9 (raw_sha256(&big), big.len() as u64) ); let mut hasher = Sha256::new(); - assert_eq!(hash_file_streamed(&single, &mut hasher)?, big.len() as u64); + hash_file_exact(&single, big.len() as u64, &mut hasher)?; assert_eq!(format!("{:x}", hasher.finalize()), raw_sha256(&big)); + for wrong in [big.len() as u64 + 1, big.len() as u64 - 1] { + let text = refusal_text(hash_file_exact(&single, wrong, &mut Sha256::new())); + assert!(text.contains("changed while hashing"), "{text}"); + } Ok(()) } @@ -14331,18 +14396,40 @@ processes 9 fn big_tree_seals_from_env() -> Result<()> { use std::time::Instant; - fn symlinks(path: &Path) -> Result { - let mut count = 0; + /// (links, links with a `..` step, regular files, files at 0555), never + /// following a link; every regular file must be 0444 or a member's 0555. + fn census( + root: &Path, + path: &Path, + members: &BTreeSet, + modes: bool, + totals: &mut [usize; 4], + ) -> Result<()> { for entry in fs::read_dir(path)? { let entry = entry?; let metadata = fs::symlink_metadata(entry.path())?; if metadata.is_dir() { - count += symlinks(&entry.path())?; + census(root, &entry.path(), members, modes, totals)?; } else if metadata.file_type().is_symlink() { - count += 1; + totals[0] += 1; + let target = fs::read_link(entry.path())?; + if target + .components() + .any(|part| matches!(part, std::path::Component::ParentDir)) + { + totals[1] += 1; + } + } else { + totals[2] += 1; + let member = members.contains(entry.path().strip_prefix(root)?); + let mode = sealed_mode(&entry.path()); + if modes { + assert_eq!(mode, if member { 0o555 } else { 0o444 }); + totals[3] += usize::from(mode == 0o555); + } } } - Ok(count) + Ok(()) } fn peak_rss() -> String { fs::read_to_string("/proc/self/status") @@ -14370,41 +14457,27 @@ processes 9 }; let started = Instant::now(); - copy_artifact(&tree, &tree, &destination)?; - println!("copy {:?} {}", started.elapsed(), peak_rss()); + let receipt = seal_artifact(&output, &tree, &tree, &staging)?; + println!("seal {:?} {}", started.elapsed(), peak_rss()); let started = Instant::now(); - let first = digest_artifact(&destination)?; - let second = digest_artifact(&destination)?; - assert_eq!(first, second); - println!("digest x2 {:?} {}", started.elapsed(), peak_rss()); - check_artifact_modes(&output, &destination)?; - let links = symlinks(&destination)?; + let again = digest_artifact(&destination)?; + assert_eq!(format!("sha256-{}", again.0), receipt.sha256); + assert_eq!(again.1, receipt.size_bytes); + println!("digest {:?} {}", started.elapsed(), peak_rss()); + let mut before = [0; 4]; + census(&destination, &destination, &output.executable_members, false, &mut before)?; let started = Instant::now(); - harden_installed_release( - &staging, - &[ArtifactReceipt { - artifact_id: "tree".into(), - destination: PathBuf::from("tree"), - sha256: format!("sha256-{}", first.0), - size_bytes: first.1, - executable: false, - executable_members: output.executable_members.clone(), - }], - )?; + harden_installed_release(&staging, &[receipt.clone()])?; println!("harden {:?} {}", started.elapsed(), peak_rss()); - for member in members { - assert_eq!(sealed_mode(&destination.join(member)), 0o555); - } - let sample = fs::read_dir(&destination)? - .map(|entry| entry.map(|entry| entry.path())) - .collect::>>()?; - for path in sample { - if fs::symlink_metadata(&path)?.is_file() { - assert_eq!(sealed_mode(&path), 0o444); - } - } - assert_eq!(symlinks(&destination)?, links); - println!("bytes {} symlinks {links}", first.1); + let mut totals = [0; 4]; + census(&destination, &destination, &output.executable_members, true, &mut totals)?; + assert_eq!(totals[0], before[0], "links changed across harden"); + assert_eq!(totals[3], members.len()); + println!( + "dotdot links {} links before {} after {} files {} exec {}", + totals[1], before[0], totals[0], totals[2], totals[3] + ); + println!("bytes {}", receipt.size_bytes); Ok(()) } diff --git a/src/host.rs b/src/host.rs index 6d05478..4c6cbec 100644 --- a/src/host.rs +++ b/src/host.rs @@ -30,14 +30,13 @@ use serde::Deserialize; use crate::control_plane::read_trust_anchor; use crate::deployment::{ - ArtifactSource, HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, LaunchArgument, OperatorBinding, TargetDeclaration, }; -use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease}; +use crate::deployment_plan::{CompiledDeploymentPlan, SealedRelease}; use crate::drivers::{ - HostWorkloadObservation, InstalledReleaseObservation, check_artifact_modes, copy_artifact, - copy_tree, digest_artifact, release_artifact, remove_tree_inside, sha256_id, - write_runtime_bundle_records, + HostWorkloadObservation, InstalledReleaseObservation, copy_tree, digest_artifact, + release_artifact, remove_tree_inside, seal_artifact, sha256_id, write_runtime_bundle_records, }; use crate::host_actuator::{ HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS, @@ -1049,39 +1048,12 @@ fn materialize( fs::create_dir_all(&staging)?; let mut artifacts = Vec::new(); for artifact in &declaration.artifacts { - let source_path = match artifact.source_kind { - ArtifactSource::RunnerOutput | ArtifactSource::WorktreeTree => { - source.join(&artifact.source) - } - }; - ensure!( - source_path.exists(), - "declared artifact output {} is absent", - artifact.id - ); - let destination = staging.join(&artifact.destination); - ensure!( - destination.starts_with(&staging), - "artifact destination escaped its staging root" - ); - copy_artifact(&source, &source_path, &destination)?; - check_artifact_modes(artifact, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact {} differs from its recipe-pinned digest", - artifact.id - ); - } - artifacts.push(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - executable_members: artifact.executable_members.clone(), - }); + artifacts.push(seal_artifact( + artifact, + &source, + &source.join(&artifact.source), + &staging, + )?); } let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?; Ok((release, staging)) From e8c4c6972881b465b30e9e1e896272a8ab7d3ceb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 23:26:50 +0200 Subject: [PATCH 083/122] Seal tests: refused links whose targets resolve (absolute, escaping, after-a-name; file and directory) Closes the live-target bypass: a judge guarded by exists() or is_dir() passed every dangling fixture. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index d82e5ba..0eb0928 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -14335,6 +14335,44 @@ processes 9 Ok(()) } + #[cfg(unix)] + #[test] + fn links_to_existing_targets_are_refused_at_the_seal() -> Result<()> { + // Every refused shape, aimed at a target that EXISTS (file and + // directory): a judge guarded by `exists()`, `is_dir()` or a file-type + // test passes the dangling fixtures and fails these. + for kind in ["file", "dir"] { + for (shape, rule) in [ + ("absolute", "absolute"), + ("escape", "escapes its artifact"), + ("after-name", "climbs after a name"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let target = match (shape, kind) { + ("absolute", "file") => source.join("upstream/bin/daemon"), + ("absolute", _) => source.join("upstream/lib"), + ("escape", "file") => { + write_file(&source.join("outside-file"), b"outside")?; + PathBuf::from("../../outside-file") + } + ("escape", _) => { + fs::create_dir_all(source.join("outside-dir"))?; + PathBuf::from("../../outside-dir") + } + (_, "file") => PathBuf::from("../bin/../NOTICE"), + _ => PathBuf::from("../bin/../lib"), + }; + let link = source.join("upstream/lib/zzcanary-link"); + std::os::unix::fs::symlink(&target, &link)?; + assert!(fs::metadata(&link).is_ok(), "{shape} {kind} link must resolve"); + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); + assert!(text.contains(rule), "{shape} {kind}: {text}"); + } + } + Ok(()) + } + #[cfg(unix)] #[test] fn streamed_digest_is_byte_compatible() -> Result<()> { From a3c6386aa4af33c7fc8a5d2fb5178bd85ef36f36 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 00:14:39 +0200 Subject: [PATCH 084/122] Grant Idunn Vili's runtime, state and release roots motion-pipeline:follow_up:idunn-unit-vili-roots: Vili's admission (motion-pipeline:cut_spec:cut-vili-admission.r1) needs its three roots writable under ProtectSystem=full, beside StreamPixels'. Lands before the d3222e5 install, which restarts idunn-yggdrasil anyway. Co-Authored-By: Claude Opus 5.5 --- deploy/idunn-yggdrasil.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index da82bd2..c63c1ed 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -104,6 +104,8 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web +# Vili (motion worker): runtime bundle, state, and releases. +ReadWritePaths=/etc/gamecult/vili/runtime /var/lib/gamecult/vili /srv/vili-idunn/releases [Install] WantedBy=multi-user.target From 112a7a5ebb78ab5ff7ab92624defb6cb99805d1b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 00:15:16 +0200 Subject: [PATCH 085/122] Idunn unit: Vili's roots are optional so a missing target cannot stop Idunn systemd refuses to start a unit whose ReadWritePaths names a missing path, and none of Vili's three roots exist on Yggdrasil yet. Idunn's startup must not depend on a managed target, so the Vili line takes the "-" prefix. Co-Authored-By: Claude Opus 5.5 --- deploy/idunn-yggdrasil.service | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index c63c1ed..1c45594 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -104,8 +104,11 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web -# Vili (motion worker): runtime bundle, state, and releases. -ReadWritePaths=/etc/gamecult/vili/runtime /var/lib/gamecult/vili /srv/vili-idunn/releases +# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a +# target whose roots are not created yet cannot stop Idunn itself from starting +# (Idunn startup never depends on a managed target). A root created after Idunn +# starts is writable only from Idunn's next restart. +ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases [Install] WantedBy=multi-user.target From 69eb3e6d6f8b077a6cb6774fcf3c2fda5dbcb639 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 14:51:10 +0200 Subject: [PATCH 086/122] deploy-provenance map: isolation credential evidence (body facts I1-I8) Imagination for cut isolation-credential-evidence: systemd 257 prints LoadCredential as [unprintable] for every unit, so observe_unit's comparison never passes a target with no extra credential and proves nothing otherwise; the rule moves to the delivered credential directory. Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-map.md | 85 +++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md index 195867d..5ec7052 100644 --- a/docs/deploy-provenance-map.md +++ b/docs/deploy-provenance-map.md @@ -199,3 +199,88 @@ on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 terminfo links (T13). If a tree holds a link the T14 rule refuses, the job prints it and stops, and the link is reported to Self, not deleted. + +## Cut `isolation-credential-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, +at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It +serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, +first deviation): every `idunn up vili` is refused before fencing. + +### Body facts (probed read-only on Yggdrasil, 2026-10-24) + +- I1. `idunn up vili` runs 6 and 7 failed with + `systemd workload isolation properties differ from the Idunn contract`, the + message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, + only one differs on the failed Vili unit: `LoadCredential`. Idunn expects + `""` because Vili has no service credential besides the presence identity + (`service_credential_names` empty); systemd shows `[unprintable]`. Every + other conjunct matches, InaccessiblePaths included (one value across all + five idunn-vili/odin units). +- I2. systemd 257.13 `systemctl show -p LoadCredential` prints + `LoadCredential=[unprintable]` for every unit, whatever the credentials: + `ssh.service` and `systemd-journald.service` (not DynamicUser, zero + LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin + unit (one). The display form carries no information, so the conjunct was + vacuous whenever credentials existed and always false when none did. Odin + and StreamPixels passed only because each carries a regular credential. +- I3. The exact value is readable: D-Bus + `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) + answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 + "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. +- I4. What the workload can actually read is the delivered directory. On the + host, `/run/credentials/` holds a directory only for units with credentials + (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID + root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` + (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` + descriptor, never a credential. +- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn + deployment transaction authority"), with its twin clause in + `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit + touched it. No ruling or finding in any campaign concerns workload isolation + verification; gamecult-chat's foreign-workloads cut touched + `CREDENTIALS_DIRECTORY` reservation, not this check. +- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both + MessagePack encodings, compact (positional) included + (`persisted_systemd_observations_decode_as_the_systemd_variant`, + `:9915-9930`), so deleting the `load_credential` field shifts every later + slot of a live incumbent's record. The field is also in the drift list + (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed + workload: changing what the field holds without leaving that list would make + the next Idunn refuse to stop the live Odin and StreamPixels incumbents. +- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 + properties for a Vili unit (presence identity only) and the Odin unit + (presence identity plus one service credential), captured 2026-10-24 into + this Imagination's scratchpad `fixtures/`. +- I8. The new rule holds for every live systemd incumbent: inside each + MainPID root the delivered entries equal its binding's `secret_files` keys + minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, + StreamPixels web 1; key names compared, no values read). An Idunn restart onto + this cut re-observes them (`:3955-3970`) and passes. + +### Decision + +A property systemd cannot display does not belong in the comparison. The +invariant it reached for, that the workload holds exactly the credentials Idunn +delivered, is observed where it is decided: the entries of +`/run/credentials/` in the MainPID's root equal Idunn's service +credential names, absent directory equal to none. That covers +`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and +needs no new program. The D-Bus read (I3) is exact but sees only one of those +properties and adds a second systemd client and parser; rejected. + +### Authority + +Owner: `observe_unit` decides the credential rule, from the delivered +directory. `LoadCredential`'s display form is no longer read or compared; the +`load_credential` field is a retired wire slot, written empty, out of the drift +list, kept only so persisted records decode. Forbidden writers: the +`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping +the slot waits on the next schema revision of the observation. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). After it, Vili's Hands reruns from step 4 of +`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. From def0b7837892c31388856b336556689ad0102e95 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 14:53:15 +0200 Subject: [PATCH 087/122] Fixtures: systemd 257 show captures and pre-cut observation records Real systemctl show output (Vili presence-only, Odin with one service credential) and the two MessagePack encodings of a pre-cut observation, as inputs for the credential-evidence cut. Nothing reads them yet. Co-Authored-By: Claude Sonnet 5.5 --- tests/fixtures/README.md | 16 +++++++++++ .../pre-cut-systemd-observation.compact.hex | 1 + .../pre-cut-systemd-observation.named.hex | 1 + ...n-presence-and-one-service-credential.show | 28 +++++++++++++++++++ .../vili-presence-identity-only.show | 28 +++++++++++++++++++ 5 files changed, 74 insertions(+) create mode 100644 tests/fixtures/pre-cut-systemd-observation.compact.hex create mode 100644 tests/fixtures/pre-cut-systemd-observation.named.hex create mode 100644 tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show create mode 100644 tests/fixtures/systemd-257/vili-presence-identity-only.show diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 70060bf..b9f2637 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -10,3 +10,19 @@ It is checked in because the TypeScript runtime emits a camelCase MessagePack map while Rust `DatabaseEntry` derives tuple payloads. Idunn must prove it can decode the producer's real wire shape rather than round-tripping a fixture it serialized itself. + +# systemd 257 `systemctl show` captures + +`systemd-257/*.show` are the verbatim output of systemd 257.13 for the 27 +properties `show_unit` requested at `112a7a5`, captured read-only on Yggdrasil +on 2026-10-24: `vili-presence-identity-only.show` (a failed idunn-vili unit whose +only credential is the presence identity) and +`odin-presence-and-one-service-credential.show` (the live Odin unit). systemd 257 +prints `LoadCredential=[unprintable]` in both. Tests read these bytes; do not edit. + +# Pre-cut systemd observation records + +`pre-cut-systemd-observation.{compact,named}.hex` are the two MessagePack +encodings of a `SystemdWorkloadObservation` with `load_credential` set to +`[unprintable]`, produced by the code at `112a7a5` (the form live incumbents +persisted). They pin that later Idunns still decode such records. diff --git a/tests/fixtures/pre-cut-systemd-observation.compact.hex b/tests/fixtures/pre-cut-systemd-observation.compact.hex new file mode 100644 index 0000000..e7d09fb --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.compact.hex @@ -0,0 +1 @@ +dc0033d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365aa4964756e6e2074657374a3696e7601a465786563a26e6fa56d69786564c3a6753631303030a6753631303030a0a0a0c3c3a9696e76697369626c65a3616c6cc3a430303037a0ad5b756e7072696e7461626c655dcd10920194cdee48cdee48cdee48cdee4894cdee48cdee48cdee48cdee4891cdee480000000000c391010708b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d3232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323290d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190 \ No newline at end of file diff --git a/tests/fixtures/pre-cut-systemd-observation.named.hex b/tests/fixtures/pre-cut-systemd-observation.named.hex new file mode 100644 index 0000000..8898adf --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.named.hex @@ -0,0 +1 @@ +de0033a4756e6974d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365b0756e69745f6465736372697074696f6eaa4964756e6e2074657374ad696e766f636174696f6e5f6964a3696e76d923657865635f6d61696e5f73746172745f74696d657374616d705f6d6f6e6f746f6e696301ac736572766963655f74797065a465786563ae726573746172745f706f6c696379a26e6fa96b696c6c5f6d6f6465a56d69786564ac64796e616d69635f75736572c3ac73797374656d645f75736572a6753631303030ad73797374656d645f67726f7570a6753631303030b4737570706c656d656e746172795f67726f757073a0b76361706162696c6974795f626f756e64696e675f736574a0b4616d6269656e745f6361706162696c6974696573a0ae707269766174655f6d6f756e7473c3ac707269766174655f70696473c3ac70726f746563745f70726f63a9696e76697369626c65ab70726f635f737562736574a3616c6cb16e6f5f6e65775f70726976696c65676573c3a5756d61736ba430303037b2696e61636365737369626c655f7061746873a0af6c6f61645f63726564656e7469616cad5b756e7072696e7461626c655da86d61696e5f706964cd1092b270726f636573735f73746172745f74696d6501ac70726f636573735f7569647394cdee48cdee48cdee48cdee48ac70726f636573735f6769647394cdee48cdee48cdee48cdee48ae70726f636573735f67726f75707391cdee48b770726f636573735f6361705f696e686572697461626c6500b570726f636573735f6361705f7065726d697474656400b570726f636573735f6361705f65666665637469766500b470726f636573735f6361705f626f756e64696e6700b370726f636573735f6361705f616d6269656e7400b970726f636573735f6e6f5f6e65775f70726976696c65676573c3b670726f636573735f6e616d6573706163655f706964739101b26d6f756e745f6e616d6573706163655f696407b07069645f6e616d6573706163655f696408aa65786563757461626c65b52f6f70742f746573742f62696e2f73657276696365b165786563757461626c655f64657669636501b065786563757461626c655f696e6f646501b165786563757461626c655f736861323536d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131b372756e74696d655f696e7374616e63655f6964d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161b1776f726b696e675f6469726563746f7279a92f6f70742f74657374ae72756e74696d655f62756e646c65a92f72756e2f74657374b3636f6d6d616e645f6c696e655f736861323536d9477368613235362d32323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232b1656e7669726f6e6d656e745f6e616d657390bb656e7669726f6e6d656e745f636f6e74726163745f736861323536d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333ad636f6e74726f6c5f67726f7570d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365b563726564656e7469616c735f6469726563746f7279c0bc706172656e745f6f6e6c795f66696c655f64657363726970746f727390bd61637469766174696f6e5f7369676e65725f6964656e746974795f6964aa61637469766174696f6ebc61637469766174696f6e5f7369676e65725f7075626c69635f6b6579dc00200101010101010101010101010101010101010101010101010101010101010101b3736572766963655f63726564656e7469616c7390 \ No newline at end of file diff --git a/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show new file mode 100644 index 0000000..21e4573 --- /dev/null +++ b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=2571 +OpenFile=/run/idunn/activation-credentials/sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/odin/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=20630000 +ControlGroup=/system.slice/idunn-odin-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.service +UMask=0007 +WorkingDirectory=/srv/odin/releases/sha256-e5c7d0b2b9c0a8ffba368aeb08abdbe37ae0d17484a785e60a66ec4a62b2f18c +CapabilityBoundingSet= +AmbientCapabilities= +User=_duc23369f1b8b167db +Group=odin +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn odin sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8 +LoadState=loaded +ActiveState=active +SubState=running +InvocationID=ada48921043449f39dacc87b14a1bf70 diff --git a/tests/fixtures/systemd-257/vili-presence-identity-only.show b/tests/fixtures/systemd-257/vili-presence-identity-only.show new file mode 100644 index 0000000..fc78fd1 --- /dev/null +++ b/tests/fixtures/systemd-257/vili-presence-identity-only.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=0 +OpenFile=/run/idunn/activation-credentials/sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/vili/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=520083420000 +ControlGroup= +UMask=0007 +WorkingDirectory=/srv/vili-idunn/releases/sha256-aeaee4e1669742572c45a3f26bc86893c44b2a4149520975eb7d8a28aa931906 +CapabilityBoundingSet= +AmbientCapabilities= +User=_du8c06c563aac6d205 +Group=vili-state +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn vili sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041 +LoadState=loaded +ActiveState=failed +SubState=failed +InvocationID=8f7a349f2f8f49fe87ac5d89239260b4 From ae6a1bea557105633b53a1bf113f0e1f01522d5c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 14:54:15 +0200 Subject: [PATCH 088/122] Idunn: judge workload credentials from the delivered directory, not LoadCredential systemd 257 displays LoadCredential as [unprintable] for every unit, so the conjunct was always false for a unit with only the presence identity and vacuous for one with credentials. Delete the show argument, the conjunct and its twin clause; the isolation conjunction becomes require_systemd_isolation_contract. The credential rule is now require_delivered_credentials: entry names of /run/credentials/ in the MainPID's root equal Idunn's launch names. load_credential stays a retired wire slot (written empty, out of the drift list) so persisted records decode. Not yet built or verified. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 298 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 253 insertions(+), 45 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 0eb0928..43af330 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -470,6 +470,9 @@ pub struct SystemdWorkloadObservation { pub no_new_privileges: bool, pub umask: String, pub inaccessible_paths: String, + /// Retired wire slot. systemd 257 displays `LoadCredential` as `[unprintable]` for + /// every unit, so it was never evidence: written empty, never compared. Kept so + /// compact (positional) MessagePack records of earlier Idunns still decode. pub load_credential: String, pub main_pid: u32, pub process_start_time: u64, @@ -2459,7 +2462,6 @@ fn changed_systemd_workload_identity_fields( no_new_privileges, umask, inaccessible_paths, - load_credential, main_pid, process_start_time, process_uids, @@ -2859,7 +2861,6 @@ impl SystemdTransientWorkloadDriver { OsString::from("--property=NoNewPrivileges"), OsString::from("--property=UMask"), OsString::from("--property=InaccessiblePaths"), - OsString::from("--property=LoadCredential"), OsString::from("--property=OpenFile"), OsString::from("--property=WorkingDirectory"), OsString::from("--property=ControlGroup"), @@ -2971,38 +2972,7 @@ impl SystemdTransientWorkloadDriver { let umask = required_systemd_property(&values, "UMask")?.to_owned(); let inaccessible_paths = required_systemd_property(&values, "InaccessiblePaths")?.to_owned(); - let load_credential = required_systemd_property(&values, "LoadCredential")?.to_owned(); - ensure!( - service_type == "exec" - && restart_policy == "no" - && kill_mode == "mixed" - // `User` is deliberately not asserted here. Idunn never emits a - // User property, and systemd reports the identity it *allocated* - // for a DynamicUser unit -- the unit name, or a generated `_du…` - // when that is too long -- so the property is never empty for a - // running workload and cannot tell an allocated identity from a - // pinned one. The invariant it was reaching for, that the workload - // does not run as a persistent named account, is carried by - // `dynamic_user` below and by Idunn never passing User at all. - && dynamic_user - && supplementary_groups.is_empty() - && capability_bounding_set.is_empty() - && ambient_capabilities.is_empty() - && private_mounts - && private_pids - && protect_proc == "invisible" - && proc_subset == "all" - && no_new_privileges - && umask == "0007" - && inaccessible_paths == self.credential_root.display().to_string() - && load_credential - == if service_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }, - "systemd workload isolation properties differ from the Idunn contract" - ); + require_systemd_isolation_contract(&values, &self.credential_root)?; let working_directory = PathBuf::from(required_systemd_property(&values, "WorkingDirectory")?); ensure!( @@ -3144,6 +3114,11 @@ impl SystemdTransientWorkloadDriver { ); Some(path) }; + require_delivered_credentials( + &process_root.join("root"), + unit, + service_credential_names, + )?; let activation_descriptor = &parent_only_file_descriptors[0]; ensure!( activation_descriptor.fd_name == IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME @@ -3251,7 +3226,7 @@ impl SystemdTransientWorkloadDriver { no_new_privileges, umask, inaccessible_paths, - load_credential, + load_credential: String::new(), main_pid, process_start_time, process_uids: process_security.uids, @@ -3465,12 +3440,7 @@ impl SystemdTransientWorkloadDriver { // workload. The same applies to `Group` when no fixed state group is // bound. In that case the numeric primary GID must match the allocated // UID; a fixed state group still requires both its exact name and GID. - let expected_load_credential = if regular_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }; - let launch_contract: [(bool, &str); 27] = [ + let launch_contract: [(bool, &str); 26] = [ ( observation.unit_description == expected_description, "unit description", @@ -3510,10 +3480,6 @@ impl SystemdTransientWorkloadDriver { observation.inaccessible_paths == self.credential_root.display().to_string(), "inaccessible paths", ), - ( - observation.load_credential == expected_load_credential, - "load credential", - ), ( observation.working_directory == installed, "working directory", @@ -7269,6 +7235,44 @@ fn path_inside_process_root(process_root: &Path, absolute_path: &Path) -> Result Ok(process_root.join(absolute_path.strip_prefix(Path::new("/"))?)) } +/// The workload holds exactly the credentials Idunn delivered: the entries of +/// `/run/credentials/` in the MainPID's root (`process_root` is that root) +/// equal `expected`, and an absent directory is the empty set. Only entry names +/// are read; a credential is never opened here. The message names no credential. +fn require_delivered_credentials( + process_root: &Path, + unit: &str, + expected: &[String], +) -> Result<()> { + let directory = path_inside_process_root( + process_root, + &Path::new("/run/credentials").join(unit), + )?; + let mut delivered = Vec::new(); + match fs::read_dir(&directory) { + Ok(entries) => { + for entry in entries { + let name = entry + .context("reading a delivered credential entry")? + .file_name() + .into_string() + .map_err(|_| anyhow!("delivered credential name is not UTF-8"))?; + delivered.push(name); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).context("reading the delivered credential directory"); + } + } + delivered.sort(); + ensure!( + delivered == expected, + "systemd delivered credentials that differ from the Idunn launch" + ); + Ok(()) +} + fn linux_process_security( status_path: &Path, expected_host_pid: u32, @@ -7484,6 +7488,56 @@ fn parse_systemd_boolean(values: &BTreeMap, name: &str) -> Resul } } +/// The isolation properties of a transient workload unit, judged from `systemctl +/// show`. `LoadCredential` is absent on purpose: systemd 257 displays it as +/// `[unprintable]` whatever the unit holds, so what the workload was given is +/// judged by `require_delivered_credentials` from the delivered directory. +fn require_systemd_isolation_contract( + values: &BTreeMap, + credential_root: &Path, +) -> Result<()> { + let service_type = required_systemd_property(values, "Type")?; + let restart_policy = required_systemd_property(values, "Restart")?; + let kill_mode = required_systemd_property(values, "KillMode")?; + let dynamic_user = parse_systemd_boolean(values, "DynamicUser")?; + let supplementary_groups = systemd_property(values, "SupplementaryGroups")?; + let capability_bounding_set = systemd_property(values, "CapabilityBoundingSet")?; + let ambient_capabilities = systemd_property(values, "AmbientCapabilities")?; + let private_mounts = parse_systemd_boolean(values, "PrivateMounts")?; + let private_pids = parse_systemd_boolean(values, "PrivatePIDs")?; + let protect_proc = required_systemd_property(values, "ProtectProc")?; + let proc_subset = required_systemd_property(values, "ProcSubset")?; + let no_new_privileges = parse_systemd_boolean(values, "NoNewPrivileges")?; + let umask = required_systemd_property(values, "UMask")?; + let inaccessible_paths = required_systemd_property(values, "InaccessiblePaths")?; + ensure!( + service_type == "exec" + && restart_policy == "no" + && kill_mode == "mixed" + // `User` is deliberately not asserted here. Idunn never emits a + // User property, and systemd reports the identity it *allocated* + // for a DynamicUser unit -- the unit name, or a generated `_du…` + // when that is too long -- so the property is never empty for a + // running workload and cannot tell an allocated identity from a + // pinned one. The invariant it was reaching for, that the workload + // does not run as a persistent named account, is carried by + // `dynamic_user` below and by Idunn never passing User at all. + && dynamic_user + && supplementary_groups.is_empty() + && capability_bounding_set.is_empty() + && ambient_capabilities.is_empty() + && private_mounts + && private_pids + && protect_proc == "invisible" + && proc_subset == "all" + && no_new_privileges + && umask == "0007" + && inaccessible_paths == credential_root.display().to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + Ok(()) +} + fn read_process_environment(path: &Path) -> Result>> { let bytes = fs::read(path) .with_context(|| format!("reading process environment {}", path.display()))?; @@ -9968,6 +10022,160 @@ processes 9 ); } + #[cfg(unix)] + fn fixture_show_properties(text: &str) -> Result> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let shown = temp.path().join("shown"); + std::fs::write(&shown, text)?; + let systemctl = temp.path().join("systemctl"); + std::fs::write( + &systemctl, + format!("#!/bin/sh\ncat '{}'\nexit 0\n", shown.display()), + )?; + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; + let driver = SystemdTransientWorkloadDriver { + systemctl_program: systemctl, + ..Default::default() + }; + Ok(driver + .show_unit("idunn-test.service")? + .context("fixture unit was not found")? + .properties) + } + + /// systemd 257 prints `LoadCredential=[unprintable]` for every unit, so the + /// isolation contract must admit the real output of a unit with no service + /// credential as well as one with one. The inputs are real captures, not a + /// re-spelling. + #[cfg(unix)] + #[test] + fn systemd_257_isolation_contract_admits_a_presence_only_target_and_one_with_service_credentials( + ) -> Result<()> { + let root = Path::new("/run/idunn/activation-credentials"); + let vili = include_str!("../tests/fixtures/systemd-257/vili-presence-identity-only.show"); + let odin = include_str!( + "../tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show" + ); + for text in [vili, odin] { + assert!(text.contains("LoadCredential=[unprintable]")); + require_systemd_isolation_contract(&fixture_show_properties(text)?, root)?; + } + let default_proc = vili.replace("ProtectProc=invisible", "ProtectProc=default"); + assert_ne!(default_proc, vili); + let other_path = vili.replace( + "/run/idunn/activation-credentials", + "/run/idunn/elsewhere", + ); + assert_ne!(other_path, vili); + for text in [default_proc, other_path] { + let error = + require_systemd_isolation_contract(&fixture_show_properties(&text)?, root) + .expect_err("a drifted isolation property must be refused"); + assert_eq!( + error.to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn delivered_credentials_are_exactly_the_launch_names() -> Result<()> { + let unit = "idunn-test.service"; + let names = |list: &[&str]| list.iter().map(|name| name.to_string()).collect::>(); + // `Some(entries)` creates the unit's directory in a fresh process root. + let check = |delivered: Option<&[&str]>, expected: &[&str]| -> Result<()> { + let temp = tempfile::tempdir()?; + if let Some(entries) = delivered { + let directory = temp.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + for entry in entries { + std::fs::write(directory.join(entry), "canary-credential-contents")?; + } + } + let result = require_delivered_credentials(temp.path(), unit, &names(expected)); + if let Err(error) = &result { + let text = format!("{error:#}"); + for canary in ["STOWAWAY", "ODIN_TOPOLOGY_IDENTITY", "canary-credential"] { + assert!(!text.contains(canary), "error echoed {canary}"); + } + assert!(!text.contains(&*temp.path().to_string_lossy())); + } + result + }; + check(None, &[])?; + check(Some(&[]), &[])?; + check(Some(&["ODIN_TOPOLOGY_IDENTITY"]), &["ODIN_TOPOLOGY_IDENTITY"])?; + check(Some(&["B", "A"]), &["A", "B"])?; + for (delivered, expected) in [ + (None, &["ODIN_TOPOLOGY_IDENTITY"][..]), + (Some(&[][..]), &["ODIN_TOPOLOGY_IDENTITY"][..]), + ( + Some(&["ODIN_TOPOLOGY_IDENTITY", "STOWAWAY"][..]), + &["ODIN_TOPOLOGY_IDENTITY"][..], + ), + (Some(&["STOWAWAY"][..]), &[][..]), + (Some(&["A"][..]), &["B"][..]), + (Some(&["A"][..]), &["A", "B"][..]), + ] { + let error = check(delivered, expected) + .expect_err("delivered credentials differing from the launch must be refused"); + assert_eq!( + error.to_string(), + "systemd delivered credentials that differ from the Idunn launch" + ); + } + // The rule reads the process root it is given, never the host's /run. + let temp = tempfile::tempdir()?; + let outside = tempfile::tempdir()?; + let directory = outside.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + std::fs::write(directory.join("A"), "")?; + require_delivered_credentials(temp.path(), unit, &[])?; + assert!(require_delivered_credentials(temp.path(), unit, &names(&["A"])).is_err()); + Ok(()) + } + + /// Live Odin and StreamPixels hold records persisted by the Idunn that + /// compared `LoadCredential`: `[unprintable]` in the retired slot. The bytes + /// are that Idunn's own output (tests/fixtures/pre-cut-*); they must decode + /// under both encodings and read as no drift against a new observation, or + /// `stop` refuses to stop the incumbents. + #[test] + fn an_incumbent_recorded_with_the_display_form_is_not_identity_drift() -> Result<()> { + let decode_hex = |text: &str| -> Vec { + (0..text.trim().len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text.trim()[at..at + 2], 16).unwrap()) + .collect() + }; + let mut recorded = systemd_audit_observation("no", 61_000); + recorded.load_credential = "[unprintable]".into(); + let observed = systemd_audit_observation("no", 61_000); + assert!(observed.load_credential.is_empty()); + for hex in [ + include_str!("../tests/fixtures/pre-cut-systemd-observation.compact.hex"), + include_str!("../tests/fixtures/pre-cut-systemd-observation.named.hex"), + ] { + let decoded: WorkloadObservation = rmp_serde::from_slice(&decode_hex(hex))?; + assert_eq!(decoded, WorkloadObservation::Systemd(recorded.clone())); + let WorkloadObservation::Systemd(prior) = decoded else { + unreachable!() + }; + assert!(changed_systemd_workload_identity_fields(&observed, &prior).is_empty()); + } + let mut drifted = observed.clone(); + drifted.main_pid += 1; + assert_eq!( + changed_systemd_workload_identity_fields(&drifted, &recorded), + vec!["main_pid"] + ); + Ok(()) + } + #[test] fn host_isolation_is_two_live_distinct_processes_on_one_host() { let live = |pid: u32, created: u64| { From dfdc226c2544150ce7b4975e1137a875c329034a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sat, 24 Oct 2026 15:14:15 +0200 Subject: [PATCH 089/122] Idunn credential rule: an unlistable credential directory is a refusal cargo-mutants showed that treating every read_dir error as the empty set survived. Test the non-NotFound case (the unit path is a file). Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 43af330..7f87896 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -10128,6 +10128,14 @@ processes 9 "systemd delivered credentials that differ from the Idunn launch" ); } + // A directory that cannot be listed is a refusal, never the empty set. + let temp = tempfile::tempdir()?; + let parent = temp.path().join("run/credentials"); + std::fs::create_dir_all(&parent)?; + std::fs::write(parent.join(unit), "")?; + let error = require_delivered_credentials(temp.path(), unit, &[]) + .expect_err("an unreadable credential directory must be refused"); + assert!(!format!("{error:#}").contains(&*temp.path().to_string_lossy())); // The rule reads the process root it is given, never the host's /run. let temp = tempfile::tempdir()?; let outside = tempfile::tempdir()?; From 78086705a05742ca94cf5308aea2f268720dd0da Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:04:37 +0100 Subject: [PATCH 090/122] deploy-provenance map: runner failure evidence (body facts R1-R8) Imagination for cut runner-failure-evidence: the docker runner discards stdout, so a step that reports on stdout fails with an empty message; the tail moves to a root-only typed evidence store read by idunn status. Co-Authored-By: Claude Opus 5.5 --- docs/deploy-provenance-map.md | 108 ++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md index 5ec7052..95c0eee 100644 --- a/docs/deploy-provenance-map.md +++ b/docs/deploy-provenance-map.md @@ -284,3 +284,111 @@ Idunn's body changes only through gamecult-ops `scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and bindings unchanged). After it, Vili's Hands reruns from step 4 of `motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. + +## Cut `runner-failure-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at +`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area +Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, +transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 +UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and +nobody could tell which recipe step failed. + +### Body facts (source read at `804ed75`; host read-only, 2026-10-25) + +- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by + `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and + `Command::output()`: stdout and stderr are each buffered whole in the + daemon's memory, unbounded. On a non-zero exit the error is + `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The + container is gone the moment it exits, so `docker ps -a` and `docker logs` + hold nothing. +- R2. The empty message: the failing process wrote nothing to stderr, and + stdout, the only stream that carried its report, is discarded at + `:2182-2193`. The same interpolation echoes whatever a step does write to + stderr into `last_error`, `pre_fencing_abort.error` and + `completion.error` (`bounded_detail`, 2048 bytes), and from there into + `idunn up`'s `deployment failed:` line and `idunn status`. +- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops + over `declaration.steps` and calls `run_in_workspace` with argv only; + `step.id` is used only in the environment precondition. External inputs + (`curl` in the runner, `:2247-2258`) fail through the same helper with the + same anonymous message. +- R4. Which step failed on 2026-10-25, inferred from the failed transaction's + staging root, which survives (926 MiB, + `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, + `node_modules/.package-lock.json` was written at 00:51:24 (so + `install-vili-dependencies`, `npm ci`, finished), nothing outside + `node_modules` is newer, and `release/vili` (the next step's output) is + absent; the python workspace holds only fetched inputs. The failing step was + `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test + reporter writes to stdout: exactly the stream R1 discards. Inference from + residue, not from output. +- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) + sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which + persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal + on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only + provider-exclusion lines. +- R6. Stores. `control.cc` (root 0640) holds commands, transactions + (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and + expiry requests; `ControlSnapshot::read` refuses any other type + (`:3420`). Records must re-encode byte-exactly (`decode_record`, + `:3717-3727`), so a new transaction field is a schema bump and a v4 lift + (`:4100-4110`). Terminal transactions move verbatim to `history.cc` + (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. + `incidents.cc` lives in the published projection directory and by its own + doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, + because a type added there stops continuity for every target on a rollback. +- R7. The daemon runs as root (`User=root`). Runner secrets reach the + container as read-only file mounts named by env (`:2041-2047`), so any text + a step derives from them can land in its output. +- R8. Staging residue: 475 transaction roots, 55 GiB under + `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces + because `materialize` returns before `remove_tree_inside`. + +### Decision + +The failure becomes a typed value at the runner, `RunnerStepFailure`: the +subject (recipe step or external input, by id), the runner id, the exit code or +signal, and the last 8 KiB of stdout and of stderr with each stream's total +byte count. Its `Display` carries the subject and status, never output. +Capture streams both pipes through bounded tails, so a verbose build no longer +buffers whole in the daemon. + +The tail lives in a new private store, `runner-evidence.cc`, beside +`control.cc` (mode 0600), one record per transaction +(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the +transaction record: that is a v5 bump whose rollback makes every older Idunn +refuse its control store (R6), the scar `incident.rs` already names. Not in +`incidents.cc`: published, and free text is barred there. Not in the journal: +journal readers are a wider set than root, so the journal line names the +transaction, the subject, the status and the store, and no output. + +No redaction. Matching secret bytes would make the daemon read every secret, +and still misses any encoding, prefix or transform of one; a claimed redaction +that leaks is worse than a stated rule. The rule instead is the reader set: the +store's only readers (root) can already read every secret the runner was +given. + +Keeping the failed container (dropping `--rm`) is worse: its output then lives +in Docker's json-file log, unbounded, untyped, outside Idunn's state and +readable by the docker group; containers accumulate with no owner to reap them; +`docker system prune` erases the evidence; and nothing ties a container to the +transaction in `idunn status`. + +### Authority + +Owner: the runner decides what failed and keeps the bounded tail; the engine's +pre-fence failure path is the one writer of `runner-evidence.cc`. +`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the +failure's `Display`. Forbidden writers: the `stderr` interpolation in +`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the +store needs no external owner; staging roots (R8) stay with +`ghostlight-verse:follow_up:idunn-generation-retention`. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). The next failed Sealing then shows its step and tail on +`sudo idunn status --command `. From ef82355bab8926362bde39e544afd3230a393488 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:11:14 +0100 Subject: [PATCH 091/122] Runner failure: a typed, bounded step failure kept in runner-evidence.cc DockerRunnerDriver::docker no longer buffers whole streams or interpolates stderr into the error. It pipes both streams through an 8 KiB tail and returns RunnerStepFailure (subject, runner, status, tails); the engine's pre-fence path is the one writer of runner-evidence.cc (0600, newest 32) and journals a line that names the store and no output. idunn status --command renders the record with control bytes escaped. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 71 ++++++++++- src/drivers.rs | 167 ++++++++++++++++++++++-- src/lib.rs | 1 + src/runner_evidence.rs | 283 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 508 insertions(+), 14 deletions(-) create mode 100644 src/runner_evidence.rs diff --git a/src/control_plane.rs b/src/control_plane.rs index 9597d43..b045e0a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -42,11 +42,12 @@ use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, RunnerStepFailure, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; +use crate::runner_evidence::{self, RunnerFailureRecord, runner_evidence_store_path}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -4877,6 +4878,24 @@ fn status( } if let Some(abort) = &transaction.pre_fencing_abort { writeln!(out, " pre-fencing abort {abort:?}")?; + // The step that failed and its output tail live in their + // own private store. An unreadable store costs its lines. + match runner_evidence::for_transaction( + &runner_evidence_store_path(store_path), + &transaction.transaction_id, + ) { + Ok(Some(record)) => { + for line in runner_evidence::render(&record) { + writeln!(out, "{line}")?; + } + } + Ok(None) => {} + Err(error) => writeln!( + out, + " runner evidence unreadable: {}", + error.root_cause() + )?, + } } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -5374,6 +5393,33 @@ impl Engine { IncidentStore::new(&self.options.incident_store) } + /// A failed runner step keeps its output tail in the runner evidence + /// store and says so on the journal, naming no output. The store write is + /// best effort: its failure is named in the line and never stops the + /// abort that follows. + fn keep_runner_failure(&self, transaction: &DeploymentTransaction, error: &anyhow::Error) { + let Some(failure) = error.downcast_ref::() else { + return; + }; + let path = runner_evidence_store_path(&self.options.state_store); + let recorded = now_millis().and_then(|now| { + runner_evidence::record( + &path, + &RunnerFailureRecord::from_failure( + &transaction.transaction_id, + &transaction.command_id, + &transaction.target, + failure, + now, + ), + ) + }); + eprintln!( + "{}", + runner_failure_journal_line(transaction, failure, &path, &recorded) + ); + } + /// Settle the outcome of one incident operation. A failure never leaves /// this function: it is traced once per episode, under the operation and /// the subject it was decided for (`*` for a pass over the whole store), @@ -6315,6 +6361,7 @@ impl Engine { if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() { + self.keep_runner_failure(&latest.value, &error); self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -10641,6 +10688,28 @@ const DETAIL_EMPTY: &str = "(no detail)"; /// only judge, and this output always satisfies it. NUL is dropped, empty /// input gets a fixed placeholder, and an over-long value keeps its head and /// its tail (where a build log's error is) cut on char boundaries. +/// The journal line for a failed runner step: the transaction, the subject and +/// status (the failure's `Display`), and where the evidence is. Never output. +fn runner_failure_journal_line( + transaction: &DeploymentTransaction, + failure: &RunnerStepFailure, + store_path: &Path, + recorded: &Result<()>, +) -> String { + let head = format!( + "Idunn transaction {} (command {}, target {}) {failure}", + transaction.transaction_id, transaction.command_id, transaction.target + ); + match recorded { + Ok(()) => format!( + "{head}; evidence in {}; read it with idunn status --command {}", + store_path.display(), + transaction.command_id + ), + Err(error) => format!("{head}; evidence was not kept: {error:#}"), + } +} + fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { diff --git a/src/drivers.rs b/src/drivers.rs index 7f87896..5c22429 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -2152,6 +2152,115 @@ pub(crate) fn docker_run_args( Ok(args) } +/// Bytes of each output stream that a failed runner step keeps. The daemon +/// never holds more than this per stream while the step runs. +pub(crate) const RUNNER_OUTPUT_TAIL_BYTES: usize = 8192; + +/// What a runner container was running: a recipe step or an external input, +/// by the id the recipe gave it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RunnerSubject { + Step(String), + ExternalInput(String), +} + +impl RunnerSubject { + pub(crate) fn id(&self) -> &str { + match self { + Self::Step(id) | Self::ExternalInput(id) => id, + } + } + + fn noun(&self) -> &'static str { + match self { + Self::Step(_) => "step", + Self::ExternalInput(_) => "external input", + } + } +} + +/// A runner container that did not exit successfully: what failed, how, and +/// the last `RUNNER_OUTPUT_TAIL_BYTES` of each stream with its total size. +/// The tails are evidence for `runner-evidence.cc`. `Display` and `Debug` +/// carry none of them, so no error chain, record or log line renders output. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RunnerStepFailure { + pub(crate) subject: RunnerSubject, + pub(crate) runner: String, + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stdout_total_bytes: u64, + pub(crate) stderr_tail: Vec, + pub(crate) stderr_total_bytes: u64, +} + +impl RunnerStepFailure { + /// The status as an operator reads it. + pub(crate) fn status(&self) -> String { + runner_status(self.exit_code, self.signal) + } +} + +/// How a container ended, as an operator reads it. +pub(crate) fn runner_status(exit_code: Option, signal: Option) -> String { + match (exit_code, signal) { + (Some(code), _) => format!("exit status {code}"), + (None, Some(signal)) => format!("signal {signal}"), + (None, None) => "an unknown status".to_owned(), + } +} + +impl std::fmt::Display for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "runner {} {} on runner {} exited with {}; its output tail is kept in runner-evidence.cc", + self.subject.noun(), + self.subject.id(), + self.runner, + self.status() + ) + } +} + +impl std::fmt::Debug for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerStepFailure") + .field("subject", &self.subject) + .field("runner", &self.runner) + .field("exit_code", &self.exit_code) + .field("signal", &self.signal) + .field("stdout_total_bytes", &self.stdout_total_bytes) + .field("stderr_total_bytes", &self.stderr_total_bytes) + .finish_non_exhaustive() + } +} + +impl std::error::Error for RunnerStepFailure {} + +/// Read `reader` to its end keeping only the last `cap` bytes and counting +/// every byte, so a verbose step never buffers whole in the daemon. +fn drain_tail(mut reader: impl Read, cap: usize) -> std::io::Result<(Vec, u64)> { + let mut tail = Vec::new(); + let mut total = 0u64; + let mut chunk = [0u8; 8192]; + loop { + let read = match reader.read(&mut chunk) { + Ok(0) => return Ok((tail, total)), + Ok(read) => read, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + total += read as u64; + tail.extend_from_slice(&chunk[..read]); + if tail.len() > cap { + tail.drain(..tail.len() - cap); + } + } +} + /// Docker is only a runner substrate. Recipe argv stays an argv vector, the /// operator binding selects the exact image/network/mount affordances, and the /// driver returns complete materialization receipts rather than build truth by @@ -2170,7 +2279,7 @@ impl Default for DockerRunnerDriver { } impl DockerRunnerDriver { - fn docker(&self, args: I) -> Result + fn docker(&self, subject: RunnerSubject, runner_id: &str, args: I) -> Result<()> where I: IntoIterator, S: AsRef, @@ -2179,23 +2288,50 @@ impl DockerRunnerDriver { self.docker_program.is_absolute(), "Docker runner program is not absolute" ); - let output = Command::new(&self.docker_program) + let mut child = Command::new(&self.docker_program) .args(args) .stdin(Stdio::null()) - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .with_context(|| format!("starting Docker runner {}", self.docker_program.display()))?; - if !output.status.success() { - bail!( - "Docker runner exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); + let stdout = child.stdout.take().context("Docker runner has no stdout pipe")?; + let stderr = child.stderr.take().context("Docker runner has no stderr pipe")?; + let stdout = thread::spawn(move || drain_tail(stdout, RUNNER_OUTPUT_TAIL_BYTES)); + let stderr = thread::spawn(move || drain_tail(stderr, RUNNER_OUTPUT_TAIL_BYTES)); + let status = child.wait().context("waiting for the Docker runner")?; + let (stdout_tail, stdout_total_bytes) = stdout + .join() + .map_err(|_| anyhow!("the runner stdout reader panicked"))? + .context("reading the runner stdout")?; + let (stderr_tail, stderr_total_bytes) = stderr + .join() + .map_err(|_| anyhow!("the runner stderr reader panicked"))? + .context("reading the runner stderr")?; + if status.success() { + return Ok(()); } - Ok(output) + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal = None; + Err(RunnerStepFailure { + subject, + runner: runner_id.to_owned(), + exit_code: status.code(), + signal, + stdout_tail, + stdout_total_bytes, + stderr_tail, + stderr_total_bytes, + } + .into()) } fn run_in_workspace( &self, + step_id: &str, + runner_id: &str, runner: &DockerRunnerBinding, workspace: &Path, working_directory: &Path, @@ -2212,8 +2348,7 @@ impl DockerRunnerDriver { build_machine_id_file(workspace)?; let spec = ContainerSpec::for_step(runner, required_environment, source_stamp)?; let args = docker_run_args(&spec, workspace, working_directory, argv)?; - self.docker(args)?; - Ok(()) + self.docker(RunnerSubject::Step(step_id.to_owned()), runner_id, args) } fn materialize_external_input( @@ -2255,7 +2390,11 @@ impl DockerRunnerDriver { input.url.clone(), ]; let args = docker_run_args(&spec, workspace, Path::new("."), &curl_argv)?; - self.docker(args)?; + self.docker( + RunnerSubject::ExternalInput(input.id.clone()), + &input.runner, + args, + )?; let bytes = fs::read(&destination).with_context(|| { format!( "reading materialized external input {}", @@ -2377,6 +2516,8 @@ impl RunnerPort for DockerRunnerDriver { ); } self.run_in_workspace( + &step.id, + &step.runner, runner, workspace, &step.working_directory, diff --git a/src/lib.rs b/src/lib.rs index e71a17f..4f365a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,4 +7,5 @@ pub mod host_actuator; #[cfg(unix)] pub mod launcher; mod incident; +mod runner_evidence; pub mod provisioning; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs new file mode 100644 index 0000000..6e305d4 --- /dev/null +++ b/src/runner_evidence.rs @@ -0,0 +1,283 @@ +//! The output tail of a failed runner step. +//! +//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the +//! last bytes of a failed runner container's stdout and stderr, one record per +//! transaction. The engine's pre-fence failure path is its only writer, and +//! `idunn status --command` its only reader. It is never `control.cc`, which +//! refuses any type it does not know, and never `incidents.cc`, which is +//! published and carries no free text. +//! +//! The tail is raw runner output. It is kept unredacted and readable by root +//! alone (mode 0600): root can already read every secret the runner was given. +//! It never enters an error, a transaction record or the journal; the render +//! below escapes control characters because the status reader is a terminal. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + TryCompareExchangeSnapshotOutcome, +}; +use serde::{Deserialize, Serialize}; + +use crate::control_plane::{decode_record, require_id, typed_envelope}; +use crate::drivers::{ + LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject, + exchange_behind_private_lock, runner_status, +}; + +const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1"; + +/// Records kept; the writer drops the oldest past this, so the store needs no +/// other owner. +pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32; + +/// `runner-evidence.cc`, beside the control store. +pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf { + state_store.with_file_name("runner-evidence.cc") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum SubjectKind { + Step, + ExternalInput, +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")] +pub(crate) struct RunnerFailureRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) transaction_id: String, + #[cultcache(key = 2)] + pub(crate) command_id: String, + #[cultcache(key = 3)] + pub(crate) target: String, + #[cultcache(key = 4)] + pub(crate) subject_kind: SubjectKind, + #[cultcache(key = 5)] + pub(crate) subject_id: String, + #[cultcache(key = 6)] + pub(crate) runner: String, + #[cultcache(key = 7)] + pub(crate) exit_code: Option, + #[cultcache(key = 8)] + pub(crate) signal: Option, + #[cultcache(key = 9)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 10)] + pub(crate) stdout_total_bytes: u64, + #[cultcache(key = 11)] + pub(crate) stderr_tail: Vec, + #[cultcache(key = 12)] + pub(crate) stderr_total_bytes: u64, + #[cultcache(key = 13)] + pub(crate) recorded_at_unix_millis: u64, +} + +/// Tails stay out of `Debug`, so a failed assertion or a logged record never +/// prints runner output. +impl std::fmt::Debug for RunnerFailureRecord { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerFailureRecord") + .field("transaction_id", &self.transaction_id) + .field("subject_id", &self.subject_id) + .finish_non_exhaustive() + } +} + +impl RunnerFailureRecord { + pub(crate) fn from_failure( + transaction_id: &str, + command_id: &str, + target: &str, + failure: &RunnerStepFailure, + now: u64, + ) -> Self { + let (subject_kind, subject_id) = match &failure.subject { + RunnerSubject::Step(id) => (SubjectKind::Step, id), + RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id), + }; + Self { + schema_version: RUNNER_FAILURE_SCHEMA.to_owned(), + transaction_id: transaction_id.to_owned(), + command_id: command_id.to_owned(), + target: target.to_owned(), + subject_kind, + subject_id: subject_id.clone(), + runner: failure.runner.clone(), + exit_code: failure.exit_code, + signal: failure.signal, + stdout_tail: failure.stdout_tail.clone(), + stdout_total_bytes: failure.stdout_total_bytes, + stderr_tail: failure.stderr_tail.clone(), + stderr_total_bytes: failure.stderr_total_bytes, + recorded_at_unix_millis: now, + } + } + + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == RUNNER_FAILURE_SCHEMA, + "runner failure schema is unsupported" + ); + require_id(&self.transaction_id, "runner failure transaction")?; + require_id(&self.command_id, "runner failure command")?; + require_id(&self.target, "runner failure target")?; + require_id(&self.subject_id, "runner failure subject")?; + require_id(&self.runner, "runner failure runner")?; + for (tail, total, stream) in [ + (&self.stdout_tail, self.stdout_total_bytes, "stdout"), + (&self.stderr_tail, self.stderr_total_bytes, "stderr"), + ] { + ensure!( + tail.len() <= RUNNER_OUTPUT_TAIL_BYTES, + "runner failure {stream} tail exceeds its bound" + ); + ensure!( + tail.len() as u64 <= total, + "runner failure {stream} tail exceeds its stream" + ); + } + Ok(()) + } + + fn envelope(&self) -> Result { + self.validate()?; + typed_envelope( + &self.transaction_id, + Self::TYPE, + RUNNER_FAILURE_SCHEMA, + self, + self.recorded_at_unix_millis, + ) + } +} + +/// One read of the store: the envelopes exactly as stored, and the records +/// decoded from them in the same order. +fn snapshot(path: &Path) -> Result<(Vec, Vec)> { + let envelopes = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading the runner evidence store")?; + let mut records = Vec::new(); + for envelope in &envelopes { + ensure!( + envelope.r#type == RunnerFailureRecord::TYPE + && envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA), + "the runner evidence store holds a document that is not a runner failure" + ); + let record: RunnerFailureRecord = decode_record(envelope)?; + record.validate()?; + ensure!( + envelope.key == record.transaction_id, + "runner evidence store key differs from the record's transaction" + ); + records.push(record); + } + Ok((envelopes, records)) +} + +/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the +/// oldest records go in the same exchange. The file ends at mode 0600. +pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result<()> { + let (envelopes, records) = snapshot(path)?; + let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes + .iter() + .zip(&records) + .filter(|(_, stored)| stored.transaction_id != record.transaction_id) + .map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone())) + .collect(); + kept.push((record.recorded_at_unix_millis, record.envelope()?)); + // Stable: records of one instant keep their stored order, so the record + // being written survives a tie. + kept.sort_by_key(|(recorded_at, _)| *recorded_at); + let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS); + let replacements: Vec<_> = kept + .into_iter() + .skip(excess) + .map(|(_, envelope)| envelope) + .collect(); + match exchange_behind_private_lock(path, &envelopes, &replacements)? { + TryCompareExchangeSnapshotOutcome::Exchanged => {} + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("the runner evidence store changed while it was being written") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + return Err(LockContended { + store: "runner evidence store", + } + .into()); + } + } + keep_private(path) +} + +#[cfg(unix)] +fn keep_private(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("making the runner evidence store private") +} + +#[cfg(not(unix))] +fn keep_private(_path: &Path) -> Result<()> { + Ok(()) +} + +/// The one reader. An absent store holds nothing. +pub(crate) fn for_transaction( + path: &Path, + transaction_id: &str, +) -> Result> { + Ok(snapshot(path)? + .1 + .into_iter() + .find(|stored| stored.transaction_id == transaction_id)) +} + +/// The record as `idunn status --command` prints it. The store keeps raw +/// bytes; every control character except newline and tab is escaped here, so +/// build output cannot drive the operator's terminal. +pub(crate) fn render(record: &RunnerFailureRecord) -> Vec { + let noun = match record.subject_kind { + SubjectKind::Step => "step", + SubjectKind::ExternalInput => "external input", + }; + let mut lines = vec![ + format!(" failed {noun} {}", escape(&record.subject_id)), + format!(" runner {}", escape(&record.runner)), + format!(" status {}", runner_status(record.exit_code, record.signal)), + ]; + for (stream, tail, total) in [ + ("stdout", &record.stdout_tail, record.stdout_total_bytes), + ("stderr", &record.stderr_tail, record.stderr_total_bytes), + ] { + lines.push(format!( + " {stream} (last {} of {total} bytes)", + tail.len() + )); + let text = escape(&String::from_utf8_lossy(tail)); + let text = text.strip_suffix('\n').unwrap_or(&text); + if !text.is_empty() { + lines.extend(text.split('\n').map(|line| format!(" | {line}"))); + } + } + lines +} + +fn escape(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + if character.is_control() && character != '\n' && character != '\t' { + escaped.push_str(&format!("\\u{{{:x}}}", u32::from(character))); + } else { + escaped.push(character); + } + } + escaped +} From 22bd904ed9d4ffdfacfdbe5d303971a40aa03b46 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:16:03 +0100 Subject: [PATCH 092/122] Runner failure tests: named step, bounded tail, never echoed, private store Drive the real DockerRunnerDriver over a fake docker program, through the engine's Sealing path: the second step is named with its exit status, both tails are kept in a 0600 store capped at 32, no error, control.cc or journal line carries the output, status escapes control bytes, an unwritable store does not stop the abort, and control.cc's record bytes equal those captured at base 804ed75. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 313 +++++++++++++++++++++++++++++++++++++++-- src/drivers.rs | 197 ++++++++++++++++++++++++++ src/runner_evidence.rs | 146 +++++++++++++++++++ 3 files changed, 647 insertions(+), 9 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index b045e0a..d927152 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5397,10 +5397,12 @@ impl Engine { /// store and says so on the journal, naming no output. The store write is /// best effort: its failure is named in the line and never stops the /// abort that follows. - fn keep_runner_failure(&self, transaction: &DeploymentTransaction, error: &anyhow::Error) { - let Some(failure) = error.downcast_ref::() else { - return; - }; + fn keep_runner_failure( + &self, + transaction: &DeploymentTransaction, + error: &anyhow::Error, + ) -> Option { + let failure = error.downcast_ref::()?; let path = runner_evidence_store_path(&self.options.state_store); let recorded = now_millis().and_then(|now| { runner_evidence::record( @@ -5414,10 +5416,7 @@ impl Engine { ), ) }); - eprintln!( - "{}", - runner_failure_journal_line(transaction, failure, &path, &recorded) - ); + Some(runner_failure_journal_line(transaction, failure, &path, &recorded)) } /// Settle the outcome of one incident operation. A failure never leaves @@ -6361,7 +6360,9 @@ impl Engine { if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() { - self.keep_runner_failure(&latest.value, &error); + if let Some(line) = self.keep_runner_failure(&latest.value, &error) { + eprintln!("{line}"); + } self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -24726,4 +24727,298 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } } + + // BEGIN runner-failure-evidence tests + // A failed runner step is named, its output tail is kept in + // runner-evidence.cc, and no output reaches an error, control.cc or the + // journal line. The runner is the real DockerRunnerDriver driving a fake + // docker program, through the engine's Sealing path. + + const RUNNER_CANARY: &str = "CANARY-7c41e0d9"; + + /// A source that freezes a fixed tree, so a deployment reaches the runner. + struct TreeSource { + recipe: String, + root: PathBuf, + } + + impl SourcePort for TreeSource { + fn resolve( + &self, + _: &OperatorBinding, + _: &str, + _: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result { + Ok(FrozenSourceReceipt { + transaction_id: transaction_id.to_owned(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: format!("sha256-{}", "a".repeat(64)), + }) + } + + fn observe_frozen( + &self, + plan: &CompiledDeploymentPlan, + receipt: &FrozenSourceReceipt, + ) -> Result { + Ok(crate::drivers::FrozenSource::fixture( + receipt.clone(), + plan.source.clone(), + plan.recipe_blob.clone(), + self.root.clone(), + )) + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// The unrouted recipe with two steps ahead of its build, and no external + /// input: the second step, `test-daemon`, is the one the fake docker fails. + fn two_step_recipe() -> String { + let recipe = unrouted_recipe(); + let (head, rest) = recipe + .split_once("[[external_inputs]]") + .expect("the recipe declares an external input"); + let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts"); + let head = head.replacen( + "[[steps]]\nid = \"build\"", + "[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\ + [[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\ + [[steps]]\nid = \"build\"", + 1, + ); + format!("{head}[[artifacts]]{tail}") + } + + /// A Sealing world whose runner is the real driver over a fake docker: the + /// first call succeeds, the second prints the canary and an escape + /// sequence on both streams and exits 3. + #[cfg(unix)] + fn failing_runner_world() -> Result<(EngineFixture, TempDir)> { + use std::os::unix::fs::PermissionsExt; + let tools = TempDir::new()?; + let docker = tools.path().join("docker"); + std::fs::write( + &docker, + format!( + "#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n" + ), + )?; + std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?; + let recipe = two_step_recipe(); + let mut world = sealing_world(&recipe)?; + let tree = tools.path().join("tree"); + std::fs::create_dir_all(&tree)?; + std::fs::write(tree.join("README"), b"source")?; + world.engine.source = Arc::new(TreeSource { recipe, root: tree }); + world.engine.docker_runner = DockerRunnerDriver { docker_program: docker }; + Ok((world, tools)) + } + + /// Scheduler ticks until the deployment ends (or the budget is spent). + fn tick_until_ended(world: &EngineFixture) -> Result { + for _ in 0..24 { + world.engine.resume_backoff.lock().unwrap().clear(); + let _ = world.engine.run_scheduler_tick(); + if latest(world)?.completion.is_some() { + break; + } + } + latest(world) + } + + fn printed_status(world: &EngineFixture) -> Result { + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?; + Ok(String::from_utf8(printed)?) + } + + fn canary_failure(stdout: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject: crate::drivers::RunnerSubject::Step("test-daemon".into()), + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stdout.to_vec(), + stderr_total_bytes: stdout.len() as u64, + } + } + + #[cfg(unix)] + #[test] + fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error); + }; + // Named: the second step, on its runner, with the exit status. + assert!( + error.contains("runner step test-daemon on runner rust exited with exit status 3"), + "{error}" + ); + // Never echoed: no error text, no control.cc byte, carries the output. + let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; + for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); + } + let control = std::fs::read(&world.state_store)?; + assert!( + !control.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), + "runner output reached control.cc" + ); + // Kept: in the private store, tails of both streams, and on status. + let evidence = runner_evidence_store_path(&world.state_store); + assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("no evidence for the failed transaction")?; + assert_eq!(record.subject_id, "test-daemon"); + assert_eq!(record.exit_code, Some(3)); + assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY)); + let printed = printed_status(&world)?; + assert!(printed.contains("failed step test-daemon"), "{printed}"); + assert!(printed.contains(RUNNER_CANARY), "{printed}"); + assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte"); + Ok(()) + } + + #[test] + fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> { + let world = EngineFixture::new()?; + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let line = world + .engine + .keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing")) + .context("a runner failure produced no journal line")?; + assert!(!line.contains(RUNNER_CANARY), "{line}"); + for needle in [ + transaction.transaction_id.as_str(), + "up-ghostlight", + "ghostlight", + "runner step test-daemon on runner rust exited with exit status 3", + "runner-evidence.cc", + "idunn status --command up-ghostlight", + ] { + assert!(line.contains(needle), "{needle} missing from {line}"); + } + // Not a runner failure: nothing is kept and nothing is said. + assert!( + world + .engine + .keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure")) + .is_none() + ); + // The line a failed write produces says so, with no output. + let unwritable = runner_failure_journal_line( + &transaction, + &failure, + Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"), + &Err(anyhow::anyhow!("disk full")), + ); + assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}"); + assert!(!unwritable.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + // A directory where the store should be: every write to it fails. + std::fs::create_dir(runner_evidence_store_path(&world.state_store))?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b""))) + .context("no journal line")?; + assert!(line.contains("evidence was not kept"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?; + let printed = printed_status(&world)?; + assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}"); + assert!(printed.contains("target service phase"), "{printed}"); + assert!(printed.contains("pre-fencing abort"), "{printed}"); + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + // The store a rolled-back Idunn reads: it refuses any foreign type. + ControlSnapshot::read(&world.state_store)?; + let envelopes = + SingleFileMessagePackBackingStore::new(&world.state_store).pull_all_read_only_snapshot()?; + assert!(!envelopes.is_empty()); + for envelope in &envelopes { + assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); + if envelope.r#type == DeploymentTransaction::TYPE { + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + } + } + Ok(()) + } + + /// The records control.cc holds re-encode byte for byte. The hex was + /// captured at base 804ed75, before this cut touched anything: a runner + /// failure adds no field, no type and no schema to the control store. + #[test] + fn control_records_encode_exactly_as_they_did_at_base() -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?; + transaction.transaction_id = "tx-pinned".into(); + transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into()); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "runner step test-daemon on runner rust exited with exit status 3".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); + let command_hex = hex(&command_envelope(&command, 100)?.payload); + let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload); + assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed"); + assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed"); + Ok(()) + } + + const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; + const PINNED_TRANSACTION_HEX: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; + // END runner-failure-evidence tests } diff --git a/src/drivers.rs b/src/drivers.rs index 5c22429..1c8d067 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -157,6 +157,24 @@ pub struct FrozenSourceReceipt { pub snapshot_sha256: String, } +#[cfg(test)] +impl FrozenSource { + /// A frozen source over an existing tree, for tests that drive a runner. + pub(crate) fn fixture( + receipt: FrozenSourceReceipt, + facts: SourceSelectionFacts, + recipe_bytes: Vec, + root: PathBuf, + ) -> Self { + Self { + receipt, + facts, + recipe_bytes, + root, + } + } +} + impl FrozenSourceReceipt { pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> { plan.validate()?; @@ -16372,4 +16390,183 @@ processes 9 ); Ok(()) } + + // BEGIN runner-failure-evidence tests + // The runner decides what failed and what it kept: these drive the real + // DockerRunnerDriver over a fake docker program, never the capture alone. + + const RUNNER_CANARY: &str = "CANARY-5d2e91b7"; + + #[cfg(unix)] + fn fake_docker(dir: &Path, body: &str) -> Result { + use std::os::unix::fs::PermissionsExt; + let program = dir.join("docker"); + fs::write(&program, format!("#!/bin/sh\n{body}\n"))?; + fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?; + Ok(DockerRunnerDriver { + docker_program: program, + }) + } + + /// One recipe step through the real `run_in_workspace` path. + #[cfg(unix)] + fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> { + let workspace = dir.join("workspace"); + fs::create_dir_all(&workspace)?; + driver.run_in_workspace( + step_id, + "rust", + &fixture_docker_runner_binding(), + &workspace, + Path::new("."), + &["cargo".to_owned(), "test".to_owned()], + &std::collections::BTreeSet::new(), + ("IDUNN_STAMP", "rev"), + ) + } + + #[cfg(unix)] + fn failure_of(result: Result<()>) -> Result { + let error = result.expect_err("the step did not fail"); + error + .downcast_ref::() + .cloned() + .with_context(|| format!("the failure is not typed: {error:#}")) + } + + #[cfg(unix)] + #[test] + fn a_failed_step_is_named_with_both_tails() -> Result<()> { + let temp = tempfile::tempdir()?; + // The first step passes, the second fails: the count is the fake's. + let driver = fake_docker( + temp.path(), + &format!( + "n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\ + echo 'stderr report' >&2\nexit 3" + ), + )?; + run_step(&driver, temp.path(), "first")?; + let failure = failure_of(run_step(&driver, temp.path(), "second"))?; + assert_eq!(failure.subject, RunnerSubject::Step("second".into())); + assert_eq!(failure.runner, "rust"); + assert_eq!(failure.exit_code, Some(3)); + assert_eq!(failure.signal, None); + assert!( + failure + .stdout_tail + .ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()), + "stdout was dropped: {:?}", + String::from_utf8_lossy(&failure.stdout_tail) + ); + assert_eq!(failure.stderr_tail, b"stderr report\n"); + assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "kill -9 $$")?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert_eq!((failure.exit_code, failure.signal), (None, Some(9))); + assert!(failure.to_string().contains("exited with signal 9"), "{failure}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_external_input_is_named_as_one() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?; + let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?; + let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?; + let input_id = declaration.external_inputs[0].id.clone(); + let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]); + fs::create_dir_all(&workspaces["rust"])?; + let error = driver + .materialize_external_input(&declaration, &binding, &workspaces, &input_id) + .expect_err("the download did not fail"); + let failure = error + .downcast_ref::() + .with_context(|| format!("the failure is not typed: {error:#}"))?; + assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone())); + assert_eq!(failure.exit_code, Some(22)); + assert!( + failure.to_string().contains(&format!("runner external input {input_id} on runner rust")), + "{failure}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn runner_output_never_reaches_the_error_text() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY)); + let error = anyhow::Error::new(failure.clone()).context("sealing"); + for text in [ + format!("{error:#}"), + format!("{error:?}"), + format!("{failure}"), + format!("{failure:?}"), + ] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { + let temp = tempfile::tempdir()?; + let data = temp.path().join("data"); + for size in [ + 0, + 1, + RUNNER_OUTPUT_TAIL_BYTES - 1, + RUNNER_OUTPUT_TAIL_BYTES, + RUNNER_OUTPUT_TAIL_BYTES + 1, + 1024 * 1024, + ] { + // Distinct neighbours, so a shifted or head-keeping tail differs. + let bytes: Vec = (0..size).map(|index| (index % 251) as u8).collect(); + fs::write(&data, &bytes)?; + let driver = fake_docker( + temp.path(), + &format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..]; + assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes"); + assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes"); + assert_eq!(failure.stdout_total_bytes, size as u64); + assert_eq!(failure.stderr_total_bytes, size as u64); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> { + let temp = tempfile::tempdir()?; + run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?; + let missing = DockerRunnerDriver { + docker_program: temp.path().join("no-such-docker"), + }; + let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran"); + assert!(error.downcast_ref::().is_none()); + assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); + Ok(()) + } + // END runner-failure-evidence tests } diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index 6e305d4..9eeab4c 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -281,3 +281,149 @@ fn escape(text: &str) -> String { } escaped } + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject, + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stderr.to_vec(), + stderr_total_bytes: stderr.len() as u64, + } + } + + fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { + RunnerFailureRecord::from_failure( + transaction, + "up-service", + "service", + &failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"), + now, + ) + } + + #[test] + fn the_evidence_store_keeps_the_newest_32() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + for index in 0..=RUNNER_EVIDENCE_RECORDS { + record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; + } + let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); + assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS); + assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); + assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); + assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); + // Rewriting a resident transaction replaces it and drops nothing. + let mut again = record_at("tx-05", 5000); + again.stdout_tail = b"rewritten\n".to_vec(); + again.stdout_total_bytes = 10; + record(&path, &again)?; + let kept = snapshot(&path)?.1; + assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS); + assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); + let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; + assert_eq!(stored.stdout_tail, b"rewritten\n"); + assert_eq!(for_transaction(&path, "tx-00")?, None); + Ok(()) + } + + #[test] + fn a_record_round_trips_byte_for_byte() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let written = RunnerFailureRecord::from_failure( + "tx-a", + "up-service", + "service", + &failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""), + 77, + ); + record(&path, &written)?; + assert_eq!(for_transaction(&path, "tx-a")?, Some(written)); + assert_eq!(for_transaction(&path, "tx-b")?, None); + Ok(()) + } + + #[test] + fn the_evidence_store_is_private() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // Whatever the umask, the mode is the store's own rule, on every write. + for index in 0..2 { + record(&path, &record_at(&format!("tx-{index}"), 10 + index))?; + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn a_tail_over_its_bound_or_its_stream_is_refused() { + let mut oversize = record_at("tx-a", 1); + oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1]; + oversize.stdout_total_bytes = u64::MAX; + assert!(oversize.validate().is_err()); + let mut impossible = record_at("tx-a", 1); + impossible.stderr_total_bytes = 1; + assert!(impossible.validate().is_err()); + let mut exact = record_at("tx-a", 1); + exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES]; + exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64; + assert!(exact.validate().is_ok()); + } + + #[test] + fn status_renders_the_tail_with_control_bytes_escaped() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec(); + stored.stdout_total_bytes = 5000; + stored.stderr_tail = b"bad\r\x07".to_vec(); + stored.stderr_total_bytes = 6; + let lines = render(&stored); + let text = lines.join("\n"); + assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); + assert!(!text.contains('\u{7}')); + assert_eq!( + lines, + vec![ + " failed step build", + " runner rust", + " status exit status 3", + " stdout (last 30 of 5000 bytes)", + " | first", + " | \\u{1b}[2Jsecond\\u{0}end\tTab", + " | last", + " stderr (last 5 of 6 bytes)", + " | bad\\u{d}\\u{7}", + ] + ); + } + + #[test] + fn an_empty_tail_renders_only_its_header() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = Vec::new(); + stored.stdout_total_bytes = 0; + let lines = render(&stored); + assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); + assert!(!lines.iter().any(|line| line == " | ")); + } + + #[test] + fn a_foreign_or_corrupt_store_is_an_error_not_an_empty_store() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 not a store")?; + assert!(for_transaction(&path, "tx-a").is_err()); + assert!(record(&path, &record_at("tx-a", 1)).is_err()); + Ok(()) + } +} From 7ef31ee69966b978eb69284023ea507a3a89087c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:18:06 +0100 Subject: [PATCH 093/122] Runner failure engine tests: declare readiness, read both control stores Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d927152..f7093c9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -24784,7 +24784,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// The unrouted recipe with two steps ahead of its build, and no external /// input: the second step, `test-daemon`, is the one the fake docker fails. fn two_step_recipe() -> String { - let recipe = unrouted_recipe(); + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); let (head, rest) = recipe .split_once("[[external_inputs]]") .expect("the recipe declares an external input"); @@ -24876,11 +24879,16 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); } - let control = std::fs::read(&world.state_store)?; - assert!( - !control.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), - "runner output reached control.cc" - ); + // The terminal transaction has moved to history.cc: both stores. + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + let bytes = std::fs::read(&store)?; + assert!(!bytes.is_empty(), "{} is empty", store.display()); + assert!( + !bytes.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), + "runner output reached {}", + store.display() + ); + } // Kept: in the private store, tails of both streams, and on status. let evidence = runner_evidence_store_path(&world.state_store); assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); @@ -24976,15 +24984,17 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide tick_until_ended(&world)?; // The store a rolled-back Idunn reads: it refuses any foreign type. ControlSnapshot::read(&world.state_store)?; - let envelopes = - SingleFileMessagePackBackingStore::new(&world.state_store).pull_all_read_only_snapshot()?; - assert!(!envelopes.is_empty()); - for envelope in &envelopes { - assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); - if envelope.r#type == DeploymentTransaction::TYPE { - assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + let mut transactions = 0; + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + for envelope in SingleFileMessagePackBackingStore::new(&store).pull_all_read_only_snapshot()? { + assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); + if envelope.r#type == DeploymentTransaction::TYPE { + transactions += 1; + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + } } } + assert_eq!(transactions, 1, "the failed transaction is in neither store"); Ok(()) } From 54a75ef9b09bd6523c665e153311bc104442d29f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 02:36:34 +0100 Subject: [PATCH 094/122] Runner failure regression: a stdout-only report is kept with an empty stderr The vili failure: node --test wrote only to stdout, stderr was empty, the step exited 1. Pin the cap at the literal 32 and the Debug impls that keep tails out of text. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 48 ++++++++++++++++++++++++++++++++++++++++++ src/runner_evidence.rs | 16 +++++++++++--- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 1c8d067..fdd4d3a 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -16568,5 +16568,53 @@ processes 9 assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); Ok(()) } + + /// The 2026-10-25 vili failure: `node --test` reported on stdout alone, + /// stderr stayed empty (the runner's /tmp is a noexec tmpfs) and the step + /// exited 1. The old capture dropped stdout and printed an empty message. + #[cfg(unix)] + #[test] + fn a_step_that_reports_only_on_stdout_keeps_that_report_and_its_name() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'not ok 1 - daemon {RUNNER_CANARY}'\necho '# fail 1'\nexit 1"), + )?; + let error = run_step(&driver, temp.path(), "test-daemon").expect_err("the step passed"); + // The error names the step and carries no output. + let text = format!("{error:#}"); + assert!(text.contains("runner step test-daemon on runner rust exited with exit status 1"), "{text}"); + assert!(!text.contains(RUNNER_CANARY) && !text.contains("not ok"), "{text}"); + // The evidence keeps the stdout tail; the empty stderr is a recorded 0. + let failure = error + .downcast_ref::() + .context("the failure is not typed")?; + let store = temp.path().join("runner-evidence.cc"); + let record = crate::runner_evidence::RunnerFailureRecord::from_failure( + "tx-vili", + "up-vili", + "vili", + failure, + 1, + ); + crate::runner_evidence::record(&store, &record)?; + let kept = crate::runner_evidence::for_transaction(&store, "tx-vili")?.context("no evidence kept")?; + assert_eq!(kept.subject_id, "test-daemon"); + assert_eq!(kept.exit_code, Some(1)); + assert_eq!( + String::from_utf8_lossy(&kept.stdout_tail), + format!("not ok 1 - daemon {RUNNER_CANARY}\n# fail 1\n") + ); + assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); + assert!(kept.stderr_tail.is_empty()); + assert_eq!(kept.stderr_total_bytes, 0); + let rendered = crate::runner_evidence::render(&kept).join("\n"); + assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); + assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); + // Debug names the failure and prints no tail. + let debug = format!("{failure:?}"); + assert!(debug.starts_with("RunnerStepFailure") && !debug.contains("not ok"), "{debug}"); + Ok(()) + } // END runner-failure-evidence tests } diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index 9eeab4c..d77961f 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -314,11 +314,12 @@ mod tests { fn the_evidence_store_keeps_the_newest_32() -> Result<()> { let temp = tempfile::tempdir()?; let path = runner_evidence_store_path(&temp.path().join("control.cc")); - for index in 0..=RUNNER_EVIDENCE_RECORDS { + // The literal, not the constant: the spec's bound is 32. + for index in 0..=32 { record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; } let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); - assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS); + assert_eq!(kept.len(), 32); assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); @@ -328,7 +329,7 @@ mod tests { again.stdout_total_bytes = 10; record(&path, &again)?; let kept = snapshot(&path)?.1; - assert_eq!(kept.len(), RUNNER_EVIDENCE_RECORDS); + assert_eq!(kept.len(), 32); assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; assert_eq!(stored.stdout_tail, b"rewritten\n"); @@ -407,6 +408,15 @@ mod tests { ); } + #[test] + fn debug_names_the_record_and_prints_no_tail() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"SECRET-OUTPUT".to_vec(); + stored.stdout_total_bytes = 13; + let debug = format!("{stored:?}"); + assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}"); + } + #[test] fn an_empty_tail_renders_only_its_header() { let mut stored = record_at("tx-a", 1); From 134c1fb5a25215763c80ef0fd3b99b96b0fd3b91 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 03:15:45 +0100 Subject: [PATCH 095/122] drain_tail tests: a scripted reader pins the Interrupted guard An interrupted read is retried with no byte lost, a persistent read error ends the drain instead of spinning, and a non-interrupted error after data is returned. Kills the three Interrupted-guard mutants (false, !=, true). Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index fdd4d3a..e0e5667 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -16525,6 +16525,65 @@ processes 9 Ok(()) } + /// A reader that answers from a script, so a read error the OS rarely + /// produces (an interrupted call) is produced on demand. + struct ScriptedReader(Vec>>); + + impl Read for ScriptedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.0.is_empty() { + return Ok(0); + } + let bytes = self.0.remove(0)?; + buffer[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + + struct BrokenReader; + + impl Read for BrokenReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("persistent read failure")) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_loses_no_byte() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + Ok(b"cd".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + ]); + let drained = drain_tail(reader, 3).expect("an interrupted read ended the drain"); + assert_eq!(drained, (b"bcd".to_vec(), 4)); + } + + #[test] + fn a_persistent_read_error_ends_the_drain_instead_of_spinning() { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(drain_tail(BrokenReader, 8).is_err()); + }); + assert_eq!( + receiver.recv_timeout(Duration::from_secs(5)), + Ok(true), + "a persistent read error spun or was swallowed" + ); + } + + #[test] + fn a_non_interrupted_read_error_after_data_is_returned_not_retried() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::BrokenPipe)), + Ok(b"cd".to_vec()), + ]); + let error = drain_tail(reader, 8).expect_err("a broken pipe was retried past"); + assert_eq!(error.kind(), ErrorKind::BrokenPipe); + } + #[cfg(unix)] #[test] fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { From 4a68df520fad6e764931a047b9aa27ad74c31e46 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 03:15:45 +0100 Subject: [PATCH 096/122] Runner evidence: set an undecodable store aside; escape bidi and pin C1 in status The writer moves a store it cannot decode (corrupt, or a newer schema) to runner-evidence.unreadable.cc, root-only, one slot replaced by the next, and opens a fresh store; the journal names the file and none of its content. A store that cannot be read at all is left in place and the write fails as before. Status escapes the bidi controls (U+061C, U+200E/F, U+202A-E, U+2066-9) in tails, subject and runner, and a test pins U+0080-U+009F. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 57 +++++++++-- src/runner_evidence.rs | 211 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 254 insertions(+), 14 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index f7093c9..dbe5096 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -47,7 +47,9 @@ use crate::drivers::{ WriteLeasePort, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; -use crate::runner_evidence::{self, RunnerFailureRecord, runner_evidence_store_path}; +use crate::runner_evidence::{ + self, Kept, RunnerFailureRecord, runner_evidence_store_path, unreadable_store_path, +}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -10695,18 +10697,27 @@ fn runner_failure_journal_line( transaction: &DeploymentTransaction, failure: &RunnerStepFailure, store_path: &Path, - recorded: &Result<()>, + recorded: &Result, ) -> String { let head = format!( "Idunn transaction {} (command {}, target {}) {failure}", transaction.transaction_id, transaction.command_id, transaction.target ); match recorded { - Ok(()) => format!( - "{head}; evidence in {}; read it with idunn status --command {}", - store_path.display(), - transaction.command_id - ), + Ok(kept) => { + let aside = match kept { + Kept::Stored => String::new(), + Kept::AfterSettingAsideAnUnreadableStore => format!( + "; an undecodable evidence store was set aside as {}", + unreadable_store_path(store_path).display() + ), + }; + format!( + "{head}; evidence in {}{aside}; read it with idunn status --command {}", + store_path.display(), + transaction.command_id + ) + } Err(error) => format!("{head}; evidence was not kept: {error:#}"), } } @@ -24963,6 +24974,38 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + #[cfg(unix)] + #[test] + fn an_undecodable_evidence_store_does_not_stop_the_next_failure_keeping_its_evidence() -> Result<()> { + let damage = format!("\u{c1} not a store {RUNNER_CANARY}-in-the-damage"); + let (world, _tools) = failing_runner_world()?; + let evidence = runner_evidence_store_path(&world.state_store); + std::fs::write(&evidence, damage.as_bytes())?; + let ended = tick_until_ended(&world)?; + // The transaction's own outcome is the runner failure, not the store. + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the damaged store changed the outcome: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + assert!(!error.contains("damage"), "{error}"); + // The damage is kept aside; the failure's evidence is in a fresh store. + assert_eq!(std::fs::read(unreadable_store_path(&evidence))?, damage.as_bytes()); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("the failure kept no evidence beside a damaged store")?; + assert_eq!(record.subject_id, "test-daemon"); + assert!(printed_status(&world)?.contains("failed step test-daemon")); + // The journal says it, naming the file and none of its content. + std::fs::write(&evidence, damage.as_bytes())?; + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"x"))) + .context("no journal line")?; + assert!(line.contains("an undecodable evidence store was set aside as"), "{line}"); + assert!(line.contains("runner-evidence.unreadable.cc"), "{line}"); + assert!(!line.contains("damage") && !line.contains("not a store"), "{line}"); + Ok(()) + } + #[cfg(unix)] #[test] fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index d77961f..cb08e0a 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -182,10 +182,44 @@ fn snapshot(path: &Path) -> Result<(Vec, Vec PathBuf { + path.with_file_name("runner-evidence.unreadable.cc") +} + +/// An undecodable store (a corrupt file, or one a newer Idunn wrote) must not +/// stop later failures from keeping their evidence. Move it aside, root-only, +/// over the previous one. Nothing of its content is named: a decode error can +/// quote stored values. A store that cannot even be read is left in place. +fn set_aside_unreadable(path: &Path) -> Result<()> { + std::fs::read(path).context("the runner evidence store could not be read")?; + let aside = unreadable_store_path(path); + std::fs::rename(path, &aside) + .context("the runner evidence store is undecodable and could not be set aside")?; + keep_private(&aside) +} + /// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the -/// oldest records go in the same exchange. The file ends at mode 0600. -pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result<()> { - let (envelopes, records) = snapshot(path)?; +/// oldest records go in the same exchange. The file ends at mode 0600. A store +/// that cannot be decoded is set aside first and the record opens a new one. +pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { + let (envelopes, records, how) = match snapshot(path) { + Ok((envelopes, records)) => (envelopes, records, Kept::Stored), + Err(_) => { + set_aside_unreadable(path)?; + (Vec::new(), Vec::new(), Kept::AfterSettingAsideAnUnreadableStore) + } + }; let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes .iter() .zip(&records) @@ -214,7 +248,8 @@ pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result<()> { .into()); } } - keep_private(path) + keep_private(path)?; + Ok(how) } #[cfg(unix)] @@ -270,10 +305,22 @@ pub(crate) fn render(record: &RunnerFailureRecord) -> Vec { lines } +/// Format characters that reorder what a terminal shows: the set the mind's +/// Title rule refuses. +fn is_bidi_control(character: char) -> bool { + matches!( + character, + '\u{61c}' | '\u{200e}' | '\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}' + ) +} + fn escape(text: &str) -> String { let mut escaped = String::with_capacity(text.len()); for character in text.chars() { - if character.is_control() && character != '\n' && character != '\t' { + if (character.is_control() || is_bidi_control(character)) + && character != '\n' + && character != '\t' + { escaped.push_str(&format!("\\u{{{:x}}}", u32::from(character))); } else { escaped.push(character); @@ -428,12 +475,162 @@ mod tests { } #[test] - fn a_foreign_or_corrupt_store_is_an_error_not_an_empty_store() -> Result<()> { + fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> { let temp = tempfile::tempdir()?; let path = runner_evidence_store_path(&temp.path().join("control.cc")); std::fs::write(&path, b"\xc1 not a store")?; assert!(for_transaction(&path, "tx-a").is_err()); - assert!(record(&path, &record_at("tx-a", 1)).is_err()); Ok(()) } + + fn files_beside(path: &Path) -> Result> { + let mut names: Vec = std::fs::read_dir(path.parent().context("no parent")?)? + .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned())) + .collect::>()?; + names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock")); + names.sort(); + Ok(names) + } + + /// A store a newer Idunn wrote: well-formed, but a schema this one does not know. + fn write_future_store(path: &Path) -> Result<()> { + let mut future = record_at("tx-future", 1); + future.schema_version = "idunn.runner_failure.v2".into(); + let envelope = typed_envelope( + &future.transaction_id, + RunnerFailureRecord::TYPE, + "idunn.runner_failure.v2", + &future, + 1, + )?; + let outcome = exchange_behind_private_lock(path, &[], &[envelope])?; + ensure!( + matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged), + "the future store was not written" + ); + Ok(()) + } + + #[test] + fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damage = b"\xc1 not a store \x00 damage"; + std::fs::write(&path, damage)?; + let kept = record(&path, &record_at("tx-a", 5))?; + assert_eq!(kept, Kept::AfterSettingAsideAnUnreadableStore); + // Kept, not deleted: the damage is where the journal says, root-only. + let aside = unreadable_store_path(&path); + assert_eq!(std::fs::read(&aside)?, damage); + assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600); + // The failure's evidence is in a fresh, private store. + assert_eq!( + for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id), + Some("tx-a".to_owned()) + ); + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + // A healthy store is not set aside again. + assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored); + assert_eq!(std::fs::read(&aside)?, damage); + Ok(()) + } + + #[test] + fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + write_future_store(&path)?; + let before = std::fs::read(&path)?; + assert!(for_transaction(&path, "tx-future").is_err()); + assert_eq!( + record(&path, &record_at("tx-a", 5))?, + Kept::AfterSettingAsideAnUnreadableStore + ); + assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); + assert!(for_transaction(&path, "tx-a")?.is_some()); + assert_eq!(for_transaction(&path, "tx-future")?, None); + Ok(()) + } + + #[test] + fn at_most_one_unreadable_store_is_kept() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"]; + for (index, damage) in damages.into_iter().enumerate() { + std::fs::write(&path, damage)?; + record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?; + assert_eq!( + std::fs::read(unreadable_store_path(&path))?, + damage, + "the newest damage is the one kept" + ); + } + assert_eq!( + files_beside(&path)?, + vec![ + "runner-evidence.cc".to_owned(), + "runner-evidence.unreadable.cc".to_owned() + ] + ); + Ok(()) + } + + #[test] + fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // A directory where the store should be: unreadable, not undecodable. + std::fs::create_dir(&path)?; + assert!(record(&path, &record_at("tx-a", 1)).is_err()); + assert!(path.is_dir()); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn status_escapes_bidi_controls_in_tails_and_in_names() { + for bidi in [ + '\u{61c}', '\u{200e}', '\u{200f}', '\u{202a}', '\u{202b}', '\u{202c}', '\u{202d}', + '\u{202e}', '\u{2066}', '\u{2067}', '\u{2068}', '\u{2069}', + ] { + let mut stored = record_at("tx-a", 1); + let tail = format!("ok {bidi}gpj.exe\n"); + stored.stdout_tail = tail.clone().into_bytes(); + stored.stdout_total_bytes = tail.len() as u64; + stored.stderr_tail = tail.clone().into_bytes(); + stored.stderr_total_bytes = tail.len() as u64; + stored.subject_id = format!("step{bidi}x"); + stored.runner = format!("rust{bidi}"); + let text = render(&stored).join("\n"); + let escaped = format!("\\u{{{:x}}}", bidi as u32); + assert!(!text.contains(bidi), "U+{:04X} reached the terminal", bidi as u32); + // Escaped in every place it can appear: two tails, the subject, the runner. + assert_eq!(text.matches(&escaped).count(), 4, "{text}"); + } + // Format characters either side of the set pass. + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = "caf\u{e9} \u{200d}\u{202f}\u{2065}\u{206a}\n".as_bytes().to_vec(); + stored.stdout_total_bytes = stored.stdout_tail.len() as u64; + let text = render(&stored).join("\n"); + assert!(text.contains("caf\u{e9} \u{200d}\u{202f}\u{2065}\u{206a}"), "{text}"); + } + + #[test] + fn status_escapes_the_c1_controls() { + for code in 0x80u32..=0x9f { + let character = char::from_u32(code).unwrap(); + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = format!("a{character}b\n").into_bytes(); + stored.stdout_total_bytes = stored.stdout_tail.len() as u64; + let text = render(&stored).join("\n"); + assert!(!text.contains(character), "U+{code:04X} reached the terminal"); + assert!(text.contains(&format!("a\\u{{{code:x}}}b")), "U+{code:04X}: {text}"); + } + // U+00A0 is not a control character and passes. + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = "a\u{a0}b\n".as_bytes().to_vec(); + stored.stdout_total_bytes = stored.stdout_tail.len() as u64; + assert!(render(&stored).join("\n").contains("a\u{a0}b")); + } } From 549f4be09328933d7e5dc55edaead1d7f474f6dc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 07:16:09 +0100 Subject: [PATCH 097/122] Runner evidence: one allowlist sink for status, quarantine only on a decode failure, a displaced set-aside is journaled Status writes everything it prints from the evidence store, including the unreadable-store line, through one function (emit): printable ASCII and tab pass, every other code point is a \u{..} escape and every invalid UTF-8 byte a \x.. escape. The bidi and C1 class lists are deleted. The writer reads the store through a seam. An I/O error leaves the store in place, keeps no evidence and names its kind; the store is set aside only when bytes that were read still read the same on a second read and do not decode. A set-aside that displaces an earlier one says so on the journal (last-written time, no content). Tests pin a failed set-aside, its no-echo, the I/O seam and the replacement. Not yet built or run; verified in the next job. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 124 +++++++++++++-- src/runner_evidence.rs | 351 ++++++++++++++++++++++++++++++----------- 2 files changed, 367 insertions(+), 108 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index dbe5096..e43a7ac 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -4882,22 +4882,11 @@ fn status( writeln!(out, " pre-fencing abort {abort:?}")?; // The step that failed and its output tail live in their // own private store. An unreadable store costs its lines. - match runner_evidence::for_transaction( + runner_evidence::show_status( + out, &runner_evidence_store_path(store_path), &transaction.transaction_id, - ) { - Ok(Some(record)) => { - for line in runner_evidence::render(&record) { - writeln!(out, "{line}")?; - } - } - Ok(None) => {} - Err(error) => writeln!( - out, - " runner evidence unreadable: {}", - error.root_cause() - )?, - } + )?; } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -10707,9 +10696,15 @@ fn runner_failure_journal_line( Ok(kept) => { let aside = match kept { Kept::Stored => String::new(), - Kept::AfterSettingAsideAnUnreadableStore => format!( - "; an undecodable evidence store was set aside as {}", - unreadable_store_path(store_path).display() + Kept::SetAside { replaced } => format!( + "; an undecodable evidence store was set aside as {}{}", + unreadable_store_path(store_path).display(), + match replaced { + Some(last_written) => format!( + ", replacing an earlier set-aside store last written at unix millis {last_written}" + ), + None => String::new(), + } ), }; format!( @@ -25020,6 +25015,101 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// Everything status prints from the evidence store goes through one + /// allowlist, so a hostile build log reaches the terminal as visible + /// escapes whatever its class. + #[cfg(unix)] + #[test] + fn status_prints_hostile_runner_output_as_visible_escapes_only() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let hostile: Vec<(&str, &str)> = vec![ + ("\u{1b}[2J", "\\u{1b}[2J"), + ("\u{7}", "\\u{7}"), + ("\r", "\\u{d}"), + ("\u{202e}", "\\u{202e}"), + ("\u{2066}", "\\u{2066}"), + ("\u{85}", "\\u{85}"), + ("\u{e0041}", "\\u{e0041}"), + ("\u{200b}", "\\u{200b}"), + ("\u{feff}", "\\u{feff}"), + ]; + let mut tail: Vec = Vec::new(); + for &(raw, _) in &hostile { + tail.extend_from_slice(b"a"); + tail.extend_from_slice(raw.as_bytes()); + tail.extend_from_slice(b"b\n"); + } + tail.extend_from_slice(b"bad \xff\xc1 bytes\n"); + runner_evidence::record( + &runner_evidence_store_path(&world.state_store), + &RunnerFailureRecord::from_failure( + &ended.transaction_id, + &ended.command_id, + &ended.target, + &canary_failure(&tail), + 9, + ), + )?; + let printed = printed_status(&world)?; + for &(raw, escaped) in &hostile { + assert!(!printed.contains(raw), "{escaped} reached the terminal raw"); + assert!(printed.contains(&format!("a{escaped}b")), "{escaped} missing: {printed}"); + } + assert!(printed.contains("bad \\xff\\xc1 bytes"), "{printed}"); + let section = printed.find(" failed step").context("no evidence section")?; + assert!(printed[section..].is_ascii(), "non-ASCII reached the terminal: {printed}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn status_prints_the_unreadable_store_line_escaped() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + let evidence = runner_evidence_store_path(&world.state_store); + // The decode error of this store quotes the stored subject kind. + let mut bytes = std::fs::read(&evidence)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker in the evidence store")?; + bytes[at..at + 4].copy_from_slice(b"\x1b[2J"); + std::fs::write(&evidence, bytes)?; + let printed = printed_status(&world)?; + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + assert!(!printed.contains('\u{1b}'), "an escape byte reached the terminal"); + assert!(printed.contains("\\u{1b}[2J"), "{printed}"); + Ok(()) + } + + #[test] + fn the_journal_line_names_a_displaced_set_aside_store_by_time_only() -> Result<()> { + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let path = Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"); + let first = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: None }), + ); + assert!(first.contains("was set aside as"), "{first}"); + assert!(!first.contains("replacing"), "{first}"); + let second = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: Some(1234) }), + ); + assert!( + second.contains("replacing an earlier set-aside store last written at unix millis 1234"), + "{second}" + ); + assert!(!second.contains(RUNNER_CANARY)); + Ok(()) + } + #[cfg(unix)] #[test] fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index cb08e0a..9e9e5ee 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -9,8 +9,9 @@ //! //! The tail is raw runner output. It is kept unredacted and readable by root //! alone (mode 0600): root can already read every secret the runner was given. -//! It never enters an error, a transaction record or the journal; the render -//! below escapes control characters because the status reader is a terminal. +//! It never enters an error, a transaction record or the journal. The status +//! reader is a terminal, so everything it prints from this store goes through +//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped. use std::path::{Path, PathBuf}; @@ -186,9 +187,11 @@ fn snapshot(path: &Path) -> Result<(Vec, Vec }, } /// Where an undecodable store is kept. One slot: the next one replaces it, so @@ -197,27 +200,62 @@ pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf { path.with_file_name("runner-evidence.unreadable.cc") } -/// An undecodable store (a corrupt file, or one a newer Idunn wrote) must not -/// stop later failures from keeping their evidence. Move it aside, root-only, -/// over the previous one. Nothing of its content is named: a decode error can -/// quote stored values. A store that cannot even be read is left in place. -fn set_aside_unreadable(path: &Path) -> Result<()> { - std::fs::read(path).context("the runner evidence store could not be read")?; +/// Moves the store over the previous set-aside one, root-only, and reports +/// what it displaced. Nothing of the store's content is named: a decode error +/// can quote stored values, and none is carried here. +fn set_aside_unreadable(path: &Path) -> Result> { let aside = unreadable_store_path(path); + let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| { + earlier + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |since| since.as_millis() as u64) + }); std::fs::rename(path, &aside) .context("the runner evidence store is undecodable and could not be set aside")?; - keep_private(&aside) + keep_private(&aside)?; + Ok(replaced) +} + +/// How the writer reads the store's bytes; tests inject an I/O error here. +type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result>; + +fn unreadable_fault(error: &std::io::Error) -> anyhow::Error { + anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind()) } /// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the -/// oldest records go in the same exchange. The file ends at mode 0600. A store -/// that cannot be decoded is set aside first and the record opens a new one. +/// oldest records go in the same exchange. The file ends at mode 0600. +/// +/// A store is set aside only when bytes that were read do not decode. An I/O +/// error leaves the store in place, keeps no evidence for that failure, and +/// the error names its kind for the journal. pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { + record_reading(path, record, &|path| std::fs::read(path)) +} + +fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result { + let first = match read(path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(unreadable_fault(&error)), + }; let (envelopes, records, how) = match snapshot(path) { Ok((envelopes, records)) => (envelopes, records, Kept::Stored), Err(_) => { - set_aside_unreadable(path)?; - (Vec::new(), Vec::new(), Kept::AfterSettingAsideAnUnreadableStore) + let Some(first) = first else { + bail!("the runner evidence store could not be decoded"); + }; + // Undecodable only if the same bytes are still there to read: a + // second read that fails, or finds other bytes, is not a verdict. + match read(path) { + Ok(second) if second == first => {} + Ok(_) => bail!("the runner evidence store changed while it was being read"), + Err(error) => return Err(unreadable_fault(&error)), + } + let replaced = set_aside_unreadable(path)?; + (Vec::new(), Vec::new(), Kept::SetAside { replaced }) } }; let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes @@ -275,58 +313,81 @@ pub(crate) fn for_transaction( .find(|stored| stored.transaction_id == transaction_id)) } -/// The record as `idunn status --command` prints it. The store keeps raw -/// bytes; every control character except newline and tab is escaped here, so -/// build output cannot drive the operator's terminal. -pub(crate) fn render(record: &RunnerFailureRecord) -> Vec { +/// The record as `idunn status --command` shows it, as raw lines. They are +/// not text yet: `show_status` is the only caller, and `emit` the only +/// writer. +fn render(record: &RunnerFailureRecord) -> Vec> { let noun = match record.subject_kind { SubjectKind::Step => "step", SubjectKind::ExternalInput => "external input", }; - let mut lines = vec![ - format!(" failed {noun} {}", escape(&record.subject_id)), - format!(" runner {}", escape(&record.runner)), - format!(" status {}", runner_status(record.exit_code, record.signal)), + let mut lines: Vec> = vec![ + format!(" failed {noun} {}", record.subject_id).into_bytes(), + format!(" runner {}", record.runner).into_bytes(), + format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(), ]; for (stream, tail, total) in [ ("stdout", &record.stdout_tail, record.stdout_total_bytes), ("stderr", &record.stderr_tail, record.stderr_total_bytes), ] { - lines.push(format!( - " {stream} (last {} of {total} bytes)", - tail.len() - )); - let text = escape(&String::from_utf8_lossy(tail)); - let text = text.strip_suffix('\n').unwrap_or(&text); + lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes()); + let text = tail.strip_suffix(b"\n").unwrap_or(tail); if !text.is_empty() { - lines.extend(text.split('\n').map(|line| format!(" | {line}"))); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); } } lines } -/// Format characters that reorder what a terminal shows: the set the mind's -/// Title rule refuses. -fn is_bidi_control(character: char) -> bool { - matches!( - character, - '\u{61c}' | '\u{200e}' | '\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}' - ) +/// What `idunn status --command` prints for a transaction's runner evidence: +/// its record, or one line naming why the store is unreadable. Nothing is +/// written when the transaction has no record. +pub(crate) fn show_status( + out: &mut impl std::io::Write, + store: &Path, + transaction_id: &str, +) -> std::io::Result<()> { + let lines = match for_transaction(store, transaction_id) { + Ok(Some(record)) => render(&record), + Ok(None) => return Ok(()), + Err(error) => { + vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()] + } + }; + emit(out, &lines) } -fn escape(text: &str) -> String { - let mut escaped = String::with_capacity(text.len()); - for character in text.chars() { - if (character.is_control() || is_bidi_control(character)) - && character != '\n' - && character != '\t' - { - escaped.push_str(&format!("\\u{{{:x}}}", u32::from(character))); - } else { - escaped.push(character); +/// The one sink: every byte the status read prints from the evidence store, +/// or from an error about it, is written here and nowhere else. An allowlist: +/// printable ASCII and tab pass; every other code point is a visible +/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline +/// inside a line is escaped too: this function writes the only ones. +fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { + for line in lines { + out.write_all(escape(line).as_bytes())?; + out.write_all(b"\n")?; + } + Ok(()) +} + +fn escape(bytes: &[u8]) -> String { + let mut shown = String::with_capacity(bytes.len()); + for chunk in bytes.utf8_chunks() { + for character in chunk.valid().chars() { + match character { + ' '..='~' | '\t' => shown.push(character), + _ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))), + } + } + for byte in chunk.invalid() { + shown.push_str(&format!("\\x{byte:02x}")); } } - escaped + shown } #[cfg(all(test, unix))] @@ -347,6 +408,17 @@ mod tests { } } + /// What status writes for a record, through the one sink. + fn shown(record: &RunnerFailureRecord) -> String { + let mut written = Vec::new(); + emit(&mut written, &render(record)).expect("a Vec takes any write"); + String::from_utf8(written).expect("the sink writes ASCII only") + } + + fn shown_lines(record: &RunnerFailureRecord) -> Vec { + shown(record).lines().map(str::to_owned).collect() + } + fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { RunnerFailureRecord::from_failure( transaction, @@ -435,7 +507,7 @@ mod tests { stored.stdout_total_bytes = 5000; stored.stderr_tail = b"bad\r\x07".to_vec(); stored.stderr_total_bytes = 6; - let lines = render(&stored); + let lines = shown_lines(&stored); let text = lines.join("\n"); assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); assert!(!text.contains('\u{7}')); @@ -469,7 +541,7 @@ mod tests { let mut stored = record_at("tx-a", 1); stored.stdout_tail = Vec::new(); stored.stdout_total_bytes = 0; - let lines = render(&stored); + let lines = shown_lines(&stored); assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); assert!(!lines.iter().any(|line| line == " | ")); } @@ -518,7 +590,7 @@ mod tests { let damage = b"\xc1 not a store \x00 damage"; std::fs::write(&path, damage)?; let kept = record(&path, &record_at("tx-a", 5))?; - assert_eq!(kept, Kept::AfterSettingAsideAnUnreadableStore); + assert_eq!(kept, Kept::SetAside { replaced: None }); // Kept, not deleted: the damage is where the journal says, root-only. let aside = unreadable_store_path(&path); assert_eq!(std::fs::read(&aside)?, damage); @@ -544,7 +616,7 @@ mod tests { assert!(for_transaction(&path, "tx-future").is_err()); assert_eq!( record(&path, &record_at("tx-a", 5))?, - Kept::AfterSettingAsideAnUnreadableStore + Kept::SetAside { replaced: None } ); assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); assert!(for_transaction(&path, "tx-a")?.is_some()); @@ -589,48 +661,145 @@ mod tests { } #[test] - fn status_escapes_bidi_controls_in_tails_and_in_names() { - for bidi in [ - '\u{61c}', '\u{200e}', '\u{200f}', '\u{202a}', '\u{202b}', '\u{202c}', '\u{202d}', - '\u{202e}', '\u{2066}', '\u{2067}', '\u{2068}', '\u{2069}', - ] { - let mut stored = record_at("tx-a", 1); - let tail = format!("ok {bidi}gpj.exe\n"); - stored.stdout_tail = tail.clone().into_bytes(); - stored.stdout_total_bytes = tail.len() as u64; - stored.stderr_tail = tail.clone().into_bytes(); - stored.stderr_total_bytes = tail.len() as u64; - stored.subject_id = format!("step{bidi}x"); - stored.runner = format!("rust{bidi}"); - let text = render(&stored).join("\n"); - let escaped = format!("\\u{{{:x}}}", bidi as u32); - assert!(!text.contains(bidi), "U+{:04X} reached the terminal", bidi as u32); - // Escaped in every place it can appear: two tails, the subject, the runner. - assert_eq!(text.matches(&escaped).count(), 4, "{text}"); + fn every_code_point_but_printable_ascii_is_a_visible_escape() { + // The allowlist, exhaustively: no class list decides anything. + for code in 0..=0x10ffffu32 { + let Some(character) = char::from_u32(code) else { + continue; + }; + let text = character.to_string(); + let shown = escape(text.as_bytes()); + if matches!(character, ' '..='~' | '\t') { + assert_eq!(shown, text, "U+{code:04X} should pass"); + } else { + assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}"); + } } - // Format characters either side of the set pass. - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = "caf\u{e9} \u{200d}\u{202f}\u{2065}\u{206a}\n".as_bytes().to_vec(); - stored.stdout_total_bytes = stored.stdout_tail.len() as u64; - let text = render(&stored).join("\n"); - assert!(text.contains("caf\u{e9} \u{200d}\u{202f}\u{2065}\u{206a}"), "{text}"); + // Invalid UTF-8 is escaped byte by byte; valid text round it passes. + assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82"); + assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}"); } #[test] - fn status_escapes_the_c1_controls() { - for code in 0x80u32..=0x9f { - let character = char::from_u32(code).unwrap(); - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = format!("a{character}b\n").into_bytes(); - stored.stdout_total_bytes = stored.stdout_tail.len() as u64; - let text = render(&stored).join("\n"); - assert!(!text.contains(character), "U+{code:04X} reached the terminal"); - assert!(text.contains(&format!("a\\u{{{code:x}}}b")), "U+{code:04X}: {text}"); - } - // U+00A0 is not a control character and passes. + fn a_newline_inside_a_line_cannot_forge_a_line() { let mut stored = record_at("tx-a", 1); - stored.stdout_tail = "a\u{a0}b\n".as_bytes().to_vec(); - stored.stdout_total_bytes = stored.stdout_tail.len() as u64; - assert!(render(&stored).join("\n").contains("a\u{a0}b")); + stored.subject_id = "build\n status exit status 0".into(); + let lines = shown_lines(&stored); + assert_eq!(lines[0], " failed step build\\u{a} status exit status 0"); + assert_eq!( + lines.iter().filter(|line| line.starts_with(" status ")).count(), + 1 + ); + } + + /// A store whose first record is well-formed except that its subject kind + /// holds `marker`: the decode error quotes it. + fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> { + record(path, &record_at("tx-a", 1))?; + let mut bytes = std::fs::read(path)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker")?; + bytes[at..at + 4].copy_from_slice(marker); + std::fs::write(path, bytes)?; + Ok(()) + } + + #[test] + fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"\x1b[2J")?; + let mut written = Vec::new(); + show_status(&mut written, &path, "tx-a")?; + assert!(!written.contains(&0x1b), "an escape byte reached the terminal"); + assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n'))); + let text = String::from_utf8(written)?; + assert!(text.starts_with(" runner evidence unreadable: "), "{text}"); + assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}"); + Ok(()) + } + + #[test] + fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"CNRY")?; + let damage = std::fs::read(&path)?; + // Sanity: this damage decodes to an error that quotes the canary. + assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY")); + // A non-empty directory where the aside goes: rename fails even as root. + let aside = unreadable_store_path(&path); + std::fs::create_dir(&aside)?; + std::fs::write(aside.join("occupied"), b"x")?; + let error = record(&path, &record_at("tx-b", 2)).unwrap_err(); + for text in [format!("{error:#}"), format!("{error:?}")] { + assert!(text.contains("could not be set aside"), "{text}"); + assert!(!text.contains("CNRY"), "stored content reached the error: {text}"); + } + // The failure cost nothing: the store is as it was, with no evidence + // written over it. + assert_eq!(std::fs::read(&path)?, damage); + assert!(aside.join("occupied").exists()); + Ok(()) + } + + #[test] + fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + record(&path, &record_at("tx-a", 1))?; + let before = std::fs::read(&path)?; + let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied); + // The first read fails on a healthy store. + let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, before, "the store changed"); + assert!(!unreadable_store_path(&path).exists()); + assert!(for_transaction(&path, "tx-b")?.is_none()); + // The second read, the one that confirms the bytes do not decode, fails. + std::fs::write(&path, b"\xc1 not a store")?; + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| { + calls.set(calls.get() + 1); + if calls.get() == 2 { + return Err(denied()); + } + std::fs::read(path) + }; + let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + // Bytes that changed between the reads are no verdict either. + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| -> std::io::Result> { + calls.set(calls.get() + 1); + let mut bytes = std::fs::read(path)?; + bytes.push(calls.get() as u8); + Ok(bytes) + }; + assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err()); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 first")?; + assert_eq!( + record(&path, &record_at("tx-a", 1))?, + Kept::SetAside { replaced: None } + ); + std::fs::write(&path, b"\xc1 second")?; + let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else { + bail!("a displaced set-aside store went unreported"); + }; + assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second"); + Ok(()) } } From 1518cfb13c46a1cad8982dc3c2b3d515fe65a885 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 25 Oct 2026 07:18:51 +0100 Subject: [PATCH 098/122] drivers test: read the kept evidence through show_status, the one status sink Fixes the test build broken by 549f4be (render is private now). Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/drivers.rs b/src/drivers.rs index e0e5667..382bd23 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -16667,7 +16667,9 @@ processes 9 assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); assert!(kept.stderr_tail.is_empty()); assert_eq!(kept.stderr_total_bytes, 0); - let rendered = crate::runner_evidence::render(&kept).join("\n"); + let mut shown = Vec::new(); + crate::runner_evidence::show_status(&mut shown, &store, "tx-vili")?; + let rendered = String::from_utf8(shown)?; assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); // Debug names the failure and prints no tail. From 024f2d7a09ec642e63f696899190b20c0d7840fe Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 29 Oct 2026 22:54:26 +0100 Subject: [PATCH 099/122] Workload: incarnation units are collected when they end, so a failed candidate no longer stays loaded and marks the host degraded Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 382bd23..3ea1145 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -605,7 +605,9 @@ pub trait WorkloadPort { /// /// Idunn launches candidates as transient units with `Restart=no`, so a /// unit that has failed, or that systemd no longer knows, will not come - /// back on its own. Past the fence every error is otherwise treated as + /// back on its own. Those units are collected once they fail + /// (`INCARNATION_UNIT_LIFECYCLE`), so a failed candidate usually reads as + /// forgotten rather than failed. Past the fence every error is otherwise treated as /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; @@ -2574,6 +2576,13 @@ impl RunnerPort for DockerRunnerDriver { } } +/// How every incarnation unit lives and dies. It runs once (`Restart=no`) +/// and systemd forgets it once it is inactive or failed (`--collect`): the +/// outcome is Idunn's to record in its own store, and a failed transient unit +/// left loaded is only clutter that marks the host degraded. +const INCARNATION_UNIT_LIFECYCLE: [&str; 3] = + ["--collect", "--property=Type=exec", "--property=Restart=no"]; + /// systemd owns process execution. This driver lowers one validated launch /// contract into a transient unit and then proves the native process and /// executable that systemd actually started. It does not decide admission, @@ -3764,8 +3773,9 @@ impl SystemdTransientWorkloadDriver { OsString::from("--no-ask-password"), OsString::from("--expand-environment=no"), OsString::from(format!("--unit={unit}")), - OsString::from("--property=Type=exec"), - OsString::from("--property=Restart=no"), + ]; + args.extend(INCARNATION_UNIT_LIFECYCLE.map(OsString::from)); + args.extend([ OsString::from("--property=KillMode=mixed"), OsString::from("--property=DynamicUser=yes"), OsString::from("--property=SupplementaryGroups="), @@ -3801,7 +3811,7 @@ impl SystemdTransientWorkloadDriver { OsString::from(format!("--working-directory={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", bundle.display())), - ]; + ]); for open_file in parent_only_open_files { args.push(OsString::from(format!("--property=OpenFile={open_file}"))); } @@ -9942,6 +9952,16 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// An incarnation unit runs once and is collected when it ends, so a + /// failed candidate does not stay loaded and mark the host degraded. + /// Collection is safe because a forgotten unit already reads as + /// permanently stopped (the test below). + #[test] + fn incarnation_units_run_once_and_are_collected() { + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--collect")); + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--property=Restart=no")); + } + /// Admission audit, claim 3: `is_permanently_stopped` now lets the /// Isolation gate skip `prove_isolation` against the incumbent. Enumerate /// what it answers `true` for, using a fake `systemctl show` that replays From e50a721dbca3324db6b9a31b4eac9746385289c9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:01 +0100 Subject: [PATCH 100/122] Steward orders: typed package, unit and deploy orders from one steward, in a sibling root unit idunn orders serve takes orders over a root-only socket from the steward uid, records order/start/outcome/post in its own orders.cc, and posts acceptance and result to #operator as a retried projection. Unit orders may act on any unit except the compiled always-refused set (idunn-*, mind-self@*, mind-persona@*), matched on a unit's Id and every name, and for stop and restart on the units systemd stops with it. A refusal names a field and a code, never the value. A started order is never run twice. Deploy orders commit through the same primitive as idunn up. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 1 + Cargo.toml | 1 + src/control_plane.rs | 112 +- src/lib.rs | 2 + src/runner_evidence.rs | 4 +- src/steward_orders.rs | 2932 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 3036 insertions(+), 16 deletions(-) create mode 100644 src/steward_orders.rs diff --git a/Cargo.lock b/Cargo.lock index dfc3087..44c9457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "libc", "rmp-serde", "serde", + "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index dcaca2e..ee797ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } +serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/src/control_plane.rs b/src/control_plane.rs index e43a7ac..cd49073 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -2934,6 +2934,10 @@ enum Command { recipe: PathBuf, binding: Option, }, + #[cfg(unix)] + OrdersServe(crate::steward_orders::ServeOptions), + #[cfg(unix)] + Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { @@ -2973,6 +2977,10 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + #[cfg(unix)] + Command::OrdersServe(options) => crate::steward_orders::serve(options), + #[cfg(unix)] + Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -3025,6 +3033,10 @@ fn parse(args: impl Iterator) -> Result { "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), + #[cfg(unix)] + "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), + #[cfg(unix)] + "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -4730,20 +4742,21 @@ fn resolve_selector( Ok(targets) } -fn submit( +/// The one commit primitive for a deployment command: `idunn up` and a steward +/// deploy order both write theirs here. Returns the commit time. +pub(crate) fn commit_deployment_command( store_path: &Path, + command_id: &str, selector: &str, requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { +) -> Result { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), + command_id: command_id.into(), kind: CommandKind::Deploy, selector: selector.into(), requested_by: requested_by.into(), @@ -4761,7 +4774,33 @@ fn submit( )?, "deployment command id collided" ); - println!("{}", command.command_id); + Ok(now) +} + +/// A command's present state as `idunn status --command` derives it, for the +/// steward order poster: `queued` and `running` are not terminal. +#[cfg(unix)] +pub(crate) fn command_state( + state_store: &Path, + command_id: &str, +) -> Result<(&'static str, String)> { + let snapshot = ControlSnapshot::read(state_store)?; + let archived = read_history_transactions(state_store); + let mut transactions = snapshot.transaction_for_command(command_id); + transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); + Ok(derived_command_status(&transactions)) +} + +fn submit( + store_path: &Path, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + let command_id = format!("up-{}", Uuid::new_v4()); + let now = commit_deployment_command(store_path, &command_id, selector, requested_by)?; + println!("{command_id}"); if !wait { return Ok(()); } @@ -4772,10 +4811,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command.command_id), + .any(|stored| stored.value.command_id == command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command.command_id); + let transactions = snapshot.transaction_for_command(&command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4800,7 +4839,7 @@ fn submit( } } -fn status( +pub(crate) fn status( out: &mut impl Write, store_path: &Path, incident_store: &Path, @@ -10592,7 +10631,13 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -fn now_millis() -> Result { +/// The control store `idunn up` and `idunn serve` use unless told otherwise. +#[cfg(unix)] +pub(crate) fn default_state_store() -> PathBuf { + RuntimeOptions::default().state_store +} + +pub(crate) fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -10612,12 +10657,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -fn string_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -fn path_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -10633,7 +10678,7 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -fn require_selector(value: &str) -> Result<()> { +pub(crate) fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } @@ -10757,7 +10802,10 @@ fn usage() -> &'static str { idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\ + idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ + idunn order --reason TEXT\n\ + idunn order show ID | idunn order list\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -11969,9 +12017,45 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["order", "run", "sh", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], + vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], + vec!["orders", "serve", "--allow-units", "nginx.service"], + vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } + // The declarative order forms parse, and only as order commands. + #[cfg(unix)] + for arguments in [ + vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r"], + vec!["order", "list"], + vec!["orders", "serve"], + ] { + let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); + assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); + } + } + + #[test] + fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + commit_deployment_command(&store, "up-one", "vili", "steward:eureka order ord-1")?; + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.commands.len(), 1); + let command = &snapshot.commands[0].value; + assert_eq!(command.command_id, "up-one"); + assert_eq!(command.selector, "vili"); + assert_eq!(command.requested_by, "steward:eureka order ord-1"); + assert!(commit_deployment_command(&store, "up-one", "other", "x").is_err()); + assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); + Ok(()) } #[test] diff --git a/src/lib.rs b/src/lib.rs index 4f365a3..b5499a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,3 +9,5 @@ pub mod launcher; mod incident; mod runner_evidence; pub mod provisioning; +#[cfg(unix)] +mod steward_orders; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index 9e9e5ee..30dd1d2 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -366,7 +366,7 @@ pub(crate) fn show_status( /// printable ASCII and tab pass; every other code point is a visible /// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline /// inside a line is escaped too: this function writes the only ones. -fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { +pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { for line in lines { out.write_all(escape(line).as_bytes())?; out.write_all(b"\n")?; @@ -374,7 +374,7 @@ fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> Ok(()) } -fn escape(bytes: &[u8]) -> String { +pub(crate) fn escape(bytes: &[u8]) -> String { let mut shown = String::with_capacity(bytes.len()); for chunk in bytes.utf8_chunks() { for character in chunk.valid().chars() { diff --git a/src/steward_orders.rs b/src/steward_orders.rs new file mode 100644 index 0000000..2aded30 --- /dev/null +++ b/src/steward_orders.rs @@ -0,0 +1,2932 @@ +//! Typed orders from Idunn's steward. +//! +//! `idunn orders serve` is a second root unit from the same binary as +//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints +//! order ids, decides acceptance and refusal, actuates, and records outcomes. +//! The survival daemon takes no part: `control.cc` never learns an order type. +//! +//! An order is a closed enum, never a command string: install packages, act on +//! a systemd unit, or deploy a target through the same commit primitive +//! `idunn up` uses. Every unit is allowed except the always-refused set, which +//! is compiled in here and read from nowhere: no binding field, record or flag +//! holds it. The `#operator` post is a retried projection of the store. +//! +//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the +//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and +//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE +//! `mind-control check`; START record before actuation, at most once; RUN; +//! RESULT outcome; POST accepted and result. +//! +//! No refusal or error here carries a value the issuer supplied: it names the +//! field and a code. A refused unit that a record names was resolved by +//! systemd and passed the unit grammar. + +use std::collections::{BTreeSet, VecDeque}; +use std::fmt; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::control_plane::{ + command_state, commit_deployment_command, decode_record, default_state_store, now_millis, + path_value, require_id, require_selector, string_value, typed_envelope, +}; +use crate::runner_evidence::{emit, escape}; + +const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; +const START_SCHEMA: &str = "idunn.steward_order_start.v1"; +const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; +const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; + +const REQUEST_FRAME_LIMIT: usize = 16 * 1024; +const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; +const TAIL_BYTES: usize = 8 * 1024; +const REASON_LIMIT: usize = 300; +const PACKAGE_LIMIT: usize = 16; +const UNIT_LIMIT: usize = 200; +const CLOSURE_LIMIT: usize = 512; +const LIST_LIMIT: usize = 50; +const UNIT_SUFFIXES: [&str; 6] = [".service", ".socket", ".timer", ".path", ".mount", ".target"]; + +/// The always-refused set: Idunn's own units and the minds' own. Matched on +/// every name a unit answers to, ASCII case-insensitively (systemd units are +/// case-sensitive, so the fold only widens the refusal). +fn is_refused_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + ["idunn-", "mind-self@", "mind-persona@"] + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +// ---------------------------------------------------------------- grammar -- + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Code { + Empty, + TooLong, + TooMany, + BadCharacter, + BadShape, + Malformed, + Unauthorized, + Unknown, + Unavailable, +} + +/// A refusal names the field and a code. It never carries the offending value. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Refusal { + pub(crate) field: String, + pub(crate) code: Code, +} + +impl Refusal { + fn new(field: &'static str, code: Code) -> Self { + Self { + field: field.to_owned(), + code, + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{} refused: {:?}", self.field, self.code) + } +} + +impl std::error::Error for Refusal {} + +fn validate_package(spec: &str) -> Result<(), Refusal> { + const FIELD: &str = "package"; + if spec.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if spec.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let (name, version) = match spec.split_once('=') { + Some((name, version)) => (name, Some(version)), + None => (spec, None), + }; + let bytes = name.as_bytes(); + if bytes.len() < 2 { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); + let rest = bytes[1..].iter().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') + }); + if !head || !rest { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + // apt reads `name-` as a removal and `name.deb` as a local file. + if name.ends_with('-') || name.ends_with(".deb") { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if let Some(version) = version { + if version.is_empty() { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if !version.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') + }) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + } + Ok(()) +} + +fn validate_unit(unit: &str) -> Result<(), Refusal> { + const FIELD: &str = "unit"; + if unit.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if unit.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let allowed = unit.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') + }); + if !allowed || unit.starts_with('-') { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + match UNIT_SUFFIXES + .iter() + .find_map(|suffix| unit.strip_suffix(suffix)) + { + Some(stem) if !stem.is_empty() => Ok(()), + _ => Err(Refusal::new(FIELD, Code::BadShape)), + } +} + +fn validate_reason(reason: &str) -> Result<(), Refusal> { + const FIELD: &str = "reason"; + if reason.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if reason.len() > REASON_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Verb { + Start, + Stop, + Restart, + Reload, +} + +impl Verb { + fn as_str(self) -> &'static str { + match self { + Verb::Start => "start", + Verb::Stop => "stop", + Verb::Restart => "restart", + Verb::Reload => "reload", + } + } + + fn parse(word: &str) -> Option { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + .into_iter() + .find(|verb| verb.as_str() == word) + } + + /// Verbs systemd propagates down the dependency graph. + fn propagates(self) -> bool { + matches!(self, Verb::Stop | Verb::Restart) + } +} + +/// The closed set of things an order can ask for. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Operation { + PackageInstall { packages: Vec }, + UnitAction { verb: Verb, unit: String }, + Deploy { selector: String }, +} + +/// One fixed argv for `systemd-run`, built from the typed operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Step { + pub(crate) argv: Vec, + pub(crate) environment: Vec<(String, String)>, +} + +impl Operation { + fn validate(&self) -> Result<(), Refusal> { + match self { + Operation::PackageInstall { packages } => { + if packages.is_empty() { + return Err(Refusal::new("package", Code::Empty)); + } + if packages.len() > PACKAGE_LIMIT { + return Err(Refusal::new("package", Code::TooMany)); + } + packages.iter().try_for_each(|package| validate_package(package)) + } + Operation::UnitAction { unit, .. } => validate_unit(unit), + Operation::Deploy { selector } => require_selector(selector) + .map_err(|_| Refusal::new("selector", Code::BadShape)), + } + } + + fn steps(&self) -> Vec { + let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); + let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; + match self { + Operation::PackageInstall { packages } => { + let mut install: Vec = words(&["apt-get", "install", "-y", "-q", "--"]); + install.extend(packages.iter().cloned()); + vec![ + Step { + argv: words(&["apt-get", "update", "-q"]), + environment: apt_environment.clone(), + }, + Step { + argv: install, + environment: apt_environment, + }, + ] + } + Operation::UnitAction { verb, unit } => vec![Step { + argv: vec![ + "systemctl".into(), + verb.as_str().into(), + "--".into(), + unit.clone(), + ], + environment: Vec::new(), + }], + Operation::Deploy { .. } => Vec::new(), + } + } + + fn describe(&self) -> String { + match self { + Operation::PackageInstall { packages } => { + format!("package-install {}", packages.join(" ")) + } + Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), + Operation::Deploy { selector } => format!("deploy {selector}"), + } + } +} + +// ---------------------------------------------------------------- records -- + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] +pub(crate) struct OrderRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) principal: String, + #[cultcache(key = 3)] + pub(crate) issuer_uid: u32, + #[cultcache(key = 4)] + pub(crate) issuer_unit: String, + #[cultcache(key = 5)] + pub(crate) issuer_invocation: String, + #[cultcache(key = 6)] + pub(crate) issued_at_unix_millis: u64, + #[cultcache(key = 7)] + pub(crate) operation: Operation, + #[cultcache(key = 8)] + pub(crate) reason: String, +} + +impl fmt::Debug for OrderRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OrderRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_start", + schema = "idunn.steward_order_start.v1" +)] +pub(crate) struct StartRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) started_at_unix_millis: u64, + /// The transient unit name, or the deployment command id. + #[cultcache(key = 3)] + pub(crate) actuation: String, +} + +impl fmt::Debug for StartRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StartRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Via { + Name, + Alias, + Propagation, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Refused { + AlwaysRefused { unit: String, via: Via }, + Braked, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Outcome { + Ran, + Failed { + exit_code: Option, + signal: Option, + }, + Refused(Refused), + Lost, + Submitted { command_id: String }, +} + +impl Outcome { + fn describe(&self) -> String { + match self { + Outcome::Ran => "ran (exit 0)".into(), + Outcome::Failed { + exit_code: Some(code), + .. + } => format!("failed (exit {code})"), + Outcome::Failed { + signal: Some(signal), + .. + } => format!("failed (signal {signal})"), + Outcome::Failed { .. } => "failed (not run to an exit)".into(), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( + "refused: always-refused unit {} reached by {}", + escape(unit.as_bytes()), + match via { + Via::Name => "name", + Via::Alias => "alias", + Via::Propagation => "propagation", + } + ), + Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Lost => "lost: the orders service restarted mid-run".into(), + Outcome::Submitted { command_id } => { + format!("submitted deployment {}", escape(command_id.as_bytes())) + } + } + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_outcome", + schema = "idunn.steward_order_outcome.v1" +)] +pub(crate) struct OutcomeRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) ended_at_unix_millis: u64, + #[cultcache(key = 3)] + pub(crate) outcome: Outcome, + #[cultcache(key = 4)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 5)] + pub(crate) stderr_tail: Vec, +} + +/// Tails stay out of `Debug`: they are raw program output. +impl fmt::Debug for OutcomeRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutcomeRecord") + .field("order_id", &self.order_id) + .field("outcome", &self.outcome) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum PostKind { + Accepted, + Result, +} + +impl PostKind { + fn as_str(self) -> &'static str { + match self { + PostKind::Accepted => "accepted", + PostKind::Result => "result", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_post", + schema = "idunn.steward_order_post.v1" +)] +pub(crate) struct PostRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `:`. + #[cultcache(key = 1)] + pub(crate) post_key: String, + #[cultcache(key = 2)] + pub(crate) event_id: String, + #[cultcache(key = 3)] + pub(crate) posted_at_unix_millis: u64, +} + +// ------------------------------------------------------------------ store -- + +/// `orders.cc`: root-only, written by `idunn orders serve` alone. +pub(crate) struct OrderStore { + directory: PathBuf, +} + +#[derive(Default)] +struct Ledger { + orders: Vec, + starts: Vec, + outcomes: Vec, + posts: Vec, +} + +impl Ledger { + fn start_of(&self, order_id: &str) -> Option<&StartRecord> { + self.starts.iter().find(|start| start.order_id == order_id) + } + + fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { + self.outcomes + .iter() + .find(|outcome| outcome.order_id == order_id) + } + + fn posted(&self, order_id: &str, kind: PostKind) -> bool { + let key = post_key(order_id, kind); + self.posts.iter().any(|post| post.post_key == key) + } +} + +fn post_key(order_id: &str, kind: PostKind) -> String { + format!("{order_id}:{}", kind.as_str()) +} + +impl OrderStore { + pub(crate) fn new(directory: &Path) -> Self { + Self { + directory: directory.to_owned(), + } + } + + fn path(&self) -> PathBuf { + self.directory.join("orders.cc") + } + + fn prepare(&self) -> Result<()> { + std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; + std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) + .context("making the orders directory private") + } + + /// Insert a record whose (type, key) is absent. `false` means it was there. + fn put(&self, key: &str, schema: &str, value: &T) -> Result { + self.prepare()?; + let path = self.path(); + let now = now_millis()?; + let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: T::TYPE.into(), + key: key.into(), + current: None, + }], + &[typed_envelope(key, T::TYPE, schema, value, now)?], + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .context("making the orders store private")?; + Ok(inserted) + } + + fn ledger(&self) -> Result { + let path = self.path(); + let mut ledger = Ledger::default(); + if !path.exists() { + return Ok(ledger); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .context("reading the orders store")?; + for envelope in &envelopes { + let schema = envelope.schema_id.as_deref(); + let key = envelope.key.as_str(); + match envelope.r#type.as_str() { + OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { + let record: OrderRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an order is stored under another key"); + ledger.orders.push(record); + } + StartRecord::TYPE if schema == Some(START_SCHEMA) => { + let record: StartRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "a start is stored under another key"); + ledger.starts.push(record); + } + OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { + let record: OutcomeRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an outcome is stored under another key"); + ledger.outcomes.push(record); + } + PostRecord::TYPE if schema == Some(POST_SCHEMA) => { + let record: PostRecord = decode_record(envelope)?; + ensure!(record.post_key == key, "a post is stored under another key"); + ledger.posts.push(record); + } + _ => bail!("the orders store holds a document that is not an order record"), + } + } + ledger + .orders + .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); + Ok(ledger) + } +} + +// ------------------------------------------------------------------ ports -- + +/// What `systemctl show` says about one unit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct UnitFacts { + pub(crate) id: String, + pub(crate) names: Vec, + pub(crate) required_by: Vec, + pub(crate) bound_by: Vec, + pub(crate) consists_of: Vec, +} + +impl UnitFacts { + fn identities(&self) -> impl Iterator { + std::iter::once(&self.id).chain(self.names.iter()) + } + + /// The units systemd stops with this one. + fn dependents(&self) -> impl Iterator { + self.required_by + .iter() + .chain(self.bound_by.iter()) + .chain(self.consists_of.iter()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RunReport { + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stderr_tail: Vec, +} + +pub(crate) trait Systemd: Send + Sync { + fn show(&self, unit: &str) -> Result; + fn invocation(&self, unit: &str) -> Option; + /// Run one step as the transient unit `unit`, to its end. + fn run(&self, unit: &str, step: &Step) -> Result; +} + +pub(crate) struct Peer { + pub(crate) uid: u32, + pub(crate) issuer_unit: String, +} + +pub(crate) trait PeerCredentials: Send + Sync { + fn peer(&self, stream: &UnixStream) -> Result; +} + +pub(crate) trait BrakeCheck: Send + Sync { + /// `true` only when the instance's brake check exits 0. + fn clear(&self, principal: &str) -> bool; +} + +pub(crate) trait Homeserver: Send + Sync { + /// Idempotent by `txn_id`; returns the event id. + fn post(&self, txn_id: &str, text: &str) -> Result; +} + +pub(crate) struct Ctx { + pub(crate) store: OrderStore, + pub(crate) state_store: PathBuf, + pub(crate) steward_uid: u32, + pub(crate) principal: String, + pub(crate) systemd: Arc, + pub(crate) peers: Arc, + pub(crate) brake: Arc, + pub(crate) homeserver: Arc, +} + +// ------------------------------------------------------------------ scope -- + +fn refused(unit: &str, via: Via) -> Refused { + Refused::AlwaysRefused { + unit: unit.to_owned(), + via, + } +} + +/// Would `verb` on `unit` reach an always-refused unit: by its name, by an +/// alias it resolves from, or, for the verbs systemd propagates, by the units +/// systemd stops with it. Resolution reads systemd before anything runs. +fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { + if is_refused_name(unit) { + return Ok(Some(refused(unit, Via::Name))); + } + let facts = systemd.show(unit)?; + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Alias))); + } + if !verb.propagates() { + return Ok(None); + } + let mut seen: BTreeSet = facts.identities().cloned().collect(); + seen.insert(unit.to_owned()); + let mut queue: VecDeque = facts.dependents().cloned().collect(); + while let Some(next) = queue.pop_front() { + if is_refused_name(&next) { + return Ok(Some(refused(&next, Via::Propagation))); + } + if !seen.insert(next.clone()) { + continue; + } + ensure!( + seen.len() <= CLOSURE_LIMIT, + "the units a stop would reach are too many to check" + ); + let facts = systemd.show(&next)?; + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Propagation))); + } + seen.extend(facts.identities().cloned()); + queue.extend(facts.dependents().cloned()); + } + Ok(None) +} + +// -------------------------------------------------------------- execution -- + +fn tail(bytes: &[u8]) -> Vec { + bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() +} + +fn conclude( + ctx: &Ctx, + order_id: &str, + outcome: Outcome, + stdout_tail: &[u8], + stderr_tail: &[u8], +) -> Result<()> { + ctx.store.put( + order_id, + OUTCOME_SCHEMA, + &OutcomeRecord { + schema_version: OUTCOME_SCHEMA.into(), + order_id: order_id.into(), + ended_at_unix_millis: now_millis()?, + outcome, + stdout_tail: tail(stdout_tail), + stderr_tail: tail(stderr_tail), + }, + )?; + Ok(()) +} + +fn order_uuid(order_id: &str) -> &str { + order_id.strip_prefix("ord-").unwrap_or(order_id) +} + +pub(crate) fn requester(principal: &str, order_id: &str) -> String { + format!("steward:{principal} order {order_id}") +} + +/// Run one accepted order. A start record is written before anything is +/// actuated and an order that has one is never run again. +pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { + let ledger = ctx.store.ledger()?; + let order = ledger + .orders + .iter() + .find(|order| order.order_id == order_id) + .context("the order is not in the store")?; + if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { + return Ok(()); + } + if order.operation.validate().is_err() { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the stored order no longer satisfies the grammar", + ); + } + if let Operation::UnitAction { verb, unit } = &order.operation { + match refusal_for(ctx.systemd.as_ref(), unit, *verb) { + Ok(None) => {} + Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), + Err(_) => { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the unit could not be resolved, so it was not acted on", + ); + } + } + } + if !ctx.brake.clear(&order.principal) { + return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); + } + let uuid = order_uuid(order_id); + let command_id = format!("up-{uuid}"); + let steps = order.operation.steps(); + let actuation = match &order.operation { + Operation::Deploy { .. } => command_id.clone(), + _ => format!("idunn-order-{uuid}"), + }; + let started = StartRecord { + schema_version: START_SCHEMA.into(), + order_id: order_id.into(), + started_at_unix_millis: now_millis()?, + actuation, + }; + if !ctx.store.put(order_id, START_SCHEMA, &started)? { + return Ok(()); + } + if let Operation::Deploy { selector } = &order.operation { + let outcome = match commit_deployment_command( + &ctx.state_store, + &command_id, + selector, + &requester(&order.principal, order_id), + ) { + Ok(_) => Outcome::Submitted { command_id }, + Err(_) => Outcome::Failed { + exit_code: None, + signal: None, + }, + }; + return conclude(ctx, order_id, outcome, b"", b""); + } + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let mut outcome = Outcome::Ran; + for (index, step) in steps.iter().enumerate() { + let unit = if steps.len() == 1 { + format!("idunn-order-{uuid}") + } else { + format!("idunn-order-{uuid}-{}", index + 1) + }; + match ctx.systemd.run(&unit, step) { + Ok(report) => { + stdout = report.stdout_tail; + stderr = report.stderr_tail; + if report.exit_code != Some(0) { + outcome = Outcome::Failed { + exit_code: report.exit_code, + signal: report.signal, + }; + break; + } + } + Err(_) => { + outcome = Outcome::Failed { + exit_code: None, + signal: None, + }; + stderr = b"the transient unit could not be started".to_vec(); + break; + } + } + } + conclude(ctx, order_id, outcome, &stdout, &stderr) +} + +/// On service start: an order with a start record and no outcome ran, or began +/// to, in a life that ended. It is Lost and never run again. Returns the +/// orders that never started, oldest first. +pub(crate) fn recover(ctx: &Ctx) -> Result> { + let ledger = ctx.store.ledger()?; + for start in &ledger.starts { + if ledger.outcome_of(&start.order_id).is_none() { + conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; + } + } + Ok(ledger + .orders + .iter() + .filter(|order| { + ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() + }) + .map(|order| order.order_id.clone()) + .collect()) +} + +// ---------------------------------------------------------------- posting -- + +struct Due { + order_id: String, + kind: PostKind, + text: String, +} + +fn quoted(reason: &str) -> String { + format!("\"{}\"", escape(reason.as_bytes())) +} + +fn accepted_text(order: &OrderRecord) -> String { + format!( + "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", + order.order_id, + escape(order.operation.describe().as_bytes()), + escape(order.issuer_unit.as_bytes()), + escape(order.issuer_invocation.as_bytes()), + quoted(&order.reason) + ) +} + +/// A deploy's command has finished once its state is neither queued nor running. +fn deploy_finished(state: &str) -> bool { + !matches!(state, "queued" | "running") +} + +fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { + let mut due = Vec::new(); + for order in &ledger.orders { + let id = &order.order_id; + if !ledger.posted(id, PostKind::Accepted) { + due.push(Due { + order_id: id.clone(), + kind: PostKind::Accepted, + text: accepted_text(order), + }); + } + if ledger.posted(id, PostKind::Result) { + continue; + } + let Some(record) = ledger.outcome_of(id) else { + continue; + }; + let mut line = record.outcome.describe(); + if let Outcome::Submitted { command_id } = &record.outcome { + match command_state(&ctx.state_store, command_id) { + Ok((state, detail)) if deploy_finished(state) => { + line = format!("{line}: {state} {}", escape(detail.as_bytes())); + } + _ => continue, + } + } + due.push(Due { + order_id: id.clone(), + kind: PostKind::Result, + text: format!( + "Steward order {id}: {line}\n{}", + escape(order.operation.describe().as_bytes()) + ), + }); + } + due +} + +/// Post what is due, in order, stopping at the first failure. Returns how many +/// went out. +pub(crate) fn post_pass(ctx: &Ctx) -> Result { + let ledger = ctx.store.ledger()?; + let mut sent = 0; + for due in due_posts(ctx, &ledger) { + let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); + let event_id = ctx.homeserver.post(&transaction, &due.text)?; + ctx.store.put( + &post_key(&due.order_id, due.kind), + POST_SCHEMA, + &PostRecord { + schema_version: POST_SCHEMA.into(), + post_key: post_key(&due.order_id, due.kind), + event_id, + posted_at_unix_millis: now_millis()?, + }, + )?; + sent += 1; + } + Ok(sent) +} + +// ----------------------------------------------------------------- frames -- + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct OrderRequest { + pub(crate) operation: Operation, + pub(crate) reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ShowRequest { + pub(crate) id: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Request { + Order(OrderRequest), + Show(ShowRequest), + List, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderView { + pub(crate) order: OrderRecord, + pub(crate) start: Option, + pub(crate) outcome: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderLine { + pub(crate) order_id: String, + pub(crate) status: String, + pub(crate) summary: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Response { + Accepted { order_id: String }, + Refused(Refusal), + Show(Box), + List(Vec), +} + +fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { + let bytes = rmp_serde::to_vec_named(value)?; + stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; + stream.write_all(&bytes)?; + Ok(stream.flush()?) +} + +fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { + let mut length = [0u8; 4]; + stream.read_exact(&mut length)?; + let length = u32::from_be_bytes(length) as usize; + ensure!(length <= limit, "a frame exceeds its limit"); + let mut bytes = vec![0u8; length]; + stream.read_exact(&mut bytes)?; + Ok(bytes) +} + +fn status_word(view: &OrderView) -> &'static str { + match (&view.outcome, &view.start) { + (Some(record), _) => match record.outcome { + Outcome::Ran => "ran", + Outcome::Failed { .. } => "failed", + Outcome::Refused(_) => "refused", + Outcome::Lost => "lost", + Outcome::Submitted { .. } => "submitted", + }, + (None, Some(_)) => "running", + (None, None) => "accepted", + } +} + +fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { + OrderView { + order: order.clone(), + start: ledger.start_of(&order.order_id).cloned(), + outcome: ledger.outcome_of(&order.order_id).cloned(), + } +} + +fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { + write_frame(stream, &response) +} + +fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { + answer(stream, Response::Refused(Refusal::new(field, code))) +} + +/// Serve one connection. The peer is authenticated before its frame is read. +pub(crate) fn handle( + ctx: &Ctx, + stream: &mut UnixStream, + queue: &mpsc::Sender, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + let peer = ctx.peers.peer(stream)?; + if peer.uid != ctx.steward_uid { + eprintln!("steward socket: refused a peer that is not the steward"); + return refuse(stream, "peer", Code::Unauthorized); + } + let request = read_frame(stream, REQUEST_FRAME_LIMIT) + .ok() + .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); + let Some(request) = request else { + eprintln!("steward socket: refused a malformed frame"); + return refuse(stream, "frame", Code::Malformed); + }; + match request { + Request::Order(request) => { + let checked = + validate_reason(&request.reason).and_then(|()| request.operation.validate()); + if let Err(refusal) = checked { + eprintln!("steward socket: refused an order at the grammar"); + return answer(stream, Response::Refused(refusal)); + } + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: format!("ord-{}", Uuid::new_v4()), + principal: ctx.principal.clone(), + issuer_uid: peer.uid, + issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), + issuer_unit: peer.issuer_unit, + issued_at_unix_millis: now_millis()?, + operation: request.operation, + reason: request.reason, + }; + match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { + Ok(true) => {} + _ => return refuse(stream, "store", Code::Unavailable), + } + queue + .send(order.order_id.clone()) + .map_err(|_| anyhow!("the executor has stopped"))?; + answer( + stream, + Response::Accepted { + order_id: order.order_id, + }, + ) + } + Request::Show(request) => { + if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) + { + return refuse(stream, "id", Code::BadShape); + } + let ledger = ctx.store.ledger()?; + match ledger.orders.iter().find(|order| order.order_id == request.id) { + Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), + None => refuse(stream, "id", Code::Unknown), + } + } + Request::List => { + let ledger = ctx.store.ledger()?; + let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); + let lines = ledger + .orders + .iter() + .skip(skip) + .map(|order| { + let view = view_of(&ledger, order); + OrderLine { + order_id: order.order_id.clone(), + status: status_word(&view).into(), + summary: order.operation.describe(), + } + }) + .collect(); + answer(stream, Response::List(lines)) + } + } +} + +// ------------------------------------------------------------- real ports -- + +/// `systemctl` and `systemd-run`, found by name. +pub(crate) struct HostSystemd; + +fn parse_names(value: &str) -> Vec { + value.split_whitespace().map(str::to_owned).collect() +} + +fn parse_show(output: &str) -> Result { + let mut facts = UnitFacts::default(); + for line in output.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + match key { + "Id" => facts.id = value.to_owned(), + "Names" => facts.names = parse_names(value), + "RequiredBy" => facts.required_by = parse_names(value), + "BoundBy" => facts.bound_by = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), + _ => {} + } + } + ensure!(!facts.id.is_empty(), "systemctl named no unit"); + Ok(facts) +} + +impl Systemd for HostSystemd { + fn show(&self, unit: &str) -> Result { + let output = Command::new("systemctl") + .args([ + "show", + "--no-pager", + "--property=Id,Names,RequiredBy,BoundBy,ConsistsOf", + "--", + unit, + ]) + .stdin(Stdio::null()) + .output() + .context("running systemctl show")?; + ensure!(output.status.success(), "systemctl show failed"); + parse_show(&String::from_utf8_lossy(&output.stdout)) + } + + fn invocation(&self, unit: &str) -> Option { + let output = Command::new("systemctl") + .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) + .arg(unit) + .stdin(Stdio::null()) + .output() + .ok()?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + (output.status.success() && !value.is_empty()).then_some(value) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + let mut command = Command::new("systemd-run"); + command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); + command.args(["-p", "RuntimeMaxSec=15min"]); + for (name, value) in &step.environment { + command.arg(format!("--setenv={name}={value}")); + } + let output = command + .arg("--") + .args(&step.argv) + .stdin(Stdio::null()) + .output() + .context("running systemd-run")?; + Ok(RunReport { + exit_code: output.status.code(), + signal: output.status.signal(), + stdout_tail: tail(&output.stdout), + stderr_tail: tail(&output.stderr), + }) + } +} + +/// The last `.service` or `.scope` in a process's cgroup path names its issuer. +fn unit_from_cgroup(text: &str) -> String { + let line = text + .lines() + .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) + .unwrap_or_default(); + let path = line.rsplit_once(':').map_or("", |(_, path)| path); + path.rsplit('/') + .find(|segment| segment.ends_with(".service") || segment.ends_with(".scope")) + .filter(|segment| segment.len() <= UNIT_LIMIT) + .map_or_else(|| "unknown".to_owned(), str::to_owned) +} + +/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. +pub(crate) struct KernelPeers; + +impl PeerCredentials for KernelPeers { + fn peer(&self, stream: &UnixStream) -> Result { + use std::os::fd::AsRawFd; + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = std::mem::size_of::() as libc::socklen_t; + // SAFETY: the descriptor is a live socket and the pointer and length + // describe `credentials`, which outlives the call. + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&mut credentials as *mut libc::ucred).cast(), + &mut length, + ) + }; + ensure!( + status == 0 && length as usize == std::mem::size_of::(), + "peer credentials are unavailable" + ); + let cgroup = + std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); + Ok(Peer { + uid: credentials.uid, + issuer_unit: unit_from_cgroup(&cgroup), + }) + } +} + +/// `mind-control --instance check`; a missing binary is a held brake. +pub(crate) struct MindControl { + pub(crate) program: PathBuf, +} + +impl BrakeCheck for MindControl { + fn clear(&self, principal: &str) -> bool { + Command::new(&self.program) + .args(["--instance", principal, "check"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} + +/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is +/// never chunked. The token is read from its credential file for each post. +pub(crate) struct Synapse { + pub(crate) address: std::net::SocketAddr, + pub(crate) room: String, + pub(crate) token_file: PathBuf, +} + +fn percent_encode(text: &str) -> String { + let mut encoded = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +impl Homeserver for Synapse { + fn post(&self, txn_id: &str, text: &str) -> Result { + let token = std::fs::read_to_string(&self.token_file) + .context("the homeserver token is unreadable")?; + let token = token.trim(); + ensure!( + !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), + "the homeserver token is not a bearer token" + ); + let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; + let head = format!( + "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ + Host: {}\r\nAuthorization: Bearer {token}\r\n\ + Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", + percent_encode(&self.room), + self.address, + body.len() + ); + let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) + .context("the homeserver is unreachable")?; + socket.set_read_timeout(Some(Duration::from_secs(10)))?; + socket.set_write_timeout(Some(Duration::from_secs(10)))?; + socket.write_all(head.as_bytes())?; + socket.write_all(&body)?; + let mut reply = Vec::new(); + socket + .take(RESPONSE_FRAME_LIMIT as u64) + .read_to_end(&mut reply) + .context("the homeserver reply was cut short")?; + let split = reply + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("the homeserver reply has no body")?; + let status = String::from_utf8_lossy(&reply[..split]); + ensure!( + status.split_whitespace().nth(1) == Some("200"), + "the homeserver refused the post" + ); + #[derive(Deserialize)] + struct Sent { + event_id: String, + } + let sent: Sent = serde_json::from_slice(&reply[split + 4..]) + .map_err(|_| anyhow!("the homeserver reply has no event id"))?; + Ok(sent.event_id) + } +} + +// ---------------------------------------------------------------- binding -- + +/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. +/// It lists no unit of any kind, and any field outside this set is refused. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct StewardBinding { + pub(crate) principal: String, + pub(crate) user: String, + pub(crate) mind_control: PathBuf, + pub(crate) room: String, + pub(crate) homeserver: String, + pub(crate) token_credential: String, +} + +impl StewardBinding { + pub(crate) fn parse(text: &str) -> Result { + let binding: Self = toml::from_str(text) + .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; + binding.validate()?; + Ok(binding) + } + + fn validate(&self) -> Result<()> { + require_id(&self.principal, "steward principal")?; + ensure!( + !self.principal.starts_with('-') && !self.principal.contains('/'), + "steward principal is invalid" + ); + let user_ok = !self.user.is_empty() + && self.user.len() <= 32 + && self.user.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) + && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); + ensure!(user_ok && self.user != "root", "steward user is invalid or root"); + ensure!( + self.mind_control.is_absolute(), + "mind_control must be an absolute path" + ); + let room_ok = self.room.starts_with('!') + && self.room.len() <= 255 + && self.room.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') + }); + ensure!(room_ok, "room must be a Matrix room id"); + self.address()?; + let credential_ok = !self.token_credential.is_empty() + && !self.token_credential.starts_with('.') + && self.token_credential.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') + }); + ensure!(credential_ok, "token_credential is not a credential name"); + Ok(()) + } + + fn address(&self) -> Result { + let address: std::net::SocketAddr = self + .homeserver + .strip_prefix("http://") + .and_then(|rest| rest.parse().ok()) + .context("homeserver must be http://:")?; + ensure!(address.ip().is_loopback(), "homeserver must be loopback"); + Ok(address) + } +} + +/// Root-admitted: owned by root and writable by no one else. +fn admitted(owner: u32, mode: u32) -> bool { + owner == 0 && mode & 0o022 == 0 +} + +fn load_binding(path: &Path) -> Result { + let metadata = std::fs::metadata(path).context("reading the steward binding")?; + ensure!( + metadata.is_file() && admitted(metadata.uid(), metadata.mode()), + "the steward binding is not a root-owned file closed to writes" + ); + let text = std::fs::read_to_string(path).context("reading the steward binding")?; + StewardBinding::parse(&text) +} + +fn user_uid(name: &str) -> Result { + let name = std::ffi::CString::new(name)?; + // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in + // `buffer`, which outlives every read of them below (none are read). + let (status, found, uid) = unsafe { + let mut passwd: libc::passwd = std::mem::zeroed(); + let mut buffer = vec![0 as libc::c_char; 4096]; + let mut result: *mut libc::passwd = std::ptr::null_mut(); + let status = libc::getpwnam_r( + name.as_ptr(), + &mut passwd, + buffer.as_mut_ptr(), + buffer.len(), + &mut result, + ); + (status, !result.is_null(), passwd.pw_uid) + }; + ensure!(status == 0 && found, "the steward user is unknown"); + ensure!(uid != 0, "the steward user resolves to root"); + Ok(uid) +} + +// ------------------------------------------------------------------ serve -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ServeOptions { + pub(crate) state_store: PathBuf, + pub(crate) store: PathBuf, + pub(crate) steward_binding: PathBuf, + pub(crate) socket: PathBuf, +} + +const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; +const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; +const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; + +pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { + ensure!( + args.next().as_deref() == Some("serve"), + "idunn orders takes the subcommand serve" + ); + let mut options = ServeOptions { + state_store: default_state_store(), + store: DEFAULT_STORE.into(), + steward_binding: DEFAULT_BINDING.into(), + socket: DEFAULT_SOCKET.into(), + }; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, + "--store" => options.store = path_value(&mut args, "--store")?, + "--steward-binding" => { + options.steward_binding = path_value(&mut args, "--steward-binding")? + } + "--socket" => options.socket = path_value(&mut args, "--socket")?, + _ => bail!("an orders serve option is not part of the grammar"), + } + } + Ok(options) +} + +/// Run a thread whose end ends the service: systemd restarts it, and the +/// start recovery settles what it left. +fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { + struct ExitOnDrop(&'static str); + impl Drop for ExitOnDrop { + fn drop(&mut self) { + eprintln!("steward orders: the {} thread ended", self.0); + std::process::exit(1); + } + } + thread::Builder::new() + .name(name.into()) + .spawn(move || { + let _guard = ExitOnDrop(name); + work(); + }) + .expect("a thread can be spawned"); +} + +fn bind_socket(path: &Path) -> Result { + if let Ok(metadata) = std::fs::symlink_metadata(path) { + use std::os::unix::fs::FileTypeExt; + ensure!( + metadata.file_type().is_socket(), + "the socket path holds something that is not a socket" + ); + std::fs::remove_file(path).context("removing a stale socket")?; + } + let listener = UnixListener::bind(path).context("binding the steward socket")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) + .context("setting the steward socket mode")?; + Ok(listener) +} + +pub(crate) fn serve(options: ServeOptions) -> Result<()> { + let binding = load_binding(&options.steward_binding)?; + let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") + .map(PathBuf::from) + .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; + let ctx = Arc::new(Ctx { + store: OrderStore::new(&options.store), + state_store: options.state_store, + steward_uid: user_uid(&binding.user)?, + principal: binding.principal.clone(), + systemd: Arc::new(HostSystemd), + peers: Arc::new(KernelPeers), + brake: Arc::new(MindControl { + program: binding.mind_control.clone(), + }), + homeserver: Arc::new(Synapse { + address: binding.address()?, + room: binding.room.clone(), + token_file: credentials.join(&binding.token_credential), + }), + }); + ctx.store.prepare()?; + let pending = recover(&ctx)?; + let listener = bind_socket(&options.socket)?; + let (queue, orders) = mpsc::channel::(); + for order_id in pending { + queue.send(order_id)?; + } + let executor = Arc::clone(&ctx); + vital("executor", move || { + for order_id in orders { + if let Err(error) = execute(&executor, &order_id) { + eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); + } + } + }); + let poster = Arc::clone(&ctx); + vital("poster", move || { + let mut failures = 0u32; + loop { + let pause = match post_pass(&poster) { + Ok(_) => { + failures = 0; + 2 + } + Err(error) => { + failures = failures.saturating_add(1); + eprintln!("steward orders: posting is delayed: {error:#}"); + (2u64 << failures.min(5)).min(60) + } + }; + thread::sleep(Duration::from_secs(pause)); + } + }); + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + if let Err(error) = handle(&ctx, &mut stream, &queue) { + eprintln!("steward socket: a connection ended in error: {error:#}"); + } + } + Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), + } + } + bail!("the steward socket stopped accepting") +} + +// ----------------------------------------------------------------- client -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OrderAction { + Issue { operation: Operation, reason: String }, + Show { id: String }, + List, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct OrderCommand { + pub(crate) socket: PathBuf, + pub(crate) action: OrderAction, +} + +pub(crate) fn parse_order(mut args: impl Iterator) -> Result { + let kind = args.next().context("idunn order requires an operation")?; + let mut socket = PathBuf::from(DEFAULT_SOCKET); + let mut reason = None; + let mut positional = Vec::new(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--socket" => socket = path_value(&mut args, "--socket")?, + "--reason" => reason = Some(string_value(&mut args, "--reason")?), + word if word.starts_with('-') => bail!("an order option is not part of the grammar"), + _ => positional.push(argument), + } + } + let action = match kind.as_str() { + "list" => { + ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); + OrderAction::List + } + "show" => { + ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); + OrderAction::Show { + id: positional.remove(0), + } + } + "package-install" | "unit" | "deploy" => { + let operation = match kind.as_str() { + "package-install" => Operation::PackageInstall { + packages: positional, + }, + "unit" => { + ensure!(positional.len() == 2, "unit takes a verb and a unit"); + let verb = Verb::parse(&positional[0]) + .context("the unit verb is not start, stop, restart or reload")?; + Operation::UnitAction { + verb, + unit: positional.remove(1), + } + } + _ => { + ensure!(positional.len() == 1, "deploy takes one selector"); + Operation::Deploy { + selector: positional.remove(0), + } + } + }; + let reason = reason.context("an order requires --reason")?; + validate_reason(&reason)?; + operation.validate()?; + OrderAction::Issue { operation, reason } + } + _ => bail!("the order operation is not part of the grammar"), + }; + Ok(OrderCommand { socket, action }) +} + +fn call(socket: &Path, request: &Request) -> Result { + let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + write_frame(&mut stream, request)?; + let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; + rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) +} + +fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); + let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); +} + +/// Raw lines for the terminal; `emit` is their only way out. +fn render_view(view: &OrderView) -> Vec> { + let order = &view.order; + let mut lines: Vec> = vec![ + format!("order {}", order.order_id).into_bytes(), + format!("status {}", status_word(view)).into_bytes(), + format!("operation {}", order.operation.describe()).into_bytes(), + format!( + "issuer {} (invocation {}, uid {})", + order.issuer_unit, order.issuer_invocation, order.issuer_uid + ) + .into_bytes(), + format!("reason {}", quoted(&order.reason)).into_bytes(), + ]; + if let Some(start) = &view.start { + lines.push(format!("started {}", start.actuation).into_bytes()); + } + if let Some(record) = &view.outcome { + lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); + render_tail(&mut lines, "stdout", &record.stdout_tail); + render_tail(&mut lines, "stderr", &record.stderr_tail); + } + lines +} + +pub(crate) fn run_order(command: OrderCommand) -> Result<()> { + let request = match command.action { + OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), + OrderAction::Show { id } => Request::Show(ShowRequest { id }), + OrderAction::List => Request::List, + }; + let mut out = std::io::stdout().lock(); + match call(&command.socket, &request)? { + Response::Accepted { order_id } => writeln!(out, "{order_id}")?, + Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), + Response::Show(view) => emit(&mut out, &render_view(&view))?, + Response::List(lines) => { + let lines: Vec> = lines + .iter() + .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) + .collect(); + emit(&mut out, &lines)? + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tempfile::TempDir; + + use super::*; + + const STEWARD: u32 = 1000; + const CANARY: &str = "CANARY-7f3a91"; + + struct FakeSystemd { + units: Mutex>, + runs: Mutex>, + report: Mutex, + } + + impl Systemd for FakeSystemd { + fn show(&self, unit: &str) -> Result { + let known = self.units.lock().unwrap().get(unit).cloned(); + Ok(known.unwrap_or_else(|| UnitFacts { + id: unit.into(), + names: vec![unit.into()], + ..UnitFacts::default() + })) + } + + fn invocation(&self, _unit: &str) -> Option { + Some("invocation-1".into()) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + self.runs.lock().unwrap().push((unit.into(), step.clone())); + Ok(self.report.lock().unwrap().clone()) + } + } + + struct FakePeers(u32); + + impl PeerCredentials for FakePeers { + fn peer(&self, _stream: &UnixStream) -> Result { + Ok(Peer { + uid: self.0, + issuer_unit: "mind-eureka-self.service".into(), + }) + } + } + + struct FakeBrake(AtomicBool); + + impl BrakeCheck for FakeBrake { + fn clear(&self, _principal: &str) -> bool { + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct FakeHome { + failing: AtomicBool, + sent: Mutex>, + } + + impl Homeserver for FakeHome { + fn post(&self, txn_id: &str, text: &str) -> Result { + ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); + let mut sent = self.sent.lock().unwrap(); + sent.push((txn_id.into(), text.into())); + Ok(format!("$event{}", sent.len())) + } + } + + struct Harness { + directory: TempDir, + ctx: Ctx, + systemd: Arc, + brake: Arc, + home: Arc, + queue: mpsc::Sender, + _orders: mpsc::Receiver, + } + + fn harness_for(peer_uid: u32) -> Harness { + let directory = TempDir::new().unwrap(); + let systemd = Arc::new(FakeSystemd { + units: Mutex::new(BTreeMap::new()), + runs: Mutex::new(Vec::new()), + report: Mutex::new(RunReport { + exit_code: Some(0), + ..RunReport::default() + }), + }); + let brake = Arc::new(FakeBrake(AtomicBool::new(true))); + let home = Arc::new(FakeHome::default()); + let (queue, orders) = mpsc::channel(); + let ctx = Ctx { + store: OrderStore::new(&directory.path().join("orders")), + state_store: directory.path().join("state").join("control.cc"), + steward_uid: STEWARD, + principal: "eureka".into(), + systemd: systemd.clone(), + peers: Arc::new(FakePeers(peer_uid)), + brake: brake.clone(), + homeserver: home.clone(), + }; + Harness { + directory, + ctx, + systemd, + brake, + home, + queue, + _orders: orders, + } + } + + fn harness() -> Harness { + harness_for(STEWARD) + } + + impl Harness { + fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { + let (mut client, mut server) = UnixStream::pair().unwrap(); + write(&mut client); + handle(&self.ctx, &mut server, &self.queue).unwrap(); + let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); + rmp_serde::from_slice(&bytes).unwrap() + } + + fn ask(&self, request: &Request) -> Response { + self.exchange(|client| write_frame(client, request).unwrap()) + } + + fn issue(&self, operation: Operation) -> Response { + self.ask(&Request::Order(OrderRequest { + operation, + reason: "probe".into(), + })) + } + + fn issue_unit(&self, verb: Verb, unit: &str) -> Response { + self.issue(Operation::UnitAction { + verb, + unit: unit.into(), + }) + } + + fn accept(&self, response: Response) -> String { + match response { + Response::Accepted { order_id } => order_id, + other => panic!("expected an accepted order, got {other:?}"), + } + } + + fn outcome(&self, order_id: &str) -> Outcome { + self.ctx + .store + .ledger() + .unwrap() + .outcome_of(order_id) + .expect("the order has an outcome") + .outcome + .clone() + } + + fn runs(&self) -> Vec<(String, Step)> { + self.systemd.runs.lock().unwrap().clone() + } + + fn describe_unit(&self, name: &str, facts: UnitFacts) { + self.systemd.units.lock().unwrap().insert(name.into(), facts); + } + } + + fn all_verbs() -> [Verb; 4] { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + } + + #[test] + fn foreign_uid_is_refused_at_the_door() { + let h = harness_for(STEWARD + 1); + let response = h.issue_unit(Verb::Restart, "nginx.service"); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + let response = h.ask(&Request::List); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); + assert!(h.runs().is_empty()); + } + + #[derive(Serialize)] + struct ForgedBody { + operation: Operation, + reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + } + + #[derive(Serialize)] + struct ForgedFrame { + order: ForgedBody, + } + + #[test] + fn order_id_and_requester_are_minted_not_declared() { + let h = harness(); + let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { + let frame = ForgedFrame { + order: ForgedBody { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into(), + }, + reason: "probe".into(), + order_id: order_id.map(str::to_owned), + requested_by: requested_by.map(str::to_owned), + command: command.map(str::to_owned), + }, + }; + h.exchange(|client| write_frame(client, &frame).unwrap()) + }; + // The same frame without the extra field is a good order. + let honest = h.accept(forge(None, None, None)); + for forged in [ + forge(Some("ord-declared"), None, None), + forge(None, Some("steward:someone-else order x"), None), + forge(None, None, Some("reboot-now")), + ] { + assert_eq!( + forged, + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + } + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); + let order = &ledger.orders[0]; + assert_eq!(order.order_id, honest); + let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); + assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); + assert_eq!(order.issuer_unit, "mind-eureka-self.service"); + assert_eq!(order.issuer_invocation, "invocation-1"); + assert_eq!(order.principal, "eureka"); + assert_eq!(order.issuer_uid, STEWARD); + assert_eq!( + requester(&order.principal, &order.order_id), + format!("steward:eureka order {}", order.order_id) + ); + } + + #[test] + fn package_names_outside_grammar_are_refused() { + let h = harness(); + let bad = [ + "-y", + "--allow-unauthenticated", + "-", + "a", + "", + "Foo", + "a/b", + "/tmp/x", + "./x", + "x.deb", + "foo.deb", + "foo bar", + "foo;reboot", + "foo$(id)", + "foo`id`", + "foo|bar", + "foo&bar", + "foo\nbar", + "foo*", + "foo?", + "pkg:amd64", + "pkg/stable", + "libc6-", + "foo=", + "foo=1 2", + "foo=1;2", + "foo=1=2", + "fo\u{ff0f}o", + "f\u{43e}o", + ]; + for name in bad { + let response = h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into(), name.into()], + }); + assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); + } + let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); + assert_eq!( + h.issue(Operation::PackageInstall { packages: many }), + Response::Refused(Refusal::new("package", Code::TooMany)) + ); + assert_eq!( + h.issue(Operation::PackageInstall { packages: vec![] }), + Response::Refused(Refusal::new("package", Code::Empty)) + ); + assert!(!h.ctx.store.path().exists(), "no refusal left a record"); + + let good = [ + "git-lfs", + "matrix-synapse-py3", + "libnss-systemd", + "bubblewrap", + "socat", + "g++", + "libstdc++6", + "python3.11", + "nginx=1.24.0-2ubuntu7", + "foo=1:2.3-4+b1~rc1", + ]; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: good.iter().map(|name| (*name).to_owned()).collect(), + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + let runs = h.runs(); + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); + let mut install = vec!["apt-get", "install", "-y", "-q", "--"]; + install.extend(good); + assert_eq!(runs[1].1.argv, install); + for (unit, step) in &runs { + assert!(unit.starts_with("idunn-order-")); + assert_eq!( + step.environment, + [("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + ); + } + assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + } + + #[test] + fn any_unit_outside_the_refused_set_runs() { + let h = harness(); + let units = [ + "wg-quick@wg0.service", + "nginx.service", + "matrix-synapse.service", + "ssh.socket", + "huginn@eureka.service", + "claude-connector.service", + "persona-matrix-mouth.service", + "mind-heartbeat.service", + "apt-daily.timer", + "multi-user.target", + // Near misses: the prefix is the rule, not a substring or a stem. + "midunn-x.service", + "not-idunn-x.service", + "idunn.service", + "mind-self.service", + "mind-selfx@a.service", + "mind-persona.service", + "mind-personal@a.service", + ]; + for verb in all_verbs() { + for unit in units { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); + let (transient, step) = h.runs().pop().unwrap(); + assert!(transient.starts_with("idunn-order-")); + assert_eq!(step.argv, ["systemctl", verb.as_str(), "--", unit]); + } + } + assert_eq!(h.runs().len(), units.len() * 4); + } + + #[test] + fn refused_set_is_refused_by_name_alias_and_propagation() { + let h = harness(); + h.describe_unit( + "backup-name.service", + UnitFacts { + id: "idunn-yggdrasil.service".into(), + names: vec!["idunn-yggdrasil.service".into(), "backup-name.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "quiet-alias.service", + UnitFacts { + id: "quiet-alias.service".into(), + names: vec!["quiet-alias.service".into(), "mind-persona@void.service".into()], + ..UnitFacts::default() + }, + ); + // fuse <- wrapper <- (bound by) mind-self@eureka + h.describe_unit( + "fuse.service", + UnitFacts { + id: "fuse.service".into(), + names: vec!["fuse.service".into()], + required_by: vec!["wrapper.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "wrapper.service", + UnitFacts { + id: "wrapper.service".into(), + names: vec!["wrapper.service".into()], + bound_by: vec!["mind-self@eureka.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "stack.target", + UnitFacts { + id: "stack.target".into(), + names: vec!["stack.target".into()], + consists_of: vec!["mind-persona@void.service".into()], + ..UnitFacts::default() + }, + ); + // A cycle that reaches an alias of a refused unit on its second hop. + h.describe_unit( + "loop-a.service", + UnitFacts { + id: "loop-a.service".into(), + names: vec!["loop-a.service".into()], + required_by: vec!["loop-b.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "loop-b.service", + UnitFacts { + id: "loop-b.service".into(), + names: vec!["loop-b.service".into()], + required_by: vec!["loop-a.service".into(), "hidden.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "hidden.service", + UnitFacts { + id: "idunn-orders.service".into(), + names: vec!["idunn-orders.service".into(), "hidden.service".into()], + ..UnitFacts::default() + }, + ); + + let direct = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-x.service", + "idunn-order-3f2c.service", + "idunn-yggdrasil.timer", + "idunn-x.socket", + "idunn-.service", + "idunn-yggdrasil.service.service", + "idunn-x@y.service", + "mind-self@eureka.service", + "mind-persona@void.service", + "mind-self@.service", + "mind-persona@a@b.service", + // Case folds: a lookalike is still refused, never run. + "IDUNN-yggdrasil.service", + "Idunn-Orders.service", + "MIND-SELF@eureka.service", + "Mind-Persona@Void.service", + ]; + for verb in all_verbs() { + for unit in direct { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + match h.outcome(&id) { + Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { + assert_eq!(hit, unit, "{verb:?} {unit}"); + assert_eq!(via, Via::Name); + } + other => panic!("{verb:?} {unit} was {other:?}"), + } + } + } + for verb in all_verbs() { + for (unit, hit) in [ + ("backup-name.service", "idunn-yggdrasil.service"), + ("quiet-alias.service", "mind-persona@void.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Alias + }), + "{verb:?} {unit}" + ); + } + } + for verb in [Verb::Stop, Verb::Restart] { + for (unit, hit) in [ + ("fuse.service", "mind-self@eureka.service"), + ("stack.target", "mind-persona@void.service"), + ("loop-a.service", "idunn-orders.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Propagation + }), + "{verb:?} {unit}" + ); + } + } + assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); + // start and reload propagate nothing, so the same units are not refused. + for verb in [Verb::Start, Verb::Reload] { + let id = h.accept(h.issue_unit(verb, "fuse.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + } + // Every refusal is recorded and has a result post due. + let ledger = h.ctx.store.ledger().unwrap(); + let refusals = ledger + .outcomes + .iter() + .filter(|record| matches!(record.outcome, Outcome::Refused(_))) + .count(); + assert_eq!(refusals, direct.len() * 4 + 8 + 6); + let due = due_posts(&h.ctx, &ledger); + let results = due + .iter() + .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) + .count(); + assert_eq!(results, refusals); + // A refused order never started. + assert!(ledger.starts.len() == 2); + } + + #[test] + fn globs_slices_and_scopes_are_not_units() { + let h = harness(); + let bad = [ + "idunn*", + "idunn-*.service", + "idunn-?ggdrasil.service", + "mind-?elf@eureka.service", + "mind-self@*.service", + "mind-[s]elf@eureka.service", + "system.slice", + "user.slice", + "session-1.scope", + "init.scope", + "idunn-yggdrasil", + "mind-self@eureka", + "nginx", + ".service", + "", + "-idunn-x.service", + "-x.service", + "/etc/systemd/system/idunn-yggdrasil.service", + "../idunn-yggdrasil.service", + "idunn-yggdrasil.service\n", + "idunn-yggdrasil.service ", + " idunn-yggdrasil.service", + "idunn-yggdrasil.service\0", + "idunn-yggdrasil.service;reboot", + "idunn\\x2dyggdrasil.service", + "idunn%2dyggdrasil.service", + // Unicode lookalikes of a refused name. + "\u{131}dunn-yggdrasil.service", + "idunn\u{2010}yggdrasil.service", + "idunn\u{ff0d}yggdrasil.service", + "idunn-yggdrasil.service\u{200b}", + "\u{200b}idunn-yggdrasil.service", + "idunn-yggdrasil\u{2024}service", + "mind-self\u{ff20}eureka.service", + ]; + for unit in bad { + for verb in all_verbs() { + assert!( + matches!(h.issue_unit(verb, unit), Response::Refused(_)), + "{unit:?} was not refused by the grammar" + ); + } + } + let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); + assert_eq!( + h.issue_unit(Verb::Start, &long), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); + } + + #[test] + fn binding_with_a_unit_list_is_refused() { + let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ + mind_control = \"/usr/local/bin/mind-control\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ + token_credential = \"idunn-matrix-token\"\n"; + let binding = StewardBinding::parse(valid).unwrap(); + assert_eq!(binding.principal, "eureka"); + assert_eq!(binding.address().unwrap().port(), 8008); + for extra in [ + "units = [\"nginx.service\"]", + "allow = [\"nginx.service\"]", + "deny = [\"idunn-*\"]", + "denylist = [\"idunn-*\"]", + "refused = [\"idunn-*\"]", + "unit = \"nginx.service\"", + "[units]\nnginx = true", + ] { + assert!( + StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), + "{extra} loaded" + ); + } + let swap = |from: &str, to: &str| valid.replace(from, to); + for broken in [ + swap("user = \"mind-eureka-self\"", "user = \"root\""), + swap("user = \"mind-eureka-self\"", "user = \"\""), + swap("/usr/local/bin/mind-control", "mind-control"), + swap("!abc:example.org", "#alias:example.org"), + swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), + swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), + swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), + swap("idunn-matrix-token", "../token"), + swap("principal = \"eureka\"\n", ""), + ] { + assert!(StewardBinding::parse(&broken).is_err()); + } + assert!(admitted(0, 0o100600)); + assert!(admitted(0, 0o100644)); + assert!(!admitted(0, 0o100664)); + assert!(!admitted(0, 0o100666)); + assert!(!admitted(0, 0o100602)); + assert!(!admitted(1000, 0o100600)); + } + + #[test] + fn held_brake_refuses_at_start() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); + assert!(h.runs().is_empty()); + // The deploy path is gated the same way and writes no command. + let deploy = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &deploy).unwrap(); + assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); + assert!(!h.ctx.state_store.exists()); + // A brake that cannot be asked is a held brake. + let directory = TempDir::new().unwrap(); + assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); + } + + #[test] + fn brake_check_is_the_exit_status_of_mind_control_check() { + let directory = TempDir::new().unwrap(); + let program = |name: &str, body: &str| { + let path = directory.path().join(name); + std::fs::write(&path, body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + MindControl { program: path } + }; + let recorder = directory.path().join("argv"); + let clear = program( + "clear", + &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), + ); + assert!(clear.clear("eureka")); + assert_eq!( + std::fs::read_to_string(&recorder).unwrap().trim(), + "--instance eureka check" + ); + assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); + } + + #[test] + fn started_order_is_never_rerun() { + let h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The first order began and its service died: a start with no outcome. + h.ctx + .store + .put( + &id, + START_SCHEMA, + &StartRecord { + schema_version: START_SCHEMA.into(), + order_id: id.clone(), + started_at_unix_millis: 1, + actuation: format!("idunn-order-{}", order_uuid(&id)), + }, + ) + .unwrap(); + let pending = recover(&h.ctx).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Lost); + assert_eq!(pending, [waiting.clone()], "an order that never started waits"); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); + // Recovery is idempotent, and a finished order is not run again. + assert_eq!(recover(&h.ctx).unwrap(), [waiting.clone()]); + execute(&h.ctx, &waiting).unwrap(); + execute(&h.ctx, &waiting).unwrap(); + assert_eq!(h.runs().len(), 1); + assert!(recover(&h.ctx).unwrap().is_empty()); + } + + #[test] + fn deploy_order_uses_the_up_primitive() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let command_id = format!("up-{}", order_uuid(&id)); + assert_eq!( + h.outcome(&id), + Outcome::Submitted { + command_id: command_id.clone() + } + ); + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); + assert!(h.runs().is_empty(), "a deploy runs no transient unit"); + let mut shown = Vec::new(); + crate::control_plane::status( + &mut shown, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + Some(&command_id), + ) + .unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); + assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); + // Re-executing writes no second command. + execute(&h.ctx, &id).unwrap(); + let mut again = Vec::new(); + crate::control_plane::status( + &mut again, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + None, + ) + .unwrap(); + assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); + // The result post waits for the transactions: queued is not finished. + let due = due_posts(&h.ctx, &ledger); + assert_eq!(due.len(), 1); + assert_eq!(due[0].kind, PostKind::Accepted); + assert!(!deploy_finished("queued") && !deploy_finished("running")); + assert!(deploy_finished("succeeded") && deploy_finished("failed")); + assert!(deploy_finished("complete")); + } + + #[test] + fn chat_outage_delays_posts_not_orders() { + let h = harness(); + h.home.failing.store(true, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + assert_eq!(h.runs().len(), 1); + assert!(post_pass(&h.ctx).is_err()); + assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); + h.home.failing.store(false, Ordering::SeqCst); + assert_eq!(post_pass(&h.ctx).unwrap(), 2); + assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); + let sent = h.home.sent.lock().unwrap().clone(); + assert_eq!(sent.len(), 2); + assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); + assert_eq!(sent[1].0, format!("idunn-{id}-result")); + assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); + assert!(sent[0].1.contains("mind-eureka-self.service")); + assert!(sent[0].1.contains("reason: \"probe\"")); + assert!(sent[1].1.contains("ran (exit 0)")); + } + + #[test] + fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { + let h = harness(); + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(100), + signal: None, + stdout_tail: vec![b'o'; 3 * TAIL_BYTES], + stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), + }; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into()], + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Failed { + exit_code: Some(100), + signal: None + } + ); + assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); + let ledger = h.ctx.store.ledger().unwrap(); + let record = ledger.outcome_of(&id).unwrap(); + assert_eq!(record.stdout_tail.len(), TAIL_BYTES); + assert!(!format!("{record:?}").contains("secret-looking")); + // Neither the post nor the journal text carries a tail. + let results: Vec<_> = due_posts(&h.ctx, &ledger) + .into_iter() + .filter(|due| due.kind == PostKind::Result) + .collect(); + assert_eq!(results.len(), 1); + assert!(results[0].text.contains("failed (exit 100)")); + assert!(!results[0].text.contains("secret-looking")); + // The steward reads them through the escape sink and nothing raw. + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("expected a show") + }; + let mut shown = Vec::new(); + emit(&mut shown, &render_view(&view)).unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert!(shown.contains("secret-looking")); + assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); + assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); + assert!(shown.contains("status failed")); + // A show of an unknown or malformed id names the field only. + assert_eq!( + h.ask(&Request::Show(ShowRequest { + id: format!("ord-{}", Uuid::new_v4()) + })), + Response::Refused(Refusal::new("id", Code::Unknown)) + ); + assert_eq!( + h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), + Response::Refused(Refusal::new("id", Code::BadShape)) + ); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("expected a list") + }; + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].status, "failed"); + } + + #[test] + fn a_refusal_or_error_never_echoes_an_input_value() { + let h = harness(); + let mut seen: Vec> = Vec::new(); + let mut note = |response: Response| { + let bytes = rmp_serde::to_vec_named(&response).unwrap(); + if let Response::Refused(refusal) = &response { + seen.push(refusal.to_string().into_bytes()); + } + seen.push(bytes); + }; + for unit in [ + format!("{CANARY}*.service"), + format!("{CANARY}/../idunn-orders.service"), + format!("idunn-{CANARY};x.service"), + format!("{CANARY}"), + ] { + note(h.issue_unit(Verb::Restart, &unit)); + } + note(h.issue(Operation::PackageInstall { + packages: vec![format!("{CANARY}/x")], + })); + note(h.issue(Operation::PackageInstall { + packages: vec![format!("pkg={CANARY} 1")], + })); + note(h.issue(Operation::Deploy { + selector: format!("{CANARY} bad"), + })); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: format!("{CANARY}\n"), + }))); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: CANARY.repeat(40), + }))); + note(h.ask(&Request::Show(ShowRequest { + id: format!("ord-{CANARY}"), + }))); + // A frame whose own keys and values are the canary, and plain garbage. + let named: std::collections::BTreeMap> = + [( + CANARY.to_owned(), + [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), + )] + .into_iter() + .collect(); + note(h.exchange(|client| write_frame(client, &named).unwrap())); + note(h.exchange(|client| { + let garbage = format!("{CANARY}{CANARY}"); + client + .write_all(&(garbage.len() as u32).to_be_bytes()) + .unwrap(); + client.write_all(garbage.as_bytes()).unwrap(); + })); + note(h.exchange(|client| { + client.write_all(&u32::MAX.to_be_bytes()).unwrap(); + })); + // The client grammar fails the same way. + let mut messages = Vec::new(); + let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); + for line in [ + words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), + words(&["unit", CANARY, "nginx.service", "--reason", "r"]), + words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), + words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), + words(&[CANARY]), + words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), + words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), + words(&["show"]), + ] { + messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); + } + for line in [ + words(&[CANARY]), + words(&["serve", &format!("--{CANARY}")]), + words(&["serve", "--store"]), + ] { + messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); + } + for bytes in &seen { + assert!( + !String::from_utf8_lossy(bytes).contains(CANARY), + "a response carried an input value" + ); + } + for message in &messages { + assert!(!message.contains(CANARY), "an error carried an input value: {message}"); + } + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn cgroup_names_the_issuer_unit() { + assert_eq!( + unit_from_cgroup("0::/system.slice/claude-connector.service\n"), + "claude-connector.service" + ); + assert_eq!( + unit_from_cgroup("0::/system.slice/huginn@eureka.service/sub/leaf\n"), + "huginn@eureka.service" + ); + assert_eq!( + unit_from_cgroup("0::/user.slice/user-1000.slice/session-3.scope\n"), + "session-3.scope" + ); + assert_eq!( + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.service\n"), + "a.service" + ); + assert_eq!(unit_from_cgroup("0::/\n"), "unknown"); + assert_eq!(unit_from_cgroup(""), "unknown"); + } + + #[test] + fn systemctl_show_output_is_read_into_facts() { + let facts = parse_show( + "Id=a.service\nNames=a.service b.service\nRequiredBy=c.service\n\ + BoundBy=\nConsistsOf=d.target e.service\nOther=x\n", + ) + .unwrap(); + assert_eq!(facts.id, "a.service"); + assert_eq!(facts.names, ["a.service", "b.service"]); + assert_eq!(facts.required_by, ["c.service"]); + assert!(facts.bound_by.is_empty()); + assert_eq!(facts.consists_of, ["d.target", "e.service"]); + assert!(parse_show("Names=a.service\n").is_err()); + } + + #[test] + fn the_store_refuses_a_second_record_and_a_foreign_document() { + let h = harness(); + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: "ord-x".into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }; + assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert_eq!( + std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); + SingleFileMessagePackBackingStore::new(&h.ctx.store.path()) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: "idunn.other".into(), + key: "k".into(), + current: None, + }], + &[foreign], + ) + .unwrap(); + assert!(h.ctx.store.ledger().is_err()); + } + + #[test] + fn synapse_post_is_one_idempotent_put_on_loopback() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (sent, requests) = mpsc::channel(); + let server = thread::spawn(move || { + for reply in [ + "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", + "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", + ] { + let (mut socket, _) = listener.accept().unwrap(); + let mut received = Vec::new(); + let mut chunk = [0u8; 1024]; + while !received.windows(4).any(|window| window == b"\r\n\r\n") + || !String::from_utf8_lossy(&received).contains("\"body\"") + { + let count = socket.read(&mut chunk).unwrap(); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); + socket.write_all(reply.as_bytes()).unwrap(); + } + }); + let synapse = Synapse { + address, + room: "!room:example.org".into(), + token_file, + }; + assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); + let request = requests.recv().unwrap(); + assert!(request.starts_with( + "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" + )); + assert!(request.contains("Authorization: Bearer tok-123\r\n")); + assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); + let error = synapse.post("txn-2", "x").unwrap_err(); + assert!(!format!("{error:#}").contains("tok-123")); + server.join().unwrap(); + assert!( + Synapse { + address, + room: "!r:e".into(), + token_file: directory.path().join("absent") + } + .post("t", "x") + .is_err() + ); + } + + #[test] + fn grammar_limits_are_exact_at_their_edges() { + let h = harness(); + let order = |operation: Operation, reason: &str| { + h.ask(&Request::Order(OrderRequest { + operation, + reason: reason.into(), + })) + }; + let packages = |names: Vec| Operation::PackageInstall { packages: names }; + let unit = |name: String| Operation::UnitAction { + verb: Verb::Start, + unit: name, + }; + // Package: two characters is the shortest name, 200 bytes the longest. + assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); + assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); + assert_eq!( + order(packages(vec!["a".repeat(201)]), "r"), + Response::Refused(Refusal::new("package", Code::TooLong)) + ); + assert_eq!( + order(packages(vec!["a".into()]), "r"), + Response::Refused(Refusal::new("package", Code::BadShape)) + ); + let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); + assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); + // Unit: 200 bytes with its suffix is the longest. + let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); + assert_eq!(longest.len(), UNIT_LIMIT); + assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); + let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); + assert_eq!( + order(unit(too_long), "r"), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + // Reason: 300 bytes is the longest, and it must be there. + assert!(matches!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), + Response::Accepted { .. } + )); + assert_eq!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), + Response::Refused(Refusal::new("reason", Code::TooLong)) + ); + assert_eq!( + order(unit("nginx.service".into()), ""), + Response::Refused(Refusal::new("reason", Code::Empty)) + ); + assert_eq!( + order(unit("nginx.service".into()), "caf\u{e9}"), + Response::Refused(Refusal::new("reason", Code::BadCharacter)) + ); + } + + #[test] + fn a_concluded_order_is_never_run_later() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + // The brake is released by someone else; the refused order stays refused. + h.brake.0.store(true, Ordering::SeqCst); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty()); + let ledger = h.ctx.store.ledger().unwrap(); + assert!(ledger.start_of(&id).is_none()); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + } + + #[test] + fn the_orders_cli_has_no_command_string() { + let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); + let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); + assert_eq!( + parsed.action, + OrderAction::Issue { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into() + }, + reason: "why".into() + } + ); + assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); + assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); + assert!(parse(&["show", "ord-x"]).is_ok()); + assert!(parse(&["list"]).is_ok()); + for refused in [ + vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], + vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], + vec!["run", "sh", "--reason", "r"], + vec!["exec", "--reason", "r"], + vec!["unit", "enable", "nginx.service", "--reason", "r"], + vec!["unit", "restart", "--reason", "r"], + vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], + vec!["unit", "restart", "nginx.service"], + vec!["unit", "restart", "nginx*", "--reason", "r"], + vec!["package-install", "--reason", "r"], + vec!["package-install", "-y", "x", "--reason", "r"], + vec!["deploy", "--reason", "r"], + vec!["deploy", "a", "b", "--reason", "r"], + vec!["list", "extra"], + vec!["show"], + vec![], + ] { + assert!(parse(&refused).is_err(), "{refused:?} parsed"); + } + } +} From 1a3634e04696bfaee40b8c4e4c317c2202e6357f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:02 +0100 Subject: [PATCH 101/122] Document idunn order and add the idunn-orders unit Co-Authored-By: Claude Sonnet 5.5 --- README.md | 29 ++++++++++++++++++++++ deploy/idunn-orders.service | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 deploy/idunn-orders.service diff --git a/README.md b/README.md index a5c48bc..6af4527 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,35 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. +### Steward orders + +A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), +takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. +It has its own store, `orders.cc`, and its own binding, `steward.toml`; the +survival daemon takes no order. There is still no command string: + +```bash +idunn order package-install [=version]... --reason TEXT # apt-get update, then install +idunn order unit --reason TEXT +idunn order deploy --reason TEXT # the same commit as `idunn up` +idunn order show # status and output tails +idunn order list +``` + +A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' +own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and +read from no configuration. A name, an alias that resolves to one of them, or a +`stop` or `restart` whose dependents systemd would stop with it is refused, +recorded and posted. Slices, scopes and globs are not units an order can name. + +An order runs once, after the steward's uid is checked and the instance's +brake check (`mind-control --instance check`) exits 0; an order +that had started when the service died is recorded as lost and never run +again. No order releases a brake. Acceptance and result are posted to +`#operator` as a retried projection of `orders.cc`: a chat outage delays the +posts and not the orders. A refusal names the field and a code, never the value +it refused. + ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service new file mode 100644 index 0000000..cd3c46b --- /dev/null +++ b/deploy/idunn-orders.service @@ -0,0 +1,48 @@ +[Unit] +Description=Idunn steward orders: typed package, unit and deploy orders from the steward +# Deliberately no After= or Requires= on any managed target, Synapse or +# idunn-yggdrasil: orders are how an operator repairs those, so none of them +# may be a condition of this unit starting. Posts are a retried projection and +# wait for Synapse on their own. + +[Service] +Type=simple + +# Root, from the same binary as idunn-yggdrasil, with its own store and socket. +# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: +# no order code runs in `idunn serve`. +# +# The unit name matches idunn-*, which is in the always-refused set, so no +# order can stop or restart this service. +User=root +# The socket is created root:idunn-steward 0660 (UMask below). The steward user +# is the only peer the service answers: it checks SO_PEERCRED on every +# connection, so the group is a second door held shut, not the lock. +Group=idunn-steward +ExecStart=/usr/local/bin/idunn orders serve \ + --store /var/lib/gamecult/idunn-orders \ + --state-store /var/lib/gamecult/idunn/control.cc \ + --steward-binding /etc/gamecult/idunn/steward.toml \ + --socket /run/idunn-orders/steward.sock +Restart=always +RestartSec=5 +TimeoutStopSec=20 +UMask=0117 + +RuntimeDirectory=idunn-orders +RuntimeDirectoryMode=0750 + +# The Synapse access token of @idunn, copied from a root-only file, and read +# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. +LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token + +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +# Its own order store, and the control store a deploy order commits a command +# to. Transient units started with systemd-run run in the manager's namespace, +# so packages and units are not limited by this list. +ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn + +[Install] +WantedBy=multi-user.target From 7d5cd85728b0d9e7dad801f48b15b358a0bd6bc3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:12 +0100 Subject: [PATCH 102/122] Reword a test comment so the no-brake-release tripwire reads clean Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 2aded30..31a2d06 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2880,7 +2880,7 @@ mod tests { let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); execute(&h.ctx, &id).unwrap(); assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - // The brake is released by someone else; the refused order stays refused. + // Someone else clears the brake; the refused order stays refused. h.brake.0.store(true, Ordering::SeqCst); execute(&h.ctx, &id).unwrap(); assert!(h.runs().is_empty()); From ec6222e2a114d3ebf5287bb326d0b5edc5ea907f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:49:26 +0100 Subject: [PATCH 103/122] Steward unit orders: refuse by the systemd transaction, install exactly the named packages An order is refused when the closure systemd would build for it reaches an always-refused unit: a start pulls Requires/Requisite/BindsTo/Wants/Upholds and stops Conflicts/ConflictedBy, a stop takes down RequiredBy/BoundBy/RequisiteOf/ ConsistsOf/PropagatesStopTo/StopPropagatedFrom, a restart does both. A refused unit already running is left alone by a start. Names are unquoted from systemctl show; a unit systemd cannot show fails the order. Package installs run apt-get --no-remove with ?exact-name() per package and a final dpkg -s step that fails unless every named package is installed. Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 676 +++++++++++++++++++++++++++++++++--------- 1 file changed, 544 insertions(+), 132 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 31a2d06..ce95337 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -213,11 +213,6 @@ impl Verb { .into_iter() .find(|verb| verb.as_str() == word) } - - /// Verbs systemd propagates down the dependency graph. - fn propagates(self) -> bool { - matches!(self, Verb::Stop | Verb::Restart) - } } /// The closed set of things an order can ask for. @@ -259,8 +254,21 @@ impl Operation { let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; match self { Operation::PackageInstall { packages } => { - let mut install: Vec = words(&["apt-get", "install", "-y", "-q", "--"]); - install.extend(packages.iter().cloned()); + // Never a removal, never a pattern: each name selects exactly + // itself, and apt aborts rather than displace an installed package. + let mut install: Vec = + words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); + let mut present: Vec = words(&["dpkg", "-s", "--"]); + for package in packages { + let (name, version) = package.split_once('=').unwrap_or((package, "")); + install.push(match version { + "" => format!("?exact-name({name})"), + version => format!("?exact-name({name})={version}"), + }); + present.push(name.to_owned()); + } + // An exact name that matches nothing installs nothing and exits 0, + // so the last step fails unless every named package is installed. vec![ Step { argv: words(&["apt-get", "update", "-q"]), @@ -270,6 +278,10 @@ impl Operation { argv: install, environment: apt_environment, }, + Step { + argv: present, + environment: Vec::new(), + }, ] } Operation::UnitAction { verb, unit } => vec![Step { @@ -595,14 +607,45 @@ impl OrderStore { // ------------------------------------------------------------------ ports -- +/// Which way systemd moves along a dependency edge when an order is given. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Edge { + /// Units a start brings up with this one. + Pulls, + /// Units a start stops: the unit's `Conflicts=` and the units that conflict with it. + Conflicts, + /// Units a stop takes down with this one. + Stops, +} + +/// The `systemctl show` properties the closure reads, and the way each one +/// moves. This is the only list: the request and the parse both read it. +const EDGES: [(&str, Edge); 13] = [ + ("Requires", Edge::Pulls), + ("Requisite", Edge::Pulls), + ("BindsTo", Edge::Pulls), + ("Wants", Edge::Pulls), + ("Upholds", Edge::Pulls), + ("Conflicts", Edge::Conflicts), + ("ConflictedBy", Edge::Conflicts), + ("RequiredBy", Edge::Stops), + ("BoundBy", Edge::Stops), + ("RequisiteOf", Edge::Stops), + ("ConsistsOf", Edge::Stops), + ("PropagatesStopTo", Edge::Stops), + ("StopPropagatedFrom", Edge::Stops), +]; + /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { pub(crate) id: String, pub(crate) names: Vec, - pub(crate) required_by: Vec, - pub(crate) bound_by: Vec, - pub(crate) consists_of: Vec, + pub(crate) not_found: bool, + pub(crate) running: bool, + pub(crate) pulls: Vec, + pub(crate) conflicts: Vec, + pub(crate) stops: Vec, } impl UnitFacts { @@ -610,12 +653,20 @@ impl UnitFacts { std::iter::once(&self.id).chain(self.names.iter()) } - /// The units systemd stops with this one. - fn dependents(&self) -> impl Iterator { - self.required_by - .iter() - .chain(self.bound_by.iter()) - .chain(self.consists_of.iter()) + fn along(&self, edge: Edge) -> &[String] { + match edge { + Edge::Pulls => &self.pulls, + Edge::Conflicts => &self.conflicts, + Edge::Stops => &self.stops, + } + } + + fn along_mut(&mut self, edge: Edge) -> &mut Vec { + match edge { + Edge::Pulls => &mut self.pulls, + Edge::Conflicts => &mut self.conflicts, + Edge::Stops => &mut self.stops, + } } } @@ -673,40 +724,95 @@ fn refused(unit: &str, via: Via) -> Refused { } } +/// The units systemd would touch for one order, read from `systemctl show`. +struct Transaction<'a> { + systemd: &'a dyn Systemd, + facts: BTreeMap, +} + +impl Transaction<'_> { + fn facts(&mut self, unit: &str) -> Result<&UnitFacts> { + if !self.facts.contains_key(unit) { + ensure!( + self.facts.len() < CLOSURE_LIMIT, + "the units an order would reach are too many to check" + ); + let facts = self.systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + self.facts.insert(unit.to_owned(), facts); + } + Ok(&self.facts[unit]) + } + + /// Every unit reached from `seeds` along `edge`, seeds included. + fn reach(&mut self, seeds: &BTreeSet, edge: Edge) -> Result> { + let mut reached = BTreeSet::new(); + let mut queue: VecDeque = seeds.iter().cloned().collect(); + while let Some(next) = queue.pop_front() { + if !reached.insert(next.clone()) { + continue; + } + queue.extend(self.facts(&next)?.along(edge).iter().cloned()); + } + Ok(reached) + } + + /// Every unit a start of `started` stops through `Conflicts=`, and what + /// that stop takes down with it. + fn conflicted(&mut self, started: &BTreeSet) -> Result> { + let mut seeds = BTreeSet::new(); + for unit in started { + seeds.extend(self.facts(unit)?.along(Edge::Conflicts).iter().cloned()); + } + self.reach(&seeds, Edge::Stops) + } +} + /// Would `verb` on `unit` reach an always-refused unit: by its name, by an -/// alias it resolves from, or, for the verbs systemd propagates, by the units -/// systemd stops with it. Resolution reads systemd before anything runs. +/// alias it resolves from, or through the transaction systemd builds for it. +/// A start pulls in what the unit requires or wants and stops what conflicts; +/// a stop takes down what depends on the unit; a restart does both. A unit +/// the transaction would stop is refused when it is refused; a unit it would +/// start is refused when it is refused and not running. Every unit in the +/// closure must be one systemd can show. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); } - let facts = systemd.show(unit)?; - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + let mut transaction = Transaction { + systemd, + facts: BTreeMap::new(), + }; + if let Some(name) = transaction + .facts(unit)? + .identities() + .find(|name| is_refused_name(name)) + { return Ok(Some(refused(name, Via::Alias))); } - if !verb.propagates() { - return Ok(None); + let root = BTreeSet::from([unit.to_owned()]); + let mut stopped = BTreeSet::new(); + let mut started = BTreeSet::new(); + match verb { + Verb::Reload => {} + Verb::Start => started = transaction.reach(&root, Edge::Pulls)?, + Verb::Stop => stopped = transaction.reach(&root, Edge::Stops)?, + Verb::Restart => { + stopped = transaction.reach(&root, Edge::Stops)?; + started = transaction.reach(&stopped, Edge::Pulls)?; + } } - let mut seen: BTreeSet = facts.identities().cloned().collect(); - seen.insert(unit.to_owned()); - let mut queue: VecDeque = facts.dependents().cloned().collect(); - while let Some(next) = queue.pop_front() { - if is_refused_name(&next) { - return Ok(Some(refused(&next, Via::Propagation))); + stopped.extend(transaction.conflicted(&started)?); + for (units, only_if_stopped) in [(&stopped, false), (&started, true)] { + for member in units { + let facts = transaction.facts(member)?; + if only_if_stopped && facts.running { + continue; + } + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Propagation))); + } } - if !seen.insert(next.clone()) { - continue; - } - ensure!( - seen.len() <= CLOSURE_LIMIT, - "the units a stop would reach are too many to check" - ); - let facts = systemd.show(&next)?; - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { - return Ok(Some(refused(name, Via::Propagation))); - } - seen.extend(facts.identities().cloned()); - queue.extend(facts.dependents().cloned()); } Ok(None) } @@ -1148,8 +1254,39 @@ pub(crate) fn handle( /// `systemctl` and `systemd-run`, found by name. pub(crate) struct HostSystemd; +/// The names in one `systemctl show` list value. systemd quotes a name that +/// holds a backslash and doubles the backslash inside the quotes. fn parse_names(value: &str) -> Vec { - value.split_whitespace().map(str::to_owned).collect() + let mut names = Vec::new(); + let mut current = String::new(); + let (mut quoted, mut open, mut escaped) = (false, false, false); + for character in value.chars() { + match character { + _ if escaped => { + current.push(character); + escaped = false; + } + '\\' if quoted => escaped = true, + '"' => { + quoted = !quoted; + open = true; + } + _ if character.is_whitespace() && !quoted => { + if open { + names.push(std::mem::take(&mut current)); + open = false; + } + } + _ => { + current.push(character); + open = true; + } + } + } + if open { + names.push(current); + } + names } fn parse_show(output: &str) -> Result { @@ -1161,10 +1298,15 @@ fn parse_show(output: &str) -> Result { match key { "Id" => facts.id = value.to_owned(), "Names" => facts.names = parse_names(value), - "RequiredBy" => facts.required_by = parse_names(value), - "BoundBy" => facts.bound_by = parse_names(value), - "ConsistsOf" => facts.consists_of = parse_names(value), - _ => {} + "LoadState" => facts.not_found = value == "not-found", + "ActiveState" => { + facts.running = matches!(value, "active" | "activating" | "reloading"); + } + _ => { + if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { + facts.along_mut(*edge).extend(parse_names(value)); + } + } } } ensure!(!facts.id.is_empty(), "systemctl named no unit"); @@ -1173,14 +1315,15 @@ fn parse_show(output: &str) -> Result { impl Systemd for HostSystemd { fn show(&self, unit: &str) -> Result { + let properties = ["Id", "Names", "LoadState", "ActiveState"] + .into_iter() + .chain(EDGES.iter().map(|(property, _)| *property)) + .collect::>() + .join(","); let output = Command::new("systemctl") - .args([ - "show", - "--no-pager", - "--property=Id,Names,RequiredBy,BoundBy,ConsistsOf", - "--", - unit, - ]) + .args(["show", "--no-pager"]) + .arg(format!("--property={properties}")) + .args(["--", unit]) .stdin(Stdio::null()) .output() .context("running systemctl show")?; @@ -1750,8 +1893,23 @@ mod tests { const STEWARD: u32 = 1000; const CANARY: &str = "CANARY-7f3a91"; + /// What `systemctl show` prints for a loaded, active unit with no edges, + /// plus the given properties (a given key replaces its default). + fn show_text(id: &str, properties: &[(&str, &str)]) -> String { + let mut text = format!("Id={id}\n"); + for (key, default) in [("Names", id), ("LoadState", "loaded"), ("ActiveState", "active")] { + if !properties.iter().any(|(given, _)| *given == key) { + text += &format!("{key}={default}\n"); + } + } + for (key, value) in properties { + text += &format!("{key}={value}\n"); + } + text + } + struct FakeSystemd { - units: Mutex>, + units: Mutex>, runs: Mutex>, report: Mutex, } @@ -1759,11 +1917,7 @@ mod tests { impl Systemd for FakeSystemd { fn show(&self, unit: &str) -> Result { let known = self.units.lock().unwrap().get(unit).cloned(); - Ok(known.unwrap_or_else(|| UnitFacts { - id: unit.into(), - names: vec![unit.into()], - ..UnitFacts::default() - })) + parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) } fn invocation(&self, _unit: &str) -> Option { @@ -1907,8 +2061,13 @@ mod tests { self.systemd.runs.lock().unwrap().clone() } - fn describe_unit(&self, name: &str, facts: UnitFacts) { - self.systemd.units.lock().unwrap().insert(name.into(), facts); + /// Tell the fake `systemctl show` what it prints for `name`. + fn describe_unit(&self, name: &str, text: String) { + self.systemd.units.lock().unwrap().insert(name.into(), text); + } + + fn describe(&self, name: &str, properties: &[(&str, &str)]) { + self.describe_unit(name, show_text(name, properties)); } } @@ -2065,19 +2224,52 @@ mod tests { execute(&h.ctx, &id).unwrap(); assert_eq!(h.outcome(&id), Outcome::Ran); let runs = h.runs(); - assert_eq!(runs.len(), 2); + assert_eq!(runs.len(), 3); assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); - let mut install = vec!["apt-get", "install", "-y", "-q", "--"]; - install.extend(good); + // Each name selects exactly itself and apt may remove nothing. + let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; + install.extend(["--no-remove".to_owned(), "--".into()]); + install.extend(good.iter().map(|spec| match spec.split_once('=') { + Some((name, version)) => format!("?exact-name({name})={version}"), + None => format!("?exact-name({spec})"), + })); assert_eq!(runs[1].1.argv, install); + assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); + assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); + // The last step fails unless every named package is installed. + let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; + present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); + assert_eq!(runs[2].1.argv, present); for (unit, step) in &runs { assert!(unit.starts_with("idunn-order-")); + let apt = step.argv[0] == "apt-get"; assert_eq!( step.environment, - [("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + if apt { + vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + } else { + Vec::new() + } ); } assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); + + // A package whose name apt would read as a regex or a conflict is + // exactly itself and never a removal. + for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { + let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); + assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); + } + // A step that failed ends the order: the absent package is a failure. + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(1), + ..RunReport::default() + }; + let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); + execute(&h.ctx, &absent).unwrap(); + assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); + assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); } #[test] @@ -2121,74 +2313,28 @@ mod tests { let h = harness(); h.describe_unit( "backup-name.service", - UnitFacts { - id: "idunn-yggdrasil.service".into(), - names: vec!["idunn-yggdrasil.service".into(), "backup-name.service".into()], - ..UnitFacts::default() - }, + show_text( + "idunn-yggdrasil.service", + &[("Names", "idunn-yggdrasil.service backup-name.service")], + ), ); - h.describe_unit( + h.describe( "quiet-alias.service", - UnitFacts { - id: "quiet-alias.service".into(), - names: vec!["quiet-alias.service".into(), "mind-persona@void.service".into()], - ..UnitFacts::default() - }, + &[("Names", "quiet-alias.service mind-persona@void.service")], ); // fuse <- wrapper <- (bound by) mind-self@eureka - h.describe_unit( - "fuse.service", - UnitFacts { - id: "fuse.service".into(), - names: vec!["fuse.service".into()], - required_by: vec!["wrapper.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "wrapper.service", - UnitFacts { - id: "wrapper.service".into(), - names: vec!["wrapper.service".into()], - bound_by: vec!["mind-self@eureka.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "stack.target", - UnitFacts { - id: "stack.target".into(), - names: vec!["stack.target".into()], - consists_of: vec!["mind-persona@void.service".into()], - ..UnitFacts::default() - }, - ); + h.describe("fuse.service", &[("RequiredBy", "wrapper.service")]); + h.describe("wrapper.service", &[("BoundBy", "mind-self@eureka.service")]); + h.describe("stack.target", &[("ConsistsOf", "mind-persona@void.service")]); // A cycle that reaches an alias of a refused unit on its second hop. - h.describe_unit( - "loop-a.service", - UnitFacts { - id: "loop-a.service".into(), - names: vec!["loop-a.service".into()], - required_by: vec!["loop-b.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "loop-b.service", - UnitFacts { - id: "loop-b.service".into(), - names: vec!["loop-b.service".into()], - required_by: vec!["loop-a.service".into(), "hidden.service".into()], - ..UnitFacts::default() - }, - ); + h.describe("loop-a.service", &[("RequiredBy", "loop-b.service")]); + h.describe("loop-b.service", &[("RequiredBy", "loop-a.service hidden.service")]); h.describe_unit( "hidden.service", - UnitFacts { - id: "idunn-orders.service".into(), - names: vec!["idunn-orders.service".into(), "hidden.service".into()], - ..UnitFacts::default() - }, + show_text( + "idunn-orders.service", + &[("Names", "idunn-orders.service hidden.service")], + ), ); let direct = [ @@ -2260,7 +2406,7 @@ mod tests { } } assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); - // start and reload propagate nothing, so the same units are not refused. + // start and reload take nothing down, so the same units are not refused. for verb in [Verb::Start, Verb::Reload] { let id = h.accept(h.issue_unit(verb, "fuse.service")); execute(&h.ctx, &id).unwrap(); @@ -2284,6 +2430,248 @@ mod tests { assert!(ledger.starts.len() == 2); } + /// Yggdrasil's `systemctl show` shapes (systemd 257, 2026-11-01), cut down + /// to the units an order can reach: seven refused units all conflict with + /// shutdown.target, and the shutdown services require it. + fn yggdrasil(h: &Harness, idunn_running: bool) { + let state = if idunn_running { "active" } else { "inactive" }; + let refused = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-e0ada640.service", + "mind-self@eureka.service", + "mind-persona@void.service", + ]; + h.describe( + "shutdown.target", + &[ + ("ActiveState", "inactive"), + ( + "RequiredBy", + "systemd-soft-reboot.service systemd-reboot.service systemd-poweroff.service", + ), + ( + "ConflictedBy", + "nginx.service idunn-yggdrasil.service idunn-orders.service \ + idunn-vili-e0ada640.service mind-self@eureka.service mind-persona@void.service", + ), + ], + ); + for refused in refused { + h.describe( + refused, + &[ + ("Requires", "sysinit.target system.slice -.mount"), + ("Wants", "tmp.mount network-online.target"), + ("Conflicts", "shutdown.target"), + ("ActiveState", if refused == "idunn-yggdrasil.service" { state } else { "inactive" }), + ], + ); + } + h.describe("nginx.service", &[("Wants", "network-online.target"), ("Conflicts", "shutdown.target")]); + for (service, target) in [ + ("systemd-poweroff.service", "poweroff.target"), + ("systemd-reboot.service", "reboot.target"), + ("systemd-soft-reboot.service", "soft-reboot.target"), + ] { + h.describe( + service, + &[ + ("Requires", "shutdown.target final.target system.slice umount.target"), + ("RequiredBy", target), + ("ActiveState", "inactive"), + ], + ); + } + for (target, service) in [ + ("poweroff.target", "systemd-poweroff.service"), + ("reboot.target", "systemd-reboot.service"), + ("soft-reboot.target", "systemd-soft-reboot.service"), + ] { + h.describe(target, &[("Requires", service), ("ActiveState", "inactive")]); + } + h.describe( + "umount.target", + &[ + ("ActiveState", "inactive"), + ("RequiredBy", "systemd-poweroff.service systemd-reboot.service"), + ("ConflictedBy", "boot-efi.mount boot.mount \"blockdev@dev-vda2.target\""), + ], + ); + h.describe( + "multi-user.target", + &[ + ("Requires", "basic.target"), + ("Wants", "cron.service ssh.service idunn-yggdrasil.service"), + ("Conflicts", "shutdown.target rescue.target rescue.service"), + ], + ); + h.describe( + "voidbot.service", + &[ + ("Requires", "docker.service -.mount sysinit.target system.slice"), + ("Wants", "idunn-yggdrasil.service network-online.target"), + ("Conflicts", "shutdown.target"), + ], + ); + } + + fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + h.outcome(&id) + } + + fn hit(outcome: &Outcome) -> Option<&str> { + match outcome { + Outcome::Refused(Refused::AlwaysRefused { unit, via: Via::Propagation }) => Some(unit.as_str()), + _ => None, + } + } + + #[test] + fn a_transaction_that_reaches_a_refused_unit_is_refused() { + let h = harness(); + yggdrasil(&h, true); + // The targets and services that power the host down reach every refused + // unit through Conflicts=, however many requirements away. + for (verb, unit) in [ + (Verb::Start, "shutdown.target"), + (Verb::Start, "poweroff.target"), + (Verb::Restart, "poweroff.target"), + (Verb::Start, "reboot.target"), + (Verb::Restart, "reboot.target"), + (Verb::Start, "soft-reboot.target"), + (Verb::Start, "systemd-soft-reboot.service"), + (Verb::Restart, "systemd-poweroff.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert!(hit(&outcome).is_some(), "{verb:?} {unit} was {outcome:?}"); + } + assert!(h.runs().is_empty(), "no refused order ran anything"); + // A stop of shutdown.target takes down what requires it and runs no start. + let outcome = outcome_of_unit(&h, Verb::Stop, "shutdown.target"); + assert_eq!(outcome, Outcome::Ran); + + // A start that wants a refused unit that is stopped would start it. + h.describe("idunn-yggdrasil.service", &[("ActiveState", "inactive"), ("Conflicts", "shutdown.target")]); + for (verb, unit) in [ + (Verb::Start, "multi-user.target"), + (Verb::Restart, "multi-user.target"), + (Verb::Start, "voidbot.service"), + (Verb::Restart, "voidbot.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert_eq!(hit(&outcome), Some("idunn-yggdrasil.service"), "{verb:?} {unit}"); + } + // The same orders start nothing while the refused unit runs. + h.describe("idunn-yggdrasil.service", &[("ActiveState", "active"), ("Conflicts", "shutdown.target")]); + for (verb, unit) in [ + (Verb::Start, "multi-user.target"), + (Verb::Restart, "voidbot.service"), + (Verb::Start, "voidbot.service"), + ] { + assert_eq!(outcome_of_unit(&h, verb, unit), Outcome::Ran, "{verb:?} {unit}"); + } + // A unit that wants only what is not refused runs, and so does a target + // that unmounts nothing a refused unit requires. + for verb in [Verb::Start, Verb::Restart, Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "nginx.service"), Outcome::Ran); + } + assert_eq!(outcome_of_unit(&h, Verb::Start, "umount.target"), Outcome::Ran); + } + + #[test] + fn the_closure_follows_every_edge_and_every_hop() { + let h = harness(); + // Start: three hops of Wants/Requires/Requisite/BindsTo/Upholds to a + // stopped refused unit. + h.describe("hop-1.service", &[("Wants", "hop-2.service")]); + h.describe("hop-2.service", &[("Requires", "hop-3.service")]); + h.describe("hop-3.service", &[("Requisite", "hop-4.service")]); + h.describe("hop-4.service", &[("BindsTo", "hop-5.service")]); + h.describe("hop-5.service", &[("Upholds", "idunn-orders.service")]); + h.describe("idunn-orders.service", &[("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "hop-1.service"); + assert_eq!(hit(&outcome), Some("idunn-orders.service")); + // Conflicts of a pulled-in unit stop it and what requires it: the + // conflict, then two hops of reverse requirement, to a refused unit. + h.describe("rival.service", &[("Conflicts", "victim.service")]); + h.describe("victim.service", &[("RequiredBy", "mid.service")]); + h.describe("mid.service", &[("BoundBy", "mind-self@eureka.service")]); + h.describe("pulls-rival.service", &[("Requires", "rival.service")]); + for unit in ["rival.service", "pulls-rival.service"] { + let outcome = outcome_of_unit(&h, Verb::Start, unit); + assert_eq!(hit(&outcome), Some("mind-self@eureka.service"), "{unit}"); + } + // The reverse Conflicts edge: a unit conflicted by a refused one. + h.describe("incumbent.service", &[("ConflictedBy", "mind-persona@void.service")]); + let outcome = outcome_of_unit(&h, Verb::Start, "incumbent.service"); + assert_eq!(hit(&outcome), Some("mind-persona@void.service")); + // Stop and restart: every stop edge, one at a time. + for (property, reached) in [ + ("RequiredBy", "idunn-vili-x.service"), + ("BoundBy", "idunn-vili-x.service"), + ("RequisiteOf", "idunn-vili-x.service"), + ("ConsistsOf", "idunn-vili-x.service"), + ("PropagatesStopTo", "mind-persona@void.service"), + ("StopPropagatedFrom", "mind-persona@void.service"), + ] { + let name = format!("edge-{property}.service"); + h.describe(&name, &[(property, reached)]); + for verb in [Verb::Stop, Verb::Restart] { + let outcome = outcome_of_unit(&h, verb, &name); + assert_eq!(hit(&outcome), Some(reached), "{verb:?} {property}"); + } + } + // A restart starts what its stopped dependents pull in. + h.describe("base.service", &[("RequiredBy", "dependent.service")]); + h.describe("dependent.service", &[("Wants", "idunn-orders.service")]); + let outcome = outcome_of_unit(&h, Verb::Restart, "base.service"); + assert_eq!(hit(&outcome), Some("idunn-orders.service")); + assert_eq!(outcome_of_unit(&h, Verb::Stop, "base.service"), Outcome::Ran); + assert_eq!(outcome_of_unit(&h, Verb::Reload, "base.service"), Outcome::Ran); + } + + #[test] + fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { + let h = harness(); + // systemctl show quotes a name with a backslash and doubles the backslash. + h.describe( + "huginn.service", + &[("RequiredBy", "\"mind-self@foo\\\\x2dbar.service\"")], + ); + let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); + assert_eq!(hit(&outcome), Some("mind-self@foo\\x2dbar.service")); + h.describe("quiet.service", &[("Wants", "\"idunn-a\\\\x2db.service\" other.service")]); + h.describe("idunn-a\\x2db.service", &[("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); + assert_eq!(hit(&outcome), Some("idunn-a\\x2db.service")); + assert!(h.runs().is_empty()); + } + + #[test] + fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { + let h = harness(); + h.describe("ghost.service", &[("LoadState", "not-found")]); + h.describe("pulls-ghost.service", &[("Wants", "ghost.service")]); + h.describe("needed-by-ghost.service", &[("RequiredBy", "ghost.service")]); + for (verb, unit) in [ + (Verb::Start, "ghost.service"), + (Verb::Start, "pulls-ghost.service"), + (Verb::Stop, "needed-by-ghost.service"), + (Verb::Restart, "needed-by-ghost.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert!( + matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), + "{verb:?} {unit}" + ); + } + assert!(h.runs().is_empty()); + } + #[test] fn globs_slices_and_scopes_are_not_units() { let h = harness(); @@ -2715,16 +3103,40 @@ mod tests { #[test] fn systemctl_show_output_is_read_into_facts() { let facts = parse_show( - "Id=a.service\nNames=a.service b.service\nRequiredBy=c.service\n\ - BoundBy=\nConsistsOf=d.target e.service\nOther=x\n", + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=inactive\n\ + Requires=r.service \"q\\\\x2dr.slice\" -.mount\nRequisite=t.service\nWants=w.service\n\ + BindsTo=b.service\nUpholds=u.service\nConflicts=x.target\nConflictedBy=y.service\n\ + RequiredBy=c.service\nBoundBy=\nRequisiteOf=ro.service\nConsistsOf=d.target e.service\n\ + PropagatesStopTo=p.service\nStopPropagatedFrom=f.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); assert_eq!(facts.names, ["a.service", "b.service"]); - assert_eq!(facts.required_by, ["c.service"]); - assert!(facts.bound_by.is_empty()); - assert_eq!(facts.consists_of, ["d.target", "e.service"]); + assert!(!facts.not_found && !facts.running); + // A quoted name holds one backslash, not two and not a quote mark. + assert_eq!( + facts.pulls, + ["r.service", "q\\x2dr.slice", "-.mount", "t.service", "w.service", "b.service", "u.service"] + ); + assert_eq!(facts.conflicts, ["x.target", "y.service"]); + assert_eq!( + facts.stops, + ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] + ); assert!(parse_show("Names=a.service\n").is_err()); + let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); + assert!(gone.not_found); + for (state, running) in [ + ("active", true), + ("activating", true), + ("reloading", true), + ("inactive", false), + ("deactivating", false), + ("failed", false), + ] { + let facts = parse_show(&format!("Id=a.service\nActiveState={state}\n")).unwrap(); + assert_eq!(facts.running, running, "{state}"); + } } #[test] From cea17c06a79f0e02dc88990805d1b4be10c31798 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:49:36 +0100 Subject: [PATCH 104/122] README: a unit order is refused by the systemd transaction; a package order installs only what it names Co-Authored-By: Claude Sonnet 5.5 --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6af4527..2bf6675 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ It has its own store, `orders.cc`, and its own binding, `steward.toml`; the survival daemon takes no order. There is still no command string: ```bash -idunn order package-install [=version]... --reason TEXT # apt-get update, then install +idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them idunn order unit --reason TEXT idunn order deploy --reason TEXT # the same commit as `idunn up` idunn order show # status and output tails @@ -75,9 +75,20 @@ idunn order list A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and -read from no configuration. A name, an alias that resolves to one of them, or a -`stop` or `restart` whose dependents systemd would stop with it is refused, -recorded and posted. Slices, scopes and globs are not units an order can name. +read from no configuration. An order is refused, recorded and posted when its +name, an alias it resolves from, or the transaction systemd would build for it +reaches one of them: a start pulls in what the unit requires or wants (a +refused unit that is already running is left alone) and stops what conflicts +with it, a stop takes down what depends on it, and a restart does both. The +transaction is read from `systemctl show`, so `start poweroff.target` is refused +because every refused unit conflicts with `shutdown.target`, not because +targets are banned. A unit `systemctl show` cannot resolve fails the order. +Slices, scopes and globs are not units an order can name. + +A package order installs the packages it names and nothing else: apt runs with +`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` +match themselves, and a last step fails the order unless every named package +is installed. An order runs once, after the steward's uid is checked and the instance's brake check (`mind-control --instance check`) exits 0; an order From be3b95bc2161239bea16d414d4b9511120ce6d9f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:50:20 +0100 Subject: [PATCH 105/122] Import BTreeMap for the transaction facts cache Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index ce95337..127e0e1 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,7 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::{BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; From 27ca875194c0126a09fc0e12d611e36a102989d4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:53:41 +0100 Subject: [PATCH 106/122] Steward orders tests: clear four clippy hits in the module Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 127e0e1..fdcaafa 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2836,11 +2836,11 @@ mod tests { .unwrap(); let pending = recover(&h.ctx).unwrap(); assert_eq!(h.outcome(&id), Outcome::Lost); - assert_eq!(pending, [waiting.clone()], "an order that never started waits"); + assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); execute(&h.ctx, &id).unwrap(); assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); // Recovery is idempotent, and a finished order is not run again. - assert_eq!(recover(&h.ctx).unwrap(), [waiting.clone()]); + assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); execute(&h.ctx, &waiting).unwrap(); execute(&h.ctx, &waiting).unwrap(); assert_eq!(h.runs().len(), 1); @@ -2995,7 +2995,7 @@ mod tests { format!("{CANARY}*.service"), format!("{CANARY}/../idunn-orders.service"), format!("idunn-{CANARY};x.service"), - format!("{CANARY}"), + CANARY.to_string(), ] { note(h.issue_unit(Verb::Restart, &unit)); } @@ -3162,7 +3162,7 @@ mod tests { 0o600 ); let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); - SingleFileMessagePackBackingStore::new(&h.ctx.store.path()) + SingleFileMessagePackBackingStore::new(h.ctx.store.path()) .compare_exchange( &[CultCacheExpectedEnvelope { r#type: "idunn.other".into(), From ca439a29ff394f51511eef095a1da234d06a6b62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 04:17:12 +0100 Subject: [PATCH 107/122] Pin the systemctl show property list and unquoted backslashes Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index fdcaafa..44f28db 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -1289,6 +1289,15 @@ fn parse_names(value: &str) -> Vec { names } +/// The properties one `systemctl show` is asked for: identity, state, edges. +fn show_properties() -> String { + ["Id", "Names", "LoadState", "ActiveState"] + .into_iter() + .chain(EDGES.iter().map(|(property, _)| *property)) + .collect::>() + .join(",") +} + fn parse_show(output: &str) -> Result { let mut facts = UnitFacts::default(); for line in output.lines() { @@ -1315,14 +1324,9 @@ fn parse_show(output: &str) -> Result { impl Systemd for HostSystemd { fn show(&self, unit: &str) -> Result { - let properties = ["Id", "Names", "LoadState", "ActiveState"] - .into_iter() - .chain(EDGES.iter().map(|(property, _)| *property)) - .collect::>() - .join(","); let output = Command::new("systemctl") .args(["show", "--no-pager"]) - .arg(format!("--property={properties}")) + .arg(format!("--property={}", show_properties())) .args(["--", unit]) .stdin(Stdio::null()) .output() @@ -3124,6 +3128,17 @@ mod tests { ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] ); assert!(parse_show("Names=a.service\n").is_err()); + // The request asks for exactly the properties the parse reads. + assert_eq!( + show_properties(), + "Id,Names,LoadState,ActiveState,Requires,Requisite,BindsTo,Wants,Upholds,\ + Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ + PropagatesStopTo,StopPropagatedFrom" + ); + // A backslash outside quotes is a backslash; inside, it escapes one character. + assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); + assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); + assert!(parse_names(" ").is_empty()); let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); for (state, running) in [ From 56edad95c1b211e982b1032a09fb29b74fa28931 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:03:17 +0100 Subject: [PATCH 108/122] Steward orders: one steward gate for fresh and recovered orders; the orders unit leaks no group or umask; root reads; kernel-attested issuer; end-to-end serve test execute refuses, with a record and a post, an order whose principal or issuing uid is not the steward bound now, before grammar, scope and brake; the brake is asked for the bound principal only. idunn-orders.service drops Group= and UMask=: bind_socket gives the socket to the steward uid at 0600 and serve sets Idunn's own umask, so a deploy order writes control.cc 0640. Root may show and list, never order. The issuer unit is the first unit under system.slice, never a name the peer chose below it. Synapse waits for a reply within one deadline. due_posts takes the deploy-state reader so a finished deploy's post is tested. Tests: the gate both ways, root, per-order posts, deploy result, intake and Synapse timeouts, parse_serve, user_uid, socket and store modes, and the whole service as root with a fake systemctl, systemd-run and homeserver. Co-Authored-By: Claude Sonnet 5.5 --- deploy/idunn-orders.service | 11 +- src/steward_orders.rs | 657 ++++++++++++++++++++++++++++++++++-- 2 files changed, 629 insertions(+), 39 deletions(-) diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service index cd3c46b..f2681c2 100644 --- a/deploy/idunn-orders.service +++ b/deploy/idunn-orders.service @@ -15,10 +15,10 @@ Type=simple # The unit name matches idunn-*, which is in the always-refused set, so no # order can stop or restart this service. User=root -# The socket is created root:idunn-steward 0660 (UMask below). The steward user -# is the only peer the service answers: it checks SO_PEERCRED on every -# connection, so the group is a second door held shut, not the lock. -Group=idunn-steward +# No Group= and no UMask=: the service gives its socket to the steward user +# (mode 0600) itself and sets Idunn's own umask for the files it writes, so +# nothing here reaches control.cc. The steward user is the only peer the +# service answers: it checks SO_PEERCRED on every connection. ExecStart=/usr/local/bin/idunn orders serve \ --store /var/lib/gamecult/idunn-orders \ --state-store /var/lib/gamecult/idunn/control.cc \ @@ -27,10 +27,9 @@ ExecStart=/usr/local/bin/idunn orders serve \ Restart=always RestartSec=5 TimeoutStopSec=20 -UMask=0117 RuntimeDirectory=idunn-orders -RuntimeDirectoryMode=0750 +RuntimeDirectoryMode=0755 # The Synapse access token of @idunn, copied from a root-only file, and read # through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 44f28db..e707307 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -381,6 +381,8 @@ pub(crate) enum Via { pub(crate) enum Refused { AlwaysRefused { unit: String, via: Via }, Braked, + /// The order's principal or issuing uid is not the steward bound now. + NotSteward, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -419,6 +421,9 @@ impl Outcome { } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Refused(Refused::NotSteward) => { + "refused: the issuer is not the current steward".into() + } Outcome::Lost => "lost: the orders service restarted mid-run".into(), Outcome::Submitted { command_id } => { format!("submitted deployment {}", escape(command_id.as_bytes())) @@ -865,6 +870,11 @@ pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { return Ok(()); } + // One gate for a fresh order and a recovered one: who may order is what the + // binding says now, not what it said when the order was accepted. + if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { + return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); + } if order.operation.validate().is_err() { return conclude( ctx, @@ -895,7 +905,7 @@ pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { } } } - if !ctx.brake.clear(&order.principal) { + if !ctx.brake.clear(&ctx.principal) { return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); } let uuid = order_uuid(order_id); @@ -1010,7 +1020,12 @@ fn deploy_finished(state: &str) -> bool { !matches!(state, "queued" | "running") } -fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { +/// What is owed to the room. `deploy_state` is a deployment command's derived +/// state and detail, read from the control store by the caller. +fn due_posts( + ledger: &Ledger, + deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, +) -> Vec { let mut due = Vec::new(); for order in &ledger.orders { let id = &order.order_id; @@ -1029,7 +1044,7 @@ fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { }; let mut line = record.outcome.describe(); if let Outcome::Submitted { command_id } = &record.outcome { - match command_state(&ctx.state_store, command_id) { + match deploy_state(command_id) { Ok((state, detail)) if deploy_finished(state) => { line = format!("{line}: {state} {}", escape(detail.as_bytes())); } @@ -1053,7 +1068,8 @@ fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { pub(crate) fn post_pass(ctx: &Ctx) -> Result { let ledger = ctx.store.ledger()?; let mut sent = 0; - for due in due_posts(ctx, &ledger) { + let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); + for due in due_posts(&ledger, &deploy_state) { let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); let event_id = ctx.homeserver.post(&transaction, &due.text)?; ctx.store.put( @@ -1173,7 +1189,9 @@ pub(crate) fn handle( stream.set_read_timeout(Some(Duration::from_secs(5)))?; stream.set_write_timeout(Some(Duration::from_secs(5)))?; let peer = ctx.peers.peer(stream)?; - if peer.uid != ctx.steward_uid { + // The steward may order and read; root, the operator, may only read. + let steward = peer.uid == ctx.steward_uid; + if !steward && peer.uid != 0 { eprintln!("steward socket: refused a peer that is not the steward"); return refuse(stream, "peer", Code::Unauthorized); } @@ -1185,6 +1203,10 @@ pub(crate) fn handle( return refuse(stream, "frame", Code::Malformed); }; match request { + Request::Order(_) if !steward => { + eprintln!("steward socket: refused an order from a peer that is not the steward"); + refuse(stream, "peer", Code::Unauthorized) + } Request::Order(request) => { let checked = validate_reason(&request.reason).and_then(|()| request.operation.validate()); @@ -1368,17 +1390,27 @@ impl Systemd for HostSystemd { } } -/// The last `.service` or `.scope` in a process's cgroup path names its issuer. +/// A system service or scope names the issuer: the first unit under +/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices +/// above it are the manager's, and what a delegated subtree adds below it is +/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. fn unit_from_cgroup(text: &str) -> String { let line = text .lines() .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) .unwrap_or_default(); let path = line.rsplit_once(':').map_or("", |(_, path)| path); - path.rsplit('/') - .find(|segment| segment.ends_with(".service") || segment.ends_with(".scope")) - .filter(|segment| segment.len() <= UNIT_LIMIT) - .map_or_else(|| "unknown".to_owned(), str::to_owned) + let unit = path.strip_prefix("/system.slice/").and_then(|under| { + for segment in under.split('/') { + if segment.ends_with(".slice") { + continue; + } + let unit = segment.ends_with(".service") || segment.ends_with(".scope"); + return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); + } + None + }); + unit.map_or_else(|| "unknown".to_owned(), str::to_owned) } /// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. @@ -1440,6 +1472,8 @@ pub(crate) struct Synapse { pub(crate) address: std::net::SocketAddr, pub(crate) room: String, pub(crate) token_file: PathBuf, + /// The whole wait for a reply, however slowly it drips. + pub(crate) reply_within: Duration, } fn percent_encode(text: &str) -> String { @@ -1474,15 +1508,26 @@ impl Homeserver for Synapse { ); let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) .context("the homeserver is unreachable")?; - socket.set_read_timeout(Some(Duration::from_secs(10)))?; - socket.set_write_timeout(Some(Duration::from_secs(10)))?; + socket.set_write_timeout(Some(self.reply_within))?; socket.write_all(head.as_bytes())?; socket.write_all(&body)?; + let deadline = std::time::Instant::now() + self.reply_within; let mut reply = Vec::new(); - socket - .take(RESPONSE_FRAME_LIMIT as u64) - .read_to_end(&mut reply) - .context("the homeserver reply was cut short")?; + let mut chunk = [0u8; 4096]; + while reply.len() < RESPONSE_FRAME_LIMIT { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|left| !left.is_zero()) + .context("the homeserver reply was too slow")?; + socket.set_read_timeout(Some(remaining))?; + let count = socket + .read(&mut chunk) + .context("the homeserver reply was cut short")?; + if count == 0 { + break; + } + reply.extend_from_slice(&chunk[..count]); + } let split = reply .windows(4) .position(|window| window == b"\r\n\r\n") @@ -1664,7 +1709,9 @@ fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { .expect("a thread can be spawned"); } -fn bind_socket(path: &Path) -> Result { +/// The socket belongs to the steward and no one else may open it; the peer +/// credential check on every connection stays the lock. +fn bind_socket(path: &Path, steward_uid: u32) -> Result { if let Ok(metadata) = std::fs::symlink_metadata(path) { use std::os::unix::fs::FileTypeExt; ensure!( @@ -1674,12 +1721,20 @@ fn bind_socket(path: &Path) -> Result { std::fs::remove_file(path).context("removing a stale socket")?; } let listener = UnixListener::bind(path).context("binding the steward socket")?; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) + std::os::unix::fs::chown(path, Some(steward_uid), None) + .context("giving the steward socket to the steward")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) .context("setting the steward socket mode")?; Ok(listener) } +/// Files this service creates in Idunn's own stores (a deploy order writes +/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. +const IDUNN_UMASK: libc::mode_t = 0o027; + pub(crate) fn serve(options: ServeOptions) -> Result<()> { + // SAFETY: umask only sets the process file-mode mask; it cannot fail. + unsafe { libc::umask(IDUNN_UMASK) }; let binding = load_binding(&options.steward_binding)?; let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") .map(PathBuf::from) @@ -1698,11 +1753,12 @@ pub(crate) fn serve(options: ServeOptions) -> Result<()> { address: binding.address()?, room: binding.room.clone(), token_file: credentials.join(&binding.token_credential), + reply_within: Duration::from_secs(10), }), }); ctx.store.prepare()?; let pending = recover(&ctx)?; - let listener = bind_socket(&options.socket)?; + let listener = bind_socket(&options.socket, ctx.steward_uid)?; let (queue, orders) = mpsc::channel::(); for order_id in pending { queue.send(order_id)?; @@ -1945,10 +2001,12 @@ mod tests { } } - struct FakeBrake(AtomicBool); + /// Whether the brake is clear, and the principals it was asked about. + struct FakeBrake(AtomicBool, Mutex>); impl BrakeCheck for FakeBrake { - fn clear(&self, _principal: &str) -> bool { + fn clear(&self, principal: &str) -> bool { + self.1.lock().unwrap().push(principal.into()); self.0.load(Ordering::SeqCst) } } @@ -1988,7 +2046,7 @@ mod tests { ..RunReport::default() }), }); - let brake = Arc::new(FakeBrake(AtomicBool::new(true))); + let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); let home = Arc::new(FakeHome::default()); let (queue, orders) = mpsc::channel(); let ctx = Ctx { @@ -2424,7 +2482,7 @@ mod tests { .filter(|record| matches!(record.outcome, Outcome::Refused(_))) .count(); assert_eq!(refusals, direct.len() * 4 + 8 + 6); - let due = due_posts(&h.ctx, &ledger); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); let results = due .iter() .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) @@ -2891,7 +2949,7 @@ mod tests { .unwrap(); assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); // The result post waits for the transactions: queued is not finished. - let due = due_posts(&h.ctx, &ledger); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); assert_eq!(due.len(), 1); assert_eq!(due[0].kind, PostKind::Accepted); assert!(!deploy_finished("queued") && !deploy_finished("running")); @@ -2948,7 +3006,7 @@ mod tests { assert_eq!(record.stdout_tail.len(), TAIL_BYTES); assert!(!format!("{record:?}").contains("secret-looking")); // Neither the post nor the journal text carries a tail. - let results: Vec<_> = due_posts(&h.ctx, &ledger) + let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) .into_iter() .filter(|due| due.kind == PostKind::Result) .collect(); @@ -3088,20 +3146,30 @@ mod tests { unit_from_cgroup("0::/system.slice/claude-connector.service\n"), "claude-connector.service" ); + // What a delegated subtree adds below the unit is the peer's to name. assert_eq!( - unit_from_cgroup("0::/system.slice/huginn@eureka.service/sub/leaf\n"), + unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), "huginn@eureka.service" ); assert_eq!( - unit_from_cgroup("0::/user.slice/user-1000.slice/session-3.scope\n"), - "session-3.scope" + unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), + "getty@tty1.service" ); assert_eq!( - unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.service\n"), - "a.service" + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), + "a.scope" ); - assert_eq!(unit_from_cgroup("0::/\n"), "unknown"); - assert_eq!(unit_from_cgroup(""), "unknown"); + // A user manager's units, and paths no manager made, name no one. + for path in [ + "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", + "0::/user.slice/user-1000.slice/session-3.scope\n", + "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", + "0::/system.slice/\n", + "0::/\n", + "", + ] { + assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); + } } #[test] @@ -3223,6 +3291,7 @@ mod tests { address, room: "!room:example.org".into(), token_file, + reply_within: Duration::from_secs(10), }; assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); let request = requests.recv().unwrap(); @@ -3238,7 +3307,8 @@ mod tests { Synapse { address, room: "!r:e".into(), - token_file: directory.path().join("absent") + token_file: directory.path().join("absent"), + reply_within: Duration::from_secs(10), } .post("t", "x") .is_err() @@ -3356,4 +3426,525 @@ mod tests { assert!(parse(&refused).is_err(), "{refused:?} parsed"); } } + + #[test] + fn an_order_runs_only_under_the_steward_bound_now() { + let mut h = harness(); + let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The binding changes while both wait: another uid, then another + // principal with the old uid. Each is refused with a record, before + // the brake is asked and before anything runs. + h.ctx.steward_uid = STEWARD + 7; + execute(&h.ctx, &first).unwrap(); + h.ctx.steward_uid = STEWARD; + h.ctx.principal = "void".into(); + execute(&h.ctx, &second).unwrap(); + for id in [&first, &second] { + assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); + assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); + } + assert!(h.runs().is_empty()); + assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); + // The refusal is posted like any other. + let ledger = h.ctx.store.ledger().unwrap(); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert!(due.iter().any(|due| due.kind == PostKind::Result + && due.text.contains("refused: the issuer is not the current steward"))); + // The other direction: an order the bound steward issued after the + // rebinding, and a recovered one under an unchanged binding, run and + // are gated by the bound principal's brake. + let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); + let resumed = harness(); + let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); + assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&resumed.ctx, &waiting).unwrap(); + execute(&h.ctx, &third).unwrap(); + assert_eq!(resumed.outcome(&waiting), Outcome::Ran); + assert_eq!(h.outcome(&third), Outcome::Ran); + assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); + assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); + } + + #[test] + fn root_reads_orders_and_never_issues_one() { + let mut h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + h.ctx.peers = Arc::new(FakePeers(0)); + let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); + assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("root could not read an order"); + }; + assert_eq!(view.order.order_id, id); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("root could not list orders"); + }; + assert_eq!(lines.len(), 1, "root's refused order left no record"); + h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); + assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); + } + + #[test] + fn every_order_posts_once_in_its_turn() { + let h = harness(); + let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] + .into_iter() + .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) + .collect(); + for id in &ids { + execute(&h.ctx, id).unwrap(); + } + assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); + assert_eq!(post_pass(&h.ctx).unwrap(), 0); + let sent = h.home.sent.lock().unwrap().clone(); + let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); + keys.sort(); + let mut expected: Vec = ids + .iter() + .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) + .collect(); + expected.sort(); + assert_eq!(keys, expected); + // An order accepted later is posted by the next pass alone. + let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + execute(&h.ctx, &late).unwrap(); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + } + + #[test] + fn a_deploys_result_is_posted_once_its_command_has_finished() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let ledger = h.ctx.store.ledger().unwrap(); + let results = |state: Option<(&'static str, &str)>| { + let owed = due_posts(&ledger, &|_| match state { + Some((state, detail)) => Ok((state, detail.to_owned())), + None => Err(anyhow!("the command cannot be read")), + }); + owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() + }; + assert!(results(Some(("queued", ""))).is_empty()); + assert!(results(Some(("running", "vili:Sealing"))).is_empty()); + assert!(results(None).is_empty()); + for state in ["succeeded", "failed"] { + let owed = results(Some((state, "vili"))); + assert_eq!(owed.len(), 1, "{state}"); + assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); + assert!(owed[0].text.contains("submitted deployment up-")); + } + } + + #[test] + fn intake_gives_up_on_a_peer_that_sends_no_frame() { + let h = harness(); + let (mut client, mut server) = UnixStream::pair().unwrap(); + // Half a length prefix, then nothing: the connection must end by itself. + client.write_all(&[0, 0]).unwrap(); + let (done, ended) = mpsc::channel(); + let worker = thread::spawn(move || { + let result = handle(&h.ctx, &mut server, &h.queue); + let _ = done.send(result.is_ok()); + h + }); + let answered = ended + .recv_timeout(Duration::from_secs(20)) + .expect("intake never gave up on a silent peer"); + assert!(answered, "a silent peer is refused as malformed, not left as an error"); + let h = worker.join().unwrap(); + assert_eq!( + rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn serve_options_are_read_from_every_flag() { + let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); + let defaults = parse(&["serve"]).unwrap(); + assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); + assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); + assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); + let given = parse(&[ + "serve", + "--state-store", + "/a/control.cc", + "--store", + "/b", + "--steward-binding", + "/c.toml", + "--socket", + "/d.sock", + ]) + .unwrap(); + assert_eq!( + given, + ServeOptions { + state_store: "/a/control.cc".into(), + store: "/b".into(), + steward_binding: "/c.toml".into(), + socket: "/d.sock".into(), + } + ); + assert!(parse(&["serve", "--store"]).is_err()); + assert!(parse(&["serve", "--other", "/x"]).is_err()); + assert!(parse(&["start"]).is_err()); + } + + #[test] + fn the_steward_user_is_a_real_user_and_never_root() { + assert!(user_uid("root").is_err()); + assert!(user_uid("no-such-user-here").is_err()); + assert!(user_uid("bad\0name").is_err()); + assert_eq!(user_uid("nobody").unwrap(), 65534); + } + + #[test] + fn the_orders_directory_is_made_private() { + let directory = TempDir::new().unwrap(); + let store = OrderStore::new(&directory.path().join("orders")); + store.prepare().unwrap(); + let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + #[test] + fn the_socket_belongs_to_the_steward_alone() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("steward.sock"); + let me = std::fs::metadata(directory.path()).unwrap().uid(); + let _listener = bind_socket(&path, me).unwrap(); + let metadata = std::fs::metadata(&path).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); + // A stale socket is replaced; anything else in the way is not. + let _again = bind_socket(&path, me).unwrap(); + let other = directory.path().join("other"); + std::fs::write(&other, "x").unwrap(); + assert!(bind_socket(&other, me).is_err()); + } + + #[test] + fn synapse_gives_up_on_a_reply_that_never_ends() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + thread::spawn(move || { + // The first connection stays silent; the second drips a byte at a time. + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + thread::spawn(move || { + let mut chunk = [0u8; 4096]; + let _ = socket.read(&mut chunk); + for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { + if index == 0 || socket.write_all(&[*byte]).is_err() { + break; + } + thread::sleep(Duration::from_millis(40)); + } + thread::sleep(Duration::from_secs(5)); + }); + } + }); + let synapse = Synapse { + address, + room: "!r:e".into(), + token_file, + reply_within: Duration::from_millis(600), + }; + for _ in 0..2 { + let started = std::time::Instant::now(); + let error = synapse.post("t", "x").unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error:#}"); + assert!(!format!("{error:#}").contains("tok-123")); + } + } + + /// The whole service, as root, against fake `systemctl`, `systemd-run` + /// and a loopback homeserver, with the steward connecting from another + /// uid. It runs only where the test process is root (the verify container). + #[test] + fn serve_end_to_end_as_root() { + // SAFETY: geteuid reads the process credentials. + if unsafe { libc::geteuid() } != 0 { + eprintln!("serve_end_to_end_as_root: skipped, not root"); + return; + } + let directory = TempDir::new().unwrap(); + let root = directory.path().to_owned(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let bin = root.join("bin"); + std::fs::create_dir(&bin).unwrap(); + let systemctl = format!( + "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ + case \"$*\" in *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + root.display() + ); + let run = format!( + "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", + root.display() + ); + for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { + let path = bin.join(name); + std::fs::write(&path, text).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let credentials = root.join("cred"); + std::fs::create_dir(&credentials).unwrap(); + std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (posted, posts) = mpsc::channel::(); + thread::spawn(move || { + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + let mut received = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + let text = String::from_utf8_lossy(&received).into_owned(); + if let Some(split) = text.find("\r\n\r\n") { + let length = text + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if received.len() >= split + 4 + length { + break; + } + } + let count = socket.read(&mut chunk).unwrap_or(0); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); + let _ = socket.write_all( + format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), + ); + } + }); + let binding = root.join("steward.toml"); + std::fs::write( + &binding, + format!( + "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ + token_credential = \"idunn-matrix-token\"\n" + ), + ) + .unwrap(); + std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); + let run_directory = root.join("run"); + std::fs::create_dir(&run_directory).unwrap(); + std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let socket = run_directory.join("steward.sock"); + let store = root.join("orders"); + let control = root.join("state").join("control.cc"); + // SAFETY: the only test that reads these variables is this one; PATH + // keeps its old entries. + unsafe { + std::env::set_var( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), + ); + std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); + } + let options = ServeOptions { + state_store: control.clone(), + store: store.clone(), + steward_binding: binding, + socket: socket.clone(), + }; + // The unit's umask must not decide the modes of what the service writes. + // SAFETY: umask only sets the process file-mode mask. + let before = unsafe { libc::umask(0o117) }; + let (ended, serve_end) = mpsc::channel::(); + thread::spawn(move || { + let result = serve(options); + let _ = ended.send(format!("{result:?}")); + }); + for _ in 0..100 { + if socket.exists() || serve_end.try_recv().is_ok() { + break; + } + thread::sleep(Duration::from_millis(100)); + } + // SAFETY: as above. + unsafe { libc::umask(before) }; + assert!(socket.exists(), "serve never bound its socket"); + let metadata = std::fs::metadata(&socket).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); + // Root is not the steward: it may read, it may not order. + let unit = |name: &str| { + Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: name.into(), + }, + reason: "end to end".into(), + }) + }; + let as_root = |request: &Request| { + let mut stream = UnixStream::connect(&socket).unwrap(); + write_frame(&mut stream, request).unwrap(); + rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + }; + assert_eq!( + as_root(&unit("nginx.service")), + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert_eq!(as_root(&Request::List), Response::List(Vec::new())); + let orders = OrderStore::new(&store); + let finished = |order_id: &str| { + for _ in 0..200 { + if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { + return record.clone(); + } + thread::sleep(Duration::from_millis(100)); + } + panic!("{order_id} never concluded"); + }; + // The steward is uid 65534 with gid 0: a service that read the gid for + // the uid would take it for root. + let first = steward_asks(&socket, &unit("nginx.service")); + let Response::Accepted { order_id: first } = first else { + panic!("the steward was not accepted: {first:?}"); + }; + assert_eq!(finished(&first).outcome, Outcome::Ran); + std::fs::write(root.join("fail"), "").unwrap(); + let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { + panic!("the second order was not accepted"); + }; + let failed = finished(&second); + assert_eq!( + failed.outcome, + Outcome::Failed { + exit_code: Some(3), + signal: None + } + ); + assert_eq!(failed.stderr_tail, b"boom\n"); + let deploy = Request::Order(OrderRequest { + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "end to end".into(), + }); + let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { + panic!("the deploy was not accepted"); + }; + assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); + let ledger = orders.ledger().unwrap(); + let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); + assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // What the service called, exactly. + let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); + assert!( + run_log.contains(&format!( + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart -- nginx.service", + order_uuid(&first) + )), + "{run_log}" + ); + let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); + assert!( + systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), + "{systemctl_log}" + ); + // Every record is posted; the deploy's result waits for its command. + let mut seen = Vec::new(); + while seen.len() < 5 { + seen.push(posts.recv_timeout(Duration::from_secs(30)).expect("five posts within 30 s")); + } + for (id, kinds) in [ + (&first, ["accepted", "result"].as_slice()), + (&second, &["accepted", "result"]), + (&third, &["accepted"]), + ] { + for kind in kinds { + let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); + assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); + } + } + // Modes: the store is private, and the deploy wrote control.cc under + // Idunn's umask, not the 0117 the process was started with. + let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; + assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); + assert_eq!(mode(&control), 0o640); + } + + /// One request as the steward: uid 65534 and gid 0, from a forked child + /// that only makes system calls. + fn steward_asks(socket: &Path, request: &Request) -> Response { + let body = rmp_serde::to_vec_named(request).unwrap(); + let mut frame = (body.len() as u32).to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + // SAFETY: plain data. + let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { + address.sun_path[index] = *byte as libc::c_char; + } + let mut fds = [0i32; 2]; + // SAFETY: `fds` holds two descriptors. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + // SAFETY: the child allocates nothing and ends with _exit. + let pid = unsafe { libc::fork() }; + if pid == 0 { + // SAFETY: only system calls on buffers made before the fork. + unsafe { + libc::close(fds[0]); + if libc::setgroups(0, std::ptr::null()) != 0 + || libc::setgid(0) != 0 + || libc::setuid(65534) != 0 + { + libc::_exit(2); + } + let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if libc::connect( + fd, + (&address as *const libc::sockaddr_un).cast(), + std::mem::size_of::() as libc::socklen_t, + ) != 0 + { + libc::_exit(3); + } + let mut offset = 0usize; + while offset < frame.len() { + let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); + if count <= 0 { + libc::_exit(4); + } + offset += count as usize; + } + let mut buffer = [0u8; 4096]; + loop { + let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); + if count <= 0 { + break; + } + libc::write(fds[1], buffer.as_ptr().cast(), count as usize); + } + libc::_exit(0); + } + } + // SAFETY: the parent's copy of the write end. + unsafe { libc::close(fds[1]) }; + let mut reply = Vec::new(); + // SAFETY: `fds[0]` is an open descriptor this function now owns. + let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; + pipe.read_to_end(&mut reply).unwrap(); + let mut status = 0; + // SAFETY: `pid` is our child. + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); + rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + } } From 572f9e6aa424661c36abf7e876a467a08d033eb8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:31:34 +0100 Subject: [PATCH 109/122] Steward orders: the end-to-end test keeps serve's umask until it has read control.cc Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index e707307..3f0d229 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3776,8 +3776,6 @@ mod tests { } thread::sleep(Duration::from_millis(100)); } - // SAFETY: as above. - unsafe { libc::umask(before) }; assert!(socket.exists(), "serve never bound its socket"); let metadata = std::fs::metadata(&socket).unwrap(); assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); @@ -3877,7 +3875,10 @@ mod tests { // Idunn's umask, not the 0117 the process was started with. let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); - assert_eq!(mode(&control), 0o640); + let control_mode = mode(&control); + // SAFETY: umask only sets the process file-mode mask. + unsafe { libc::umask(before) }; + assert_eq!(control_mode, 0o640); } /// One request as the steward: uid 65534 and gid 0, from a forked child From 7e15bf1a7b5e9003fd44025dd9e5fae05f18b2b3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:33:23 +0100 Subject: [PATCH 110/122] Steward orders: the Synapse deadline test fails instead of hanging when the deadline is gone Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 3f0d229..261cb24 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3658,11 +3658,20 @@ mod tests { token_file, reply_within: Duration::from_millis(600), }; + let synapse = Arc::new(synapse); for _ in 0..2 { let started = std::time::Instant::now(); - let error = synapse.post("t", "x").unwrap_err(); - assert!(started.elapsed() < Duration::from_secs(3), "{error:#}"); - assert!(!format!("{error:#}").contains("tok-123")); + let (done, ended) = mpsc::channel(); + let poster = Arc::clone(&synapse); + thread::spawn(move || { + let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); + }); + let error = ended + .recv_timeout(Duration::from_secs(4)) + .expect("the post never gave up") + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error}"); + assert!(!error.contains("tok-123")); } } From 8a1beb5c873c96b2741ffb64e7cfb958adbdcd4f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:35:30 +0100 Subject: [PATCH 111/122] README: an order is gated by the steward bound at execution; the socket is the steward's; root reads Co-Authored-By: Claude Sonnet 5.5 --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2bf6675..5240106 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,12 @@ match themselves, and a last step fails the order unless every named package is installed. An order runs once, after the steward's uid is checked and the instance's -brake check (`mind-control --instance check`) exits 0; an order -that had started when the service died is recorded as lost and never run -again. No order releases a brake. Acceptance and result are posted to +brake check (`mind-control --instance check`) exits 0. The check +that the order's principal and uid are the bound steward's runs again at every +execution, so an order accepted before `steward.toml` changed is refused and +recorded, not run. The socket belongs to the steward user alone (mode 0600); +root may `show` and `list` orders but never issue one. An order that had +started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to `#operator` as a retried projection of `orders.cc`: a chat outage delays the posts and not the orders. A refusal names the field and a code, never the value it refused. From 9653d270e5d47b6bc7b17a6680666e5529b26e07 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:16:04 +0100 Subject: [PATCH 112/122] Steward orders: pin the ledger schema guards, invocation ids and post backoff; tests fail fast instead of waiting Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 91 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 261cb24..1394891 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -1690,6 +1690,16 @@ pub(crate) fn parse_serve(mut args: impl Iterator) -> Result u64 { + if failures == 0 { + 2 + } else { + (2u64 << failures.min(5)).min(60) + } +} + /// Run a thread whose end ends the service: systemd restarts it, and the /// start recovery settles what it left. fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { @@ -1775,18 +1785,14 @@ pub(crate) fn serve(options: ServeOptions) -> Result<()> { vital("poster", move || { let mut failures = 0u32; loop { - let pause = match post_pass(&poster) { - Ok(_) => { - failures = 0; - 2 - } + match post_pass(&poster) { + Ok(_) => failures = 0, Err(error) => { failures = failures.saturating_add(1); eprintln!("steward orders: posting is delayed: {error:#}"); - (2u64 << failures.min(5)).min(60) } - }; - thread::sleep(Duration::from_secs(pause)); + } + thread::sleep(Duration::from_secs(post_pause(failures))); } }); for stream in listener.incoming() { @@ -2077,6 +2083,7 @@ mod tests { impl Harness { fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); write(&mut client); handle(&self.ctx, &mut server, &self.queue).unwrap(); let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); @@ -3543,6 +3550,7 @@ mod tests { fn intake_gives_up_on_a_peer_that_sends_no_frame() { let h = harness(); let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); // Half a length prefix, then nothing: the connection must end by itself. client.write_all(&[0, 0]).unwrap(); let (done, ended) = mpsc::channel(); @@ -3563,6 +3571,63 @@ mod tests { assert!(!h.ctx.store.path().exists()); } + #[test] + fn a_record_under_another_schema_is_not_read() { + let put = |h: &Harness, schema: &str, key: &str| match schema { + _ if schema.contains("start") => h.ctx.store.put( + key, + schema, + &StartRecord { + schema_version: schema.into(), + order_id: key.into(), + started_at_unix_millis: 1, + actuation: "idunn-order-x".into(), + }, + ), + _ if schema.contains("outcome") => h.ctx.store.put( + key, + schema, + &OutcomeRecord { + schema_version: schema.into(), + order_id: key.into(), + ended_at_unix_millis: 1, + outcome: Outcome::Lost, + stdout_tail: Vec::new(), + stderr_tail: Vec::new(), + }, + ), + _ => h.ctx.store.put( + key, + schema, + &PostRecord { + schema_version: schema.into(), + post_key: key.into(), + event_id: "$e".into(), + posted_at_unix_millis: 1, + }, + ), + }; + for (current, key) in [ + (START_SCHEMA, "ord-a"), + (OUTCOME_SCHEMA, "ord-a"), + (POST_SCHEMA, "ord-a:accepted"), + ] { + let h = harness(); + assert!(put(&h, current, key).unwrap()); + assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); + let h = harness(); + let other = current.replace(".v1", ".v0"); + assert!(put(&h, &other, key).unwrap()); + assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); + } + } + + #[test] + fn posts_are_retried_after_a_widening_pause() { + let pauses: Vec = (0..9).map(post_pause).collect(); + assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); + } + #[test] fn serve_options_are_read_from_every_flag() { let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); @@ -3692,7 +3757,7 @@ mod tests { std::fs::create_dir(&bin).unwrap(); let systemctl = format!( "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ - case \"$*\" in *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", root.display() ); let run = format!( @@ -3800,6 +3865,7 @@ mod tests { }; let as_root = |request: &Request| { let mut stream = UnixStream::connect(&socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); write_frame(&mut stream, request).unwrap(); rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() }; @@ -3851,6 +3917,10 @@ mod tests { let ledger = orders.ledger().unwrap(); let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // An invocation id is what a succeeding systemctl prints, if anything. + assert_eq!(HostSystemd.invocation("quiet.service"), None); + assert_eq!(HostSystemd.invocation("broken.service"), None); + assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); // What the service called, exactly. let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); assert!( @@ -3868,7 +3938,7 @@ mod tests { // Every record is posted; the deploy's result waits for its command. let mut seen = Vec::new(); while seen.len() < 5 { - seen.push(posts.recv_timeout(Duration::from_secs(30)).expect("five posts within 30 s")); + seen.push(posts.recv_timeout(Duration::from_secs(15)).expect("five posts within 15 s")); } for (id, kinds) in [ (&first, ["accepted", "result"].as_slice()), @@ -3911,6 +3981,7 @@ mod tests { // SAFETY: only system calls on buffers made before the fork. unsafe { libc::close(fds[0]); + libc::alarm(10); if libc::setgroups(0, std::ptr::null()) != 0 || libc::setgid(0) != 0 || libc::setuid(65534) != 0 From caacbb4bbaa4de31ffa63f76c1d4eec53a5080ce Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:35:31 +0100 Subject: [PATCH 113/122] Steward orders: the end-to-end test waits 10 s for its posts Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 1394891..688edc8 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3938,7 +3938,7 @@ mod tests { // Every record is posted; the deploy's result waits for its command. let mut seen = Vec::new(); while seen.len() < 5 { - seen.push(posts.recv_timeout(Duration::from_secs(15)).expect("five posts within 15 s")); + seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); } for (id, kinds) in [ (&first, ["accepted", "result"].as_slice()), From d9710c988a0a09adb74bb7f3ae2580ed881c9ef0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:35:43 +0100 Subject: [PATCH 114/122] Steward orders: the ledger reads an order only under its own schema too Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 688edc8..1d72e0e 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3573,8 +3573,25 @@ mod tests { #[test] fn a_record_under_another_schema_is_not_read() { - let put = |h: &Harness, schema: &str, key: &str| match schema { - _ if schema.contains("start") => h.ctx.store.put( + let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { + "order" => h.ctx.store.put( + key, + schema, + &OrderRecord { + schema_version: schema.into(), + order_id: key.into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }, + ), + "start" => h.ctx.store.put( key, schema, &StartRecord { @@ -3584,7 +3601,7 @@ mod tests { actuation: "idunn-order-x".into(), }, ), - _ if schema.contains("outcome") => h.ctx.store.put( + "outcome" => h.ctx.store.put( key, schema, &OutcomeRecord { @@ -3607,17 +3624,18 @@ mod tests { }, ), }; - for (current, key) in [ - (START_SCHEMA, "ord-a"), - (OUTCOME_SCHEMA, "ord-a"), - (POST_SCHEMA, "ord-a:accepted"), + for (kind, current, key) in [ + ("order", ORDER_SCHEMA, "ord-a"), + ("start", START_SCHEMA, "ord-a"), + ("outcome", OUTCOME_SCHEMA, "ord-a"), + ("post", POST_SCHEMA, "ord-a:accepted"), ] { let h = harness(); - assert!(put(&h, current, key).unwrap()); + assert!(put(&h, kind, current, key).unwrap()); assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); let h = harness(); let other = current.replace(".v1", ".v0"); - assert!(put(&h, &other, key).unwrap()); + assert!(put(&h, kind, &other, key).unwrap()); assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); } } From d29bf93be60fdf08412d43299f28e0fbe40016b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:26:29 +0100 Subject: [PATCH 115/122] Steward orders: refuse a start or stop whose unit ends by emergency action or isolates; boot and recovery units are never started from stopped refusal_for now reads SuccessAction, FailureAction, StartLimitAction and JobTimeoutAction (each mapped to the target it runs, so reboot, poweroff, halt, kexec, soft-reboot and exit, with -force and -immediate) and OnFailure and OnSuccess as units a touched unit fires, followed through what they pull in and what those fire. An action it does not know fails the order closed. A stopped DefaultDependencies=no unit in the transaction is refused (Via:: EarlyBoot): rescue.service ends by isolating default.target, which no edge describes. Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 175 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 1d72e0e..b9d64b7 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -374,6 +374,10 @@ pub(crate) enum Via { Name, Alias, Propagation, + /// A unit the order would start from a stopped state that is boot or + /// recovery machinery (`DefaultDependencies=no`): such a unit can end by + /// isolating a target, which no dependency edge describes. + EarlyBoot, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -418,6 +422,7 @@ impl Outcome { Via::Name => "name", Via::Alias => "alias", Via::Propagation => "propagation", + Via::EarlyBoot => "early-boot", } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), @@ -641,6 +646,32 @@ const EDGES: [(&str, Edge); 13] = [ ("StopPropagatedFrom", Edge::Stops), ]; +/// The properties naming what systemd does when a unit ends or its job times +/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). +const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; + +/// The properties naming units systemd starts when a unit ends. +const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; + +/// The target an emergency action runs, `None` for `none`. An action this +/// check does not know is an error: the unit is not checked, so not run. +fn action_target(action: &str) -> Result> { + let base = ["-force", "-immediate"] + .iter() + .find_map(|suffix| action.strip_suffix(suffix)) + .unwrap_or(action); + Ok(match base { + "" | "none" => None, + "reboot" => Some("reboot.target"), + "poweroff" => Some("poweroff.target"), + "halt" => Some("halt.target"), + "kexec" => Some("kexec.target"), + "soft-reboot" => Some("soft-reboot.target"), + "exit" => Some("exit.target"), + _ => bail!("a unit has an emergency action this check does not know"), + }) +} + /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { @@ -648,6 +679,11 @@ pub(crate) struct UnitFacts { pub(crate) names: Vec, pub(crate) not_found: bool, pub(crate) running: bool, + /// `DefaultDependencies=no`: boot, recovery and shutdown machinery. + pub(crate) early_boot: bool, + /// What systemd starts when this unit ends or times out: the targets of + /// its emergency actions and its `OnFailure=` and `OnSuccess=` units. + pub(crate) fires: Vec, pub(crate) pulls: Vec, pub(crate) conflicts: Vec, pub(crate) stops: Vec, @@ -762,6 +798,27 @@ impl Transaction<'_> { Ok(reached) } + /// Every unit the emergency actions and `OnFailure=`/`OnSuccess=` of + /// `touched` would start, with what each pulls in, and in turn what those + /// that are not running would set off. A touched unit ends, so it fires. + fn fired(&mut self, touched: &BTreeSet) -> Result> { + let mut fired = BTreeSet::new(); + let mut seen = BTreeSet::new(); + let mut queue: VecDeque = touched.iter().cloned().collect(); + while let Some(unit) = queue.pop_front() { + if !seen.insert(unit.clone()) { + continue; + } + let seeds: BTreeSet = self.facts(&unit)?.fires.iter().cloned().collect(); + for reached in self.reach(&seeds, Edge::Pulls)? { + if fired.insert(reached.clone()) && !self.facts(&reached)?.running { + queue.push_back(reached); + } + } + } + Ok(fired) + } + /// Every unit a start of `started` stops through `Conflicts=`, and what /// that stop takes down with it. fn conflicted(&mut self, started: &BTreeSet) -> Result> { @@ -807,6 +864,13 @@ fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result Result Vec { /// The properties one `systemctl show` is asked for: identity, state, edges. fn show_properties() -> String { - ["Id", "Names", "LoadState", "ActiveState"] + ["Id", "Names", "LoadState", "ActiveState", "DefaultDependencies"] .into_iter() + .chain(ACTIONS) + .chain(ON_END) .chain(EDGES.iter().map(|(property, _)| *property)) .collect::>() .join(",") @@ -1333,6 +1402,9 @@ fn parse_show(output: &str) -> Result { "ActiveState" => { facts.running = matches!(value, "active" | "activating" | "reloading"); } + "DefaultDependencies" => facts.early_boot = value == "no", + _ if ACTIONS.contains(&key) => facts.fires.extend(action_target(value)?.map(str::to_owned)), + _ if ON_END.contains(&key) => facts.fires.extend(parse_names(value)), _ => { if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { facts.along_mut(*edge).extend(parse_names(value)); @@ -2719,6 +2791,107 @@ mod tests { assert!(h.runs().is_empty()); } + #[test] + fn a_unit_whose_end_reboots_the_host_is_refused_by_its_properties() { + let h = harness(); + yggdrasil(&h, true); + // Each emergency action property alone, in each of systemd's spellings, + // on a unit no name list knows: every verb that starts or stops it. + for (property, action) in [ + ("SuccessAction", "reboot"), + ("FailureAction", "reboot"), + ("StartLimitAction", "poweroff-force"), + ("JobTimeoutAction", "reboot-immediate"), + ("SuccessAction", "poweroff"), + ] { + let name = format!("storagetm-{property}.service"); + h.describe(&name, &[(property, action), ("ActiveState", "inactive")]); + for verb in [Verb::Start, Verb::Restart, Verb::Stop] { + let outcome = outcome_of_unit(&h, verb, &name); + assert!(hit(&outcome).is_some(), "{verb:?} {property}={action} {outcome:?}"); + } + } + // OnFailure= and OnSuccess= start the units they name, and what those + // set off in turn. + for property in ["OnFailure", "OnSuccess"] { + let name = format!("handled-{property}.service"); + h.describe(&name, &[(property, "reboot.target"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, &name); + assert!(hit(&outcome).is_some(), "{property} {outcome:?}"); + } + h.describe("on-failure-chain.service", &[("OnFailure", "handler.service"), ("ActiveState", "inactive")]); + h.describe("handler.service", &[("FailureAction", "reboot"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "on-failure-chain.service"); + assert!(hit(&outcome).is_some(), "{outcome:?}"); + // A unit a start pulls in fires its action too. + h.describe("pulls-storagetm.service", &[("Wants", "storagetm.service")]); + h.describe("storagetm.service", &[("SuccessAction", "reboot"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "pulls-storagetm.service"); + assert!(hit(&outcome).is_some(), "{outcome:?}"); + // A running unit the order does not stop is not ended by it, and an + // action of none ends nothing. + h.describe("pulls-running.service", &[("Wants", "running-action.service")]); + h.describe("running-action.service", &[("SuccessAction", "reboot")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "pulls-running.service"), Outcome::Ran); + h.describe("quiet-actions.service", &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", "")]); + for verb in [Verb::Start, Verb::Restart, Verb::Stop] { + assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); + } + // An action this check does not know is not checked, so it does not run. + h.describe("odd-action.service", &[("SuccessAction", "implode")]); + let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); + execute(&h.ctx, &id).unwrap(); + assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); + assert!(h.runs().is_empty()); + } + + #[test] + fn a_stopped_boot_or_recovery_unit_is_never_started() { + let h = harness(); + yggdrasil(&h, true); + // rescue.service ends by isolating default.target, which stops every + // unit outside it; no edge says so, its DefaultDependencies=no does. + h.describe( + "rescue.service", + &[("DefaultDependencies", "no"), ("ActiveState", "inactive")], + ); + h.describe("rescue.target", &[("Requires", "rescue.service"), ("ActiveState", "inactive")]); + h.describe("pulls-rescue.service", &[("Wants", "rescue.target")]); + for (verb, unit) in [ + (Verb::Start, "rescue.service"), + (Verb::Restart, "rescue.service"), + (Verb::Start, "rescue.target"), + (Verb::Restart, "rescue.target"), + (Verb::Start, "pulls-rescue.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert!( + matches!( + outcome, + Outcome::Refused(Refused::AlwaysRefused { ref unit, via: Via::EarlyBoot }) + if unit == "rescue.service" + ), + "{verb:?} {unit} was {outcome:?}" + ); + } + assert!(h.runs().is_empty()); + // Stopping it, or starting one whose defaults are on, is an ordinary order. + assert_eq!(outcome_of_unit(&h, Verb::Stop, "rescue.service"), Outcome::Ran); + h.describe("ordinary.service", &[("DefaultDependencies", "yes"), ("ActiveState", "inactive")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "ordinary.service"), Outcome::Ran); + // A boot unit that is already running is left alone by a start. + h.describe("journald.service", &[("DefaultDependencies", "no")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); + } + + #[test] + fn show_asks_for_every_property_the_refusal_reads() { + let asked = show_properties(); + for property in ["DefaultDependencies", "SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction", "OnFailure", "OnSuccess"] { + assert!(asked.split(',').any(|asked| asked == property), "{property}"); + } + } + #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); From 631fb7836b6e875fcbfa64fcc01b0ea78a4a08fd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:29:13 +0100 Subject: [PATCH 116/122] Steward orders: pin the new show properties; the unknown-action probe counts runs instead of expecting none Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index b9d64b7..884afda 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2838,11 +2838,12 @@ mod tests { assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); } // An action this check does not know is not checked, so it does not run. + let ran = h.runs().len(); h.describe("odd-action.service", &[("SuccessAction", "implode")]); let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); execute(&h.ctx, &id).unwrap(); assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); - assert!(h.runs().is_empty()); + assert_eq!(h.runs().len(), ran); } #[test] @@ -2884,14 +2885,6 @@ mod tests { assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); } - #[test] - fn show_asks_for_every_property_the_refusal_reads() { - let asked = show_properties(); - for property in ["DefaultDependencies", "SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction", "OnFailure", "OnSuccess"] { - assert!(asked.split(',').any(|asked| asked == property), "{property}"); - } - } - #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); @@ -3379,8 +3372,9 @@ mod tests { // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), - "Id,Names,LoadState,ActiveState,Requires,Requisite,BindsTo,Wants,Upholds,\ - Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ + "Id,Names,LoadState,ActiveState,DefaultDependencies,SuccessAction,FailureAction,\ + StartLimitAction,JobTimeoutAction,OnFailure,OnSuccess,Requires,Requisite,BindsTo,\ + Wants,Upholds,Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ PropagatesStopTo,StopPropagatedFrom" ); // A backslash outside quotes is a backslash; inside, it escapes one character. From 879b76a8425df76ba18f17a78fff488e87e1bc0a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:50:23 +0100 Subject: [PATCH 117/122] Steward orders: pin every emergency action spelling to its target Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 884afda..69eea41 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2846,6 +2846,31 @@ mod tests { assert_eq!(h.runs().len(), ran); } + #[test] + fn every_emergency_action_runs_the_target_it_names() { + for (action, target) in [ + ("none", None), + ("", None), + ("reboot", Some("reboot.target")), + ("reboot-force", Some("reboot.target")), + ("reboot-immediate", Some("reboot.target")), + ("poweroff", Some("poweroff.target")), + ("poweroff-force", Some("poweroff.target")), + ("poweroff-immediate", Some("poweroff.target")), + ("halt", Some("halt.target")), + ("halt-force", Some("halt.target")), + ("kexec", Some("kexec.target")), + ("kexec-immediate", Some("kexec.target")), + ("soft-reboot", Some("soft-reboot.target")), + ("soft-reboot-force", Some("soft-reboot.target")), + ("exit", Some("exit.target")), + ("exit-force", Some("exit.target")), + ] { + assert_eq!(action_target(action).unwrap(), target, "{action}"); + } + assert!(action_target("implode").is_err()); + } + #[test] fn a_stopped_boot_or_recovery_unit_is_never_started() { let h = harness(); From 4894ad20df0f98fa308044d82e1f9fa21468ddf7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:52:41 +0100 Subject: [PATCH 118/122] README: an order's transaction follows emergency actions and OnFailure/OnSuccess; stopped boot units are refused Co-Authored-By: Claude Opus 5.5 --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5240106..28c1e94 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,14 @@ refused unit that is already running is left alone) and stops what conflicts with it, a stop takes down what depends on it, and a restart does both. The transaction is read from `systemctl show`, so `start poweroff.target` is refused because every refused unit conflicts with `shutdown.target`, not because -targets are banned. A unit `systemctl show` cannot resolve fails the order. +targets are banned. The units a start or stop ends or times out are followed too: +the target each of `SuccessAction=`, `FailureAction=`, `StartLimitAction=` and +`JobTimeoutAction=` runs (`reboot`, `poweroff`, `halt`, `kexec`, `soft-reboot`, +`exit`, with `-force` and `-immediate`), and the units `OnFailure=` and +`OnSuccess=` name, with what each pulls in; an action it does not know fails the +order. A start refuses a stopped `DefaultDependencies=no` unit (boot and recovery +machinery, such as `rescue.service`, which ends by isolating the default +target), whatever pulls it in. A unit `systemctl show` cannot resolve fails the order. Slices, scopes and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with From a859914b0bb481ee9abeb999d73753f9313c28a0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:40:58 +0100 Subject: [PATCH 119/122] Steward orders: run every unit verb as one job and check only the named unit Delete the transaction model (Edge, EDGES, action_target, Transaction, the early-boot rule). Every unit verb runs systemctl --job-mode=ignore-requirements -- ; refusal_for reads one systemctl show of the ordered unit: its names, the host-ending names, what fires when it ends, and who is bound to it. Targets are no longer orderable. Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 806 +++++++++++------------------------------- 1 file changed, 207 insertions(+), 599 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 69eea41..d5fdcfe 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,7 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::BTreeMap; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -58,9 +58,8 @@ const TAIL_BYTES: usize = 8 * 1024; const REASON_LIMIT: usize = 300; const PACKAGE_LIMIT: usize = 16; const UNIT_LIMIT: usize = 200; -const CLOSURE_LIMIT: usize = 512; const LIST_LIMIT: usize = 50; -const UNIT_SUFFIXES: [&str; 6] = [".service", ".socket", ".timer", ".path", ".mount", ".target"]; +const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; /// The always-refused set: Idunn's own units and the minds' own. Matched on /// every name a unit answers to, ASCII case-insensitively (systemd units are @@ -72,6 +71,21 @@ fn is_refused_name(name: &str) -> bool { .any(|prefix| name.starts_with(prefix)) } +/// Units whose binaries suspend, hibernate or open recovery: no property +/// reveals it, so they are refused by name, on every name they answer to. +const HOST_NAMES: [&str; 6] = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", +]; + +fn is_host_name(name: &str) -> bool { + HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) +} + // ---------------------------------------------------------------- grammar -- #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -288,6 +302,7 @@ impl Operation { argv: vec![ "systemctl".into(), verb.as_str().into(), + "--job-mode=ignore-requirements".into(), "--".into(), unit.clone(), ], @@ -373,11 +388,9 @@ impl fmt::Debug for StartRecord { pub(crate) enum Via { Name, Alias, - Propagation, - /// A unit the order would start from a stopped state that is boot or - /// recovery machinery (`DefaultDependencies=no`): such a unit can end by - /// isolating a target, which no dependency edge describes. - EarlyBoot, + Host, + OnEnd, + Bound, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -421,8 +434,9 @@ impl Outcome { match via { Via::Name => "name", Via::Alias => "alias", - Via::Propagation => "propagation", - Via::EarlyBoot => "early-boot", + Via::Host => "host", + Via::OnEnd => "on-end", + Via::Bound => "bound", } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), @@ -617,35 +631,6 @@ impl OrderStore { // ------------------------------------------------------------------ ports -- -/// Which way systemd moves along a dependency edge when an order is given. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Edge { - /// Units a start brings up with this one. - Pulls, - /// Units a start stops: the unit's `Conflicts=` and the units that conflict with it. - Conflicts, - /// Units a stop takes down with this one. - Stops, -} - -/// The `systemctl show` properties the closure reads, and the way each one -/// moves. This is the only list: the request and the parse both read it. -const EDGES: [(&str, Edge); 13] = [ - ("Requires", Edge::Pulls), - ("Requisite", Edge::Pulls), - ("BindsTo", Edge::Pulls), - ("Wants", Edge::Pulls), - ("Upholds", Edge::Pulls), - ("Conflicts", Edge::Conflicts), - ("ConflictedBy", Edge::Conflicts), - ("RequiredBy", Edge::Stops), - ("BoundBy", Edge::Stops), - ("RequisiteOf", Edge::Stops), - ("ConsistsOf", Edge::Stops), - ("PropagatesStopTo", Edge::Stops), - ("StopPropagatedFrom", Edge::Stops), -]; - /// The properties naming what systemd does when a unit ends or its job times /// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; @@ -653,62 +638,24 @@ const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction" /// The properties naming units systemd starts when a unit ends. const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; -/// The target an emergency action runs, `None` for `none`. An action this -/// check does not know is an error: the unit is not checked, so not run. -fn action_target(action: &str) -> Result> { - let base = ["-force", "-immediate"] - .iter() - .find_map(|suffix| action.strip_suffix(suffix)) - .unwrap_or(action); - Ok(match base { - "" | "none" => None, - "reboot" => Some("reboot.target"), - "poweroff" => Some("poweroff.target"), - "halt" => Some("halt.target"), - "kexec" => Some("kexec.target"), - "soft-reboot" => Some("soft-reboot.target"), - "exit" => Some("exit.target"), - _ => bail!("a unit has an emergency action this check does not know"), - }) -} - /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { pub(crate) id: String, pub(crate) names: Vec, pub(crate) not_found: bool, - pub(crate) running: bool, - /// `DefaultDependencies=no`: boot, recovery and shutdown machinery. - pub(crate) early_boot: bool, - /// What systemd starts when this unit ends or times out: the targets of - /// its emergency actions and its `OnFailure=` and `OnSuccess=` units. - pub(crate) fires: Vec, - pub(crate) pulls: Vec, - pub(crate) conflicts: Vec, - pub(crate) stops: Vec, + /// The properties that make the unit act when it ends: each emergency + /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. + pub(crate) on_end: Vec, + /// The units bound to this one at runtime (`BoundBy=`): systemd stops + /// them when this unit stops, whatever the job mode. + pub(crate) bound_by: Vec, } impl UnitFacts { fn identities(&self) -> impl Iterator { std::iter::once(&self.id).chain(self.names.iter()) } - - fn along(&self, edge: Edge) -> &[String] { - match edge { - Edge::Pulls => &self.pulls, - Edge::Conflicts => &self.conflicts, - Edge::Stops => &self.stops, - } - } - - fn along_mut(&mut self, edge: Edge) -> &mut Vec { - match edge { - Edge::Pulls => &mut self.pulls, - Edge::Conflicts => &mut self.conflicts, - Edge::Stops => &mut self.stops, - } - } } #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -765,126 +712,31 @@ fn refused(unit: &str, via: Via) -> Refused { } } -/// The units systemd would touch for one order, read from `systemctl show`. -struct Transaction<'a> { - systemd: &'a dyn Systemd, - facts: BTreeMap, -} - -impl Transaction<'_> { - fn facts(&mut self, unit: &str) -> Result<&UnitFacts> { - if !self.facts.contains_key(unit) { - ensure!( - self.facts.len() < CLOSURE_LIMIT, - "the units an order would reach are too many to check" - ); - let facts = self.systemd.show(unit)?; - ensure!(!facts.not_found, "systemd has no such unit to check"); - self.facts.insert(unit.to_owned(), facts); - } - Ok(&self.facts[unit]) - } - - /// Every unit reached from `seeds` along `edge`, seeds included. - fn reach(&mut self, seeds: &BTreeSet, edge: Edge) -> Result> { - let mut reached = BTreeSet::new(); - let mut queue: VecDeque = seeds.iter().cloned().collect(); - while let Some(next) = queue.pop_front() { - if !reached.insert(next.clone()) { - continue; - } - queue.extend(self.facts(&next)?.along(edge).iter().cloned()); - } - Ok(reached) - } - - /// Every unit the emergency actions and `OnFailure=`/`OnSuccess=` of - /// `touched` would start, with what each pulls in, and in turn what those - /// that are not running would set off. A touched unit ends, so it fires. - fn fired(&mut self, touched: &BTreeSet) -> Result> { - let mut fired = BTreeSet::new(); - let mut seen = BTreeSet::new(); - let mut queue: VecDeque = touched.iter().cloned().collect(); - while let Some(unit) = queue.pop_front() { - if !seen.insert(unit.clone()) { - continue; - } - let seeds: BTreeSet = self.facts(&unit)?.fires.iter().cloned().collect(); - for reached in self.reach(&seeds, Edge::Pulls)? { - if fired.insert(reached.clone()) && !self.facts(&reached)?.running { - queue.push_back(reached); - } - } - } - Ok(fired) - } - - /// Every unit a start of `started` stops through `Conflicts=`, and what - /// that stop takes down with it. - fn conflicted(&mut self, started: &BTreeSet) -> Result> { - let mut seeds = BTreeSet::new(); - for unit in started { - seeds.extend(self.facts(unit)?.along(Edge::Conflicts).iter().cloned()); - } - self.reach(&seeds, Edge::Stops) - } -} - -/// Would `verb` on `unit` reach an always-refused unit: by its name, by an -/// alias it resolves from, or through the transaction systemd builds for it. -/// A start pulls in what the unit requires or wants and stops what conflicts; -/// a stop takes down what depends on the unit; a restart does both. A unit -/// the transaction would stop is refused when it is refused; a unit it would -/// start is refused when it is refused and not running. Every unit in the -/// closure must be one systemd can show. +/// Would `verb` on `unit` be refused. systemd builds the transaction and the +/// order tells it to build one job (`--job-mode=ignore-requirements`), so this +/// reads only what that job mode cannot remove from the one named unit: the +/// name it answers to, what fires when it ends, and who is bound to it. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); } - let mut transaction = Transaction { - systemd, - facts: BTreeMap::new(), - }; - if let Some(name) = transaction - .facts(unit)? - .identities() - .find(|name| is_refused_name(name)) + let facts = systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + for name in facts.identities() { + if is_refused_name(name) { + return Ok(Some(refused(name, Via::Alias))); + } + if is_host_name(name) { + return Ok(Some(refused(name, Via::Host))); + } + } + if verb != Verb::Reload && !facts.on_end.is_empty() { + return Ok(Some(refused(unit, Via::OnEnd))); + } + if matches!(verb, Verb::Stop | Verb::Restart) + && let Some(bound) = facts.bound_by.first() { - return Ok(Some(refused(name, Via::Alias))); - } - let root = BTreeSet::from([unit.to_owned()]); - let mut stopped = BTreeSet::new(); - let mut started = BTreeSet::new(); - match verb { - Verb::Reload => {} - Verb::Start => started = transaction.reach(&root, Edge::Pulls)?, - Verb::Stop => stopped = transaction.reach(&root, Edge::Stops)?, - Verb::Restart => { - stopped = transaction.reach(&root, Edge::Stops)?; - started = transaction.reach(&stopped, Edge::Pulls)?; - } - } - let mut touched = stopped.clone(); - for member in &started { - if !transaction.facts(member)?.running { - touched.insert(member.clone()); - } - } - started.extend(transaction.fired(&touched)?); - stopped.extend(transaction.conflicted(&started)?); - for (units, only_if_stopped) in [(&stopped, false), (&started, true)] { - for member in units { - let facts = transaction.facts(member)?; - if only_if_stopped && facts.running { - continue; - } - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { - return Ok(Some(refused(name, Via::Propagation))); - } - if only_if_stopped && facts.early_boot { - return Ok(Some(refused(&facts.id, Via::EarlyBoot))); - } - } + return Ok(Some(refused(bound, Via::Bound))); } Ok(None) } @@ -1378,13 +1230,14 @@ fn parse_names(value: &str) -> Vec { names } -/// The properties one `systemctl show` is asked for: identity, state, edges. +/// The properties one `systemctl show` is asked for: identity, what fires at +/// the unit's end, and who is bound to it. fn show_properties() -> String { - ["Id", "Names", "LoadState", "ActiveState", "DefaultDependencies"] + ["Id", "Names", "LoadState"] .into_iter() .chain(ACTIONS) .chain(ON_END) - .chain(EDGES.iter().map(|(property, _)| *property)) + .chain(["BoundBy"]) .collect::>() .join(",") } @@ -1399,17 +1252,16 @@ fn parse_show(output: &str) -> Result { "Id" => facts.id = value.to_owned(), "Names" => facts.names = parse_names(value), "LoadState" => facts.not_found = value == "not-found", - "ActiveState" => { - facts.running = matches!(value, "active" | "activating" | "reloading"); - } - "DefaultDependencies" => facts.early_boot = value == "no", - _ if ACTIONS.contains(&key) => facts.fires.extend(action_target(value)?.map(str::to_owned)), - _ if ON_END.contains(&key) => facts.fires.extend(parse_names(value)), - _ => { - if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { - facts.along_mut(*edge).extend(parse_names(value)); + "BoundBy" => facts.bound_by = parse_names(value), + _ if ACTIONS.contains(&key) => { + if !matches!(value, "" | "none") { + facts.on_end.push(key.to_owned()); } } + _ if ON_END.contains(&key) => { + facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); + } + _ => {} } } ensure!(!facts.id.is_empty(), "systemctl named no unit"); @@ -2031,11 +1883,11 @@ mod tests { const STEWARD: u32 = 1000; const CANARY: &str = "CANARY-7f3a91"; - /// What `systemctl show` prints for a loaded, active unit with no edges, + /// What `systemctl show` prints for a loaded unit that acts on nothing, /// plus the given properties (a given key replaces its default). fn show_text(id: &str, properties: &[(&str, &str)]) -> String { let mut text = format!("Id={id}\n"); - for (key, default) in [("Names", id), ("LoadState", "loaded"), ("ActiveState", "active")] { + for (key, default) in [("Names", id), ("LoadState", "loaded")] { if !properties.iter().any(|(given, _)| *given == key) { text += &format!("{key}={default}\n"); } @@ -2049,11 +1901,13 @@ mod tests { struct FakeSystemd { units: Mutex>, runs: Mutex>, + shows: Mutex>, report: Mutex, } impl Systemd for FakeSystemd { fn show(&self, unit: &str) -> Result { + self.shows.lock().unwrap().push(unit.into()); let known = self.units.lock().unwrap().get(unit).cloned(); parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) } @@ -2119,6 +1973,7 @@ mod tests { let systemd = Arc::new(FakeSystemd { units: Mutex::new(BTreeMap::new()), runs: Mutex::new(Vec::new()), + shows: Mutex::new(Vec::new()), report: Mutex::new(RunReport { exit_code: Some(0), ..RunReport::default() @@ -2426,7 +2281,6 @@ mod tests { "persona-matrix-mouth.service", "mind-heartbeat.service", "apt-daily.timer", - "multi-user.target", // Near misses: the prefix is the rule, not a substring or a stem. "midunn-x.service", "not-idunn-x.service", @@ -2443,14 +2297,17 @@ mod tests { assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); let (transient, step) = h.runs().pop().unwrap(); assert!(transient.starts_with("idunn-order-")); - assert_eq!(step.argv, ["systemctl", verb.as_str(), "--", unit]); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] + ); } } assert_eq!(h.runs().len(), units.len() * 4); } #[test] - fn refused_set_is_refused_by_name_alias_and_propagation() { + fn refused_set_is_refused_by_name_and_alias() { let h = harness(); h.describe_unit( "backup-name.service", @@ -2463,21 +2320,6 @@ mod tests { "quiet-alias.service", &[("Names", "quiet-alias.service mind-persona@void.service")], ); - // fuse <- wrapper <- (bound by) mind-self@eureka - h.describe("fuse.service", &[("RequiredBy", "wrapper.service")]); - h.describe("wrapper.service", &[("BoundBy", "mind-self@eureka.service")]); - h.describe("stack.target", &[("ConsistsOf", "mind-persona@void.service")]); - // A cycle that reaches an alias of a refused unit on its second hop. - h.describe("loop-a.service", &[("RequiredBy", "loop-b.service")]); - h.describe("loop-b.service", &[("RequiredBy", "loop-a.service hidden.service")]); - h.describe_unit( - "hidden.service", - show_text( - "idunn-orders.service", - &[("Names", "idunn-orders.service hidden.service")], - ), - ); - let direct = [ "idunn-yggdrasil.service", "idunn-orders.service", @@ -2528,31 +2370,7 @@ mod tests { ); } } - for verb in [Verb::Stop, Verb::Restart] { - for (unit, hit) in [ - ("fuse.service", "mind-self@eureka.service"), - ("stack.target", "mind-persona@void.service"), - ("loop-a.service", "idunn-orders.service"), - ] { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - assert_eq!( - h.outcome(&id), - Outcome::Refused(Refused::AlwaysRefused { - unit: hit.into(), - via: Via::Propagation - }), - "{verb:?} {unit}" - ); - } - } assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); - // start and reload take nothing down, so the same units are not refused. - for verb in [Verb::Start, Verb::Reload] { - let id = h.accept(h.issue_unit(verb, "fuse.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran); - } // Every refusal is recorded and has a result post due. let ledger = h.ctx.store.ledger().unwrap(); let refusals = ledger @@ -2560,7 +2378,7 @@ mod tests { .iter() .filter(|record| matches!(record.outcome, Outcome::Refused(_))) .count(); - assert_eq!(refusals, direct.len() * 4 + 8 + 6); + assert_eq!(refusals, direct.len() * 4 + 8); let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); let results = due .iter() @@ -2568,93 +2386,7 @@ mod tests { .count(); assert_eq!(results, refusals); // A refused order never started. - assert!(ledger.starts.len() == 2); - } - - /// Yggdrasil's `systemctl show` shapes (systemd 257, 2026-11-01), cut down - /// to the units an order can reach: seven refused units all conflict with - /// shutdown.target, and the shutdown services require it. - fn yggdrasil(h: &Harness, idunn_running: bool) { - let state = if idunn_running { "active" } else { "inactive" }; - let refused = [ - "idunn-yggdrasil.service", - "idunn-orders.service", - "idunn-vili-e0ada640.service", - "mind-self@eureka.service", - "mind-persona@void.service", - ]; - h.describe( - "shutdown.target", - &[ - ("ActiveState", "inactive"), - ( - "RequiredBy", - "systemd-soft-reboot.service systemd-reboot.service systemd-poweroff.service", - ), - ( - "ConflictedBy", - "nginx.service idunn-yggdrasil.service idunn-orders.service \ - idunn-vili-e0ada640.service mind-self@eureka.service mind-persona@void.service", - ), - ], - ); - for refused in refused { - h.describe( - refused, - &[ - ("Requires", "sysinit.target system.slice -.mount"), - ("Wants", "tmp.mount network-online.target"), - ("Conflicts", "shutdown.target"), - ("ActiveState", if refused == "idunn-yggdrasil.service" { state } else { "inactive" }), - ], - ); - } - h.describe("nginx.service", &[("Wants", "network-online.target"), ("Conflicts", "shutdown.target")]); - for (service, target) in [ - ("systemd-poweroff.service", "poweroff.target"), - ("systemd-reboot.service", "reboot.target"), - ("systemd-soft-reboot.service", "soft-reboot.target"), - ] { - h.describe( - service, - &[ - ("Requires", "shutdown.target final.target system.slice umount.target"), - ("RequiredBy", target), - ("ActiveState", "inactive"), - ], - ); - } - for (target, service) in [ - ("poweroff.target", "systemd-poweroff.service"), - ("reboot.target", "systemd-reboot.service"), - ("soft-reboot.target", "systemd-soft-reboot.service"), - ] { - h.describe(target, &[("Requires", service), ("ActiveState", "inactive")]); - } - h.describe( - "umount.target", - &[ - ("ActiveState", "inactive"), - ("RequiredBy", "systemd-poweroff.service systemd-reboot.service"), - ("ConflictedBy", "boot-efi.mount boot.mount \"blockdev@dev-vda2.target\""), - ], - ); - h.describe( - "multi-user.target", - &[ - ("Requires", "basic.target"), - ("Wants", "cron.service ssh.service idunn-yggdrasil.service"), - ("Conflicts", "shutdown.target rescue.target rescue.service"), - ], - ); - h.describe( - "voidbot.service", - &[ - ("Requires", "docker.service -.mount sysinit.target system.slice"), - ("Wants", "idunn-yggdrasil.service network-online.target"), - ("Conflicts", "shutdown.target"), - ], - ); + assert!(ledger.starts.is_empty()); } fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { @@ -2663,277 +2395,171 @@ mod tests { h.outcome(&id) } - fn hit(outcome: &Outcome) -> Option<&str> { + fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { match outcome { - Outcome::Refused(Refused::AlwaysRefused { unit, via: Via::Propagation }) => Some(unit.as_str()), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { + Some(unit.as_str()) + } _ => None, } } - #[test] - fn a_transaction_that_reaches_a_refused_unit_is_refused() { - let h = harness(); - yggdrasil(&h, true); - // The targets and services that power the host down reach every refused - // unit through Conflicts=, however many requirements away. - for (verb, unit) in [ - (Verb::Start, "shutdown.target"), - (Verb::Start, "poweroff.target"), - (Verb::Restart, "poweroff.target"), - (Verb::Start, "reboot.target"), - (Verb::Restart, "reboot.target"), - (Verb::Start, "soft-reboot.target"), - (Verb::Start, "systemd-soft-reboot.service"), - (Verb::Restart, "systemd-poweroff.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert!(hit(&outcome).is_some(), "{verb:?} {unit} was {outcome:?}"); - } - assert!(h.runs().is_empty(), "no refused order ran anything"); - // A stop of shutdown.target takes down what requires it and runs no start. - let outcome = outcome_of_unit(&h, Verb::Stop, "shutdown.target"); - assert_eq!(outcome, Outcome::Ran); - - // A start that wants a refused unit that is stopped would start it. - h.describe("idunn-yggdrasil.service", &[("ActiveState", "inactive"), ("Conflicts", "shutdown.target")]); - for (verb, unit) in [ - (Verb::Start, "multi-user.target"), - (Verb::Restart, "multi-user.target"), - (Verb::Start, "voidbot.service"), - (Verb::Restart, "voidbot.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert_eq!(hit(&outcome), Some("idunn-yggdrasil.service"), "{verb:?} {unit}"); - } - // The same orders start nothing while the refused unit runs. - h.describe("idunn-yggdrasil.service", &[("ActiveState", "active"), ("Conflicts", "shutdown.target")]); - for (verb, unit) in [ - (Verb::Start, "multi-user.target"), - (Verb::Restart, "voidbot.service"), - (Verb::Start, "voidbot.service"), - ] { - assert_eq!(outcome_of_unit(&h, verb, unit), Outcome::Ran, "{verb:?} {unit}"); - } - // A unit that wants only what is not refused runs, and so does a target - // that unmounts nothing a refused unit requires. - for verb in [Verb::Start, Verb::Restart, Verb::Stop, Verb::Reload] { - assert_eq!(outcome_of_unit(&h, verb, "nginx.service"), Outcome::Ran); - } - assert_eq!(outcome_of_unit(&h, Verb::Start, "umount.target"), Outcome::Ran); - } - - #[test] - fn the_closure_follows_every_edge_and_every_hop() { - let h = harness(); - // Start: three hops of Wants/Requires/Requisite/BindsTo/Upholds to a - // stopped refused unit. - h.describe("hop-1.service", &[("Wants", "hop-2.service")]); - h.describe("hop-2.service", &[("Requires", "hop-3.service")]); - h.describe("hop-3.service", &[("Requisite", "hop-4.service")]); - h.describe("hop-4.service", &[("BindsTo", "hop-5.service")]); - h.describe("hop-5.service", &[("Upholds", "idunn-orders.service")]); - h.describe("idunn-orders.service", &[("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "hop-1.service"); - assert_eq!(hit(&outcome), Some("idunn-orders.service")); - // Conflicts of a pulled-in unit stop it and what requires it: the - // conflict, then two hops of reverse requirement, to a refused unit. - h.describe("rival.service", &[("Conflicts", "victim.service")]); - h.describe("victim.service", &[("RequiredBy", "mid.service")]); - h.describe("mid.service", &[("BoundBy", "mind-self@eureka.service")]); - h.describe("pulls-rival.service", &[("Requires", "rival.service")]); - for unit in ["rival.service", "pulls-rival.service"] { - let outcome = outcome_of_unit(&h, Verb::Start, unit); - assert_eq!(hit(&outcome), Some("mind-self@eureka.service"), "{unit}"); - } - // The reverse Conflicts edge: a unit conflicted by a refused one. - h.describe("incumbent.service", &[("ConflictedBy", "mind-persona@void.service")]); - let outcome = outcome_of_unit(&h, Verb::Start, "incumbent.service"); - assert_eq!(hit(&outcome), Some("mind-persona@void.service")); - // Stop and restart: every stop edge, one at a time. - for (property, reached) in [ - ("RequiredBy", "idunn-vili-x.service"), - ("BoundBy", "idunn-vili-x.service"), - ("RequisiteOf", "idunn-vili-x.service"), - ("ConsistsOf", "idunn-vili-x.service"), - ("PropagatesStopTo", "mind-persona@void.service"), - ("StopPropagatedFrom", "mind-persona@void.service"), - ] { - let name = format!("edge-{property}.service"); - h.describe(&name, &[(property, reached)]); - for verb in [Verb::Stop, Verb::Restart] { - let outcome = outcome_of_unit(&h, verb, &name); - assert_eq!(hit(&outcome), Some(reached), "{verb:?} {property}"); - } - } - // A restart starts what its stopped dependents pull in. - h.describe("base.service", &[("RequiredBy", "dependent.service")]); - h.describe("dependent.service", &[("Wants", "idunn-orders.service")]); - let outcome = outcome_of_unit(&h, Verb::Restart, "base.service"); - assert_eq!(hit(&outcome), Some("idunn-orders.service")); - assert_eq!(outcome_of_unit(&h, Verb::Stop, "base.service"), Outcome::Ran); - assert_eq!(outcome_of_unit(&h, Verb::Reload, "base.service"), Outcome::Ran); - } - #[test] fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { let h = harness(); // systemctl show quotes a name with a backslash and doubles the backslash. h.describe( "huginn.service", - &[("RequiredBy", "\"mind-self@foo\\\\x2dbar.service\"")], + &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], ); let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); - assert_eq!(hit(&outcome), Some("mind-self@foo\\x2dbar.service")); - h.describe("quiet.service", &[("Wants", "\"idunn-a\\\\x2db.service\" other.service")]); - h.describe("idunn-a\\x2db.service", &[("ActiveState", "inactive")]); + assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); + h.describe( + "quiet.service", + &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], + ); let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); - assert_eq!(hit(&outcome), Some("idunn-a\\x2db.service")); + assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); assert!(h.runs().is_empty()); } #[test] - fn a_unit_whose_end_reboots_the_host_is_refused_by_its_properties() { + fn unit_orders_run_as_one_job() { let h = harness(); - yggdrasil(&h, true); - // Each emergency action property alone, in each of systemd's spellings, - // on a unit no name list knows: every verb that starts or stops it. - for (property, action) in [ - ("SuccessAction", "reboot"), - ("FailureAction", "reboot"), - ("StartLimitAction", "poweroff-force"), - ("JobTimeoutAction", "reboot-immediate"), - ("SuccessAction", "poweroff"), - ] { - let name = format!("storagetm-{property}.service"); - h.describe(&name, &[(property, action), ("ActiveState", "inactive")]); - for verb in [Verb::Start, Verb::Restart, Verb::Stop] { - let outcome = outcome_of_unit(&h, verb, &name); - assert!(hit(&outcome).is_some(), "{verb:?} {property}={action} {outcome:?}"); - } + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); + let (_, step) = h.runs().pop().unwrap(); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] + ); } - // OnFailure= and OnSuccess= start the units they name, and what those - // set off in turn. - for property in ["OnFailure", "OnSuccess"] { - let name = format!("handled-{property}.service"); - h.describe(&name, &[(property, "reboot.target"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, &name); - assert!(hit(&outcome).is_some(), "{property} {outcome:?}"); - } - h.describe("on-failure-chain.service", &[("OnFailure", "handler.service"), ("ActiveState", "inactive")]); - h.describe("handler.service", &[("FailureAction", "reboot"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "on-failure-chain.service"); - assert!(hit(&outcome).is_some(), "{outcome:?}"); - // A unit a start pulls in fires its action too. - h.describe("pulls-storagetm.service", &[("Wants", "storagetm.service")]); - h.describe("storagetm.service", &[("SuccessAction", "reboot"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "pulls-storagetm.service"); - assert!(hit(&outcome).is_some(), "{outcome:?}"); - // A running unit the order does not stop is not ended by it, and an - // action of none ends nothing. - h.describe("pulls-running.service", &[("Wants", "running-action.service")]); - h.describe("running-action.service", &[("SuccessAction", "reboot")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "pulls-running.service"), Outcome::Ran); - h.describe("quiet-actions.service", &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", "")]); - for verb in [Verb::Start, Verb::Restart, Verb::Stop] { - assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); - } - // An action this check does not know is not checked, so it does not run. - let ran = h.runs().len(); - h.describe("odd-action.service", &[("SuccessAction", "implode")]); - let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); - execute(&h.ctx, &id).unwrap(); - assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); - assert_eq!(h.runs().len(), ran); } #[test] - fn every_emergency_action_runs_the_target_it_names() { - for (action, target) in [ - ("none", None), - ("", None), - ("reboot", Some("reboot.target")), - ("reboot-force", Some("reboot.target")), - ("reboot-immediate", Some("reboot.target")), - ("poweroff", Some("poweroff.target")), - ("poweroff-force", Some("poweroff.target")), - ("poweroff-immediate", Some("poweroff.target")), - ("halt", Some("halt.target")), - ("halt-force", Some("halt.target")), - ("kexec", Some("kexec.target")), - ("kexec-immediate", Some("kexec.target")), - ("soft-reboot", Some("soft-reboot.target")), - ("soft-reboot-force", Some("soft-reboot.target")), - ("exit", Some("exit.target")), - ("exit-force", Some("exit.target")), - ] { - assert_eq!(action_target(action).unwrap(), target, "{action}"); - } - assert!(action_target("implode").is_err()); - } - - #[test] - fn a_stopped_boot_or_recovery_unit_is_never_started() { + fn the_rule_reads_only_the_named_unit() { let h = harness(); - yggdrasil(&h, true); - // rescue.service ends by isolating default.target, which stops every - // unit outside it; no edge says so, its DefaultDependencies=no does. + // Whatever the unit lists, no other unit is looked at. h.describe( - "rescue.service", - &[("DefaultDependencies", "no"), ("ActiveState", "inactive")], + "plain.service", + &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], ); - h.describe("rescue.target", &[("Requires", "rescue.service"), ("ActiveState", "inactive")]); - h.describe("pulls-rescue.service", &[("Wants", "rescue.target")]); - for (verb, unit) in [ - (Verb::Start, "rescue.service"), - (Verb::Restart, "rescue.service"), - (Verb::Start, "rescue.target"), - (Verb::Restart, "rescue.target"), - (Verb::Start, "pulls-rescue.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert!( - matches!( - outcome, - Outcome::Refused(Refused::AlwaysRefused { ref unit, via: Via::EarlyBoot }) - if unit == "rescue.service" - ), - "{verb:?} {unit} was {outcome:?}" + h.describe("other.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); + } + + #[test] + fn a_unit_that_acts_when_it_ends_is_refused() { + let h = harness(); + let properties = [ + ("SuccessAction", "reboot"), + ("FailureAction", "poweroff-force"), + ("StartLimitAction", "soft-reboot-immediate"), + ("JobTimeoutAction", "exit"), + ("FailureAction", "a-spelling-nobody-knows"), + ("OnFailure", "rescue.service"), + ("OnSuccess", "cleanup.service"), + ]; + for (property, value) in properties { + h.describe("ender.service", &[(property, value)]); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "ender.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "ender.service".into(), + via: Via::OnEnd + }), + "{verb:?} {property}={value}" + ); + } + assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); + } + // `none` and an empty list say the unit does nothing at its end. + h.describe( + "calm.service", + &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.runs().len(), 11); + } + + #[test] + fn host_names_are_refused() { + let h = harness(); + let names = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", + ]; + h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); + for verb in all_verbs() { + for name in names { + assert_eq!( + outcome_of_unit(&h, verb, name), + Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), + "{verb:?} {name}" + ); + } + assert_eq!( + outcome_of_unit(&h, verb, "sleepy.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "systemd-hibernate.service".into(), + via: Via::Host + }), + "{verb:?}" ); } assert!(h.runs().is_empty()); - // Stopping it, or starting one whose defaults are on, is an ordinary order. - assert_eq!(outcome_of_unit(&h, Verb::Stop, "rescue.service"), Outcome::Ran); - h.describe("ordinary.service", &[("DefaultDependencies", "yes"), ("ActiveState", "inactive")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "ordinary.service"), Outcome::Ran); - // A boot unit that is already running is left alone by a start. - h.describe("journald.service", &[("DefaultDependencies", "no")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); + assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); + } + + #[test] + fn a_unit_others_are_bound_to_only_starts() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service second.service")]); + for verb in [Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "base.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "first.service".into(), + via: Via::Bound + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Start, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); + } } #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); h.describe("ghost.service", &[("LoadState", "not-found")]); - h.describe("pulls-ghost.service", &[("Wants", "ghost.service")]); - h.describe("needed-by-ghost.service", &[("RequiredBy", "ghost.service")]); - for (verb, unit) in [ - (Verb::Start, "ghost.service"), - (Verb::Start, "pulls-ghost.service"), - (Verb::Stop, "needed-by-ghost.service"), - (Verb::Restart, "needed-by-ghost.service"), - ] { - let id = h.accept(h.issue_unit(verb, unit)); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "ghost.service")); execute(&h.ctx, &id).unwrap(); assert!( matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), - "{verb:?} {unit}" + "{verb:?}" ); } assert!(h.runs().is_empty()); } #[test] - fn globs_slices_and_scopes_are_not_units() { + fn globs_slices_scopes_and_targets_are_not_units() { let h = harness(); let bad = [ "idunn*", @@ -2943,6 +2569,8 @@ mod tests { "mind-self@*.service", "mind-[s]elf@eureka.service", "system.slice", + "poweroff.target", + "multi-user.target", "user.slice", "session-1.scope", "init.scope", @@ -3373,34 +3001,24 @@ mod tests { #[test] fn systemctl_show_output_is_read_into_facts() { let facts = parse_show( - "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=inactive\n\ - Requires=r.service \"q\\\\x2dr.slice\" -.mount\nRequisite=t.service\nWants=w.service\n\ - BindsTo=b.service\nUpholds=u.service\nConflicts=x.target\nConflictedBy=y.service\n\ - RequiredBy=c.service\nBoundBy=\nRequisiteOf=ro.service\nConsistsOf=d.target e.service\n\ - PropagatesStopTo=p.service\nStopPropagatedFrom=f.service\nOther=x\n", + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ + SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ + OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ + Requires=r.service\nConflicts=x.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); assert_eq!(facts.names, ["a.service", "b.service"]); - assert!(!facts.not_found && !facts.running); + assert!(!facts.not_found); + assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); // A quoted name holds one backslash, not two and not a quote mark. - assert_eq!( - facts.pulls, - ["r.service", "q\\x2dr.slice", "-.mount", "t.service", "w.service", "b.service", "u.service"] - ); - assert_eq!(facts.conflicts, ["x.target", "y.service"]); - assert_eq!( - facts.stops, - ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] - ); + assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); assert!(parse_show("Names=a.service\n").is_err()); // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), - "Id,Names,LoadState,ActiveState,DefaultDependencies,SuccessAction,FailureAction,\ - StartLimitAction,JobTimeoutAction,OnFailure,OnSuccess,Requires,Requisite,BindsTo,\ - Wants,Upholds,Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ - PropagatesStopTo,StopPropagatedFrom" + "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ + OnFailure,OnSuccess,BoundBy" ); // A backslash outside quotes is a backslash; inside, it escapes one character. assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); @@ -3408,17 +3026,7 @@ mod tests { assert!(parse_names(" ").is_empty()); let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); - for (state, running) in [ - ("active", true), - ("activating", true), - ("reloading", true), - ("inactive", false), - ("deactivating", false), - ("failed", false), - ] { - let facts = parse_show(&format!("Id=a.service\nActiveState={state}\n")).unwrap(); - assert_eq!(facts.running, running, "{state}"); - } + } } #[test] From 8ebe0208a65bb2191acfeec7d22ffe337795a385 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:41:56 +0100 Subject: [PATCH 120/122] Steward orders: pin the one-job rule on the real unit set and a real user manager Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 118 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index d5fdcfe..7f8ced7 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3027,7 +3027,6 @@ mod tests { let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); } - } #[test] fn the_store_refuses_a_second_record_and_a_foreign_document() { @@ -3846,4 +3845,121 @@ mod tests { assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() } + + fn host_refusal(verb: Verb, unit: &str) -> Option { + refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) + } + + #[test] + #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] + fn real_host_unit_set() { + for name in [ + "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", + "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", + "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", + "wg-quick@wg-gamecult", "cron", "apt-daily", + ] { + let unit = format!("{name}.service"); + for verb in all_verbs() { + assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); + } + } + for name in [ + "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", + "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", + "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", + "systemd-storagetm", + ] { + let unit = format!("{name}.service"); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + host_refusal(verb, &unit), + Some(refused(&unit, Via::OnEnd)), + "{verb:?} {unit}" + ); + } + } + for unit in HOST_NAMES { + for verb in all_verbs() { + assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); + } + } + for unit in [ + "systemd-networkd.service", + "user-runtime-dir@1000.service", + "user-runtime-dir@105.service", + "user-runtime-dir@978.service", + ] { + assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); + assert!( + matches!( + host_refusal(Verb::Stop, unit), + Some(Refused::AlwaysRefused { via: Via::Bound, .. }) + ), + "{unit}" + ); + } + let listing = Command::new("systemctl") + .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) + .output() + .unwrap(); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let units: Vec<&str> = listing + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|unit| validate_unit(unit).is_ok()) + .collect(); + for verb in all_verbs() { + let mut counts = BTreeMap::new(); + for unit in &units { + let key = match refusal_for(&HostSystemd, unit, verb) { + Ok(None) => "ok".to_owned(), + Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), + Ok(Some(other)) => format!("{other:?}"), + Err(_) => "unresolved".to_owned(), + }; + *counts.entry(key).or_insert(0usize) += 1; + } + println!("{verb:?} over {} units: {counts:?}", units.len()); + } + } + + #[test] + #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] + fn one_job_argv_against_a_real_manager() { + let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); + std::fs::create_dir_all(&directory).unwrap(); + let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); + let body = "[Service]\nExecStart=/bin/sleep 600\n"; + std::fs::write(directory.join(a), body).unwrap(); + std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); + let user = |args: &[&str]| { + let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); + (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) + }; + let outcome = std::panic::catch_unwind(|| { + let one_job = |verb: Verb, unit: &str| { + let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); + let mut argv = step.argv.clone(); + argv.insert(1, "--user".into()); + let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); + assert!(status.success(), "{verb:?} {unit}"); + }; + assert!(user(&["daemon-reload"]).0); + // A start of b leaves a running, though b conflicts with it. + assert!(user(&["start", a]).0); + one_job(Verb::Start, b); + assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); + // A restart of b, with both running, leaves a running. + one_job(Verb::Restart, b); + assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); + }); + user(&["stop", a, b]); + std::fs::remove_file(directory.join(a)).unwrap(); + std::fs::remove_file(directory.join(b)).unwrap(); + user(&["daemon-reload"]); + outcome.unwrap(); + } } From fb1e7b5dd9a0e9f0a77febd66cb43d5ce2f995f4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:42:52 +0100 Subject: [PATCH 121/122] Steward orders: README states the one-job rule; pin the end-to-end argv Co-Authored-By: Claude Opus 5.5 --- README.md | 35 +++++++++++++++++------------------ src/steward_orders.rs | 2 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 28c1e94..3a695a4 100644 --- a/README.md +++ b/README.md @@ -73,24 +73,23 @@ idunn order show # status and outp idunn order list ``` -A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' -own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and -read from no configuration. An order is refused, recorded and posted when its -name, an alias it resolves from, or the transaction systemd would build for it -reaches one of them: a start pulls in what the unit requires or wants (a -refused unit that is already running is left alone) and stops what conflicts -with it, a stop takes down what depends on it, and a restart does both. The -transaction is read from `systemctl show`, so `start poweroff.target` is refused -because every refused unit conflicts with `shutdown.target`, not because -targets are banned. The units a start or stop ends or times out are followed too: -the target each of `SuccessAction=`, `FailureAction=`, `StartLimitAction=` and -`JobTimeoutAction=` runs (`reboot`, `poweroff`, `halt`, `kexec`, `soft-reboot`, -`exit`, with `-force` and `-immediate`), and the units `OnFailure=` and -`OnSuccess=` name, with what each pulls in; an action it does not know fails the -order. A start refuses a stopped `DefaultDependencies=no` unit (boot and recovery -machinery, such as `rescue.service`, which ends by isolating the default -target), whatever pulls it in. A unit `systemctl show` cannot resolve fails the order. -Slices, scopes and globs are not units an order can name. +A unit order acts on exactly the unit it names: every verb runs +`systemctl --job-mode=ignore-requirements -- `, so systemd builds a +transaction of one job. It does not start what the unit requires or restart what +is part of it: order the dependency first, and order `postgresql@16-main.service` +rather than the `postgresql.service` wrapper. Targets are not orderable. A unit +order may act on any unit except Idunn's own (`idunn-*`) and the minds' own +(`mind-self@*`, `mind-persona@*`), and the units that end or take the host +(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, +`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled +into the binary and read from no configuration. An order is refused, recorded +and posted when the ordered unit, or an alias it resolves from, is one of them; +when it has an emergency action (`SuccessAction=`, `FailureAction=`, +`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` +or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another +unit is `BoundBy=` it, because systemd stops that unit with it whatever the job +mode. A unit `systemctl show` cannot resolve fails the order. Slices, scopes, +targets and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with `--no-remove`, each name is passed as an exact-name pattern so `.` and `+` diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 7f8ced7..3b40c88 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3742,7 +3742,7 @@ mod tests { let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); assert!( run_log.contains(&format!( - "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart -- nginx.service", + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", order_uuid(&first) )), "{run_log}" From 77e2295e9705c2a9eebb46f27ac40f34b7b49c2a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 23:32:30 +0100 Subject: [PATCH 122/122] Steward orders: refuse Upholds and ConsistsOf units, pin the rule's gaps, README names the residuals A unit with Upholds= is refused for start and restart (systemd would start the upheld unit in a full transaction). A wrapper with ConsistsOf= is refused for stop, restart and reload with a typed Refused::Wrapper naming the first member, so it cannot report ran while touching nothing. Tests pin empty BoundBy, a single show per refusal, no show for a refused name and each posted reason. README: the one-job rule's residual full transactions. Removes the unused BTreeMap import and the never_loop in the host actuator test. Co-Authored-By: Claude Opus 5.5 --- README.md | 15 ++++- src/host_actuator.rs | 2 +- src/steward_orders.rs | 126 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3a695a4..8c6935c 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,16 @@ idunn order show # status and outp idunn order list ``` -A unit order acts on exactly the unit it names: every verb runs +A unit order submits exactly one job, for the unit it names: every verb runs `systemctl --job-mode=ignore-requirements -- `, so systemd builds a transaction of one job. It does not start what the unit requires or restart what is part of it: order the dependency first, and order `postgresql@16-main.service` -rather than the `postgresql.service` wrapper. Targets are not orderable. A unit +rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is +not: systemd still runs full transactions of its own for `Restart=` after a +failure, for a `StopWhenUnneeded=` dependency left without a user, and for a +`Conflicts=` partner of a unit that fails, so an ordered stop of a unit +that something restarts does not last, and a start of a `Restart=always` +unit can report failure and be active seconds later. A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' own (`mind-self@*`, `mind-persona@*`), and the units that end or take the host (`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, @@ -88,7 +93,11 @@ when it has an emergency action (`SuccessAction=`, `FailureAction=`, `StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another unit is `BoundBy=` it, because systemd stops that unit with it whatever the job -mode. A unit `systemctl show` cannot resolve fails the order. Slices, scopes, +mode; for a start or restart, when it has an `Upholds=` unit, which systemd +starts in a full transaction whatever the job mode; or, for a stop, restart or +reload, when it has a `ConsistsOf=` member, a wrapper such as +`postgresql.service` whose verbs would exit 0 and touch nothing. A unit +`systemctl show` cannot resolve fails the order. Slices, scopes, targets and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with diff --git a/src/host_actuator.rs b/src/host_actuator.rs index f7bf4c7..8fd776c 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1151,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - while let Some(frame) = client.receive_once()? { + if let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 3b40c88..d4be1ee 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,6 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::BTreeMap; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -391,12 +390,16 @@ pub(crate) enum Via { Host, OnEnd, Bound, + Upholds, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub(crate) enum Refused { AlwaysRefused { unit: String, via: Via }, + /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload + /// of the wrapper exits 0 and leaves the member alone. + Wrapper { member: String }, Braked, /// The order's principal or issuing uid is not the steward bound now. NotSteward, @@ -437,8 +440,13 @@ impl Outcome { Via::Host => "host", Via::OnEnd => "on-end", Via::Bound => "bound", + Via::Upholds => "upholds", } ), + Outcome::Refused(Refused::Wrapper { member }) => format!( + "refused: wrapper unit, it only wraps {}", + escape(member.as_bytes()) + ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), Outcome::Refused(Refused::NotSteward) => { "refused: the issuer is not the current steward".into() @@ -650,6 +658,12 @@ pub(crate) struct UnitFacts { /// The units bound to this one at runtime (`BoundBy=`): systemd stops /// them when this unit stops, whatever the job mode. pub(crate) bound_by: Vec, + /// The units this one upholds (`Upholds=`): systemd starts them in a full + /// transaction whenever this unit is active, whatever the job mode. + pub(crate) upholds: Vec, + /// The units this one groups (`ConsistsOf=`): the verbs reach them only + /// through propagation, which the job mode removes. + pub(crate) consists_of: Vec, } impl UnitFacts { @@ -715,7 +729,8 @@ fn refused(unit: &str, via: Via) -> Refused { /// Would `verb` on `unit` be refused. systemd builds the transaction and the /// order tells it to build one job (`--job-mode=ignore-requirements`), so this /// reads only what that job mode cannot remove from the one named unit: the -/// name it answers to, what fires when it ends, and who is bound to it. +/// name it answers to, what fires when it ends, who is bound to it, what it upholds and +/// what it only wraps. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); @@ -738,6 +753,16 @@ fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result Vec { } /// The properties one `systemctl show` is asked for: identity, what fires at -/// the unit's end, and who is bound to it. +/// the unit's end, who is bound to it, and what it upholds or groups. fn show_properties() -> String { ["Id", "Names", "LoadState"] .into_iter() .chain(ACTIONS) .chain(ON_END) - .chain(["BoundBy"]) + .chain(["BoundBy", "Upholds", "ConsistsOf"]) .collect::>() .join(",") } @@ -1253,6 +1278,8 @@ fn parse_show(output: &str) -> Result { "Names" => facts.names = parse_names(value), "LoadState" => facts.not_found = value == "not-found", "BoundBy" => facts.bound_by = parse_names(value), + "Upholds" => facts.upholds = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), _ if ACTIONS.contains(&key) => { if !matches!(value, "" | "none") { facts.on_end.push(key.to_owned()); @@ -2543,6 +2570,91 @@ mod tests { } } + #[test] + fn a_unit_that_upholds_others_is_refused_where_it_would_start() { + let h = harness(); + h.describe("holder.service", &[("Upholds", "held.service")]); + for verb in [Verb::Start, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "holder.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "holder.service".into(), + via: Via::Upholds + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { + let h = harness(); + h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); + for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { + assert_eq!( + outcome_of_unit(&h, verb, "wrapper.service"), + Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); + } + + #[test] + fn empty_dependency_lists_name_no_one() { + let h = harness(); + // systemctl prints every property it is asked for, empty or not. + h.describe( + "alone.service", + &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service")]); + h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); + assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); + h.systemd.shows.lock().unwrap().clear(); + assert_eq!( + hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), + Some("idunn-ghost.service") + ); + assert!(h.systemd.shows.lock().unwrap().is_empty()); + } + + #[test] + fn each_refusal_is_posted_under_its_own_reason() { + for (via, word) in [ + (Via::Name, "name"), + (Via::Alias, "alias"), + (Via::Host, "host"), + (Via::OnEnd, "on-end"), + (Via::Bound, "bound"), + (Via::Upholds, "upholds"), + ] { + assert_eq!( + Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) + .describe(), + format!("refused: always-refused unit u.service reached by {word}") + ); + } + assert_eq!( + Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), + "refused: wrapper unit, it only wraps m.service" + ); + } + #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); @@ -3004,7 +3116,7 @@ mod tests { "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ - Requires=r.service\nConflicts=x.service\nOther=x\n", + Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); @@ -3013,12 +3125,14 @@ mod tests { assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); // A quoted name holds one backslash, not two and not a quote mark. assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); + assert_eq!(facts.upholds, ["u.service"]); + assert_eq!(facts.consists_of, ["m.service", "n.service"]); assert!(parse_show("Names=a.service\n").is_err()); // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ - OnFailure,OnSuccess,BoundBy" + OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" ); // A backslash outside quotes is a backslash; inside, it escapes one character. assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]);