From d91ecba759b898114332c0db02cfd60a00fd90ea Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:07:13 +0200 Subject: [PATCH 01/47] Cut 2 negative tests, failing at 6138329: restart beside a building deployment, profile freeze over a crash loop a_dead_incumbent_is_restarted_alongside_a_deployment_that_is_building and a_profile_deploy_freezes_over_a_crash_looping_target use only base API and pin the states cut 2 (handover-phase-split) ends. They fail until the predicate split and the open gates land. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 100 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e077373..d67d9d3 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16091,6 +16091,99 @@ mod tests { Ok(()) } + /// The 2026-10-18 shape at the engine: a dead incumbent with a deployment + /// building over it. Continuity restarts the incumbent beside the build + /// and neither touches the other (handover-phase-split). + #[test] + fn a_dead_incumbent_is_restarted_alongside_a_deployment_that_is_building() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + let stored_deploy = |world: &EngineFixture| -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == deploy.transaction_id) + .context("the deployment left the live set") + }; + let before = stored_deploy(&world)?; + + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1, "no restart beside the building deployment"); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 2); + assert_eq!(stored_deploy(&world)?.envelope, before.envelope); + + for _ in 0..2 { + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + let after = stored_deploy(&world)?; + assert_eq!(after.envelope, before.envelope); + assert!(after.value.pre_fencing_abort.is_none()); + } + Ok(()) + } + + /// A `profile:` command over two targets, one of them crash-looping with a + /// live continuity, freezes at once: freeze refuses only a second live + /// deployment of a target (follow-up profile-deploy-joint-gap). + #[test] + fn a_profile_deploy_freezes_over_a_crash_looping_target() -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + workload.kill(); + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + let continuity = ControlSnapshot::read(&world.state_store)?.transactions.remove(0); + + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + for target in ["service", "other"] { + let paired = binding.replacen( + "target = \"service\"\n", + &format!("target = \"{target}\"\nprofiles = [\"pair\"]\n"), + 1, + ); + std::fs::write(world.root.join(format!("bindings/{target}.toml")), paired)?; + } + queue_deploy_of(&world, "up-pair", "profile:pair")?; + assert!(world.engine.freeze_one_queued_command()?, "the profile command stayed queued"); + + let live = ControlSnapshot::read(&world.state_store)?.transactions; + let mut frozen = live + .iter() + .filter(|stored| stored.value.command_id == "up-pair") + .map(|stored| (stored.value.ordinal, stored.value.target.as_str())) + .collect::>(); + frozen.sort(); + assert_eq!(frozen, [(0, "other"), (1, "service")]); + let kept = live + .iter() + .find(|stored| stored.value.transaction_id == continuity.value.transaction_id) + .context("the continuity left the live set")?; + assert_eq!(kept.envelope, continuity.envelope); + + // A live deployment of `service` still refuses a second command for it. + queue_deploy_of(&world, "up-service-second", "service")?; + assert!(!world.engine.freeze_one_queued_command()?); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| transaction.command_id != "up-service-second") + ); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -16923,11 +17016,16 @@ mod tests { ); std::fs::create_dir_all(world.root.join("bindings"))?; std::fs::write(world.root.join("bindings/service.toml"), binding)?; + queue_deploy_of(world, command_id, "service") + } + + /// Queue a Deploy command for `selector`; the bindings are the caller's. + fn queue_deploy_of(world: &EngineFixture, command_id: &str, selector: &str) -> Result<()> { let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), command_id: command_id.into(), kind: CommandKind::Deploy, - selector: "service".into(), + selector: selector.into(), requested_by: "operator".into(), requested_at_unix_millis: now_millis()?, }; From b4bb158707bd20ff56b4200910adfcf26a65724b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:11:36 +0200 Subject: [PATCH 02/47] Cut 2, commit 1: split the ownership predicate and the one-transaction-per-target validation owns_target_authority and blocks_new_target_mutation answered four questions with one phase-blind rule. They become is_live, owns_target (derived from the phase and takes_target_at) and drains_incumbent, each read by the consumer that asks that question: sibling order, validation, status, ports, freeze and the supervision gates. Validation now holds, per target, at most one owner, one live Deploy and one live Continuity, each refused with its own message, so a continuity beside a pre-handover deployment is readable. Freeze and the mint gate are still conservative (any live or draining transaction refuses), so no intermediate commit lets two transactions coexist unguarded. seeded_transaction mints a fresh command id when its default is taken, so a world can hold two of one kind. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 216 +++++++++++++++++++++++++++++++++---------- 1 file changed, 165 insertions(+), 51 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d67d9d3..16ea8d7 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,18 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// The phase whose entry is the handover of a target: Starting when the rollout +/// stops the incumbent first (it is stopped there by design), else Fencing. The +/// one spelling of the rule; a free function so the stop-then-start arm is +/// testable without a host-actuator fixture. +fn handover_phase(stops_incumbent_first: bool) -> DeploymentPhase { + if stops_incumbent_first { + DeploymentPhase::Starting + } else { + DeploymentPhase::Fencing + } +} + impl DeploymentTransaction { /// The one writer of `phase`. It stamps the update time and the phase /// deadline together, so a record outside Fencing..=Committing carries no @@ -1891,21 +1903,36 @@ impl DeploymentTransaction { } } - /// Complete transactions retain retryable cleanup work, but admission has - /// already transferred current-incarnation authority to AdmittedGeneration. - fn owns_target_authority(&self) -> bool { + /// The phase whose entry is the handover: the one transaction that makes + /// it takes the target's incumbent slot from continuity supervision. + fn takes_target_at(&self) -> DeploymentPhase { + handover_phase(self.rollout_stops_incumbent_first()) + } + + /// Not Complete: the transaction still holds its own candidate slot. + /// Read by sibling order inside one command, freeze (a live Deploy + /// refuses a second), the mint gate (a live Continuity refuses a second), + /// candidate-port reservation and validation. + fn is_live(&self) -> bool { self.phase != DeploymentPhase::Complete } + /// From the handover until Complete the transaction owns the target's + /// incumbent slot: it fences, leases, routes and commits. Derived from the + /// phase, never stored. Read by validation (one owner), supervision (which + /// defers to an owner), the handover CAS, a waiting deployment and status. + fn owns_target(&self) -> bool { + (self.takes_target_at()..DeploymentPhase::Complete).contains(&self.phase) + } + /// Admission has moved to the new generation at Complete, but an exact /// draining incumbent still reserves its process and candidate endpoint. - /// No later mutation for the same target may overlap that cleanup. - fn blocks_new_target_mutation(&self) -> bool { - self.owns_target_authority() - || self - .post_commit_cleanup - .as_ref() - .is_some_and(|cleanup| !cleanup.is_complete()) + /// Read by the mint gate, the handover, candidate-port reservation and + /// status: no later mutation of the target may overlap that cleanup. + fn drains_incumbent(&self) -> bool { + self.post_commit_cleanup + .as_ref() + .is_some_and(|cleanup| !cleanup.is_complete()) } /// Where what this transaction has collected disagrees with what its own @@ -3242,16 +3269,27 @@ impl ControlSnapshot { "transaction kind differs from its immutable command" ); } - let mut live_targets = BTreeSet::new(); + // Per target: one owner of the incumbent slot, one live Deploy, one + // live Continuity. A continuity may sit beside a deployment that has + // not yet taken the target (handover-phase-split). + let mut owners = BTreeSet::new(); + let mut live_deploys = BTreeSet::new(); + let mut live_continuities = BTreeSet::new(); for transaction in self .transactions .iter() - .filter(|stored| stored.value.owns_target_authority()) + .filter(|stored| stored.value.is_live()) { + let target = transaction.value.target.as_str(); ensure!( - live_targets.insert(transaction.value.target.as_str()), - "multiple transactions claim current-incarnation authority for one target" + !transaction.value.owns_target() || owners.insert(target), + "multiple transactions own the incumbent slot of one target" ); + let (seen, kind) = match transaction.value.command_kind { + CommandKind::Deploy => (&mut live_deploys, "deployments"), + CommandKind::Continuity => (&mut live_continuities, "continuity transactions"), + }; + ensure!(seen.insert(target), "multiple live {kind} for one target"); if let Some(fencing) = transaction .value .fencing @@ -3381,7 +3419,7 @@ impl ControlSnapshot { fn has_earlier_authority_sibling(&self, transaction: &DeploymentTransaction) -> bool { self.transactions.iter().any(|stored| { - stored.value.owns_target_authority() + stored.value.is_live() && stored.value.command_id == transaction.command_id && stored.value.ordinal < transaction.ordinal }) @@ -4608,9 +4646,10 @@ fn status( } writeln!( out, - " terminal {} owns-target {}", + " terminal {} owns-target {} drains {}", transaction.is_terminal(), - transaction.blocks_new_target_mutation() + transaction.owns_target(), + transaction.drains_incumbent() )?; } } @@ -5934,7 +5973,7 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -5995,23 +6034,19 @@ impl Engine { &anyhow::Error::new(disagreement), ); } - let blocker = snapshot.transactions.iter().find(|stored| { - stored.value.target == current.value.target - && stored.value.blocks_new_target_mutation() - }); - if blocker.is_some_and(|stored| { - let owns_from = if stored.value.rollout_stops_incumbent_first() { - // The incumbent is stopped at Starting by design; a - // continuity restart in that window would fight the - // candidate for the same host resources. - DeploymentPhase::Starting - } else { - DeploymentPhase::Fencing - }; - stored.value.phase >= owns_from && stored.value.phase < DeploymentPhase::Complete - }) { + let on_target = || { + snapshot + .transactions + .iter() + .filter(|stored| stored.value.target == current.value.target) + }; + // A transaction that owns the incumbent slot fences, stops or + // replaces this generation: nothing here touches it meanwhile. + if on_target().any(|stored| stored.value.owns_target()) { continue; } + let blocked = on_target() + .any(|stored| stored.value.is_live() || stored.value.drains_incumbent()); match self.supervise_admitted_route(&snapshot, current) { Ok(true) => { progressed = true; @@ -6056,9 +6091,7 @@ impl Engine { // the admitted one leaves the projection naming an incarnation that // no longer exists -- which the workload rightly refuses to adopt. // That record is published from observation once the process is up. - if blocker.is_none() - || blocker.is_some_and(|stored| stored.value.phase == DeploymentPhase::Complete) - { + if !on_target().any(|stored| stored.value.is_live()) { let repair = (|| -> Result<()> { let topology = self.topology(); let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; @@ -6228,7 +6261,7 @@ impl Engine { // Continuity defers to any live transaction for its target and // never aborts one (handover-phase-split). - if blocker.is_some() { + if blocked { continue; } if supervision.restarts_page(now) { @@ -7994,7 +8027,7 @@ impl Engine { snapshot .transactions .iter() - .filter(|stored| stored.value.blocks_new_target_mutation()) + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) .filter_map(|stored| stored.value.plan.as_ref()), ) { @@ -11595,8 +11628,9 @@ mod tests { }); assert!(!transaction.is_terminal()); - assert!(!transaction.owns_target_authority()); - assert!(transaction.blocks_new_target_mutation()); + assert!(!transaction.is_live()); + assert!(!transaction.owns_target()); + assert!(transaction.drains_incumbent()); Ok(()) } @@ -11835,7 +11869,7 @@ mod tests { ); assert_eq!(lifted.phase_deadline, None); assert_eq!(lifted.lease_adoption, None); - assert_eq!(lifted.owns_target_authority(), legacy.phase != DeploymentPhase::Complete); + assert_eq!(lifted.is_live(), legacy.phase != DeploymentPhase::Complete); } assert_eq!( phases.iter().map(|(_, phase)| *phase).collect::>(), @@ -12457,16 +12491,23 @@ mod tests { declare_odin(&mut expected); let now = now_millis()?; + // A second seeding of one kind into one world is a second command. + let base_id = format!( + "{}-service{}", + match kind { + CommandKind::Deploy => "up", + CommandKind::Continuity => "continuity", + }, + if incumbent.is_some() { "-again" } else { "" } + ); + let known = ControlSnapshot::read(&world.state_store)?.commands; + let command_id = (1..) + .map(|n| if n == 1 { base_id.clone() } else { format!("{base_id}-{n}") }) + .find(|id| known.iter().all(|stored| stored.value.command_id != *id)) + .context("no free command id")?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!( - "{}-service{}", - match kind { - CommandKind::Deploy => "up", - CommandKind::Continuity => "continuity", - }, - if incumbent.is_some() { "-again" } else { "" } - ), + command_id, kind, selector: "service".into(), requested_by: "test".into(), @@ -14220,7 +14261,7 @@ mod tests { assert!(lifted.is_terminal(), "{name}: the lift reopened a finished record"); assert_eq!(lifted.phase, archived.phase, "{name}"); assert_eq!(lifted.completion, archived.completion, "{name}"); - assert!(!lifted.owns_target_authority(), "{name}"); + assert!(!lifted.is_live(), "{name}"); } else { assert_eq!(abort_owed(&lifted), Some(CleanupEvidence::Pending), "{name}"); assert!(!lifted.is_terminal(), "{name}"); @@ -16184,6 +16225,79 @@ mod tests { Ok(()) } + /// The stored transaction `id`, moved into `phase` in place. + fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { + let stored = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.transaction_id == id) + .context("no such live transaction")?; + let mut next = stored.value.clone(); + next.enter_phase(phase, now_millis()?); + replace_transaction(&world.state_store, &stored, &next) + } + + #[test] + fn the_handover_phase_is_starting_only_for_stop_then_start() -> Result<()> { + assert_eq!(handover_phase(true), DeploymentPhase::Starting); + assert_eq!(handover_phase(false), DeploymentPhase::Fencing); + + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let mut seeded = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!(seeded.takes_target_at(), DeploymentPhase::Fencing); + // The target is owned from the handover to Complete, and not after; + // it is live until Complete. + for (phase, owns) in [ + (DeploymentPhase::Sealing, false), + (DeploymentPhase::Starting, false), + (DeploymentPhase::Warming, false), + (DeploymentPhase::Fencing, true), + (DeploymentPhase::Leasing, true), + (DeploymentPhase::AwaitingReady, true), + (DeploymentPhase::Routing, true), + (DeploymentPhase::Committing, true), + (DeploymentPhase::Complete, false), + ] { + seeded.phase = phase; + assert_eq!(seeded.owns_target(), owns, "{phase:?}"); + assert_eq!(seeded.is_live(), phase != DeploymentPhase::Complete, "{phase:?}"); + } + Ok(()) + } + + #[test] + fn the_store_holds_one_deploy_one_continuity_and_one_owner_per_target() -> Result<()> { + let seed = |first: (CommandKind, DeploymentPhase), second: (CommandKind, DeploymentPhase)| { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let incumbent = admit_incumbent(&world)?; + for (kind, phase) in [first, second] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + if phase != DeploymentPhase::Warming { + rephase(&world, &seeded.transaction_id, phase)?; + } + } + Ok::<_, anyhow::Error>(world) + }; + use CommandKind::{Continuity, Deploy}; + use DeploymentPhase::{Fencing, Warming}; + + // A pre-handover deployment beside an owning continuity, and the + // reverse: one owner, one of each kind. + for (first, second) in [((Deploy, Warming), (Continuity, Fencing)), ((Continuity, Warming), (Deploy, Fencing))] { + let world = seed(first, second)?; + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 2); + } + let refused = |first, second| -> Result { + let world = seed(first, second)?; + Ok(format!("{:#}", ControlSnapshot::read(&world.state_store).err().context("two were accepted")?)) + }; + assert!(refused((Deploy, Fencing), (Continuity, Fencing))?.contains("own the incumbent slot")); + assert!(refused((Deploy, Warming), (Deploy, Warming))?.contains("multiple live deployments")); + assert!(refused((Continuity, Warming), (Continuity, Warming))?.contains("multiple live continuity transactions")); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -17207,7 +17321,7 @@ mod tests { )); assert!(finished.is_terminal()); // The target is released and nothing of the incumbent was touched. - assert!(!finished.blocks_new_target_mutation()); + assert!(!finished.is_live() && !finished.drains_incumbent()); assert_eq!(incumbent_envelope(&world)?, before); assert!( every_transaction(&world)? From 755306d654c3c4151f9eb3ad16d137e37d411b54 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:13:38 +0200 Subject: [PATCH 03/47] Cut 2, commit 2: one phase-entry seam and one handover CAS Engine::enter is the only writer of a move into the next phase; transition and the Sealing tail both call it. Entering the phase where a transaction takes its target (takes_target_at: Fencing, or Starting for stop-then-start) goes through Engine::take_target, one CAS that expects the transaction's own envelope, the target's admitted envelope as read (or its absence) and every other non-terminal transaction of the target, and writes only the transaction. Of two entrants from one view exactly one wins; a losing deployment waits with the owner named, a losing continuity is superseded. exact_incumbent now says what a mismatch is: a fault, not a race. Test helper: the brake release is per transaction (a world can hold two deployments). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 332 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 314 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 16ea8d7..5abe6ef 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,20 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// What `Engine::take_target` decided about one transaction's entry into the +/// phase where it takes its target. +#[derive(Debug, PartialEq, Eq)] +enum Handover { + /// The CAS wrote the entry: the transaction owns the target. + Took, + /// The CAS lost to a concurrent write; nothing was written. + Lost, + /// Another transaction owns or drains the target; nothing was written. + OwnedBy(String), + /// The admitted generation is no longer the one the transaction names. + Moved, +} + /// The phase whose entry is the handover of a target: Starting when the rollout /// stops the incumbent first (it is stopped there by design), else Fencing. The /// one spelling of the rule; a free function so the stop-then-start arm is @@ -3383,6 +3397,17 @@ impl ControlSnapshot { .find(|stored| stored.value.target == target) } + /// The transaction, other than `except`, that holds the target's + /// incumbent slot or is still draining it. The one lookup supervision, a + /// waiting deployment and the handover share. + fn target_owner(&self, target: &str, except: Option<&str>) -> Option<&Stored> { + self.transactions.iter().find(|stored| { + stored.value.target == target + && Some(stored.value.transaction_id.as_str()) != except + && (stored.value.owns_target() || stored.value.drains_incumbent()) + }) + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -6838,7 +6863,9 @@ impl Engine { })?; next.enter_phase(DeploymentPhase::Starting, now); next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) + // Starting is the handover for stop-then-start, so this write goes + // through the seam that decides it. + self.enter(current, next) } fn advance_starting(&self, current: &Stored) -> Result<()> { @@ -7798,6 +7825,10 @@ impl Engine { Ok(()) } + /// The admitted generation this transaction was proven against. A + /// pre-handover deployment is rebased onto the admitted generation before + /// any step reads its incumbent, and the handover pins it after, so a + /// mismatch here is a fault, not a race. fn exact_incumbent<'a>( &self, snapshot: &'a ControlSnapshot, @@ -7809,7 +7840,9 @@ impl Engine { (Some(expected), Some(current)) if current.value.generation_id == *expected => { Ok(Some(current)) } - _ => bail!("target incumbent changed after transaction creation"), + _ => bail!( + "admitted generation changed under a transaction that owns the target or names it as its source" + ), } } @@ -8949,14 +8982,108 @@ impl Engine { current: &Stored, next_phase: DeploymentPhase, ) -> Result<()> { - ensure!( - next_phase as u8 == current.value.phase as u8 + 1, - "deployment phase transition is not adjacent" - ); let mut next = current.value.clone(); next.enter_phase(next_phase, now_millis()?); next.last_error = None; - replace_transaction(&self.options.state_store, current, &next) + self.enter(current, next) + } + + /// The one phase-entry seam. Every move into the next phase is written + /// here, and entering the phase where the transaction takes its target + /// (`takes_target_at`) is decided by `take_target`: that CAS is the + /// handover, and nothing else writes that entry. + fn enter( + &self, + current: &Stored, + next: DeploymentTransaction, + ) -> Result<()> { + ensure!( + next.phase as u8 == current.value.phase as u8 + 1, + "deployment phase transition is not adjacent" + ); + if next.phase != next.takes_target_at() { + return replace_transaction(&self.options.state_store, current, &next); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let target = ¤t.value.target; + match (self.take_target(&snapshot, current, &next)?, current.value.command_kind) { + // Lost to a concurrent writer: the next step reads the new world. + (Handover::Took | Handover::Lost, _) => Ok(()), + // Wait, never abort: a pre-fencing error would end the deployment. + (Handover::OwnedBy(owner), CommandKind::Deploy) => self.record_gate_wait( + current, + &format!( + "target {target} is owned by {owner}; this deployment fences what is admitted when it completes" + ), + ), + // The next step rebases onto the generation now admitted. + (Handover::Moved, CommandKind::Deploy) => Ok(()), + (Handover::OwnedBy(owner), CommandKind::Continuity) => { + bail!("continuity superseded: target {target} is owned by {owner}") + } + (Handover::Moved, CommandKind::Continuity) => { + bail!("continuity superseded: the generation it restarts is no longer admitted") + } + } + } + + /// The handover: one CAS that writes `next` (the entry into + /// `takes_target_at`) only if nothing else moved. It expects the + /// transaction's own envelope, the target's admitted envelope as read + /// (or its absence), and every other non-terminal transaction of the + /// target as read, so of two transactions entering from one view exactly + /// one succeeds. + fn take_target( + &self, + snapshot: &ControlSnapshot, + current: &Stored, + next: &DeploymentTransaction, + ) -> Result { + let target = current.value.target.as_str(); + let id = current.value.transaction_id.as_str(); + if let Some(owner) = snapshot.target_owner(target, Some(id)) { + return Ok(Handover::OwnedBy(owner.value.transaction_id.clone())); + } + let admitted = snapshot.admitted_for(target); + if admitted.map(|stored| stored.value.generation_id.as_str()) + != current.value.incumbent_generation_id.as_deref() + { + return Ok(Handover::Moved); + } + next.validate()?; + let mut expected = vec![ + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: id.into(), + current: Some(current.envelope.clone()), + }, + CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: target.into(), + current: admitted.map(|stored| stored.envelope.clone()), + }, + ]; + expected.extend( + snapshot + .transactions + .iter() + .filter(|stored| { + stored.value.target == target + && stored.value.transaction_id != id + && !stored.value.is_terminal() + }) + .map(|stored| CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: stored.value.transaction_id.clone(), + current: Some(stored.envelope.clone()), + }), + ); + let took = SingleFileMessagePackBackingStore::new(&self.options.state_store) + .compare_exchange( + &expected, + &[transaction_envelope(next, next.updated_at_unix_millis)?], + )?; + Ok(if took { Handover::Took } else { Handover::Lost }) } /// Every post-fencing phase ends. Before a live transaction past the @@ -12329,10 +12456,17 @@ mod tests { IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, }; - let signer = enroll_service_identity_at::( - &world.root.join("identities/brake-operator.cc"), - )?; - export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + // One operator per world, however many deployments it releases. + let signer_path = world.root.join("identities/brake-operator.cc"); + let signer = if signer_path.exists() { + open_service_identity_at::(&signer_path)? + } else { + let signer = + enroll_service_identity_at::(&signer_path)?; + export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + signer + }; + let authorization_id = format!("authorization-{transaction_id}"); let mut record = IdunnDeploymentBrakeRecord { schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), @@ -12343,7 +12477,7 @@ mod tests { reason: "test rollout".into(), observed_at_unix_millis: now, expires_at_unix_millis: Some(now + 600_000), - authorization_id: Some("authorization-test".into()), + authorization_id: Some(authorization_id.clone()), authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), authorized_release_id: Some(expected.sealed_release_id.clone()), authorized_deployment_id: Some(transaction_id.to_owned()), @@ -12363,7 +12497,7 @@ mod tests { record.validate()?; let canonical_brake_bytes = rmp_serde::to_vec(&record)?; Ok(DeploymentAuthorization { - authorization_id: "authorization-test".into(), + authorization_id, brake_sha256: sha256_id(&canonical_brake_bytes), canonical_brake_bytes, authorized_at_unix_millis: now, @@ -16225,16 +16359,178 @@ mod tests { Ok(()) } - /// The stored transaction `id`, moved into `phase` in place. - fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { - let stored = ControlSnapshot::read(&world.state_store)? + /// The live transaction `id`. + fn resident_id(world: &EngineFixture, id: &str) -> Result> { + ControlSnapshot::read(&world.state_store)? .transactions .into_iter() .find(|stored| stored.value.transaction_id == id) - .context("no such live transaction")?; + .context("no such live transaction") + } + + /// `stored` one phase on, as the seam would be asked to write it. + fn entering(stored: &Stored, phase: DeploymentPhase) -> Result { let mut next = stored.value.clone(); next.enter_phase(phase, now_millis()?); - replace_transaction(&world.state_store, &stored, &next) + Ok(next) + } + + /// Drop every live transaction but `keep`. + fn clear_other_live_transactions(world: &EngineFixture, keep: &str) -> Result<()> { + let others = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .filter(|stored| stored.value.transaction_id != keep) + .map(|stored| stored.envelope) + .collect::>(); + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).delete_batch_if_unchanged(&others)?); + Ok(()) + } + + /// The stored transaction `id`, moved into `phase` in place. + fn rephase(world: &EngineFixture, id: &str, phase: DeploymentPhase) -> Result<()> { + let stored = resident_id(world, id)?; + replace_transaction(&world.state_store, &stored, &entering(&stored, phase)?) + } + + /// A Deploy at `deploy_phase` and a Continuity at `continuity_phase`, both + /// over one incumbent of `service`, and that incumbent. + fn deploy_beside_continuity( + world: &EngineFixture, + deploy_phase: DeploymentPhase, + continuity_phase: DeploymentPhase, + ) -> Result<(DeploymentTransaction, DeploymentTransaction, AdmittedGeneration)> { + let incumbent = admit_incumbent(world)?; + let deploy = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let continuity = seeded_transaction( + world, + DeploymentPhase::Warming, + CommandKind::Continuity, + Some(&incumbent), + )?; + for (seeded, phase) in [(&deploy, deploy_phase), (&continuity, continuity_phase)] { + if phase != DeploymentPhase::Warming { + rephase(world, &seeded.transaction_id, phase)?; + } + } + Ok((deploy, continuity, incumbent)) + } + + #[test] + fn exactly_one_transaction_takes_a_target_from_one_view() -> Result<()> { + let take = |snapshot: &ControlSnapshot, world: &EngineFixture, id: &str| -> Result { + let stored = resident_id(world, id)?; + world.engine.take_target(snapshot, &stored, &entering(&stored, DeploymentPhase::Fencing)?) + }; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let view = ControlSnapshot::read(&world.state_store)?; + + assert_eq!(take(&view, &world, &continuity.transaction_id)?, Handover::Took); + // The second claimant read the same world: its CAS loses. + assert_eq!(take(&view, &world, &deploy.transaction_id)?, Handover::Lost); + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!(after.transactions.iter().filter(|stored| stored.value.owns_target()).count(), 1); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Warming); + // A fresh view names the owner instead of racing it. + assert_eq!( + take(&after, &world, &deploy.transaction_id)?, + Handover::OwnedBy(continuity.transaction_id.clone()) + ); + + // The reverse world: the deployment wins and the continuity loses. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let view = ControlSnapshot::read(&world.state_store)?; + assert_eq!(take(&view, &world, &deploy.transaction_id)?, Handover::Took); + assert_eq!(take(&view, &world, &continuity.transaction_id)?, Handover::Lost); + Ok(()) + } + + #[test] + fn the_handover_loses_when_the_admitted_generation_moved_after_it_was_read() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, _, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + // Only the continuity is a rival here; take it out of the guards' way. + clear_other_live_transactions(&world, &deploy.transaction_id)?; + let view = ControlSnapshot::read(&world.state_store)?; + let stored = resident_id(&world, &deploy.transaction_id)?; + + // Same generation, new envelope: a refresh landed after the read. + edit_incumbent(&world, |generation| generation.odin_publisher_sequence_cursor += 1)?; + assert_eq!( + world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, + Handover::Lost + ); + // A view that already shows another generation is Moved, not raced. + let mut other = stored.clone(); + other.value.incumbent_generation_id = Some("generation-elsewhere".into()); + assert_eq!( + world.engine.take_target( + &ControlSnapshot::read(&world.state_store)?, + &other, + &entering(&other, DeploymentPhase::Fencing)? + )?, + Handover::Moved + ); + Ok(()) + } + + #[test] + fn the_seam_makes_a_deployment_wait_and_a_continuity_step_aside() -> Result<()> { + // A deployment ready to fence while a continuity owns the target waits + // where it is, with the owner named and no abort. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let stored = resident_id(&world, &deploy.transaction_id)?; + world.engine.enter(&stored, entering(&stored, DeploymentPhase::Fencing)?)?; + let waiting = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.pre_fencing_abort.is_none()); + assert!( + waiting.last_error.as_deref().is_some_and(|note| note.contains(&continuity.transaction_id)), + "{:?}", + waiting.last_error + ); + // A deployment whose generation moved is left for its rebase. + let world_two = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy_two, _, _) = + deploy_beside_continuity(&world_two, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + clear_other_live_transactions(&world_two, &deploy_two.transaction_id)?; + let mut stored_two = resident_id(&world_two, &deploy_two.transaction_id)?; + stored_two.value.incumbent_generation_id = Some("generation-elsewhere".into()); + let before = ControlSnapshot::read(&world_two.state_store)?.transactions[0].envelope.clone(); + world_two.engine.enter(&stored_two, entering(&stored_two, DeploymentPhase::Fencing)?)?; + assert_eq!(ControlSnapshot::read(&world_two.state_store)?.transactions[0].envelope, before); + + // A continuity in the same position is superseded, by either cause. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Fencing, DeploymentPhase::Warming)?; + let stored = resident_id(&world, &continuity.transaction_id)?; + let error = world + .engine + .enter(&stored, entering(&stored, DeploymentPhase::Fencing)?) + .expect_err("a continuity entered a target another transaction owns"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + assert!(format!("{error:#}").contains(&deploy.transaction_id), "{error:#}"); + clear_other_live_transactions(&world, &continuity.transaction_id)?; + let mut stored = resident_id(&world, &continuity.transaction_id)?; + stored.value.incumbent_generation_id = Some("generation-elsewhere".into()); + let error = world + .engine + .enter(&stored, entering(&stored, DeploymentPhase::Fencing)?) + .expect_err("a continuity entered over a generation that moved"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + // Phases are entered one at a time. + let stored = resident_id(&world, &continuity.transaction_id)?; + assert!(world.engine.enter(&stored, entering(&stored, DeploymentPhase::Routing)?).is_err()); + Ok(()) } #[test] From 9139d7edb8964827ee9910f2f164bf76b93eecda Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:16:05 +0200 Subject: [PATCH 04/47] Cut 2, commit 3: rebase before the handover, wait in Warming, supersede a continuity settle_before_handover runs after the two abort dispatches and before any pre-handover step. A Deploy rebases onto the admitted generation when it differs from the one its evidence was proven against: incumbent_generation_id is rewritten, warming, route_preflight and isolation are dropped and the Odin cursor is raised, nothing else. In Warming it waits (a gate note, no error, so no abort) while another transaction owns or drains the target. A Continuity is never rebased: when its target is owned or its generation replaced it ends itself through the ordinary pre-fencing abort. Tests drive a real continuity restart to Complete beside a deployment and check the deployment rebases onto it and fences it. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 306 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 294 insertions(+), 12 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5abe6ef..1229bdd 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1686,6 +1686,11 @@ struct DeploymentTransaction { lease_adoption: Option, } +/// The note a deployment keeps while another transaction owns its target. +fn waiting_for_owner(target: &str, owner: &str) -> String { + format!("target {target} is owned by {owner}; this deployment fences what is admitted when it completes") +} + /// What `Engine::take_target` decided about one transaction's entry into the /// phase where it takes its target. #[derive(Debug, PartialEq, Eq)] @@ -6738,6 +6743,11 @@ impl Engine { if current.value.pre_fencing_abort.is_some() && current.value.completion.is_none() { return self.advance_pre_fencing_abort(current); } + if current.value.phase < current.value.takes_target_at() + && self.settle_before_handover(current)? + { + return Ok(()); + } match current.value.phase { DeploymentPhase::Sealing => self.advance_sealing(current), DeploymentPhase::Starting => self.advance_starting(current), @@ -8988,6 +8998,92 @@ impl Engine { self.enter(current, next) } + /// Before its handover a Deploy is proven against the generation that was + /// admitted when the evidence was gathered. When another generation has + /// been admitted since (continuity restarted the incumbent), rewrite the + /// incumbent and drop exactly the evidence that names the old one: the + /// warming (its lease), the route preflight (its route) and the isolation + /// (its workload). A continuity is never rebased. Returns whether it wrote. + fn rebase_onto_admitted(&self, current: &Stored) -> Result { + let held = ¤t.value; + if held.command_kind != CommandKind::Deploy + || held.pre_fencing_abort.is_some() + || held.phase >= held.takes_target_at() + { + return Ok(false); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let admitted = snapshot.admitted_for(&held.target); + let admitted_id = admitted.map(|stored| stored.value.generation_id.as_str()); + if admitted_id == held.incumbent_generation_id.as_deref() { + return Ok(false); + } + self.persist_same_phase(current, |next| { + next.incumbent_generation_id = admitted_id.map(str::to_owned); + next.warming = None; + next.route_preflight = None; + next.isolation = None; + next.odin_publisher_sequence_cursor = next + .odin_publisher_sequence_cursor + .max(admitted.map_or(0, |stored| stored.value.odin_publisher_sequence_cursor)); + Ok(()) + })?; + Ok(true) + } + + /// What a transaction that has not yet taken its target does first, when + /// the world moved under it. `true` means this decided the step. + /// + /// A Deploy rebases onto the generation now admitted; in Warming, whose + /// steps read the incumbent an owner is fencing, it waits while another + /// transaction owns or drains the target. Both return Ok: a pre-fencing + /// error is an abort, and no deployment is aborted for continuity. A + /// Continuity restarts one generation and no other, so it ends itself + /// (through the ordinary pre-fencing abort) when its target is taken or + /// its generation replaced. + fn settle_before_handover(&self, current: &Stored) -> Result { + let held = ¤t.value; + match held.command_kind { + CommandKind::Deploy => { + if self.rebase_onto_admitted(current)? { + return Ok(true); + } + if held.phase != DeploymentPhase::Warming { + return Ok(false); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id)) + else { + return Ok(false); + }; + self.record_gate_wait( + current, + &waiting_for_owner(&held.target, &owner.value.transaction_id), + )?; + Ok(true) + } + CommandKind::Continuity => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(owner) = snapshot.target_owner(&held.target, Some(&held.transaction_id)) + { + bail!( + "continuity superseded: target {} is owned by {}", + held.target, + owner.value.transaction_id + ); + } + ensure!( + snapshot + .admitted_for(&held.target) + .map(|stored| stored.value.generation_id.as_str()) + == held.incumbent_generation_id.as_deref(), + "continuity superseded: the generation it restarts is no longer admitted" + ); + Ok(false) + } + } + } + /// The one phase-entry seam. Every move into the next phase is written /// here, and entering the phase where the transaction takes its target /// (`takes_target_at`) is decided by `take_target`: that CAS is the @@ -9010,12 +9106,9 @@ impl Engine { // Lost to a concurrent writer: the next step reads the new world. (Handover::Took | Handover::Lost, _) => Ok(()), // Wait, never abort: a pre-fencing error would end the deployment. - (Handover::OwnedBy(owner), CommandKind::Deploy) => self.record_gate_wait( - current, - &format!( - "target {target} is owned by {owner}; this deployment fences what is admitted when it completes" - ), - ), + (Handover::OwnedBy(owner), CommandKind::Deploy) => { + self.record_gate_wait(current, &waiting_for_owner(target, &owner)) + } // The next step rebases onto the generation now admitted. (Handover::Moved, CommandKind::Deploy) => Ok(()), (Handover::OwnedBy(owner), CommandKind::Continuity) => { @@ -12863,23 +12956,47 @@ mod tests { } } - /// One engine step at a time until `stop` holds, recording each phase - /// entered and checking after every step that the deadline on the record - /// is exactly the current post-fencing phase's, and absent outside one. + /// The transaction wherever it now lives: live, or archived once terminal. + fn latest_id(world: &EngineFixture, id: &str) -> Result { + match resident_id(world, id) { + Ok(stored) => Ok(stored.value), + Err(_) => read_history_transactions(&world.state_store) + .into_iter() + .find(|transaction| transaction.transaction_id == id) + .context("the transaction is in neither the live set nor history"), + } + } + + /// `drive_id` for the world's only transaction. fn drive( world: &EngineFixture, stop: impl Fn(&DeploymentTransaction) -> bool, + ) -> Result> { + match resident(world) { + Ok(sole) => drive_id(world, &sole.value.transaction_id, stop), + Err(_) => Ok(Vec::new()), + } + } + + /// One engine step of transaction `id` at a time until `stop` holds, + /// recording each phase entered and checking after every step that the + /// deadline on the record is exactly the current post-fencing phase's, and + /// absent outside one. + fn drive_id( + world: &EngineFixture, + id: &str, + stop: impl Fn(&DeploymentTransaction) -> bool, ) -> Result> { let mut phases = Vec::new(); for _ in 0..40 { - let Ok(current) = resident(world) else { + let Ok(current) = resident_id(world, id) else { return Ok(phases); }; if stop(¤t.value) { return Ok(phases); } world.engine.advance_transaction(¤t)?; - let after = latest(world)?; + let after = latest_id(world, id)?; match &after.phase_deadline { Some(deadline) => assert_eq!(deadline.phase, after.phase), None => assert!( @@ -16461,7 +16578,7 @@ mod tests { let stored = resident_id(&world, &deploy.transaction_id)?; // Same generation, new envelope: a refresh landed after the read. - edit_incumbent(&world, |generation| generation.odin_publisher_sequence_cursor += 1)?; + edit_incumbent(&world, |generation| generation.admitted_at_unix_millis += 1)?; assert_eq!( world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, Handover::Lost @@ -16594,6 +16711,171 @@ mod tests { Ok(()) } + /// The continuity transaction minted over `service`, whatever its state. + fn the_continuity(world: &EngineFixture) -> Result> { + ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .find(|stored| stored.value.command_kind == CommandKind::Continuity) + .context("no live continuity") + } + + #[test] + fn a_deployment_rebases_onto_the_generation_continuity_admitted_mid_build_and_fences_it() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + age_incumbent(&world, 500_000)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let deploy_id = deploy.transaction_id.as_str(); + workload.kill(); + assert!(world.engine.supervise_one_admitted_generation()?); + let continuity_id = the_continuity(&world)?.value.transaction_id; + + // Continuity restarts the incumbent beside the build, to Complete. + workload.revive(); + drive_id(&world, &continuity_id, |t| t.phase == DeploymentPhase::Warming)?; + odin_republishes(&world, &latest_id(&world, &continuity_id)?, seeded_sequence(Some(&incumbent)) + 10)?; + drive_id(&world, &continuity_id, |t| t.is_terminal())?; + let restarted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the restart admitted nothing")? + .value + .clone(); + assert_ne!(restarted.generation_id, incumbent.generation_id); + assert_eq!(restarted.transaction_id, continuity_id); + let untouched = resident_id(&world, deploy_id)?; + assert_eq!(untouched.value.incumbent_generation_id, Some(incumbent.generation_id.clone())); + + // The deployment's next step is the rebase and nothing else. + world.engine.advance_transaction(&untouched)?; + let rebased = resident_id(&world, deploy_id)?.value; + assert_eq!(rebased.incumbent_generation_id, Some(restarted.generation_id.clone())); + assert_eq!(rebased.phase, DeploymentPhase::Warming); + assert!(rebased.warming.is_none() && rebased.route_preflight.is_none() && rebased.isolation.is_none()); + assert!(rebased.pre_fencing_abort.is_none()); + assert!(rebased.odin_publisher_sequence_cursor >= restarted.odin_publisher_sequence_cursor); + assert_eq!(rebased.expected, untouched.value.expected); + + // It then fences the generation continuity admitted, and replaces it. + odin_reports_ready(&world, &deploy, restarted.odin_publisher_sequence_cursor + 10)?; + drive_id(&world, deploy_id, |t| t.is_terminal())?; + let landed = latest_id(&world, deploy_id)?; + assert!(matches!(landed.completion, Some(TransactionCompletion::Admitted { .. })), "{landed:?}"); + assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); + assert!(matches!( + landed.post_commit_cleanup.map(|cleanup| cleanup.incumbent), + Some( + IncumbentCleanupEvidence::Pending { generation_id, .. } + | IncumbentCleanupEvidence::Complete { generation_id } + ) if generation_id == restarted.generation_id + )); + let now_admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("the deployment admitted nothing")? + .value + .transaction_id + .clone(); + assert_eq!(now_admitted, deploy_id); + Ok(()) + } + + /// Kill the Warming step with the workload: if the gate did not hold the + /// deployment, running the step would be a pre-fencing error and an abort. + #[test] + fn a_deployment_waits_in_warming_while_a_continuity_owns_the_target() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Leasing)?; + workload.kill(); + let deploy_id = deploy.transaction_id.as_str(); + let before = resident_id(&world, deploy_id)?; + + for _ in 0..3 { + let stored = resident_id(&world, deploy_id)?; + world.engine.advance_transaction(&stored)?; + } + let waiting = resident_id(&world, deploy_id)?; + assert_eq!(waiting.value.phase, DeploymentPhase::Warming); + assert!(waiting.value.pre_fencing_abort.is_none()); + assert_eq!(waiting.value.warming, before.value.warming); + assert_eq!(waiting.value.isolation, before.value.isolation); + assert!( + waiting.value.last_error.as_deref().is_some_and(|note| note.contains(&continuity.transaction_id)), + "{:?}", + waiting.value.last_error + ); + + // The build overlaps: a deployment still in Sealing is not gated. + rephase(&world, deploy_id, DeploymentPhase::Sealing)?; + let sealing = resident_id(&world, deploy_id)?; + world.engine.advance_transaction(&sealing)?; + let after = resident_id(&world, deploy_id)?.value; + assert!( + after.last_error.as_deref().is_some_and(|note| !note.contains(&continuity.transaction_id)), + "Sealing was held by the owner: {:?}", + after.last_error + ); + + // The owner completes having moved the generation: the next step is the + // rebase, and with the target free the deployment goes on. + rephase(&world, deploy_id, DeploymentPhase::Warming)?; + workload.revive(); + clear_other_live_transactions(&world, deploy_id)?; + edit_incumbent(&world, |generation| generation.generation_id = "generation-restarted".into())?; + world.engine.advance_transaction(&resident_id(&world, deploy_id)?)?; + let rebased = resident_id(&world, deploy_id)?.value; + assert_eq!(rebased.incumbent_generation_id.as_deref(), Some("generation-restarted")); + assert!(rebased.warming.is_none()); + Ok(()) + } + + #[test] + fn a_continuity_whose_target_was_taken_or_generation_replaced_aborts_itself_and_is_not_rebased() -> Result<()> { + let resume = |world: &EngineFixture, id: &str| -> Result<()> { + for _ in 0..12 { + let stored = resident_id(world, id); + let Ok(stored) = stored else { break }; + world.engine.resume_candidate(&stored)?; + world.engine.resume_backoff.lock().unwrap().clear(); + } + Ok(()) + }; + // Taken: the deployment owns the target; the continuity ends itself and + // the deployment goes on, untouched. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Fencing, DeploymentPhase::Warming)?; + let owner = resident_id(&world, &deploy.transaction_id)?; + resume(&world, &continuity.transaction_id)?; + let ended = latest_id(&world, &continuity.transaction_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.envelope, owner.envelope); + + // Replaced: the generation it restarts is gone. It is not rebased. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, incumbent) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + clear_other_live_transactions(&world, &continuity.transaction_id)?; + edit_incumbent(&world, |generation| generation.generation_id = "generation-elsewhere".into())?; + resume(&world, &continuity.transaction_id)?; + let ended = latest_id(&world, &continuity.transaction_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert_eq!(ended.incumbent_generation_id, Some(incumbent.generation_id)); + let _ = deploy; + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; From 278cc5c408ed3a7ac9b1f066eec65e044c0f3826 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:20:30 +0200 Subject: [PATCH 05/47] Cut 2, commit 4: open the gates; fold in the sustained-reset test reach Freeze refuses only a second live Deploy of a target, so a profile command freezes over crash-looping targets. Supervision defers to an owner, a drain or its own live continuity, never to a deployment that has not taken the target, and the admitted Expected repair runs beside one. Cut 1's interim test (no continuity beside a pre-fencing deployment) goes with the state it asserted. Fold-in c1-sustained-reset-test-reach: the lower edge to the millisecond, the newest attempt (not the oldest) deciding the reset, and the mid-wait crossing recorded in continuity_backoff's doc and pinned. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 245 +++++++++++++++++++++++++++---------------- 1 file changed, 155 insertions(+), 90 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 1229bdd..01005b1 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1244,7 +1244,9 @@ impl TargetSupervision { /// run: the generation has lived the sustained time and no attempt, charged /// or the one re-materialization, has been minted since it was admitted. /// That rule decides first: a sustained death never began a wait, so a - /// standing anchor, however it came to stand, has no say in it. + /// standing anchor, however it came to stand, has no say in it. A wait + /// already running when the generation reaches the sustained time ends + /// there instead: an accepted shortening, always under the cap. fn continuity_backoff(&self, now: u64, admitted_at: u64) -> ContinuityBackoff { let streak = self.continuity_attempt_streak; let sustained = self @@ -5680,8 +5682,8 @@ impl Engine { let continuity_progress = self.supervise_one_admitted_generation()?; // Freezing is not gated on the others' progress: a queued command // whose target is free must not wait behind an unrelated target's - // wedge. `freeze_command` already refuses a busy target, so this is - // still at most one freeze per tick and never a second claimant. + // wedge. `freeze_command` refuses only a second live deployment of a + // target, and this freezes at most one command per tick. let froze = self.freeze_one_queued_command()?; Ok(transaction_progress || continuity_progress || froze) } @@ -5963,8 +5965,10 @@ impl Engine { Ok(false) } - /// Freeze one queued command into its transactions. `Ok(false)` means its - /// target is busy and nothing was written. + /// Freeze one queued command into its transactions. `Ok(false)` means a + /// live deployment already exists for one of its targets and nothing was + /// written. Continuity, owners and drains never refuse a freeze: a + /// deployment that has not taken its target owns nothing. fn freeze_command( &self, snapshot: &ControlSnapshot, @@ -6003,7 +6007,9 @@ impl Engine { let busy = snapshot .transactions .iter() - .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) + .filter(|stored| { + stored.value.command_kind == CommandKind::Deploy && stored.value.is_live() + }) .map(|stored| stored.value.target.as_str()) .collect::>(); if targets.iter().any(|target| busy.contains(target.as_str())) { @@ -6075,8 +6081,13 @@ impl Engine { if on_target().any(|stored| stored.value.owns_target()) { continue; } - let blocked = on_target() - .any(|stored| stored.value.is_live() || stored.value.drains_incumbent()); + // Continuity defers to an owner, a drain or its own live attempt, + // never to a deployment that has not taken the target, and aborts + // nothing (handover-phase-split). + let mint_blocked = snapshot.target_owner(¤t.value.target, None).is_some() + || on_target().any(|stored| { + stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity + }); match self.supervise_admitted_route(&snapshot, current) { Ok(true) => { progressed = true; @@ -6121,28 +6132,26 @@ impl Engine { // the admitted one leaves the projection naming an incarnation that // no longer exists -- which the workload rightly refuses to adopt. // That record is published from observation once the process is up. - if !on_target().any(|stored| stored.value.is_live()) { - let repair = (|| -> Result<()> { - let topology = self.topology(); - let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; - if !topology.admitted_expected_projection_is_exact( - ¤t.value.expected, - &provider_anchor, - )? { - ensure!( - topology.publish_expected(¤t.value.expected, &provider_anchor)? - == current.value.expected.canonical_sha256()?, - "admitted Expected projection repair differs" - ); - } - Ok(()) - })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target + let repair = (|| -> Result<()> { + let topology = self.topology(); + let provider_anchor = self.provider_anchor_for_plan(¤t.value.plan)?; + if !topology.admitted_expected_projection_is_exact( + ¤t.value.expected, + &provider_anchor, + )? { + ensure!( + topology.publish_expected(¤t.value.expected, &provider_anchor)? + == current.value.expected.canonical_sha256()?, + "admitted Expected projection repair differs" ); } + Ok(()) + })(); + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -6289,9 +6298,7 @@ impl Engine { } let continuity_key = format!("continuity:{}", current.value.target); - // Continuity defers to any live transaction for its target and - // never aborts one (handover-phase-split). - if blocked { + if mint_blocked { continue; } if supervision.restarts_page(now) { @@ -13829,6 +13836,9 @@ mod tests { alive: std::sync::atomic::AtomicBool, rebooted: std::sync::atomic::AtomicBool, silent: std::sync::atomic::AtomicBool, + /// Runtime instances that stay alive when the rest are killed: a + /// candidate building beside a dead incumbent. + spared: std::sync::Mutex>, } impl SwitchWorkload { @@ -13837,9 +13847,15 @@ mod tests { alive: std::sync::atomic::AtomicBool::new(true), rebooted: std::sync::atomic::AtomicBool::new(false), silent: std::sync::atomic::AtomicBool::new(false), + spared: std::sync::Mutex::new(Vec::new()), }) } + /// `kill` leaves this runtime instance running. + fn spare(&self, runtime_instance_id: &str) { + self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); + } + /// The host cannot be asked: observing fails as `HostUnobservable` /// whatever the workload is doing. fn silence(&self, silent: bool) { @@ -13914,8 +13930,14 @@ mod tests { } .into()); } + let spared = match prior { + WorkloadObservation::Systemd(observed) => { + self.spared.lock().unwrap().contains(&observed.runtime_instance_id) + } + WorkloadObservation::Host(_) => false, + }; ensure!( - self.alive.load(std::sync::atomic::Ordering::SeqCst), + spared || self.alive.load(std::sync::atomic::Ordering::SeqCst), "the workload is gone" ); Ok(prior.clone()) @@ -16329,53 +16351,6 @@ mod tests { ); } - /// The 2026-10-18 state: a dead incumbent that has paged, with a deployment - /// waiting before Fencing. Continuity never stops and never yields a - /// deployment (budget-never-stop, handover-phase-split), so the window - /// sliding must not turn the deployment into a casualty. - #[test] - fn a_deployment_before_fencing_over_a_dead_incumbent_that_paged_is_never_yielded_and_lands() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let redeploy = - seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; - workload.kill(); - set_meters( - &world, - &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - world.engine.supervise_one_admitted_generation()?; - // The oldest leaves the window: base would now yield the deployment. - set_meters( - &world, - &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, - )?; - for _ in 0..3 { - let _ = world.engine.supervise_one_admitted_generation()?; - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, redeploy.transaction_id); - assert!( - after.pre_fencing_abort.is_none(), - "the deployment was yielded: {:?}", - after.pre_fencing_abort - ); - assert_eq!( - continuity_mints(&world)?, - 0, - "continuity minted beside a pre-fencing deployment" - ); - } - workload.revive(); - odin_reports_ready(&world, &redeploy, seeded_sequence(Some(&incumbent)) + 1)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - let landed = latest(&world)?; - assert_eq!(landed.transaction_id, redeploy.transaction_id); - assert_eq!(landed.phase, DeploymentPhase::Complete); - assert!(landed.pre_fencing_abort.is_none()); - Ok(()) - } - /// The incumbent was admitted this long ago, so an anchor written "in the /// past" is not older than its generation. fn age_incumbent(world: &EngineFixture, millis: u64) -> Result<()> { @@ -16788,7 +16763,7 @@ mod tests { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; let (deploy, continuity, _) = - deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Leasing)?; + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; workload.kill(); let deploy_id = deploy.transaction_id.as_str(); let before = resident_id(&world, deploy_id)?; @@ -16838,7 +16813,9 @@ mod tests { for _ in 0..12 { let stored = resident_id(world, id); let Ok(stored) = stored else { break }; - world.engine.resume_candidate(&stored)?; + // The last step archives the record, which the scheduler's own + // progress read reports as an error; the record is what counts. + let _ = world.engine.resume_candidate(&stored); world.engine.resume_backoff.lock().unwrap().clear(); } Ok(()) @@ -16876,6 +16853,62 @@ mod tests { Ok(()) } + /// A dead incumbent with a deployment building beside it (its candidate + /// alive) whose Warming evidence is complete, and that deployment. + fn dead_incumbent_with_a_deployment_ready_to_fence( + workload: &SwitchWorkload, + world: &EngineFixture, + ) -> Result { + let incumbent = admit_incumbent(world)?; + age_incumbent(world, 500_000)?; + let deploy = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let candidate = deploy.activation.as_ref().context("no activation")?; + workload.spare(&candidate.runtime_instance_id); + workload.kill(); + Ok(deploy) + } + + #[test] + fn a_continuity_minted_as_a_deployment_takes_the_target_is_resolved_next_pass() -> Result<()> { + // (a) The deployment fences first in the tick, so supervision reads its + // ownership and mints nothing. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + assert!(world.engine.run_scheduler_tick()?); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + assert_eq!(continuity_mints(&world)?, 0); + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0); + + // (b) Supervision minted from the older view: the continuity starts at + // Sealing and owns nothing, the deployment takes the target, and the + // continuity is superseded at its next step, never the deployment. + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + assert!(world.engine.supervise_one_admitted_generation()?); + let continuity_id = the_continuity(&world)?.value.transaction_id; + assert_eq!(resident_id(&world, &continuity_id)?.value.phase, DeploymentPhase::Sealing); + world.engine.advance_transaction(&resident_id(&world, &deploy.transaction_id)?)?; + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + world.engine.resume_backoff.lock().unwrap().clear(); + let snapshot = ControlSnapshot::read(&world.state_store)?; + assert!(snapshot.transactions.iter().filter(|stored| stored.value.owns_target()).count() <= 1); + } + let ended = latest_id(&world, &continuity_id)?; + assert!( + matches!(&ended.completion, Some(TransactionCompletion::FailedBeforeFencing { error }) if error.contains("continuity superseded")), + "{:?}", + ended.completion + ); + assert!(latest_id(&world, &deploy.transaction_id)?.pre_fencing_abort.is_none()); + Ok(()) + } + /// Put the target's backoff anchor `ago` milliseconds back. fn anchor_ago(world: &EngineFixture, ago: u64) -> Result<()> { let mut meters = meters_of(world, "service")?; @@ -16980,11 +17013,19 @@ mod tests { meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS), ContinuityBackoff::Due { streak: 0 } ); - // A moment short of it. - assert_eq!( - meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + 10_000), - ContinuityBackoff::Begin - ); + // A moment short of it, down to the millisecond. + for short in [10_000, 1_000, 1] { + assert_eq!( + meters.continuity_backoff(now, now - CONTINUITY_SUSTAINED_RUN_MILLIS + short), + ContinuityBackoff::Begin, + "{short} ms short" + ); + } + // The newest attempt decides, not the oldest: one before admission and + // one after it, on a generation that has lived long enough. + meters.continuity_restarts = vec![now - 800_000, now - 100_000]; + assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); + meters.continuity_restarts = vec![1_000]; // Attempts after admission failed on this generation: it has not run well. meters.continuity_restarts = vec![now - 100_000]; assert_eq!(meters.continuity_backoff(now, now - 700_000), ContinuityBackoff::Begin); @@ -17008,6 +17049,29 @@ mod tests { assert_eq!(fresh.continuity_backoff(now, now - 1_000), ContinuityBackoff::Due { streak: 0 }); } + #[test] + fn a_wait_that_crosses_the_sustained_run_ends_early_and_only_by_less_than_the_cap() { + let admitted = 50_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.continuity_attempt_streak = 7; + meters.continuity_restarts = vec![admitted - 1_000]; + // The wait began when the generation was 400 s old and runs 300 s. + meters.continuity_backoff_from_unix_millis = Some(admitted + 400_000); + assert_eq!(meters.backoff_due(admitted), Some(admitted + 700_000)); + assert_eq!( + meters.continuity_backoff(admitted + CONTINUITY_SUSTAINED_RUN_MILLIS - 1, admitted), + ContinuityBackoff::Waiting + ); + assert_eq!( + meters.continuity_backoff(admitted + CONTINUITY_SUSTAINED_RUN_MILLIS, admitted), + ContinuityBackoff::Due { streak: 0 } + ); + assert!( + CONTINUITY_SUSTAINED_RUN_MILLIS - 400_000 < CONTINUITY_BACKOFF_CEILING_MILLIS, + "the shortening is under the cap" + ); + } + #[test] fn a_continuity_wait_runs_from_when_the_target_was_found_dead_and_free() -> Result<()> { let workload = SwitchWorkload::new(); @@ -17057,10 +17121,10 @@ mod tests { all.iter().any(|transaction| transaction.command_id == "up-service-queued"), "the queued deployment never froze" ); - assert_eq!(continuity_mints(&world)?, 0); + // Freeze needs no gap (a deployment before its handover owns nothing), + // and continuity may restart the incumbent beside it after its wait. for _ in 0..3 { let _ = world.engine.run_scheduler_tick(); - assert_eq!(continuity_mints(&world)?, 0); let aborts = every_transaction(&world)? .into_iter() .filter_map(|transaction| transaction.pre_fencing_abort) @@ -17864,10 +17928,11 @@ mod tests { for _ in 0..4 { let _ = world.engine.supervise_one_admitted_generation()?; } - let after = resident(&world)?.value; - assert_eq!(after.transaction_id, held.transaction_id); + let after = resident_id(&world, &held.transaction_id)?.value; assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); assert!(after.completion.is_none()); + // Continuity restarts the declared incumbent beside it, once. + assert_eq!(continuity_mints(&world)?, 1); Ok(()) } From a7bfbc2296ac5a5a42ebf9d0e599d113ead9b5cf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:22:58 +0200 Subject: [PATCH 06/47] Cut 2 tests: a launching switch workload, so a real continuity restart runs beside a deployment Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 01005b1..e2518f4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13839,6 +13839,9 @@ mod tests { /// Runtime instances that stay alive when the rest are killed: a /// candidate building beside a dead incumbent. spared: std::sync::Mutex>, + /// Whether a start launches a process (a fresh private identity) + /// instead of refusing like `StillWorkload`. + launches: std::sync::atomic::AtomicBool, } impl SwitchWorkload { @@ -13848,9 +13851,15 @@ mod tests { rebooted: std::sync::atomic::AtomicBool::new(false), silent: std::sync::atomic::AtomicBool::new(false), spared: std::sync::Mutex::new(Vec::new()), + launches: std::sync::atomic::AtomicBool::new(false), }) } + /// From now on a start launches a process distinct from every recorded one. + fn launching(&self) { + self.launches.store(true, std::sync::atomic::Ordering::SeqCst); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -13897,6 +13906,9 @@ mod tests { expected: &IdunnExpectedIncarnationRecord, launch: IdunnRuntimeActivationLaunch, ) -> Result { + if self.launches.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(launch.activation().clone()); + } StillWorkload.prepare_activation(plan, expected, launch) } fn start_prepared( @@ -13907,7 +13919,25 @@ mod tests { expected: &IdunnExpectedIncarnationRecord, activation: &IdunnRuntimeActivationRecord, ) -> Result { - StillWorkload.start_prepared(plan, release, installed, expected, activation) + if !self.launches.load(std::sync::atomic::Ordering::SeqCst) { + return StillWorkload.start_prepared(plan, release, installed, expected, activation); + } + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("recorded transaction has no workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + // Another UID and other namespaces than any recorded unit. + observed.process_uids[0] += 1_000_000; + observed.pid_namespace_id += 1_000_000; + observed.mount_namespace_id += 1_000_000; + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + Ok(workload) } fn discard_prepared( &self, @@ -16704,6 +16734,7 @@ mod tests { let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; let deploy_id = deploy.transaction_id.as_str(); + workload.launching(); workload.kill(); assert!(world.engine.supervise_one_admitted_generation()?); let continuity_id = the_continuity(&world)?.value.transaction_id; From 1953bf233b3aa366eb555e113029ae73a2bec4cc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:57:31 +0200 Subject: [PATCH 07/47] Cut 2: one per-target lookup, and tests that kill the survivors it exposed ControlSnapshot::on_target is the one place a transaction is filed under a target; target_owner, the supervision gates and the handover read it. Tests: ownership is per target and names only a holder; a draining target is not restarted until its cleanup ends; the admitted Expected is repaired beside a building deployment; a live continuity alone holds back a second mint. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 109 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e2518f4..63bfe73 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3408,13 +3408,23 @@ impl ControlSnapshot { /// incumbent slot or is still draining it. The one lookup supervision, a /// waiting deployment and the handover share. fn target_owner(&self, target: &str, except: Option<&str>) -> Option<&Stored> { - self.transactions.iter().find(|stored| { - stored.value.target == target - && Some(stored.value.transaction_id.as_str()) != except + self.on_target(target).find(|stored| { + Some(stored.value.transaction_id.as_str()) != except && (stored.value.owns_target() || stored.value.drains_incumbent()) }) } + /// Every transaction of the target, in whatever state. + fn on_target<'a>( + &'a self, + target: &str, + ) -> impl Iterator> + use<'a> { + let target = target.to_owned(); + self.transactions + .iter() + .filter(move |stored| stored.value.target == target) + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -6070,22 +6080,19 @@ impl Engine { &anyhow::Error::new(disagreement), ); } - let on_target = || { - snapshot - .transactions - .iter() - .filter(|stored| stored.value.target == current.value.target) - }; // A transaction that owns the incumbent slot fences, stops or // replaces this generation: nothing here touches it meanwhile. - if on_target().any(|stored| stored.value.owns_target()) { + if snapshot + .on_target(¤t.value.target) + .any(|stored| stored.value.owns_target()) + { continue; } // Continuity defers to an owner, a drain or its own live attempt, // never to a deployment that has not taken the target, and aborts // nothing (handover-phase-split). let mint_blocked = snapshot.target_owner(¤t.value.target, None).is_some() - || on_target().any(|stored| { + || snapshot.on_target(¤t.value.target).any(|stored| { stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity }); match self.supervise_admitted_route(&snapshot, current) { @@ -9165,13 +9172,8 @@ impl Engine { ]; expected.extend( snapshot - .transactions - .iter() - .filter(|stored| { - stored.value.target == target - && stored.value.transaction_id != id - && !stored.value.is_terminal() - }) + .on_target(target) + .filter(|stored| stored.value.transaction_id != id && !stored.value.is_terminal()) .map(|stored| CultCacheExpectedEnvelope { r#type: DeploymentTransaction::TYPE.into(), key: stored.value.transaction_id.clone(), @@ -16415,6 +16417,8 @@ mod tests { assert_eq!(stored_deploy(&world)?.envelope, before.envelope); for _ in 0..2 { + // Due at once again (streak 0), so only the live attempt holds it back. + set_meters(&world, &TargetSupervision::new("service"))?; let _ = world.engine.supervise_one_admitted_generation()?; assert_eq!(continuity_mints(&world)?, 1); let after = stored_deploy(&world)?; @@ -16884,6 +16888,75 @@ mod tests { Ok(()) } + #[test] + fn ownership_is_read_per_target_and_names_only_a_transaction_that_holds_it() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let mut view = ControlSnapshot::read(&world.state_store)?; + assert_eq!(view.on_target("service").count(), 2); + assert_eq!(view.on_target("elsewhere").count(), 0); + // The pre-handover deployment holds nothing; the continuity holds the slot. + let owner = view.target_owner("service", None).context("no owner")?; + assert_eq!(owner.value.transaction_id, continuity.transaction_id); + assert!(view.target_owner("service", Some(&continuity.transaction_id)).is_none()); + assert!(view.target_owner("elsewhere", None).is_none()); + // The same records filed under another target are that target's. + for stored in &mut view.transactions { + stored.value.target = "elsewhere".into(); + } + assert_eq!(view.on_target("service").count(), 0); + assert!(view.target_owner("service", None).is_none()); + assert_eq!(view.on_target("elsewhere").count(), 2); + assert!(view.target_owner("elsewhere", None).is_some()); + let _ = deploy; + Ok(()) + } + + /// A deployment that has committed still drains the old incumbent; the + /// target is not restarted over that cleanup, and is once it ends. + #[test] + fn a_target_draining_its_old_incumbent_is_not_restarted_until_the_drain_ends() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let deploy = + seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &deploy, seeded_sequence(Some(&incumbent)) + 1)?; + drive_id(&world, &deploy.transaction_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, &deploy.transaction_id)?.value; + assert!(draining.drains_incumbent() && !draining.owns_target() && !draining.is_live()); + + age_incumbent(&world, 500_000)?; + workload.kill(); + for _ in 0..3 { + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "restarted over a draining incumbent"); + } + drive_id(&world, &deploy.transaction_id, |t| t.is_terminal())?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(continuity_mints(&world)?, 1); + Ok(()) + } + + /// A deployment that has not taken the target no longer suppresses the + /// repair of the admitted Expected (only the Expected, only when inexact). + #[test] + fn the_admitted_expected_is_repaired_beside_a_deployment_that_is_building() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let store = world.engine.options.topology_store.clone(); + if store.exists() { + std::fs::remove_file(&store)?; + } + assert_eq!(projected_under(&world, &incumbent.expected)?, Vec::::new()); + + let _ = world.engine.supervise_one_admitted_generation()?; + assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From dd8707b5a26755e76b39d5bbaa5d9c1f962dcac9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:14:46 +0200 Subject: [PATCH 08/47] Cut 2: pin the candidate-port reservation and the rebase guards directly candidate_ports_in_use is the one reading of which ports admitted, live and draining transactions hold. Tests pin it, select_candidate_port's routed and unrouted answers, and the three conditions under which a transaction is not rebased. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 115 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 18 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 63bfe73..0e1683e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3425,6 +3425,22 @@ impl ControlSnapshot { .filter(move |stored| stored.value.target == target) } + /// The candidate ports taken by admitted generations and by transactions + /// that are live or still draining; a finished one has released its port. + fn candidate_ports_in_use(&self) -> BTreeSet { + self.admitted + .iter() + .map(|stored| &stored.value.plan) + .chain( + self.transactions + .iter() + .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) + .filter_map(|stored| stored.value.plan.as_ref()), + ) + .filter_map(|plan| plan.candidate_port) + .collect() + } + fn supervision_for(&self, target: &str) -> Option<&Stored> { self.targets .iter() @@ -8074,24 +8090,7 @@ impl Engine { let Some(route) = &binding.route else { return Ok(None); }; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let mut used = BTreeSet::new(); - for plan in snapshot - .admitted - .iter() - .map(|stored| &stored.value.plan) - .chain( - snapshot - .transactions - .iter() - .filter(|stored| stored.value.is_live() || stored.value.drains_incumbent()) - .filter_map(|stored| stored.value.plan.as_ref()), - ) - { - if let Some(port) = plan.candidate_port { - used.insert(port); - } - } + let used = ControlSnapshot::read(&self.options.state_store)?.candidate_ports_in_use(); (route.private_port_start..=route.private_port_end) .find(|port| !used.contains(port)) .map(Some) @@ -16957,6 +16956,86 @@ mod tests { Ok(()) } + #[test] + fn a_candidate_port_is_held_by_the_admitted_a_live_and_a_draining_transaction_only() -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + let mut view = ControlSnapshot::read(&world.state_store)?; + assert!(view.candidate_ports_in_use().is_empty()); + + // Ports are set in memory: the fixture plans are unrouted. + view.admitted[0].value.plan.candidate_port = Some(18000); + for stored in &mut view.transactions { + let port = if stored.value.transaction_id == deploy.transaction_id { 18001 } else { 18002 }; + stored.value.plan.as_mut().context("no plan")?.candidate_port = Some(port); + } + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001, 18002])); + // Complete with cleanup still pending drains: its port stays held. + let finished = view + .transactions + .iter_mut() + .find(|stored| stored.value.transaction_id == continuity.transaction_id) + .context("no continuity")?; + finished.value.phase = DeploymentPhase::Complete; + finished.value.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Pending, + }); + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001, 18002])); + let finished = view + .transactions + .iter_mut() + .find(|stored| stored.value.transaction_id == continuity.transaction_id) + .context("no continuity")?; + finished.value.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: SourceCleanupEvidence::Complete, + }); + assert_eq!(view.candidate_ports_in_use(), BTreeSet::from([18000, 18001])); + + // A routed binding gets the first free port of its range; an unrouted one none. + let routed = OperatorBinding::parse(BINDING)?; + assert_eq!(world.engine.select_candidate_port(&routed)?, Some(routed.route.as_ref().context("route")?.private_port_start)); + let (head, tail) = BINDING.split_once("[route]").context("route")?; + let unrouted = OperatorBinding::parse(&format!("{head}[brakes]{}", tail.split_once("[brakes]").context("brakes")?.1))?; + assert_eq!(world.engine.select_candidate_port(&unrouted)?, None); + Ok(()) + } + + #[test] + fn only_a_deployment_before_its_handover_is_rebased() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Warming)?; + // Another generation is admitted under both. + edit_incumbent(&world, |generation| generation.generation_id = "generation-restarted".into())?; + let rebase = |id: &str| -> Result { + world.engine.rebase_onto_admitted(&resident_id(&world, id)?) + }; + // A continuity restarts one generation and no other. + assert!(!rebase(&continuity.transaction_id)?); + assert_ne!( + resident_id(&world, &continuity.transaction_id)?.value.incumbent_generation_id.as_deref(), + Some("generation-restarted") + ); + // A deployment already ending is left to end. + let ending = resident_id(&world, &deploy.transaction_id)?; + let mut aborting = ending.value.clone(); + aborting.pre_fencing_abort = Some(pre_fencing_abort_intent(&aborting, "cancelled by operator")); + replace_transaction(&world.state_store, &ending, &aborting)?; + assert!(!rebase(&deploy.transaction_id)?); + let ending = resident_id(&world, &deploy.transaction_id)?; + let mut live = ending.value.clone(); + live.pre_fencing_abort = None; + replace_transaction(&world.state_store, &ending, &live)?; + // Past its handover the incumbent is pinned, whatever is admitted. + rephase(&world, &deploy.transaction_id, DeploymentPhase::Fencing)?; + assert!(!rebase(&deploy.transaction_id)?); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From 6ea0d127d8ea2242625922f083fe29e32f66d461 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:15:19 +0200 Subject: [PATCH 09/47] Cut 2 test: a finished transaction retired mid-handover is not a lost race Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0e1683e..0b20e33 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17036,6 +17036,33 @@ mod tests { Ok(()) } + #[test] + fn the_handover_does_not_wait_on_a_finished_transaction_being_retired() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + // A committed transaction left resident, finished and not yet retired. + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, seeded_sequence(None) + 1)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + let incumbent = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + let deploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let view = ControlSnapshot::read(&world.state_store)?; + assert!(view.transactions.iter().any(|stored| stored.value.is_terminal())); + + // It is retired between the read and the handover; that is no race. + assert!(world.engine.retire_one_terminal_transaction()?); + let stored = resident_id(&world, &deploy.transaction_id)?; + assert_eq!( + world.engine.take_target(&view, &stored, &entering(&stored, DeploymentPhase::Fencing)?)?, + Handover::Took + ); + Ok(()) + } + /// A dead incumbent with a deployment building beside it (its candidate /// alive) whose Warming evidence is complete, and that deployment. fn dead_incumbent_with_a_deployment_ready_to_fence( From 94d3b53b55007063c60f8f5e184f1043ed6a622e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:00:49 +0200 Subject: [PATCH 10/47] Cut 2 fixes: stop-then-start handover fixture, supervision owner skip, rebase preflight, archive-as-progress - Add an engine-level stop-then-start fixture (route-less host-actuator plan) and a test that two such transactions never both enter Starting, in either order; it kills enter taking the CAS only at Fencing and takes_target_at ignoring the rollout. Discharges follow-up stop-first-handover-fixture. - Pin supervision's skip of an owned target: a fenced generation is not refreshed from Odin, and a stop-first owner's stopped incumbent is not restarted. - The rebase test now carries a route preflight, so keeping it dies. - The step that ends a transaction archives it inside its write; the progress read now treats a terminal archived record as progress, not a fault. - enter's doc no longer claims every phase move goes through it. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 364 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 333 insertions(+), 31 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0b20e33..108d5f7 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5894,11 +5894,22 @@ impl Engine { } } let after = ControlSnapshot::read(&self.options.state_store)?; - let live = after + let Some(live) = after .transactions .iter() .find(|stored| stored.value.transaction_id == current.value.transaction_id) - .context("transaction disappeared while checking scheduler progress")?; + else { + // The step that ends a transaction archives it inside its own + // write, so it is gone from the live set because it finished: that + // is the step's progress. Gone from both is a fault. + ensure!( + read_history_transactions(&self.options.state_store) + .iter() + .any(|archived| archived.transaction_id == id && archived.is_terminal()), + "transaction disappeared while checking scheduler progress" + ); + return Ok(true); + }; Ok(state_advanced(¤t.value, &live.value)) } @@ -9097,10 +9108,12 @@ impl Engine { } } - /// The one phase-entry seam. Every move into the next phase is written - /// here, and entering the phase where the transaction takes its target - /// (`takes_target_at`) is decided by `take_target`: that CAS is the - /// handover, and nothing else writes that entry. + /// The seam for the forward moves that can be the handover (`transition` + /// and the end of Sealing). Entering the phase where the transaction takes + /// its target (`takes_target_at`) is decided by `take_target`: that CAS is + /// the handover, and nothing else writes that entry. The other writes + /// (Starting to Warming, completions, aborts, same-phase evidence) go to + /// `replace_transaction` directly because none of them enters that phase. fn enter( &self, current: &Stored, @@ -12605,6 +12618,27 @@ mod tests { }) } + /// `binding` with its provider trust anchor and signer pointed at this + /// world's provider identity, enrolled on first use. + fn bind_provider(world: &EngineFixture, binding: &str) -> Result { + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = if provider_path.exists() { + open_service_identity_at::(&provider_path)? + } else { + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + provider + }; + Ok(binding + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id)) + } + /// What makes a target Odin: it provides the rendezvous. fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { @@ -12657,32 +12691,19 @@ mod tests { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, }; - let provider_path = world.root.join("identities/provider.cc"); - let provider_anchor = world.root.join("identities/provider-anchor.cc"); - let provider = if provider_path.exists() { - open_service_identity_at::(&provider_path)? - } else { - let provider = - enroll_service_identity_at::(&provider_path)?; - export_service_identity_trust_anchor(&provider, &provider_anchor)?; - provider - }; - let (binding_head, binding_tail) = BINDING .split_once("[route]") .context("binding has no route table")?; - let binding = format!( - "{binding_head}[brakes]{}", - binding_tail - .split_once("[brakes]") - .context("binding has no brakes table")? - .1 - ) - .replace( - "/etc/gamecult/trust/service.cc", - &provider_anchor.display().to_string(), - ) - .replace("service-runtime-signer", &provider.entry().identity_id); + let binding = bind_provider( + world, + &format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ), + )?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -13987,17 +14008,25 @@ mod tests { /// Admit a generation for "service" through the real phase machine and /// retire its transaction, leaving the world with an incumbent. fn admit_incumbent(world: &EngineFixture) -> Result { + Ok(admit_seeded_incumbent(world)?.0) + } + + /// `admit_incumbent`, and the transaction that was admitted. + fn admit_seeded_incumbent( + world: &EngineFixture, + ) -> Result<(AdmittedGeneration, DeploymentTransaction)> { let seeded = transaction_at(world, DeploymentPhase::Fencing)?; odin_reports_ready(world, &seeded, seeded_sequence(None) + 1)?; drive(world, |transaction| { transaction.phase == DeploymentPhase::Complete })?; assert!(world.engine.retire_one_terminal_transaction()?); - Ok(ControlSnapshot::read(&world.state_store)? + let admitted = ControlSnapshot::read(&world.state_store)? .admitted_for("service") .context("commit wrote no admitted generation")? .value - .clone()) + .clone(); + Ok((admitted, seeded)) } #[test] @@ -16686,6 +16715,260 @@ mod tests { Ok(()) } + /// A route-less host-actuator target that rolls out stop-then-start: the + /// seeded systemd binding with its runner, workload and rollout swapped. + fn stop_first_binding(world: &EngineFixture) -> Result { + use crate::deployment_plan::tests::BINDING; + const HOST: &str = r#"[runners.rust] +driver = "host-native" +affordances = ["source-read", "artifact-write"] +allowed_programs = ["cargo"] + +[workload] +driver = "host-actuator" +host = "yggdrasil" +host_trust_anchor_store = "/etc/gamecult/trust/host-actuator.cc" +release_root = "/srv/service/releases" +runtime_root = "/run/gamecult/idunn/service" +state_root = "/var/lib/gamecult/service" + +[workload.argument_bindings] +state_root = "/var/lib/gamecult/service" + +[workload.secret_files] +GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" + +"#; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; + let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; + let (identity, rest) = rest.split_once("[route]").context("no route")?; + let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; + bind_provider( + world, + &format!("{head}{HOST}[runtime_identity]{identity}[brakes]{brakes}") + .replace("candidate-then-promote", "stop-then-start"), + ) + } + + /// The seeded transaction `id`, rewritten as the Sealing transaction of a + /// stop-then-start target: the host-actuator plan, release and Expected, + /// and none of the evidence that follows Sealing. + fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + use crate::deployment_plan::tests::{ + RECIPE, artifact_receipt, external_input_receipt, source, + }; + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let binding = stop_first_binding(world)?; + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", "GAMECULT_IDUNN_RUNTIME_BUNDLE", "GAMECULT_RUNTIME_PRESENCE_IDENTITY"]"#, + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + &format!("service-{id}"), + None, + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let mut expected = release.expected_projection(&plan)?; + declare_odin(&mut expected); + let now = now_millis()?; + if next.command_kind == CommandKind::Deploy { + next.frozen_source = Some(FrozenSourceReceipt { + transaction_id: id.into(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: sha256_id(b"frozen source snapshot"), + }); + next.deployment_authorization = + Some(deployment_authorization(world, &expected, id, now)?); + } + next.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + next.sealed_release = Some(release); + next.expected = Some(expected); + next.plan = Some(plan); + next.expected_publication_sha256 = None; + next.activation_publication_sha256 = None; + next.activation = None; + next.workload = None; + next.warming = None; + next.isolation = None; + next.enter_phase(DeploymentPhase::Sealing, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// An admitted incumbent of `service`, and beside it a Deploy and a + /// Continuity (their ids, in that order), both stop-then-start at Sealing. + fn stop_first_world( + workload: Arc, + ) -> Result<(EngineFixture, String, String)> { + let world = EngineFixture::with_workload(workload)?; + let incumbent = admit_incumbent(&world)?; + let mut ids = Vec::new(); + for kind in [CommandKind::Deploy, CommandKind::Continuity] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + make_stop_first(&world, &seeded.transaction_id)?; + ids.push(seeded.transaction_id); + } + let continuity = ids.pop().context("no continuity")?; + let deploy = ids.pop().context("no deploy")?; + Ok((world, deploy, continuity)) + } + + /// Stop-then-start takes the target at Starting, by the same CAS as + /// Fencing: of two such transactions at Sealing, whichever enters Starting + /// first owns the target, and the other neither enters nor aborts. + #[test] + fn two_stop_then_start_transactions_never_both_enter_starting() -> Result<()> { + use DeploymentPhase::{Sealing, Starting}; + for deploy_first in [true, false] { + let (world, deploy, continuity) = stop_first_world(Arc::new(StillWorkload))?; + let (first, second) = if deploy_first { + (deploy.clone(), continuity.clone()) + } else { + (continuity.clone(), deploy.clone()) + }; + let stored = resident_id(&world, &first)?; + assert_eq!(stored.value.takes_target_at(), Starting); + assert!(!stored.value.owns_target()); + world.engine.enter(&stored, entering(&stored, Starting)?)?; + let owner = resident_id(&world, &first)?.value; + assert_eq!(owner.phase, Starting); + assert!(owner.owns_target(), "a stop-first transaction at Starting does not own the target"); + + let rival = resident_id(&world, &second)?; + let outcome = world.engine.enter(&rival, entering(&rival, Starting)?); + if deploy_first { + let error = outcome.expect_err("a continuity entered Starting over an owner"); + assert!(format!("{error:#}").contains("continuity superseded"), "{error:#}"); + assert!(format!("{error:#}").contains(&deploy), "{error:#}"); + } else { + outcome?; + let waiting = resident_id(&world, &second)?.value; + assert!( + waiting.last_error.as_deref().is_some_and(|note| note.contains(&continuity)), + "{:?}", + waiting.last_error + ); + } + // The store still reads: one owner, and the rival held at Sealing. + let after = ControlSnapshot::read(&world.state_store)?; + assert_eq!(after.transactions.iter().filter(|stored| stored.value.owns_target()).count(), 1); + let held = resident_id(&world, &second)?.value; + assert_eq!(held.phase, Sealing); + assert!(held.pre_fencing_abort.is_none()); + } + Ok(()) + } + + /// A stop-first deployment that has entered Starting has stopped the + /// incumbent by design; supervision must read it as the owner and not + /// restart what it stopped. + #[test] + fn supervision_does_not_restart_the_incumbent_a_stop_first_owner_stopped() -> Result<()> { + let workload = SwitchWorkload::new(); + let (world, deploy, _) = stop_first_world(workload.clone())?; + clear_other_live_transactions(&world, &deploy)?; + age_incumbent(&world, 500_000)?; + let stored = resident_id(&world, &deploy)?; + world.engine.enter(&stored, entering(&stored, DeploymentPhase::Starting)?)?; + workload.kill(); + + let before = incumbent_envelope(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 0, "supervision restarted what the owner stopped"); + assert_eq!(incumbent_envelope(&world)?, before); + + // Released, the same dead incumbent is restarted. + clear_other_live_transactions(&world, "")?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(continuity_mints(&world)?, 1); + Ok(()) + } + + /// An owner fencing the incumbent is the only writer of it: supervision + /// skips the generation, so it neither refreshes it from Odin nor repairs + /// or retires around it. + #[test] + fn supervision_leaves_a_generation_its_owner_is_fencing_alone() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (incumbent, admitted) = admit_seeded_incumbent(&world)?; + let owner = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + rephase(&world, &owner.transaction_id, DeploymentPhase::Fencing)?; + // Odin has a newer reading of the incumbent than the one it holds. + odin_republishes(&world, &admitted, seeded_sequence(None) + 5)?; + + let before = incumbent_envelope(&world)?; + world.engine.supervise_one_admitted_generation()?; + assert_eq!(incumbent_envelope(&world)?, before, "supervision refreshed a generation an owner is fencing"); + + // The same world with no owner takes the reading, so the skip alone held it. + clear_other_live_transactions(&world, "")?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_ne!(incumbent_envelope(&world)?, before); + Ok(()) + } + + /// The step that ends a transaction archives it inside its own write. It + /// left the live set because it finished: progress, not a fault. + #[test] + fn the_step_that_ends_a_transaction_reports_progress_and_no_fault() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let ended = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let id = ended.transaction_id; + let mut last = None; + for _ in 0..20 { + let Ok(current) = resident_id(&world, &id) else { + break; + }; + let moved = world.engine.resume_candidate(¤t)?; + if resident_id(&world, &id).is_err() { + last = Some(moved); + break; + } + } + assert_eq!(last, Some(true), "the ending step reported no progress"); + assert!(latest_id(&world, &id)?.is_terminal()); + assert!(world.engine.fault_reports.lock().unwrap().is_empty()); + + // Through the tick, the way the scheduler runs it: no fault is kept. + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let ended = transaction_at(&world, DeploymentPhase::Warming)?; + world + .engine + .begin_pre_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + for _ in 0..20 { + if resident_id(&world, &ended.transaction_id).is_err() { + break; + } + world.engine.resume_one_transaction()?; + } + assert!(latest_id(&world, &ended.transaction_id)?.is_terminal()); + assert!(world.engine.fault_reports.lock().unwrap().is_empty()); + Ok(()) + } + #[test] fn the_store_holds_one_deploy_one_continuity_and_one_owner_per_target() -> Result<()> { let seed = |first: (CommandKind, DeploymentPhase), second: (CommandKind, DeploymentPhase)| { @@ -16737,6 +17020,24 @@ mod tests { let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; let deploy_id = deploy.transaction_id.as_str(); + // The build recorded a route preflight against the incumbent it saw. + let building = resident_id(&world, deploy_id)?; + let mut preflighted = building.value.clone(); + preflighted.route_preflight = Some(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: building + .value + .activation + .as_ref() + .context("no activation")? + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(b"membership"), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }); + replace_transaction(&world.state_store, &building, &preflighted)?; workload.launching(); workload.kill(); assert!(world.engine.supervise_one_admitted_generation()?); @@ -16758,6 +17059,7 @@ mod tests { assert_eq!(untouched.value.incumbent_generation_id, Some(incumbent.generation_id.clone())); // The deployment's next step is the rebase and nothing else. + assert!(untouched.value.route_preflight.is_some() && untouched.value.warming.is_some()); world.engine.advance_transaction(&untouched)?; let rebased = resident_id(&world, deploy_id)?.value; assert_eq!(rebased.incumbent_generation_id, Some(restarted.generation_id.clone())); From 9ea6c719217a21cc7b7c7f9d344807b1e2031a71 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:01:59 +0200 Subject: [PATCH 11/47] Cut 2 test: the stop-first fixture builds without a network input A host-native runner has no network profile, so the host-actuator recipe drops the external input the systemd recipe fetches. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 108d5f7..3932388 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -16755,7 +16755,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// and none of the evidence that follows Sealing. fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { use crate::deployment_plan::tests::{ - RECIPE, artifact_receipt, external_input_receipt, source, + RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); @@ -16769,6 +16769,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, r#"["GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", "GAMECULT_IDUNN_RUNTIME_BUNDLE", "GAMECULT_RUNTIME_PRESENCE_IDENTITY"]"#, ); + // A host-native runner has no network profile to fetch an input with. + let (before_inputs, inputs) = recipe + .split_once("[[external_inputs]]") + .context("recipe has no external inputs")?; + let recipe = format!( + "{before_inputs}[[artifacts]]{}", + inputs.split_once("[[artifacts]]").context("recipe has no artifacts")?.1 + ); let plan = compile_deployment_plan( recipe.as_bytes(), binding.as_bytes(), @@ -16781,7 +16789,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let release = SealedRelease::new( &plan, vec![artifact_receipt()], - vec![external_input_receipt()], + Vec::new(), 120, )?; let mut expected = release.expected_projection(&plan)?; From 3265ccb575eec95293c020e074ed3970c73efe48 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:15:07 +0200 Subject: [PATCH 12/47] Cut c2b test: an abort stops a candidate that holds its lease before revoking it Negative test for ghostlight-abort-lease-before-stop: the candidate holds the lease lock until stopped. Recorded failing at this commit; the fix follows. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3932388..947a031 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13864,6 +13864,11 @@ mod tests { /// Whether a start launches a process (a fresh private identity) /// instead of refusing like `StillWorkload`. launches: std::sync::atomic::AtomicBool, + /// Files whose advisory locks stand for a process that holds a lease + /// for its lifetime: `stop` is what releases them. + holds: std::sync::Mutex>, + /// How many times the process was stopped. + stops: std::sync::atomic::AtomicUsize, } impl SwitchWorkload { @@ -13874,6 +13879,8 @@ mod tests { silent: std::sync::atomic::AtomicBool::new(false), spared: std::sync::Mutex::new(Vec::new()), launches: std::sync::atomic::AtomicBool::new(false), + holds: std::sync::Mutex::new(Vec::new()), + stops: std::sync::atomic::AtomicUsize::new(0), }) } @@ -13882,6 +13889,24 @@ mod tests { self.launches.store(true, std::sync::atomic::Ordering::SeqCst); } + /// The process takes a shared lock on `store`'s authority lock, as a + /// candidate holding its lifetime write lease does, until stopped. + #[cfg(unix)] + fn hold_lease_lock(&self, store: &std::path::Path) { + use std::os::unix::io::AsRawFd; + let mut lock = store.as_os_str().to_os_string(); + lock.push(".lock"); + let file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(lock) + .unwrap(); + // SAFETY: flock on a descriptor this function owns. + assert_eq!(unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) }, 0); + self.holds.lock().unwrap().push(file); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -13995,6 +14020,8 @@ mod tests { Ok(prior.clone()) } fn stop(&self, _: &WorkloadObservation) -> Result<()> { + self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.holds.lock().unwrap().clear(); Ok(()) } fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { @@ -21881,6 +21908,34 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// ghostlight-abort-lease-before-stop. A candidate that holds its + /// lifetime write lease keeps the lease's shared lock until it is + /// stopped, so the abort stops it before revoking; revoking first + /// fails on the lock on every attempt and the abort never ends. + #[cfg(unix)] + #[test] + fn an_abort_stops_a_candidate_that_holds_its_lease_before_revoking_it() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + let lease_path = routed.world.root.join("lease/process-write-lease.cc"); + routed.workload.hold_lease_lock(&lease_path); + let driver = CultCacheWriteLeaseDriver::new("service", &lease_path); + + pass_deadline(&routed.world)?; + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + let stops = routed.workload.stops.load(Ordering::SeqCst); + let finished = routed.resume_until_terminal()?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.last_error + ); + assert!(driver.observe_empty()?, "the abort left the lease on disk"); + assert!(routed.workload.stops.load(Ordering::SeqCst) > stops, "the candidate was never stopped"); + Ok(()) + } + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word /// carries the granted lease, and that Ready records the adoption, /// sourced from Odin's topology and naming the presence Odin observed. From ffb22e368b5cfd1ad74760184387b2e52b9f2d30 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:15:47 +0200 Subject: [PATCH 13/47] Cut c2b: the post-fencing abort stops its candidate before revoking the lifetime-held lease ruling c2b-abort-stops-before-revoke; fixes follow-up ghostlight-abort-lease-before-stop. Mirrors advance_fencing. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 947a031..e745693 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -9355,8 +9355,9 @@ impl Engine { } /// Withdraw the candidate in the one order that never leaves two writers: - /// take away its route, then its write lease, then the process itself, and - /// only then reconcile the projection. + /// take away its route, then stop the process and withdraw its write lease + /// (the lease is lifetime-held, so the holder stops first), then clean up, + /// and only then reconcile the projection. fn advance_post_fencing_abort(&self, current: &Stored) -> Result<()> { let abort = required( ¤t.value.post_fencing_abort, @@ -9396,6 +9397,15 @@ impl Engine { .process_write_lease .context("leased target has no write-lease binding")? .record_path; + // The candidate holds its lifetime write lease; revoking it under + // a live holder fails on the shared lock forever. A stopped + // process writes nothing, so stop first (as advance_fencing does). + if let Some(workload) = ¤t.value.workload { + let plan = required(¤t.value.plan, "abandoned candidate plan")?; + self.workload_for(plan)?.stop(workload).context( + "stopping the abandoned candidate before revoking its lifetime-held write lease", + )?; + } let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); driver .revoke_exact(Some(lease)) From 86926c4e426f4473f5557f93e216ce94f9b7c266 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:23:59 +0200 Subject: [PATCH 14/47] Cut c2b: one derivation of every deadline; Warming, aborts and drains are timed WARMING_DEADLINE_SECONDS and PRE_FENCING_ABORT_DEADLINE_SECONDS are Idunn constants (no PhaseDeadlines field: plan ids stay). DEADLINE_PHASES is the one spelling of the deadline range. deadline_for_current_activity derives the duration of a phase, an abort from its beginning, or a drain to its retire; retime is the only non-test writer. leaves_to_operator is derived from it. Validation asks for an activity, not a post-fencing phase. Records written by this cut are forward-only (ruling c2b-forward-only-records). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 310 ++++++++++++++++++++++++++++++++++++----- src/deployment_plan.rs | 7 + 2 files changed, 283 insertions(+), 34 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e745693..c9179c6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -35,7 +35,8 @@ use crate::deployment::{ capability_compatible, }; use crate::deployment_plan::{ - CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, + CompiledDeploymentPlan, DependencyProviderAuthority, PRE_FENCING_ABORT_DEADLINE_SECONDS, + SealedRelease, WARMING_DEADLINE_SECONDS, compile_deployment_plan, }; use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, @@ -925,8 +926,15 @@ impl LeaseAdoptionEvidence { } } -/// When the transaction entered its current post-fencing phase and when that -/// phase must end. Durations come from the plan's frozen `PhaseDeadlines`. +/// The phases whose entry starts a deadline: the one spelling of it. +const DEADLINE_PHASES: std::ops::RangeInclusive = + DeploymentPhase::Warming..=DeploymentPhase::Committing; + +/// What a live record is doing now and when that must end: a deadline phase +/// from its entry, an abort from its beginning, a drain to its retire. The +/// post-fencing durations come from the plan's frozen `PhaseDeadlines`; the +/// rest are Idunn's. `DeploymentTransaction::deadline_for_current_activity` +/// is the one derivation. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct PhaseDeadline { @@ -936,7 +944,8 @@ struct PhaseDeadline { } impl PhaseDeadline { - /// `None` for a phase that has no deadline (pre-fencing and Complete). + /// `None` for a phase that has no deadline of its own (Sealing, Starting + /// and Complete). fn entering( phase: DeploymentPhase, plan: &CompiledDeploymentPlan, @@ -944,6 +953,7 @@ impl PhaseDeadline { ) -> Option { let deadlines = plan.phase_deadlines(); let seconds = match phase { + DeploymentPhase::Warming => WARMING_DEADLINE_SECONDS, DeploymentPhase::Fencing => deadlines.fencing_seconds, DeploymentPhase::Leasing => deadlines.leasing_seconds, DeploymentPhase::AwaitingReady => deadlines.awaiting_ready_seconds, @@ -959,9 +969,12 @@ impl PhaseDeadline { } } -/// How the deadline resolver ends a post-fencing phase. +/// How the deadline resolver ends a deadline phase. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum PhaseEnd { + /// Before the fence the transaction owns only its candidate: it ends + /// through its own pre-fencing abort. + AbortBeforeFencing, /// Abort through the post-fencing path. `lease_not_adopted` names a /// granted lease with no adoption evidence. Abort { lease_not_adopted: bool }, @@ -1720,25 +1733,112 @@ fn handover_phase(stops_incumbent_first: bool) -> DeploymentPhase { } impl DeploymentTransaction { - /// The one writer of `phase`. It stamps the update time and the phase - /// deadline together, so a record outside Fencing..=Committing carries no - /// deadline by construction. No other code assigns `phase`. + /// The one writer of `phase`. It stamps the update time and, through + /// `retime`, the deadline of what the record now does. No other code + /// assigns `phase`. fn enter_phase(&mut self, phase: DeploymentPhase, now: u64) { self.phase = phase; self.updated_at_unix_millis = now; - self.phase_deadline = self - .plan - .as_ref() - .and_then(|plan| PhaseDeadline::entering(phase, plan, now)); + self.retime(now); } - /// A live transaction past the fence that is not already being aborted: - /// the only kind whose phase a deadline, or an operator's expiry, ends. + /// Whether the abort of the pre-fence path still has cleanup to do. + fn pre_fencing_abort_pending(&self) -> bool { + self.completion.is_none() + && self + .pre_fencing_abort + .as_ref() + .is_some_and(|abort| !abort.is_complete()) + } + + /// The one derivation of every duration the record is held to. (1) A + /// pending pre-fencing abort is timed from now by Idunn's constant. (2) A + /// draining Complete is timed from its incumbent's retire time (the route + /// promotion plus the drain, or now when nothing was promoted) for the + /// plan's committing duration. (3) Any other Complete has none. (4) The + /// rest take their phase's duration from now, so a post-fencing abort gets + /// the frozen duration of the phase it aborts, counted from its beginning. + fn deadline_for_current_activity(&self, now: u64) -> Option { + if self.pre_fencing_abort_pending() { + return Some(PhaseDeadline { + phase: self.phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: now.saturating_add( + u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000, + ), + }); + } + if self.phase == DeploymentPhase::Complete { + if !matches!(self.completion, Some(TransactionCompletion::Admitted { .. })) + || !self.drains_incumbent() + { + return None; + } + let plan = self.plan.as_ref()?; + let from = match self + .routing + .as_ref() + .and_then(RoutingEvidence::promoted_at_unix_millis) + { + Some(promoted_at) => { + route_drain_deadline(promoted_at, plan.parsed_inputs().ok()?.1.rollout.drain_seconds) + .ok()? + } + None => now, + }; + return Some(PhaseDeadline { + phase: self.phase, + entered_at_unix_millis: now, + deadline_at_unix_millis: from + .max(now) + .saturating_add(u64::from(plan.phase_deadlines().committing_seconds) * 1000), + }); + } + PhaseDeadline::entering(self.phase, self.plan.as_ref()?, now) + } + + /// Stamp the deadline of what the record does now. The only non-test + /// writer that sets one: phase entry, an abort's beginning, the commit to a + /// draining Complete and the resolver's lift call it. + fn retime(&mut self, now: u64) { + self.phase_deadline = self.deadline_for_current_activity(now); + } + + /// Whether a deadline can describe the record's current activity: a + /// deadline phase, a pending pre-fencing abort, or an admitted Complete. + fn carries_a_deadline(&self) -> bool { + DEADLINE_PHASES.contains(&self.phase) + || self.pre_fencing_abort_pending() + || (self.phase == DeploymentPhase::Complete + && matches!(self.completion, Some(TransactionCompletion::Admitted { .. }))) + } + + /// A live transaction in a deadline phase that is not already being + /// aborted: the only kind whose phase a deadline, or an operator's expiry, + /// ends. fn resolves_by_deadline(&self) -> bool { self.completion.is_none() && self.pre_fencing_abort.is_none() && self.post_fencing_abort.is_none() - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing).contains(&self.phase) + && DEADLINE_PHASES.contains(&self.phase) + } + + /// Whether Idunn has run out of ways to end this record and leaves it to + /// the operator: its deadline has passed in an abort that cannot finish, a + /// drain whose stop keeps failing, or a stateful deploy. Derived, never + /// stored. The step goes on retrying and the target stays owned. + fn leaves_to_operator(&self, now: u64) -> bool { + if self.is_terminal() { + return false; + } + let Some(deadline) = &self.phase_deadline else { + return false; + }; + now >= deadline.deadline_at_unix_millis + && (self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some() + || self.drains_incumbent() + || (self.resolves_by_deadline() && self.phase_end() == PhaseEnd::OperatorRequired)) } /// What an ended post-fencing phase resolves to (Q3). A candidate that was @@ -1750,6 +1850,9 @@ impl DeploymentTransaction { /// new state contract, and restoring the incumbent over that is not /// Idunn's call: it is left for the operator. fn phase_end(&self) -> PhaseEnd { + if self.phase < DeploymentPhase::Fencing { + return PhaseEnd::AbortBeforeFencing; + } let leasing = self.leasing.as_ref(); let granted = leasing.and_then(LeasingEvidence::lease).is_some(); let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); @@ -2150,11 +2253,10 @@ impl DeploymentTransaction { if let Some(deadline) = &self.phase_deadline { ensure!( deadline.phase == self.phase - && (DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&deadline.phase) + && self.carries_a_deadline() && deadline.entered_at_unix_millis > 0 && deadline.deadline_at_unix_millis > deadline.entered_at_unix_millis, - "phase deadline does not describe the current post-fencing phase" + "phase deadline does not describe the record's current activity" ); } if let Some(adoption) = &self.lease_adoption { @@ -3012,7 +3114,7 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> ); ensure!( current.value.resolves_by_deadline(), - "only a live transaction past the fence, and not already aborting, can be expired; before the fence, use idunn cancel" + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" ); let now = now_millis()?; let request = ExpiryRequest { @@ -7849,6 +7951,9 @@ impl Engine { generation_id: generation.generation_id.clone(), }); complete.post_commit_cleanup = Some(post_commit_cleanup); + // enter_phase ran before the cleanup was set: a draining record + // carries its drain's deadline. + complete.retime(now); complete.validate()?; let admitted_expected = CultCacheExpectedEnvelope { @@ -9205,9 +9310,10 @@ impl Engine { /// command an operator asked to expire, is resolved as `phase_end` says. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// - /// The deadline is written only by `enter_phase`, so nothing here or in a - /// resume path extends it. A record lifted from before deadlines existed - /// has none; its phase is timed once, from the first time this sees it. + /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the + /// commit and this lift), so nothing in a step or a resume path extends one. + /// A record lifted from before deadlines existed has none; what it does is + /// timed once, from the first time this sees it. fn resolve_phase_deadline( &self, current: &Stored, @@ -9250,6 +9356,10 @@ impl Engine { self.begin_post_fencing_abort(current, error)?; Ok(Some(true)) } + PhaseEnd::AbortBeforeFencing => { + self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; + Ok(Some(true)) + } PhaseEnd::OperatorRequired => { self.report_once( &format!("deadline:{}", transaction.transaction_id), @@ -9313,8 +9423,10 @@ impl Engine { "pre-fencing abort intent is already durable" ); let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); + let now = now_millis()?; self.persist_same_phase(current, |next| { next.pre_fencing_abort = Some(abort); + next.retime(now); Ok(()) }) } @@ -9348,8 +9460,10 @@ impl Engine { "post-fencing abort intent is already durable" ); let abort = post_fencing_abort_intent(¤t.value, &truncate(&format!("{error:#}"), 2048)); + let now = now_millis()?; self.persist_same_phase(current, |next| { next.post_fencing_abort = Some(abort); + next.retime(now); Ok(()) }) } @@ -12391,11 +12505,12 @@ mod tests { } #[test] - fn phase_deadlines_follow_the_plan_and_only_cover_post_fencing_phases() -> Result<()> { + fn phase_deadlines_follow_the_plan_and_cover_warming_through_committing() -> Result<()> { let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; let plan = &generation.plan; let defaults = plan.phase_deadlines(); for (phase, seconds) in [ + (DeploymentPhase::Warming, WARMING_DEADLINE_SECONDS), (DeploymentPhase::Fencing, defaults.fencing_seconds), (DeploymentPhase::Leasing, defaults.leasing_seconds), (DeploymentPhase::AwaitingReady, defaults.awaiting_ready_seconds), @@ -12410,7 +12525,6 @@ mod tests { for phase in [ DeploymentPhase::Sealing, DeploymentPhase::Starting, - DeploymentPhase::Warming, DeploymentPhase::Complete, ] { assert!(PhaseDeadline::entering(phase, plan, 1_000).is_none()); @@ -12467,7 +12581,7 @@ mod tests { } #[test] - fn a_deadline_must_describe_the_current_post_fencing_phase() -> Result<()> { + fn a_deadline_must_describe_the_records_current_activity() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; let plan = transaction.plan.clone().unwrap(); let now = transaction.updated_at_unix_millis; @@ -12482,10 +12596,25 @@ mod tests { backwards.phase_deadline.as_mut().unwrap().deadline_at_unix_millis = now; assert!(backwards.validate().is_err()); // Entry is not ordered against the wall-clock stamp of later writes. - let mut entered_after_the_stamp = transaction; + let mut entered_after_the_stamp = transaction.clone(); entered_after_the_stamp.phase_deadline = PhaseDeadline::entering(DeploymentPhase::AwaitingReady, &plan, now + 1); entered_after_the_stamp.validate()?; + + // The activity rule: Warming carries a deadline; Sealing only while + // its abort is pending; Complete only once admitted. + let mut other = transaction.clone(); + other.phase = DeploymentPhase::Warming; + assert!(other.carries_a_deadline()); + other.phase = DeploymentPhase::Sealing; + assert!(!other.carries_a_deadline(), "Sealing has no activity to time"); + other.pre_fencing_abort = Some(pre_fencing_abort_intent(&other, "refused")); + assert!(other.carries_a_deadline()); + other.pre_fencing_abort = None; + other.phase = DeploymentPhase::Complete; + assert!(!other.carries_a_deadline()); + other.completion = Some(TransactionCompletion::Admitted { generation_id: "g".into() }); + assert!(other.carries_a_deadline()); Ok(()) } @@ -13019,8 +13148,8 @@ mod tests { /// One engine step of transaction `id` at a time until `stop` holds, /// recording each phase entered and checking after every step that the - /// deadline on the record is exactly the current post-fencing phase's, and - /// absent outside one. + /// deadline on the record is exactly the current phase's, and absent + /// outside a deadline phase. fn drive_id( world: &EngineFixture, id: &str, @@ -13038,10 +13167,12 @@ mod tests { let after = latest_id(world, id)?; match &after.phase_deadline { Some(deadline) => assert_eq!(deadline.phase, after.phase), + // A waiter's Warming deadline is disarmed while another + // transaction owns the target. None => assert!( - !(DeploymentPhase::Fencing..=DeploymentPhase::Committing) - .contains(&after.phase), - "a post-fencing record lost its deadline" + after.phase == DeploymentPhase::Warming + || !DEADLINE_PHASES.contains(&after.phase), + "a deadline-phase record lost its deadline" ), } if phases.last() != Some(&after.phase) { @@ -13087,15 +13218,28 @@ mod tests { } #[test] - fn a_pre_fence_abort_runs_to_complete_without_a_deadline() -> Result<()> { + fn a_pre_fence_abort_is_timed_from_its_beginning_and_ends_without_a_deadline() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; transaction_at(&world, DeploymentPhase::Warming)?; let warming = resident(&world)?; - assert_eq!(warming.value.phase_deadline, None); + let warming_deadline = warming.value.phase_deadline.context("Warming has no deadline")?; + assert_eq!( + warming_deadline.deadline_at_unix_millis - warming_deadline.entered_at_unix_millis, + u64::from(WARMING_DEADLINE_SECONDS) * 1000 + ); + let before = now_millis()?; world .engine .begin_pre_fencing_abort(&warming, anyhow!("candidate refused"))?; + let aborting = latest(&world)?; + let deadline = aborting.phase_deadline.context("the abort has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Warming); + assert!(deadline.entered_at_unix_millis >= before); + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + u64::from(PRE_FENCING_ABORT_DEADLINE_SECONDS) * 1000 + ); drive(&world, |transaction| transaction.completion.is_some())?; let finished = latest(&world)?; assert_eq!(finished.phase, DeploymentPhase::Complete); @@ -13139,6 +13283,96 @@ mod tests { Ok(()) } + #[test] + fn an_abort_is_timed_from_its_beginning_with_its_phases_duration() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + odin_reports_ready(&world, &seeded, 5)?; + drive(&world, |transaction| { + transaction.phase == DeploymentPhase::Routing + })?; + // The phase's own deadline is long past when the abort begins. + let routing = resident(&world)?; + let mut stale = routing.value.clone(); + let deadline = stale.phase_deadline.as_mut().context("Routing has no deadline")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, &routing, &stale)?; + + let before = now_millis()?; + world + .engine + .begin_post_fencing_abort(&resident(&world)?, anyhow!("candidate refused"))?; + let aborting = latest(&world)?; + let deadline = aborting.phase_deadline.context("the abort has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Routing); + assert!(deadline.entered_at_unix_millis >= before); + let routing_millis = + u64::from(aborting.plan.as_ref().unwrap().phase_deadlines().routing_seconds) * 1000; + assert_eq!( + deadline.deadline_at_unix_millis - deadline.entered_at_unix_millis, + routing_millis + ); + assert!(!aborting.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(aborting.leaves_to_operator(deadline.deadline_at_unix_millis)); + Ok(()) + } + + #[test] + fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + let (_, mut draining) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; + let plan = draining.plan.clone().unwrap(); + let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; + let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; + assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); + draining.phase = DeploymentPhase::Complete; + draining.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-new".into(), + }); + draining.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Pending { + generation_id: "generation-old".into(), + workload: workload(1001, 40, 50), + }, + source: SourceCleanupEvidence::Complete, + }); + let promoted_at = 10_000_000_u64; + match draining.routing.as_mut() { + Some(RoutingEvidence::Promoted { promoted_at_unix_millis, .. }) => { + *promoted_at_unix_millis = promoted_at; + } + other => bail!("the fixture is not promoted: {other:?}"), + } + + // Before the retire time: the retire time plus the committing duration. + draining.retime(promoted_at + 1); + let deadline = draining.phase_deadline.context("a drain has no deadline")?; + assert_eq!(deadline.phase, DeploymentPhase::Complete); + assert_eq!(deadline.entered_at_unix_millis, promoted_at + 1); + assert_eq!(deadline.deadline_at_unix_millis, promoted_at + drain + committing); + assert!(!draining.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(draining.leaves_to_operator(deadline.deadline_at_unix_millis)); + + // After it: the committing duration from now. + let late = promoted_at + drain + 5_000; + draining.retime(late); + assert_eq!(draining.phase_deadline.unwrap().deadline_at_unix_millis, late + committing); + + // Nothing promoted: from now. + draining.routing = Some(RoutingEvidence::SkippedUnrouted); + draining.retime(promoted_at); + assert_eq!(draining.phase_deadline.unwrap().deadline_at_unix_millis, promoted_at + committing); + + // A finished drain, and any other Complete, carry none. + draining.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::Complete { generation_id: "generation-old".into() }, + source: SourceCleanupEvidence::Complete, + }); + draining.retime(promoted_at); + assert_eq!(draining.phase_deadline, None); + Ok(()) + } + #[test] fn entering_a_phase_owns_the_deadline_and_a_stepped_clock_cannot_wedge_it() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; @@ -16277,6 +16511,7 @@ mod tests { let stored = resident(&world)?; let mut starting = seeded.clone(); starting.phase = DeploymentPhase::Starting; + starting.phase_deadline = None; starting.expected_publication_sha256 = None; starting.activation = None; starting.workload = None; @@ -16314,6 +16549,7 @@ mod tests { let stored = resident(world)?; let mut starting = seeded.clone(); starting.phase = DeploymentPhase::Starting; + starting.phase_deadline = None; starting.expected_publication_sha256 = None; starting.activation = None; starting.workload = None; @@ -21645,8 +21881,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[test] fn an_operator_expiry_ends_the_phase_through_the_resolver() -> Result<()> { let early = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + // Warming is a deadline phase: the operator can expire it, and + // the pre-fencing abort then ends it. Once aborting it cannot be + // expired again. + expire(&early.world.state_store, "continuity-service", "operator")?; + early.resume()?; + assert!(early.transaction()?.pre_fencing_abort.is_some(), "the expiry did not end Warming"); let refused = expire(&early.world.state_store, "continuity-service", "operator") - .expect_err("a transaction before the fence was expired"); + .expect_err("an aborting transaction was expired"); assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 603c9d2..1363c48 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -475,6 +475,13 @@ pub const DEFAULT_AWAITING_READY_DEADLINE_SECONDS: u32 = 300; pub const DEFAULT_ROUTING_DEADLINE_SECONDS: u32 = 120; pub const DEFAULT_COMMITTING_DEADLINE_SECONDS: u32 = 120; +/// Idunn's own deadlines, not binding-declared: a new `PhaseDeadlines` field +/// would change every v3 plan id (`recomputed_plan_id` re-encodes the plan). +/// Warming is timed for every transaction, owner or not. +pub const WARMING_DEADLINE_SECONDS: u32 = 600; +/// A pre-fencing abort is timed from its beginning. +pub const PRE_FENCING_ABORT_DEADLINE_SECONDS: u32 = 300; + /// The resolved per-phase deadline durations a plan carries. Entering a /// post-fencing phase stamps its deadline from them, and the deadline /// resolver ends the phase once it passes. From a14055115cbf0f06dbcd653ebbc71383b6ecd044 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:37:21 +0200 Subject: [PATCH 15/47] Cut c2b: owner deadlines; a waiter never waits on an owner Idunn will not finish wait_for_owner disarms a waiter Warming deadline (the owner bounds the wait) and ends the waiter through its own pre-fencing abort when the owner leaves to the operator (owner-left-to-operator); it never aborts the owner. The resolver times Warming for every transaction, lifts a deadline onto a record that needs one, skips an owned waiter, and leaves an abort or drain past its deadline to the operator (report only; the incident follows). Rulings c2b-waiter-ends-on- operator-owner, c2b-warming-deadline, c2b-abort-and-drain-deadlines. Negative tests recorded failing at 9ea6c71. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 408 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 383 insertions(+), 25 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c9179c6..d408ee6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -8372,6 +8372,49 @@ impl Engine { } } + /// A deployment waits on the transaction that owns or drains its target. + /// The owner bounds the wait, so the waiter's own Warming deadline is + /// disarmed (the resolver re-arms it when the target is free). When the + /// owner's deadline has passed in a state Idunn will not end by itself, + /// the waiter ends through its own pre-fencing abort (the caller's Err + /// path): it never aborts the owner. + fn wait_for_owner( + &self, + current: &Stored, + owner_id: &str, + ) -> Result<()> { + let held = ¤t.value; + let now = now_millis()?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if snapshot.transactions.iter().any(|stored| { + stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) + }) { + bail!( + "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", + held.target + ); + } + let detail = waiting_for_owner(&held.target, owner_id); + if held.last_error.as_deref() == Some(detail.as_str()) && held.phase_deadline.is_none() { + return Ok(()); + } + let mut next = held.clone(); + next.last_error = Some(detail); + next.phase_deadline = None; + next.updated_at_unix_millis = now; + replace_transaction(&self.options.state_store, current, &next) + } + + /// Idunn has run out of ways to end this record. It stays owned and its + /// step keeps retrying; the operator is told. + fn leave_to_operator(&self, transaction: &DeploymentTransaction, cause: &str) { + let id = &transaction.transaction_id; + self.report_once( + &format!("deadline:{id}"), + format!("Idunn leaves {id} to the operator: {cause}; it keeps retrying the step"), + ); + } + fn record_gate_wait( &self, current: &Stored, @@ -9185,11 +9228,8 @@ impl Engine { else { return Ok(false); }; - self.record_gate_wait( - current, - &waiting_for_owner(&held.target, &owner.value.transaction_id), - )?; - Ok(true) + self.wait_for_owner(current, &owner.value.transaction_id) + .map(|()| true) } CommandKind::Continuity => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; @@ -9238,7 +9278,7 @@ impl Engine { (Handover::Took | Handover::Lost, _) => Ok(()), // Wait, never abort: a pre-fencing error would end the deployment. (Handover::OwnedBy(owner), CommandKind::Deploy) => { - self.record_gate_wait(current, &waiting_for_owner(target, &owner)) + self.wait_for_owner(current, &owner) } // The next step rebases onto the generation now admitted. (Handover::Moved, CommandKind::Deploy) => Ok(()), @@ -9305,33 +9345,61 @@ impl Engine { Ok(if took { Handover::Took } else { Handover::Lost }) } - /// Every post-fencing phase ends. Before a live transaction past the - /// fence runs its step, a phase whose deadline has passed, or whose - /// command an operator asked to expire, is resolved as `phase_end` says. + /// Every live record is held to a deadline. Before a transaction runs its + /// step, a deadline phase whose deadline has passed, or whose command an + /// operator asked to expire, is resolved as `phase_end` says; an abort or + /// a drain past its deadline is left to the operator and its step retries. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the - /// commit and this lift), so nothing in a step or a resume path extends one. - /// A record lifted from before deadlines existed has none; what it does is - /// timed once, from the first time this sees it. + /// commit and this lift), so nothing in a step or a resume path extends + /// one. A record with none whose activity needs one is timed once, from + /// the first time this sees it: that is also how a waiter is re-armed once + /// the target is free. fn resolve_phase_deadline( &self, current: &Stored, ) -> Result> { let transaction = ¤t.value; - if !transaction.resolves_by_deadline() { + if transaction.is_terminal() { + return Ok(None); + } + // An owned waiter is timed by its owner. + if transaction.phase < DeploymentPhase::Fencing + && transaction.pre_fencing_abort.is_none() + && ControlSnapshot::read(&self.options.state_store)? + .target_owner(&transaction.target, Some(&transaction.transaction_id)) + .is_some() + { return Ok(None); } let now = now_millis()?; let Some(deadline) = transaction.phase_deadline else { let mut next = transaction.clone(); - next.enter_phase(transaction.phase, now); + next.retime(now); if next.phase_deadline.is_none() { return Ok(None); } replace_transaction(&self.options.state_store, current, &next)?; return Ok(Some(true)); }; + if !transaction.resolves_by_deadline() { + if transaction.leaves_to_operator(now) { + let what = if transaction.drains_incumbent() { + "draining its incumbent" + } else { + "its abort" + }; + self.leave_to_operator( + transaction, + &format!( + "{what} in the {:?} phase passed its deadline at unix ms {}", + transaction.phase, deadline.deadline_at_unix_millis + ), + ); + } + return Ok(None); + } let requested = ControlSnapshot::read(&self.options.state_store)? .expiry_requests .iter() @@ -9347,6 +9415,10 @@ impl Engine { return Ok(None); }; match transaction.phase_end() { + PhaseEnd::AbortBeforeFencing => { + self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; + Ok(Some(true)) + } PhaseEnd::Abort { lease_not_adopted } => { let error = if lease_not_adopted { anyhow!("lease-not-adopted: {cause}, and the granted write lease has no adoption evidence") @@ -9356,17 +9428,10 @@ impl Engine { self.begin_post_fencing_abort(current, error)?; Ok(Some(true)) } - PhaseEnd::AbortBeforeFencing => { - self.begin_pre_fencing_abort(current, anyhow!("{cause}"))?; - Ok(Some(true)) - } PhaseEnd::OperatorRequired => { - self.report_once( - &format!("deadline:{}", transaction.transaction_id), - format!( - "Idunn leaves {} to the operator: {cause}, and its stateful candidate was issued a write lease", - transaction.transaction_id - ), + self.leave_to_operator( + transaction, + &format!("{cause}, and its stateful candidate was issued a write lease"), ); Ok(None) } @@ -14113,6 +14178,8 @@ mod tests { holds: std::sync::Mutex>, /// How many times the process was stopped. stops: std::sync::atomic::AtomicUsize, + /// Runtime instances whose stop fails, whatever is asked. + failing_stops: std::sync::Mutex>, } impl SwitchWorkload { @@ -14125,6 +14192,7 @@ mod tests { launches: std::sync::atomic::AtomicBool::new(false), holds: std::sync::Mutex::new(Vec::new()), stops: std::sync::atomic::AtomicUsize::new(0), + failing_stops: std::sync::Mutex::new(Vec::new()), }) } @@ -14151,6 +14219,11 @@ mod tests { self.holds.lock().unwrap().push(file); } + /// From now on stopping this runtime instance fails. + fn fail_stop_of(&self, runtime_instance_id: &str) { + self.failing_stops.lock().unwrap().push(runtime_instance_id.to_owned()); + } + /// `kill` leaves this runtime instance running. fn spare(&self, runtime_instance_id: &str) { self.spared.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -14263,8 +14336,14 @@ mod tests { ); Ok(prior.clone()) } - fn stop(&self, _: &WorkloadObservation) -> Result<()> { + fn stop(&self, stopping: &WorkloadObservation) -> Result<()> { self.stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if let WorkloadObservation::Systemd(observed) = stopping { + ensure!( + !self.failing_stops.lock().unwrap().contains(&observed.runtime_instance_id), + "the unit would not stop" + ); + } self.holds.lock().unwrap().clear(); Ok(()) } @@ -16844,6 +16923,254 @@ mod tests { Ok((deploy, continuity, incumbent)) } + /// One scheduler pass over every live transaction, with no resume backoff + /// standing between it and the step. + fn resume_pass(world: &EngineFixture) -> Result { + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_one_transaction() + } + + /// The runtime instance whose workload `id` runs. + fn runtime_instance_of(world: &EngineFixture, id: &str) -> Result { + match resident_id(world, id)?.value.workload { + Some(WorkloadObservation::Systemd(observed)) => Ok(observed.runtime_instance_id), + other => bail!("transaction {id} has no systemd workload: {other:?}"), + } + } + + /// The deadline of transaction `id` is already behind it. + fn pass_deadline_of(world: &EngineFixture, id: &str) -> Result<()> { + let current = resident_id(world, id)?; + let mut next = current.value.clone(); + let deadline = next.phase_deadline.as_mut().context("no deadline to pass")?; + deadline.entered_at_unix_millis = 1; + deadline.deadline_at_unix_millis = 2; + replace_transaction(&world.state_store, ¤t, &next) + } + + /// NEGATIVE (a). A continuity that owns the target is aborting and its + /// abort step fails on every attempt. Its deadline passes: Idunn leaves it + /// to the operator and keeps it owned, and the deployment waiting behind it + /// ends itself before fencing instead of waiting forever. + #[test] + fn a_waiting_deployment_ends_when_its_owners_abort_is_left_to_the_operator() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + workload.fail_stop_of(&runtime_instance_of(&world, owner_id)?); + world + .engine + .begin_post_fencing_abort(&resident_id(&world, owner_id)?, anyhow!("candidate refused"))?; + + // Before the deadline the deployment waits on a bounded owner. + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, deploy_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); + assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); + + pass_deadline_of(&world, owner_id)?; + for _ in 0..10 { + resume_pass(&world)?; + } + let owner = resident_id(&world, owner_id)?.value; + assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); + assert!(owner.owns_target()); + let ended = latest_id(&world, deploy_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); + assert!(ended.fencing.is_none()); + Ok(()) + } + + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop + /// keeps failing. A deployment waits behind it until the drain's deadline, + /// then ends itself; the drain stays and the target stays owned. + #[test] + fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; + let drainer_id = drainer.transaction_id.as_str(); + drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, drainer_id)?.value; + assert!(draining.drains_incumbent(), "the incumbent was already retired"); + assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); + match &draining.workload { + Some(WorkloadObservation::Systemd(_)) => {} + other => bail!("no workload: {other:?}"), + } + let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = + &draining.post_commit_cleanup + else { + bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); + }; + let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; + workload.fail_stop_of(&old.runtime_instance_id); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; + let waiter_id = waiter.transaction_id.as_str(); + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, waiter_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); + assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); + + pass_deadline_of(&world, drainer_id)?; + for _ in 0..10 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + let ended = latest_id(&world, waiter_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(drainer_id), "{error}"); + Ok(()) + } + + /// A stop-then-start transaction held in Warming with a started unit, as + /// the Starting step leaves it (this world serves no host actuator to + /// start one). + fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let expected = next.expected.clone().context("no Expected")?; + let now = now_millis()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&next.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + next.expected_publication_sha256 = Some(expected.canonical_sha256()?); + next.activation_publication_sha256 = Some(activation.canonical_sha256()?); + next.activation = Some(activation); + next.workload = Some(workload); + next.enter_phase(DeploymentPhase::Warming, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting + /// and waits in Warming for an Odin word that never comes. Its Warming + /// deadline ends it, and the deployment waiting at its Sealing tail takes + /// the target. + #[test] + fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( + ) -> Result<()> { + use DeploymentPhase::{Starting, Warming}; + let (world, deploy, continuity) = stop_first_world(SwitchWorkload::new())?; + warm_stop_first(&world, &continuity)?; + let owner = resident_id(&world, &continuity)?; + assert_eq!(owner.value.phase, Warming); + assert!(owner.value.owns_target() && owner.value.phase_deadline.is_some()); + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, Starting)?)?; + let held = resident_id(&world, &deploy)?.value; + assert_eq!(held.phase, DeploymentPhase::Sealing); + assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); + + pass_deadline_of(&world, &continuity)?; + world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; + let aborting = resident_id(&world, &continuity)?.value; + let abort = aborting.pre_fencing_abort.clone().context("the deadline did not end Warming")?; + assert!(abort.error.contains("Warming phase passed its deadline"), "{}", abort.error); + // This world serves no host actuator, so the abandoned unit cannot be + // stopped here; the abort's cleanup is recorded done. + let mut cleaned = aborting.clone(); + let intent = cleaned.pre_fencing_abort.as_mut().unwrap(); + intent.candidate_cleanup = CleanupEvidence::Complete; + intent.topology_reconciliation = CleanupEvidence::Complete; + replace_transaction(&world.state_store, &resident_id(&world, &continuity)?, &cleaned)?; + for _ in 0..12 { + if resident_id(&world, &continuity).is_err() { + break; + } + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; + } + let ended = latest_id(&world, &continuity)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("Warming phase passed its deadline"), "{error}"); + + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, Starting)?)?; + let took = resident_id(&world, &deploy)?.value; + assert_eq!(took.phase, Starting); + assert!(took.owns_target() && took.pre_fencing_abort.is_none()); + Ok(()) + } + + #[test] + fn a_waiters_warming_deadline_is_disarmed_while_an_owner_holds_the_target_and_rearmed_when_it_ends( + ) -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + rephase(&world, deploy_id, DeploymentPhase::Warming)?; + assert!(resident_id(&world, deploy_id)?.value.phase_deadline.is_some()); + + resume_pass(&world)?; + let waiting = resident_id(&world, deploy_id)?.value; + assert_eq!(waiting.phase_deadline, None, "the owner does not bound the waiter"); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); + // Held for as long as the owner stands: a deadline that fell due + // meanwhile would otherwise abort it. + for _ in 0..3 { + resume_pass(&world)?; + } + assert!(resident_id(&world, deploy_id)?.value.pre_fencing_abort.is_none()); + + // The owner retires without moving the generation. The waiter's own + // entry is long past, but it is timed from now. + clear_other_live_transactions(&world, deploy_id)?; + let stored = resident_id(&world, deploy_id)?; + let mut aged = stored.value.clone(); + aged.created_at_unix_millis = 1; + aged.updated_at_unix_millis = 1; + replace_transaction(&world.state_store, &stored, &aged)?; + let retired = now_millis()?; + resume_pass(&world)?; + let rearmed = resident_id(&world, deploy_id)?.value; + let deadline = rearmed.phase_deadline.context("the waiter was not re-armed")?; + assert_eq!(deadline.phase, DeploymentPhase::Warming); + assert!(deadline.entered_at_unix_millis >= retired); + assert!(rearmed.pre_fencing_abort.is_none()); + Ok(()) + } + #[test] fn exactly_one_transaction_takes_a_target_from_one_view() -> Result<()> { let take = |snapshot: &ControlSnapshot, world: &EngineFixture, id: &str| -> Result { @@ -21876,6 +22203,37 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// A candidate no one ever answers for is ended by its Warming + /// deadline, or by the operator's expiry: it owns only itself. + #[test] + fn a_candidate_that_never_correlates_aborts_before_fencing_at_its_warming_deadline() -> Result<()> { + for by_operator in [false, true] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + for _ in 0..3 { + routed.resume()?; + } + let waiting = routed.transaction()?; + assert!(waiting.pre_fencing_abort.is_none(), "it ended before its deadline"); + assert!(waiting.phase_deadline.is_some()); + assert_eq!(waiting.phase, DeploymentPhase::Warming); + + let cause = if by_operator { + expire(&routed.world.state_store, "continuity-service", "operator")?; + "expired by the operator" + } else { + pass_deadline(&routed.world)?; + "Warming phase passed its deadline" + }; + routed.resume()?; + let abort = routed.transaction()?.pre_fencing_abort.context("it was not aborted")?; + assert!(abort.error.contains(cause), "{}", abort.error); + let ended = routed.resume_until_terminal()?; + assert!(matches!(ended.completion, Some(TransactionCompletion::FailedBeforeFencing { .. }))); + } + Ok(()) + } + /// `idunn expire` writes a request and nothing else; the resolver ends /// the phase from it, and the spent request is deleted. #[test] From 126d710de3edb997e1a40ce2415e6ba80a920af1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:39:42 +0200 Subject: [PATCH 16/47] Cut c2b: operator-required incident, opened at the decision site and closed when spent IncidentCondition::OperatorRequired (subject: transaction id) opens in leave_to_operator, only for a record that leaves_to_operator, and closes as recovered in close_spent_operator_incidents once no transaction in the snapshot is left to the operator. Incident writes are never progress. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 90 +++++++++++++++++++++++++++++++++++++++++++- src/incident.rs | 27 ++++++++++--- 2 files changed, 110 insertions(+), 7 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d408ee6..8732e4c 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5880,6 +5880,8 @@ impl Engine { fn resume_one_transaction(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut progressed = self.retire_spent_expiry_requests(&snapshot)?; + // Incident writes are never progress. + self.close_spent_operator_incidents(&snapshot, now_millis()?); let mut candidates = snapshot .transactions .iter() @@ -8406,9 +8408,19 @@ impl Engine { } /// Idunn has run out of ways to end this record. It stays owned and its - /// step keeps retrying; the operator is told. + /// step keeps retrying; the operator is told. The incident comes first and + /// the trace second, and only for a record that is left to the operator + /// (an operator's expiry of a stateful deploy is reported, not paged). fn leave_to_operator(&self, transaction: &DeploymentTransaction, cause: &str) { let id = &transaction.transaction_id; + let now = now_millis(); + if now.as_ref().is_ok_and(|now| transaction.leaves_to_operator(*now)) { + self.record_incident( + "open", + id, + now.and_then(|now| self.incidents().open(IncidentCondition::OperatorRequired, id, now)), + ); + } self.report_once( &format!("deadline:{id}"), format!("Idunn leaves {id} to the operator: {cause}; it keeps retrying the step"), @@ -9438,6 +9450,24 @@ impl Engine { } } + /// Close the operator-required incident of every transaction that is no + /// longer left to the operator: it ended, or its step went on. A + /// transaction that is still left keeps its incident open. + fn close_spent_operator_incidents(&self, snapshot: &ControlSnapshot, now: u64) { + let store = self.incidents(); + let close = || -> Result<()> { + for subject in store.open_subjects(IncidentCondition::OperatorRequired)? { + if !snapshot.transactions.iter().any(|stored| { + stored.value.transaction_id == subject && stored.value.leaves_to_operator(now) + }) { + store.close(IncidentCondition::OperatorRequired, &subject, CloseReason::Recovered, now)?; + } + } + Ok(()) + }; + self.record_incident("close", "*", close()); + } + /// Delete expiry requests whose command has no phase left to end: it was /// resolved, it finished, or its command is gone. fn retire_spent_expiry_requests(&self, snapshot: &ControlSnapshot) -> Result { @@ -14219,6 +14249,11 @@ mod tests { self.holds.lock().unwrap().push(file); } + /// Every stop works again. + fn mend_stops(&self) { + self.failing_stops.lock().unwrap().clear(); + } + /// From now on stopping this runtime instance fails. fn fail_stop_of(&self, runtime_instance_id: &str) { self.failing_stops.lock().unwrap().push(runtime_instance_id.to_owned()); @@ -16964,7 +16999,9 @@ mod tests { .engine .begin_post_fencing_abort(&resident_id(&world, owner_id)?, anyhow!("candidate refused"))?; - // Before the deadline the deployment waits on a bounded owner. + // Before the deadline the deployment waits on a bounded owner, and no + // one is paged. + assert!(world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); for _ in 0..3 { resume_pass(&world)?; } @@ -16980,6 +17017,11 @@ mod tests { let owner = resident_id(&world, owner_id)?.value; assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); assert!(owner.owns_target()); + // Exactly one incident, however many passes ran. + assert_eq!( + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [owner_id] + ); let ended = latest_id(&world, deploy_id)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); @@ -16989,6 +17031,46 @@ mod tests { Ok(()) } + #[test] + fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let id = seeded.transaction_id.as_str(); + workload.fail_stop_of(&runtime_instance_of(&world, id)?); + world.engine.begin_post_fencing_abort(&resident_id(&world, id)?, anyhow!("candidate refused"))?; + pass_deadline_of(&world, id)?; + let open = || world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired); + for _ in 0..4 { + resume_pass(&world)?; + } + assert_eq!(open()?, [id]); + let records = world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + + // Still left to the operator: still open, and not written again. + resume_pass(&world)?; + assert_eq!(world.engine.incidents().read()?, records); + + // The step succeeds and the transaction ends; the next pass closes it. + workload.mend_stops(); + for _ in 0..12 { + if resident_id(&world, id).is_err() { + break; + } + resume_pass(&world)?; + } + assert!(latest_id(&world, id)?.is_terminal()); + resume_pass(&world)?; + assert!(open()?.is_empty()); + let closed = world.engine.incidents().read()?; + assert_eq!(closed.len(), 1); + assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + resume_pass(&world)?; + assert_eq!(world.engine.incidents().read()?, closed, "a spent incident was written again"); + Ok(()) + } + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop /// keeps failing. A deployment waits behind it until the drain's deadline, /// then ends itself; the drain stays and the target stays owned. @@ -17037,6 +17119,10 @@ mod tests { resume_pass(&world)?; } assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + assert_eq!( + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [drainer_id] + ); let ended = latest_id(&world, waiter_id)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); diff --git a/src/incident.rs b/src/incident.rs index 1d7f095..a0941ff 100644 --- a/src/incident.rs +++ b/src/incident.rs @@ -32,6 +32,9 @@ pub(crate) const INCIDENT_RETENTION_MILLIS: u64 = 7 * 24 * 60 * 60 * 1000; #[serde(rename_all = "kebab-case")] pub(crate) enum IncidentCondition { ContinuityExhausted, + /// A transaction past its deadline in a state Idunn will not end by + /// itself. Subject: the transaction id. + OperatorRequired, } impl IncidentCondition { @@ -40,6 +43,7 @@ impl IncidentCondition { pub(crate) const fn name(self) -> &'static str { match self { Self::ContinuityExhausted => "continuity-exhausted", + Self::OperatorRequired => "operator-required", } } @@ -49,6 +53,7 @@ impl IncidentCondition { pub(crate) const fn ends_with_admission(self) -> bool { match self { Self::ContinuityExhausted => true, + Self::OperatorRequired => false, } } } @@ -271,6 +276,17 @@ impl IncidentStore { Ok(true) } + /// The subjects of every open incident of `condition`. + pub(crate) fn open_subjects(&self, condition: IncidentCondition) -> Result> { + Ok(self + .snapshot()? + .records + .into_iter() + .filter(|record| record.is_open() && record.condition == condition) + .map(|record| record.subject) + .collect()) + } + /// Close the open incident for `(condition, subject)`, if there is one. /// `Ok(true)` means this call wrote the closure. pub(crate) fn close( @@ -468,11 +484,12 @@ pub(crate) mod tests { ); // The wire spelling readers match on is the spelling the key and // status use. - let payload = rmp_serde::to_vec(&IncidentCondition::ContinuityExhausted).unwrap(); - assert_eq!( - rmp_serde::from_slice::(&payload).unwrap(), - IncidentCondition::ContinuityExhausted.name() - ); + for condition in [IncidentCondition::ContinuityExhausted, IncidentCondition::OperatorRequired] { + let payload = rmp_serde::to_vec(&condition).unwrap(); + assert_eq!(rmp_serde::from_slice::(&payload).unwrap(), condition.name()); + } + assert_eq!(IncidentCondition::OperatorRequired.name(), "operator-required"); + assert!(!IncidentCondition::OperatorRequired.ends_with_admission()); let payload = rmp_serde::to_vec(&CloseReason::NoLongerAdmitted).unwrap(); assert_eq!( rmp_serde::from_slice::(&payload).unwrap(), From 0bfadcfc717908be51933255c75b0cae0c5dd9e2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 16:10:11 +0200 Subject: [PATCH 17/47] Cut c2b test: pin the stateful-deploy arm of leaves_to_operator and the incident close predicate Survivors of cargo mutants --in-diff on 9ea6c71..126d710: the OperatorRequired clause of leaves_to_operator (now a table beside phase_end) and the close predicate (a vanished subject closes while another transaction is still left). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8732e4c..3169542 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13948,6 +13948,15 @@ mod tests { transaction.leasing = leasing.clone(); transaction.lease_adoption = lease_adoption.clone(); assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); + // Only a stateful deploy is left to the operator when its phase ends. + transaction.retime(1_000); + let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; + assert_eq!( + transaction.leaves_to_operator(due), + expected == PhaseEnd::OperatorRequired, + "{kind:?} {leasing:?} {lease_adoption:?}" + ); + assert!(!transaction.leaves_to_operator(due - 1)); let recovery = if kind == Continuity && lease_adoption.is_some() { TerminalRecovery::RestartAdmitted } else { @@ -17048,6 +17057,13 @@ mod tests { let records = world.engine.incidents().read()?; assert_eq!(records.len(), 1); + // An incident whose transaction is gone closes even while another is + // still left to the operator. + world.engine.incidents().open(IncidentCondition::OperatorRequired, "tx-vanished", now_millis()?)?; + resume_pass(&world)?; + assert_eq!(open()?, [id]); + let records = world.engine.incidents().read()?; + // Still left to the operator: still open, and not written again. resume_pass(&world)?; assert_eq!(world.engine.incidents().read()?, records); @@ -17064,8 +17080,8 @@ mod tests { resume_pass(&world)?; assert!(open()?.is_empty()); let closed = world.engine.incidents().read()?; - assert_eq!(closed.len(), 1); - assert_eq!(closed[0].close_reason, Some(CloseReason::Recovered)); + assert!(closed.iter().all(|record| record.close_reason == Some(CloseReason::Recovered))); + assert_eq!(closed.iter().filter(|record| record.subject == id).count(), 1); resume_pass(&world)?; assert_eq!(world.engine.incidents().read()?, closed, "a spent incident was written again"); Ok(()) From 21849eb09753cccf0286105a4ff1e91ce30f533f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:48:03 +0200 Subject: [PATCH 18/47] Cut c2b fix: a drain whose stored plan no longer parses is bounded, not unbounded deadline_for_current_activity's drain arm returned None when parsed_inputs failed (the running binary re-parses and re-admits the frozen binding), which left the drain with no deadline, unpaged, and disarmed every waiter behind it. The retire time is now the promotion plus the drain when the binding parses, and otherwise now; the frozen committing duration from it bounds the drain. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3169542..0f4385b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1775,17 +1775,20 @@ impl DeploymentTransaction { return None; } let plan = self.plan.as_ref()?; - let from = match self + // The retire time needs the binding's drain, which the running + // binary re-parses and re-admits: a plan frozen by an older binary + // may not parse now. That must not leave a drain with no deadline, + // so a retire time that cannot be derived is now, and the + // committing duration from it bounds the drain. + let from = self .routing .as_ref() .and_then(RoutingEvidence::promoted_at_unix_millis) - { - Some(promoted_at) => { - route_drain_deadline(promoted_at, plan.parsed_inputs().ok()?.1.rollout.drain_seconds) - .ok()? - } - None => now, - }; + .and_then(|promoted_at| { + let (_, binding) = plan.parsed_inputs().ok()?; + route_drain_deadline(promoted_at, binding.rollout.drain_seconds).ok() + }) + .unwrap_or(now); return Some(PhaseDeadline { phase: self.phase, entered_at_unix_millis: now, From a254a3e6a5f767201f07ee7d9892d821076cfe24 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:49:14 +0200 Subject: [PATCH 19/47] Cut c2b fix: pin the forever-waits at the waiter, the pre-fence abort deadline, the stateful incident lifecycle and the archive vanish Negative tests (a)(b)(c) are one marked harness that ages a deadline only when one exists and ends on the waiter's own record, so at the base they fail at the waiter assertion (still waiting), not at setup. (c) now runs the Odin gate-wait it names: a stop-then-start continuity on a systemd unit observes and waits, where the host-actuator step of a world with no actuator errs at once. New: a stop-first owner's wedged pre-fence abort pages and ends its waiter; a waiter's own wedged pre-fence abort pages beside a standing owner; a stateful deploy's incident is silent under an early expiry, one per episode, and closes while the record goes on; an unparsable drain plan is bounded; a transaction gone from both live set and history stays a fault. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 635 +++++++++++++++++++++++++++++++------------ 1 file changed, 467 insertions(+), 168 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 0f4385b..43810c4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13417,12 +13417,24 @@ mod tests { } #[test] - fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + fn a_drain_whose_stored_plan_no_longer_parses_is_still_bounded() -> Result<()> { + let (mut draining, promoted_at) = promoted_draining_record()?; + let committing = u64::from(draining.plan.as_ref().unwrap().phase_deadlines().committing_seconds) * 1000; + // A binding the running binary refuses: the drain seconds cannot be read. + draining.plan.as_mut().unwrap().binding_blob = b"not an operator binding".to_vec(); + assert!(draining.plan.as_ref().unwrap().parsed_inputs().is_err()); + draining.retime(promoted_at + 1); + let deadline = draining.phase_deadline.context("an unparsable plan left the drain with no deadline")?; + assert_eq!(deadline.deadline_at_unix_millis, promoted_at + 1 + committing); + assert!(!draining.leaves_to_operator(deadline.deadline_at_unix_millis - 1)); + assert!(draining.leaves_to_operator(deadline.deadline_at_unix_millis)); + Ok(()) + } + + /// A Complete record draining its incumbent, whose route was promoted at + /// the returned time. + fn promoted_draining_record() -> Result<(DeploymentTransaction, u64)> { let (_, mut draining) = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?; - let plan = draining.plan.clone().unwrap(); - let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; - let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; - assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); draining.phase = DeploymentPhase::Complete; draining.completion = Some(TransactionCompletion::Admitted { generation_id: "generation-new".into(), @@ -13441,6 +13453,16 @@ mod tests { } other => bail!("the fixture is not promoted: {other:?}"), } + Ok((draining, promoted_at)) + } + + #[test] + fn a_drain_is_timed_to_its_retire_time_plus_committing() -> Result<()> { + let (mut draining, promoted_at) = promoted_draining_record()?; + let plan = draining.plan.clone().unwrap(); + let drain = u64::from(plan.parsed_inputs()?.1.rollout.drain_seconds) * 1000; + let committing = u64::from(plan.phase_deadlines().committing_seconds) * 1000; + assert!(drain > 0 && committing > 0, "the fixture must make both terms matter"); // Before the retire time: the retire time plus the committing duration. draining.retime(promoted_at + 1); @@ -15789,6 +15811,104 @@ mod tests { Ok(()) } + /// A workload port whose observation deletes every live transaction from + /// the control store once armed: the step ends, and the record is simply + /// gone, archived nowhere. + struct VanishingWorkload { + store: Mutex>, + armed: std::sync::atomic::AtomicBool, + } + + impl WorkloadPort for VanishingWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + StillWorkload.install(plan, release) + } + fn prepare_activation( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + launch: IdunnRuntimeActivationLaunch, + ) -> Result { + StillWorkload.prepare_activation(plan, expected, launch) + } + fn start_prepared( + &self, + plan: &CompiledDeploymentPlan, + release: &SealedRelease, + installed: &crate::drivers::InstalledReleaseObservation, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result { + StillWorkload.start_prepared(plan, release, installed, expected, activation) + } + fn discard_prepared( + &self, + plan: &CompiledDeploymentPlan, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + StillWorkload.discard_prepared(plan, expected, activation) + } + fn observe( + &self, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + prior: &WorkloadObservation, + ) -> Result { + if self.armed.load(std::sync::atomic::Ordering::SeqCst) { + let path = self.store.lock().unwrap().clone().context("no store")?; + let gone = ControlSnapshot::read(&path)? + .transactions + .into_iter() + .map(|stored| stored.envelope) + .collect::>(); + assert!(SingleFileMessagePackBackingStore::new(&path).delete_batch_if_unchanged(&gone)?); + } + StillWorkload.observe(expected, activation, prior) + } + fn stop(&self, observation: &WorkloadObservation) -> Result<()> { + StillWorkload.stop(observation) + } + fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result { + StillWorkload.is_permanently_stopped(observation) + } + fn lost_to_host_reboot(&self, observation: &WorkloadObservation, at: u64) -> Result { + StillWorkload.lost_to_host_reboot(observation, at) + } + } + + /// The step that ends a transaction archives it in its own write, so a + /// record gone from the live set is finished only if history holds it. One + /// that vanished from both is a fault, never progress. + #[test] + fn a_transaction_gone_from_both_the_live_set_and_history_is_a_fault() -> Result<()> { + let workload = Arc::new(VanishingWorkload { + store: Mutex::new(None), + armed: std::sync::atomic::AtomicBool::new(false), + }); + let world = EngineFixture::with_workload(workload.clone())?; + *workload.store.lock().unwrap() = Some(world.state_store.clone()); + let live = transaction_at(&world, DeploymentPhase::Warming)?; + let id = live.transaction_id.as_str(); + // The Warming step waits for Odin; waiting again rewrites nothing, so + // the step succeeds without touching the record it will find missing. + world.engine.resume_candidate(&resident_id(&world, id)?)?; + assert!(resident_id(&world, id)?.value.last_error.is_some(), "the Warming step did not wait"); + workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); + world.engine.resume_backoff.lock().unwrap().clear(); + let error = world + .engine + .resume_candidate(&resident_id(&world, id)?) + .expect_err("a record gone from both stores was read as a finished one"); + assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); + assert!(read_history_transactions(&world.state_store).is_empty()); + Ok(()) + } + // ---- fix 2: supervision owns the Expected-only precondition ---- #[test] @@ -16970,6 +17090,13 @@ mod tests { Ok((deploy, continuity, incumbent)) } + // BEGIN c2b-forever-wait harness + // The forever-waits of cut c2b, driven through the API that exists at + // 9ea6c71 (the base): each test ends on the waiter's own record, so at the + // base it fails at the waiter assertion (still waiting), never at setup. + // Lines ending in `head-only` read what only the cut adds (the incident + // store, `leaves_to_operator`); the base probe strips them. + /// One scheduler pass over every live transaction, with no resume backoff /// standing between it and the step. fn resume_pass(world: &EngineFixture) -> Result { @@ -16995,6 +17122,32 @@ mod tests { replace_transaction(&world.state_store, ¤t, &next) } + /// The deadline of `id`, when it has one, is already behind it. A record + /// that has none is left alone: at the base nothing times it, and the + /// scenario must still run to the waiter's assertion. + fn age_deadline_of(world: &EngineFixture, id: &str) -> Result<()> { + if resident_id(world, id)?.value.phase_deadline.is_some() { + pass_deadline_of(world, id)?; + } + Ok(()) + } + + /// The deployment `waiter_id` ended before fencing, with a cause naming + /// the owner it will not wait for. + fn assert_ended_for_owner(world: &EngineFixture, waiter_id: &str, owner_id: &str) -> Result<()> { + let ended = latest_id(world, waiter_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); + assert!(ended.fencing.is_none()); + Ok(()) + } + + fn operator_required(world: &EngineFixture) -> Result> { + world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) // head-only + } + /// NEGATIVE (a). A continuity that owns the target is aborting and its /// abort step fails on every attempt. Its deadline passes: Idunn leaves it /// to the operator and keeps it owned, and the deployment waiting behind it @@ -17013,36 +17166,266 @@ mod tests { // Before the deadline the deployment waits on a bounded owner, and no // one is paged. - assert!(world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); + assert!(operator_required(&world)?.is_empty()); // head-only for _ in 0..3 { resume_pass(&world)?; } let waiting = resident_id(&world, deploy_id)?.value; assert!(waiting.pre_fencing_abort.is_none()); assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(owner_id))); - assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); + assert!(!resident_id(&world, owner_id)?.value.leaves_to_operator(now_millis()?)); // head-only - pass_deadline_of(&world, owner_id)?; - for _ in 0..10 { + age_deadline_of(&world, owner_id)?; + for _ in 0..20 { resume_pass(&world)?; } let owner = resident_id(&world, owner_id)?.value; assert!(!owner.is_terminal() && owner.post_fencing_abort.is_some(), "the owner was ended"); assert!(owner.owns_target()); // Exactly one incident, however many passes ran. - assert_eq!( - world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, - [owner_id] - ); - let ended = latest_id(&world, deploy_id)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); + assert_eq!(operator_required(&world)?, [owner_id]); // head-only + assert_ended_for_owner(&world, deploy_id, owner_id) + } + + /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop + /// keeps failing. A deployment waits behind it until the drain's deadline, + /// then ends itself; the drain stays and the target stays owned. + #[test] + fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; + odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; + let drainer_id = drainer.transaction_id.as_str(); + drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; + let draining = resident_id(&world, drainer_id)?.value; + assert!(draining.drains_incumbent(), "the incumbent was already retired"); + assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); // head-only + let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = + &draining.post_commit_cleanup + else { + bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); }; - assert!(error.contains("owner-left-to-operator") && error.contains(owner_id), "{error}"); - assert!(ended.fencing.is_none()); + let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; + workload.fail_stop_of(&old.runtime_instance_id); + + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; + let waiter_id = waiter.transaction_id.as_str(); + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, waiter_id)?.value; + assert!(waiting.pre_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); + assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); + assert!(operator_required(&world)?.is_empty()); // head-only + + age_deadline_of(&world, drainer_id)?; + for _ in 0..20 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); + assert_eq!(operator_required(&world)?, [drainer_id]); // head-only + assert_ended_for_owner(&world, waiter_id, drainer_id) + } + + /// A stop-then-start transaction held in Warming with a started unit, as + /// the Starting step leaves it. + fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + let stored = resident_id(world, id)?; + let mut next = stored.value.clone(); + let expected = next.expected.clone().context("no Expected")?; + let now = now_millis()?; + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&next.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? + .1 + .workload + .context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + next.expected_publication_sha256 = Some(expected.canonical_sha256()?); + next.activation_publication_sha256 = Some(activation.canonical_sha256()?); + next.activation = Some(activation); + next.workload = Some(workload); + next.enter_phase(DeploymentPhase::Warming, now); + replace_transaction(&world.state_store, &stored, &next) + } + + /// The stop-then-start binding of `stop_first_binding`, on a systemd unit + /// instead of a host actuator: its Warming step observes the unit through + /// the world's workload port and waits for Odin's word, where the + /// host-actuator step of a world that serves no actuator errs at once. + fn stop_first_systemd_binding(world: &EngineFixture) -> Result { + use crate::deployment_plan::tests::BINDING; + let host_runner = "[runners.rust]\ndriver = \"host-native\"\naffordances = [\"source-read\", \"artifact-write\"]\nallowed_programs = [\"cargo\"]\n\n"; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; + let (_, rest) = rest.split_once("[workload]").context("no workload")?; + let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; + let (identity, rest) = rest.split_once("[route]").context("no route")?; + let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; + bind_provider( + world, + &format!("{head}{host_runner}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") + .replace("candidate-then-promote", "stop-then-start"), + ) + } + + /// A stop-then-start deployment and continuity over one incumbent, on a + /// systemd unit. The continuity is held in Warming with a started unit and + /// the deployment, one `enter` later, waits at its Sealing tail on it. + fn warm_stop_first_owner( + workload: Arc, + ) -> Result<(EngineFixture, String, String)> { + let world = EngineFixture::with_workload(workload)?; + let incumbent = admit_incumbent(&world)?; + let binding = stop_first_systemd_binding(&world)?; + let mut ids = Vec::new(); + for kind in [CommandKind::Deploy, CommandKind::Continuity] { + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; + make_stop_first(&world, &seeded.transaction_id, &binding)?; + ids.push(seeded.transaction_id); + } + let continuity = ids.pop().context("no continuity")?; + let deploy = ids.pop().context("no deploy")?; + warm_stop_first(&world, &continuity)?; + let owner = resident_id(&world, &continuity)?; + assert_eq!(owner.value.phase, DeploymentPhase::Warming); + assert!(owner.value.owns_target()); + let waiting = resident_id(&world, &deploy)?; + world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?)?; + let held = resident_id(&world, &deploy)?.value; + assert_eq!(held.phase, DeploymentPhase::Sealing); + assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); + Ok((world, deploy, continuity)) + } + + /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting + /// and waits in Warming for an Odin word that never comes: its step runs, + /// observes the unit and gate-waits. Its Warming deadline ends it, and the + /// deployment waiting at its Sealing tail takes the target. + #[test] + fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( + ) -> Result<()> { + let (world, deploy, continuity) = warm_stop_first_owner(SwitchWorkload::new())?; + // The gate-wait is the Warming step itself, not an error of it. + for _ in 0..3 { + resume_pass(&world)?; + } + let waiting = resident_id(&world, &continuity)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.pre_fencing_abort.is_none(), "the Warming step failed: {:?}", waiting.last_error); + assert!(waiting.last_error.is_some(), "the Warming step did not wait for Odin"); + assert!(resident_id(&world, &deploy)?.value.last_error.as_deref().is_some_and(|n| n.contains(&continuity))); + + age_deadline_of(&world, &continuity)?; + for _ in 0..12 { + if resident_id(&world, &continuity).is_err() { + break; + } + resume_pass(&world)?; + } + let ended = latest_id(&world, &continuity)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("Warming phase passed its deadline"), "{error}"); + assert!(operator_required(&world)?.is_empty()); // head-only + + // The pass that retired the owner, or the next, moves the waiter to + // Starting; stop there, before its Starting step runs. + for _ in 0..3 { + if resident_id(&world, &deploy)?.value.phase >= DeploymentPhase::Starting { + break; + } + resume_pass(&world)?; + } + let took = resident_id(&world, &deploy)?.value; + assert_eq!(took.phase, DeploymentPhase::Starting); + assert!(took.owns_target() && took.pre_fencing_abort.is_none()); Ok(()) } + /// The wedged pre-fence abort of a stop-then-start owner is paged too: the + /// waiter behind it ends, and one incident opens. + #[test] + fn a_waiting_deployment_ends_when_a_stop_first_owners_prefence_abort_is_left_to_the_operator( + ) -> Result<()> { + let workload = SwitchWorkload::new(); + let (world, deploy, owner) = warm_stop_first_owner(workload.clone())?; + workload.fail_stop_of(&runtime_instance_of(&world, &owner)?); + world + .engine + .begin_pre_fencing_abort(&resident_id(&world, &owner)?, anyhow!("candidate refused"))?; + for _ in 0..3 { + resume_pass(&world)?; + } + // Before the abort's deadline: it waits, unpaged. + let aborting = resident_id(&world, &owner)?.value; + assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); + assert!(operator_required(&world)?.is_empty()); // head-only + assert!(resident_id(&world, &deploy)?.value.pre_fencing_abort.is_none()); + + age_deadline_of(&world, &owner)?; + for _ in 0..20 { + resume_pass(&world)?; + } + let aborting = resident_id(&world, &owner)?.value; + assert!(!aborting.is_terminal() && aborting.owns_target(), "the owner was ended"); + assert_eq!(operator_required(&world)?, [owner.as_str()]); // head-only + assert_ended_for_owner(&world, &deploy, &owner) + } + + /// A waiter whose own pre-fence abort is wedged, beside an owner it will + /// not outlast, is paged by its own deadline: the owner's standing does not + /// excuse it. + #[test] + fn a_waiters_wedged_prefence_abort_is_paged_beside_a_standing_owner() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, _) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.as_str(), continuity.transaction_id.as_str()); + workload.fail_stop_of(&runtime_instance_of(&world, deploy_id)?); + world + .engine + .begin_pre_fencing_abort(&resident_id(&world, deploy_id)?, anyhow!("candidate refused"))?; + for _ in 0..3 { + resume_pass(&world)?; + } + let aborting = resident_id(&world, deploy_id)?.value; + assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); + assert!(!operator_required(&world)?.iter().any(|subject| subject == deploy_id)); // head-only + + age_deadline_of(&world, deploy_id)?; + for _ in 0..3 { + resume_pass(&world)?; + } + assert!(!resident_id(&world, deploy_id)?.value.is_terminal()); + assert!(operator_required(&world)?.iter().any(|subject| subject == deploy_id), "{owner_id}"); // head-only + Ok(()) + } + // END c2b-forever-wait harness + #[test] fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { let workload = SwitchWorkload::new(); @@ -17090,153 +17473,6 @@ mod tests { Ok(()) } - /// NEGATIVE (b). An admitted transaction drains its incumbent and the stop - /// keeps failing. A deployment waits behind it until the drain's deadline, - /// then ends itself; the drain stays and the target stays owned. - #[test] - fn a_waiting_deployment_ends_when_a_drain_whose_stop_keeps_failing_is_left_to_the_operator( - ) -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let incumbent = admit_incumbent(&world)?; - let drainer = seeded_transaction(&world, DeploymentPhase::Fencing, CommandKind::Deploy, Some(&incumbent))?; - odin_reports_ready(&world, &drainer, seeded_sequence(Some(&incumbent)) + 1)?; - let drainer_id = drainer.transaction_id.as_str(); - drive_id(&world, drainer_id, |t| t.phase == DeploymentPhase::Complete)?; - let draining = resident_id(&world, drainer_id)?.value; - assert!(draining.drains_incumbent(), "the incumbent was already retired"); - assert!(draining.phase_deadline.is_some(), "a draining record has no deadline"); - match &draining.workload { - Some(WorkloadObservation::Systemd(_)) => {} - other => bail!("no workload: {other:?}"), - } - let Some(PostCommitCleanup { incumbent: IncumbentCleanupEvidence::Pending { workload: old, .. }, .. }) = - &draining.post_commit_cleanup - else { - bail!("no pending incumbent: {:?}", draining.post_commit_cleanup); - }; - let WorkloadObservation::Systemd(old) = old else { bail!("incumbent is not a systemd unit") }; - workload.fail_stop_of(&old.runtime_instance_id); - - let admitted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("nothing admitted")? - .value - .clone(); - let waiter = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&admitted))?; - let waiter_id = waiter.transaction_id.as_str(); - for _ in 0..3 { - resume_pass(&world)?; - } - let waiting = resident_id(&world, waiter_id)?.value; - assert!(waiting.pre_fencing_abort.is_none()); - assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(drainer_id)), "{:?}", waiting.last_error); - assert_eq!(waiting.phase_deadline, None, "the waiter's deadline is armed under a bounded owner"); - - pass_deadline_of(&world, drainer_id)?; - for _ in 0..10 { - resume_pass(&world)?; - } - assert!(!resident_id(&world, drainer_id)?.value.is_terminal(), "the drain was ended"); - assert_eq!( - world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, - [drainer_id] - ); - let ended = latest_id(&world, waiter_id)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the waiting deployment did not end: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("owner-left-to-operator") && error.contains(drainer_id), "{error}"); - Ok(()) - } - - /// A stop-then-start transaction held in Warming with a started unit, as - /// the Starting step leaves it (this world serves no host actuator to - /// start one). - fn warm_stop_first(world: &EngineFixture, id: &str) -> Result<()> { - let stored = resident_id(world, id)?; - let mut next = stored.value.clone(); - let expected = next.expected.clone().context("no Expected")?; - let now = now_millis()?; - let launch = IdunnRuntimeActivationLaunch::issue( - &expected, - runtime_instance_id(&next.transaction_id)?, - now, - &world.engine.idunn_signer, - )?; - let mut seed = Vec::new(); - let activation = launch.write_credential(&mut seed)?; - let mut workload = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)? - .1 - .workload - .context("no recorded workload")?; - match &mut workload { - WorkloadObservation::Systemd(observed) => { - observed.runtime_instance_id = activation.runtime_instance_id.clone(); - observed.executable_sha256 = expected.artifact_sha256.clone(); - } - WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), - } - next.expected_publication_sha256 = Some(expected.canonical_sha256()?); - next.activation_publication_sha256 = Some(activation.canonical_sha256()?); - next.activation = Some(activation); - next.workload = Some(workload); - next.enter_phase(DeploymentPhase::Warming, now); - replace_transaction(&world.state_store, &stored, &next) - } - - /// NEGATIVE (c). A stop-then-start continuity takes the target at Starting - /// and waits in Warming for an Odin word that never comes. Its Warming - /// deadline ends it, and the deployment waiting at its Sealing tail takes - /// the target. - #[test] - fn a_stop_first_owner_that_never_warms_ends_at_its_warming_deadline_and_the_waiter_takes_the_target( - ) -> Result<()> { - use DeploymentPhase::{Starting, Warming}; - let (world, deploy, continuity) = stop_first_world(SwitchWorkload::new())?; - warm_stop_first(&world, &continuity)?; - let owner = resident_id(&world, &continuity)?; - assert_eq!(owner.value.phase, Warming); - assert!(owner.value.owns_target() && owner.value.phase_deadline.is_some()); - let waiting = resident_id(&world, &deploy)?; - world.engine.enter(&waiting, entering(&waiting, Starting)?)?; - let held = resident_id(&world, &deploy)?.value; - assert_eq!(held.phase, DeploymentPhase::Sealing); - assert!(held.last_error.as_deref().is_some_and(|note| note.contains(&continuity))); - - pass_deadline_of(&world, &continuity)?; - world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; - let aborting = resident_id(&world, &continuity)?.value; - let abort = aborting.pre_fencing_abort.clone().context("the deadline did not end Warming")?; - assert!(abort.error.contains("Warming phase passed its deadline"), "{}", abort.error); - // This world serves no host actuator, so the abandoned unit cannot be - // stopped here; the abort's cleanup is recorded done. - let mut cleaned = aborting.clone(); - let intent = cleaned.pre_fencing_abort.as_mut().unwrap(); - intent.candidate_cleanup = CleanupEvidence::Complete; - intent.topology_reconciliation = CleanupEvidence::Complete; - replace_transaction(&world.state_store, &resident_id(&world, &continuity)?, &cleaned)?; - for _ in 0..12 { - if resident_id(&world, &continuity).is_err() { - break; - } - world.engine.resume_backoff.lock().unwrap().clear(); - world.engine.resume_candidate(&resident_id(&world, &continuity)?)?; - } - let ended = latest_id(&world, &continuity)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the owner did not end: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("Warming phase passed its deadline"), "{error}"); - - let waiting = resident_id(&world, &deploy)?; - world.engine.enter(&waiting, entering(&waiting, Starting)?)?; - let took = resident_id(&world, &deploy)?.value; - assert_eq!(took.phase, Starting); - assert!(took.owns_target() && took.pre_fencing_abort.is_none()); - Ok(()) - } - #[test] fn a_waiters_warming_deadline_is_disarmed_while_an_owner_holds_the_target_and_rearmed_when_it_ends( ) -> Result<()> { @@ -17458,13 +17694,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// The seeded transaction `id`, rewritten as the Sealing transaction of a /// stop-then-start target: the host-actuator plan, release and Expected, /// and none of the evidence that follows Sealing. - fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { + fn make_stop_first(world: &EngineFixture, id: &str, binding: &str) -> Result<()> { use crate::deployment_plan::tests::{ RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); - let binding = stop_first_binding(world)?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -17533,11 +17768,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ) -> Result<(EngineFixture, String, String)> { let world = EngineFixture::with_workload(workload)?; let incumbent = admit_incumbent(&world)?; + let binding = stop_first_binding(&world)?; let mut ids = Vec::new(); for kind in [CommandKind::Deploy, CommandKind::Continuity] { let seeded = seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id)?; + make_stop_first(&world, &seeded.transaction_id, &binding)?; ids.push(seeded.transaction_id); } let continuity = ids.pop().context("no continuity")?; @@ -22592,6 +22828,69 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// The incident of a stateful deployment left to the operator is one + /// per episode. An operator's expiry before the deadline reports the + /// hold and pages no one; the deadline opens one incident; and the + /// incident closes as soon as the record is no longer left, while the + /// transaction goes on. + #[test] + fn a_stateful_deploys_incident_is_one_per_episode_and_follows_its_deadline() -> Result<()> { + let deploy = + build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false, true, CommandKind::Deploy)?; + deploy.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let held = deploy.transaction()?; + let id = held.transaction_id.as_str(); + let open = || deploy.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired); + let now = now_millis()?; + let request = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: held.command_id.clone(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + assert!(SingleFileMessagePackBackingStore::new(&deploy.world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: request.command_id.clone(), + current: None, + }], + &[expiry_request_envelope(&request, now)?], + )?); + + // The operator asked, the deadline has not passed: nobody is paged, + // however many passes see the request. + for _ in 0..4 { + deploy.resume()?; + } + assert!(open()?.is_empty(), "an expiry before the deadline paged the operator"); + assert!(deploy.world.engine.incidents().read()?.is_empty()); + assert!(deploy.transaction()?.post_fencing_abort.is_none(), "a leased deployment was aborted"); + + // The deadline passes: one incident, not one per pass. + pass_deadline(&deploy.world)?; + for _ in 0..4 { + deploy.resume()?; + } + assert_eq!(open()?, [id]); + assert_eq!(deploy.world.engine.incidents().read()?.len(), 1); + + // The record goes on with a deadline in the future (its step + // advanced): the incident closes while the transaction lives. + let current = resident(&deploy.world)?; + let mut next = current.value.clone(); + let due = next.phase_deadline.as_mut().context("no deadline")?; + due.entered_at_unix_millis = now_millis()?; + due.deadline_at_unix_millis = due.entered_at_unix_millis + 3_600_000; + replace_transaction(&deploy.world.state_store, ¤t, &next)?; + deploy.resume()?; + assert!(!deploy.transaction()?.is_terminal()); + assert!(open()?.is_empty(), "the incident outlived the episode"); + let records = deploy.world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + Ok(()) + } + /// B5 Soul A. The physical grant lands before `Granted` is durable. A /// continuity that dies in that window and passes its deadline is /// aborted from `Prepared`; the abort still withdraws the lease on From 8b6c539a6a1ba89e2714bb347633077e0bbaa568 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:51:02 +0200 Subject: [PATCH 20/47] Cut c2b fix: the stop-first harness owner runs on a systemd unit --- src/control_plane.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 43810c4..c8b0cd6 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15897,7 +15897,8 @@ mod tests { // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. world.engine.resume_candidate(&resident_id(&world, id)?)?; - assert!(resident_id(&world, id)?.value.last_error.is_some(), "the Warming step did not wait"); + let waited = resident_id(&world, id)?.value; + assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?} {:?}", waited.phase, waited.warming); workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); world.engine.resume_backoff.lock().unwrap().clear(); let error = world @@ -17276,15 +17277,13 @@ mod tests { /// host-actuator step of a world that serves no actuator errs at once. fn stop_first_systemd_binding(world: &EngineFixture) -> Result { use crate::deployment_plan::tests::BINDING; - let host_runner = "[runners.rust]\ndriver = \"host-native\"\naffordances = [\"source-read\", \"artifact-write\"]\nallowed_programs = [\"cargo\"]\n\n"; - let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; - let (_, rest) = rest.split_once("[workload]").context("no workload")?; + let (head, rest) = BINDING.split_once("[workload]").context("no workload")?; let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; bind_provider( world, - &format!("{head}{host_runner}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") + &format!("{head}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") .replace("candidate-then-promote", "stop-then-start"), ) } @@ -17680,7 +17679,6 @@ state_root = "/var/lib/gamecult/service" GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" "#; - let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; From c6805d48a79252ed2493ac62b260e86da43c2f6a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:51:38 +0200 Subject: [PATCH 21/47] Cut c2b fix: restore stop_first_binding's runner split --- src/control_plane.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index c8b0cd6..8e43d81 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17679,6 +17679,7 @@ state_root = "/var/lib/gamecult/service" GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-identity.cc" "#; + let (head, rest) = BINDING.split_once("[runners.rust]").context("no runner")?; let (_, rest) = rest.split_once("[runtime_identity]").context("no identity")?; let (identity, rest) = rest.split_once("[route]").context("no route")?; let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; From 2ff57533b112f36e1486cae801f1dc6b82cdf564 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:53:58 +0200 Subject: [PATCH 22/47] Cut c2b fix: a test world runs a host-actuator target's workload through its own port --- src/control_plane.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8e43d81..e7a8009 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5039,6 +5039,10 @@ struct Engine { docker_runner: DockerRunnerDriver, systemd_workload: Arc, host_actuators: Option, + /// A test world that serves no host actuator runs a host-actuator + /// binding's workload through this port instead. + #[cfg(test)] + host_workload: Mutex>>, history_report: ReportOnce, /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. @@ -5140,6 +5144,8 @@ impl Engine { docker_runner: DockerRunnerDriver::default(), systemd_workload, host_actuators, + #[cfg(test)] + host_workload: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), resume_backoff: Mutex::default(), @@ -5375,9 +5381,15 @@ impl Engine { let (_, binding) = plan.parsed_inputs()?; Ok(match &binding.workload { WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), - WorkloadBinding::HostActuator(_) => Arc::new(HostActuatorWorkloadDriver { - access: self.host_access()?, - }), + WorkloadBinding::HostActuator(_) => { + #[cfg(test)] + if let Some(port) = self.host_workload.lock().expect("host workload mutex").clone() { + return Ok(port); + } + Arc::new(HostActuatorWorkloadDriver { + access: self.host_access()?, + }) + } }) } @@ -15892,7 +15904,8 @@ mod tests { }); let world = EngineFixture::with_workload(workload.clone())?; *workload.store.lock().unwrap() = Some(world.state_store.clone()); - let live = transaction_at(&world, DeploymentPhase::Warming)?; + let incumbent = admit_incumbent(&world)?; + let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Continuity, Some(&incumbent))?; let id = live.transaction_id.as_str(); // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. From 95e807661867d3f3afb4975e2fd8012246a8c768 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:55:04 +0200 Subject: [PATCH 23/47] Cut c2b fix: the stop-first owner harness uses the host-actuator world with its workload port --- src/control_plane.rs | 39 +++++++-------------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e7a8009..dacf5d9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17284,41 +17284,16 @@ mod tests { replace_transaction(&world.state_store, &stored, &next) } - /// The stop-then-start binding of `stop_first_binding`, on a systemd unit - /// instead of a host actuator: its Warming step observes the unit through - /// the world's workload port and waits for Odin's word, where the - /// host-actuator step of a world that serves no actuator errs at once. - fn stop_first_systemd_binding(world: &EngineFixture) -> Result { - use crate::deployment_plan::tests::BINDING; - let (head, rest) = BINDING.split_once("[workload]").context("no workload")?; - let (workload, rest) = rest.split_once("[runtime_identity]").context("no identity")?; - let (identity, rest) = rest.split_once("[route]").context("no route")?; - let (_, brakes) = rest.split_once("[brakes]").context("no brakes")?; - bind_provider( - world, - &format!("{head}[workload]{workload}[runtime_identity]{identity}[brakes]{brakes}") - .replace("candidate-then-promote", "stop-then-start"), - ) - } - /// A stop-then-start deployment and continuity over one incumbent, on a /// systemd unit. The continuity is held in Warming with a started unit and /// the deployment, one `enter` later, waits at its Sealing tail on it. fn warm_stop_first_owner( workload: Arc, ) -> Result<(EngineFixture, String, String)> { - let world = EngineFixture::with_workload(workload)?; - let incumbent = admit_incumbent(&world)?; - let binding = stop_first_systemd_binding(&world)?; - let mut ids = Vec::new(); - for kind in [CommandKind::Deploy, CommandKind::Continuity] { - let seeded = - seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id, &binding)?; - ids.push(seeded.transaction_id); - } - let continuity = ids.pop().context("no continuity")?; - let deploy = ids.pop().context("no deploy")?; + let (world, deploy, continuity) = stop_first_world(workload.clone())?; + // This world serves no host actuator: the target's host workload runs + // through the same port as everything else here. + *world.engine.host_workload.lock().unwrap() = Some(workload); // seam warm_stop_first(&world, &continuity)?; let owner = resident_id(&world, &continuity)?; assert_eq!(owner.value.phase, DeploymentPhase::Warming); @@ -17706,12 +17681,13 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// The seeded transaction `id`, rewritten as the Sealing transaction of a /// stop-then-start target: the host-actuator plan, release and Expected, /// and none of the evidence that follows Sealing. - fn make_stop_first(world: &EngineFixture, id: &str, binding: &str) -> Result<()> { + fn make_stop_first(world: &EngineFixture, id: &str) -> Result<()> { use crate::deployment_plan::tests::{ RECIPE, artifact_receipt, source, }; let stored = resident_id(world, id)?; let mut next = stored.value.clone(); + let binding = stop_first_binding(world)?; let recipe = RECIPE .split("[[dependencies]]") .next() @@ -17780,12 +17756,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ) -> Result<(EngineFixture, String, String)> { let world = EngineFixture::with_workload(workload)?; let incumbent = admit_incumbent(&world)?; - let binding = stop_first_binding(&world)?; let mut ids = Vec::new(); for kind in [CommandKind::Deploy, CommandKind::Continuity] { let seeded = seeded_transaction(&world, DeploymentPhase::Warming, kind, Some(&incumbent))?; - make_stop_first(&world, &seeded.transaction_id, &binding)?; + make_stop_first(&world, &seeded.transaction_id)?; ids.push(seeded.transaction_id); } let continuity = ids.pop().context("no continuity")?; From 9844806eb125159bcca9d99fbc2963134c5915cb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:56:28 +0200 Subject: [PATCH 24/47] Cut c2b fix: the waiter's own entry attempt drives the owner harness --- src/control_plane.rs | 62 +++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index dacf5d9..c5a51a8 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -15902,21 +15902,20 @@ mod tests { store: Mutex::new(None), armed: std::sync::atomic::AtomicBool::new(false), }); - let world = EngineFixture::with_workload(workload.clone())?; + let (world, _deploy, owner) = stop_first_world(workload.clone())?; + *world.engine.host_workload.lock().unwrap() = Some(workload.clone()); *workload.store.lock().unwrap() = Some(world.state_store.clone()); - let incumbent = admit_incumbent(&world)?; - let live = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Continuity, Some(&incumbent))?; - let id = live.transaction_id.as_str(); + warm_stop_first(&world, &owner)?; // The Warming step waits for Odin; waiting again rewrites nothing, so // the step succeeds without touching the record it will find missing. - world.engine.resume_candidate(&resident_id(&world, id)?)?; - let waited = resident_id(&world, id)?.value; - assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?} {:?}", waited.phase, waited.warming); + world.engine.resume_candidate(&resident_id(&world, &owner)?)?; + let waited = resident_id(&world, &owner)?.value; + assert!(waited.last_error.is_some(), "the Warming step did not wait: {:?}", waited.phase); workload.armed.store(true, std::sync::atomic::Ordering::SeqCst); world.engine.resume_backoff.lock().unwrap().clear(); let error = world .engine - .resume_candidate(&resident_id(&world, id)?) + .resume_candidate(&resident_id(&world, &owner)?) .expect_err("a record gone from both stores was read as a finished one"); assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); assert!(read_history_transactions(&world.state_store).is_empty()); @@ -17158,6 +17157,32 @@ mod tests { Ok(()) } + /// What the scheduler does for a deployment waiting at its Sealing tail: it + /// asks to enter Starting, and a refusal ends it before fencing. (A resume + /// pass would stop first at the deployment brake this world never releases.) + fn step_waiter(world: &EngineFixture, id: &str) -> Result<()> { + let waiting = resident_id(world, id)?; + if waiting.value.pre_fencing_abort.is_some() { + return Ok(()); + } + if let Err(error) = world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?) { + world.engine.begin_pre_fencing_abort(&resident_id(world, id)?, error)?; + } + Ok(()) + } + + /// Passes of the scheduler over the owner, each followed by the waiter's + /// own attempt to enter Starting. + fn run_owner_and_waiter(world: &EngineFixture, waiter: &str, passes: usize) -> Result<()> { + for _ in 0..passes { + resume_pass(world)?; + if resident_id(world, waiter).is_ok() { + step_waiter(world, waiter)?; + } + } + Ok(()) + } + fn operator_required(world: &EngineFixture) -> Result> { world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) // head-only } @@ -17315,9 +17340,7 @@ mod tests { ) -> Result<()> { let (world, deploy, continuity) = warm_stop_first_owner(SwitchWorkload::new())?; // The gate-wait is the Warming step itself, not an error of it. - for _ in 0..3 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 3)?; let waiting = resident_id(&world, &continuity)?.value; assert_eq!(waiting.phase, DeploymentPhase::Warming); assert!(waiting.pre_fencing_abort.is_none(), "the Warming step failed: {:?}", waiting.last_error); @@ -17329,7 +17352,7 @@ mod tests { if resident_id(&world, &continuity).is_err() { break; } - resume_pass(&world)?; + run_owner_and_waiter(&world, &deploy, 1)?; } let ended = latest_id(&world, &continuity)?; let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { @@ -17340,12 +17363,7 @@ mod tests { // The pass that retired the owner, or the next, moves the waiter to // Starting; stop there, before its Starting step runs. - for _ in 0..3 { - if resident_id(&world, &deploy)?.value.phase >= DeploymentPhase::Starting { - break; - } - resume_pass(&world)?; - } + step_waiter(&world, &deploy)?; let took = resident_id(&world, &deploy)?.value; assert_eq!(took.phase, DeploymentPhase::Starting); assert!(took.owns_target() && took.pre_fencing_abort.is_none()); @@ -17363,9 +17381,7 @@ mod tests { world .engine .begin_pre_fencing_abort(&resident_id(&world, &owner)?, anyhow!("candidate refused"))?; - for _ in 0..3 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 3)?; // Before the abort's deadline: it waits, unpaged. let aborting = resident_id(&world, &owner)?.value; assert!(aborting.pre_fencing_abort.is_some() && !aborting.is_terminal()); @@ -17373,9 +17389,7 @@ mod tests { assert!(resident_id(&world, &deploy)?.value.pre_fencing_abort.is_none()); age_deadline_of(&world, &owner)?; - for _ in 0..20 { - resume_pass(&world)?; - } + run_owner_and_waiter(&world, &deploy, 20)?; let aborting = resident_id(&world, &owner)?.value; assert!(!aborting.is_terminal() && aborting.owns_target(), "the owner was ended"); assert_eq!(operator_required(&world)?, [owner.as_str()]); // head-only From 8230c544acf3a45f0874c3906e368a94fce63c59 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:58:13 +0200 Subject: [PATCH 25/47] Cut c2b fix: a pre-fencing abort keeps its deadline through its last cleanup write (a Sealing waiter's abort could not finish) --- src/control_plane.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c5a51a8..52842ea 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1808,10 +1808,13 @@ impl DeploymentTransaction { } /// Whether a deadline can describe the record's current activity: a - /// deadline phase, a pending pre-fencing abort, or an admitted Complete. + /// deadline phase, a pre-fencing abort not yet ended, or an admitted Complete. fn carries_a_deadline(&self) -> bool { + // An abort carries its deadline until the record ends, including the + // write that completes its last cleanup: a Sealing or Starting record + // has no phase duration to fall back on, and that write is validated. DEADLINE_PHASES.contains(&self.phase) - || self.pre_fencing_abort_pending() + || (self.completion.is_none() && self.pre_fencing_abort.is_some()) || (self.phase == DeploymentPhase::Complete && matches!(self.completion, Some(TransactionCompletion::Admitted { .. }))) } @@ -15918,7 +15921,7 @@ mod tests { .resume_candidate(&resident_id(&world, &owner)?) .expect_err("a record gone from both stores was read as a finished one"); assert!(format!("{error:#}").contains("disappeared while checking scheduler progress"), "{error:#}"); - assert!(read_history_transactions(&world.state_store).is_empty()); + assert!(read_history_transactions(&world.state_store).iter().all(|archived| archived.transaction_id != owner)); Ok(()) } @@ -17162,7 +17165,7 @@ mod tests { /// pass would stop first at the deployment brake this world never releases.) fn step_waiter(world: &EngineFixture, id: &str) -> Result<()> { let waiting = resident_id(world, id)?; - if waiting.value.pre_fencing_abort.is_some() { + if waiting.value.pre_fencing_abort.is_some() || waiting.value.phase >= DeploymentPhase::Starting { return Ok(()); } if let Err(error) = world.engine.enter(&waiting, entering(&waiting, DeploymentPhase::Starting)?) { From 58f213fe1525c9f3acacbe40c705e55a03a4d791 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:24:22 +0200 Subject: [PATCH 26/47] Cut c2c: boot lift returns a wedged continuity abort to forward retry A pre-c2c Idunn aborted a continuity at its phase deadline, which stops the only running copy of the admitted body. At boot, a continuity abort that has not stopped its running candidate (candidate_cleanup Pending, lease_withdrawal not Complete, workload observed, issued lease exact on disk) is withdrawn and the record retimed, the exact inverse of begin_post_fencing_abort. Negative tests carry the live shapes of tx-4eaed669 (Routing, adopted lease) and tx-039b91e9 (AwaitingReady, granted lease); the lift's refusals are pinned per predicate input. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 301 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 52842ea..092324d 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5626,6 +5626,79 @@ struct ProjectionReconciliation { } impl Engine { + /// Once at boot: withdraw the abort of a continuity that has not stopped + /// its running candidate, and retime the record so its phase retries + /// forward. A pre-c2c Idunn aborted a continuity at its deadline; that + /// abort stops the only running copy of the admitted body (ruling + /// c2c-boot-returns-wedged-aborts-forward). The result is the exact inverse + /// of `begin_post_fencing_abort`. + /// + /// A continuity abort now begins only over a permanently stopped + /// candidate, so this predicate cannot match a new record: it is a + /// migration with a deletion line (follow-up c2c-delete-boot-lift). A + /// record it cannot decide is reported and left aborting. + fn return_wedged_continuity_aborts_forward(&self) -> Result> { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let mut returned = Vec::new(); + for stored in &snapshot.transactions { + let transaction = &stored.value; + let Some(abort) = &transaction.post_fencing_abort else { + continue; + }; + if transaction.command_kind != CommandKind::Continuity + || transaction.completion.is_some() + || abort.candidate_cleanup != CleanupEvidence::Pending + || abort.lease_withdrawal == CleanupEvidence::Complete + { + continue; + } + let id = &transaction.transaction_id; + let lifted = (|| -> Result { + let plan = required(&transaction.plan, "transaction plan")?; + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(plan)? + .observe(expected, activation, workload)?; + if let Some(lease) = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + { + let path = plan + .parsed_inputs()? + .1 + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + if !CultCacheWriteLeaseDriver::new(&transaction.target, path) + .observe_exact(lease)? + { + return Ok(false); + } + } + let now = now_millis()?; + let mut next = transaction.clone(); + next.post_fencing_abort = None; + next.last_error = Some(abort.error.clone()); + next.updated_at_unix_millis = now; + next.retime(now); + replace_transaction(&self.options.state_store, stored, &next)?; + Ok(true) + })(); + match lifted { + Ok(true) => returned.push(id.clone()), + Ok(false) => {} + Err(error) => eprintln!( + "Idunn could not decide whether continuity {id} may retry forward; its abort is left running: {error:#}" + ), + } + } + Ok(returned) + } + /// Once at boot: demote the activation a failed continuity left standing, /// where the projection still names exactly the activation that failed /// transaction issued. A continuity abort before the single resolution @@ -5739,6 +5812,16 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; + match engine.return_wedged_continuity_aborts_forward() { + Ok(returned) => { + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + } + Err(error) => eprintln!("Idunn boot could not lift wedged continuity aborts: {error:#}"), + } match engine.reconcile_failed_continuity_projections() { Ok(outcome) => { for transaction_id in &outcome.demoted { @@ -22362,6 +22445,18 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// Scheduler passes until `done`, at most twelve. + #[cfg(unix)] + fn drive_until_resuming(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..12 { + if done(&self.transaction()?) { + return Ok(()); + } + self.resume()?; + } + bail!("the transaction never reached the awaited state: {:?}", self.transaction()?.last_error) + } + fn resume_until_terminal(&self) -> Result { for _ in 0..12 { if latest(&self.world)?.completion.is_some() { @@ -22954,6 +23049,212 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// What a pre-c2c Idunn left when a continuity's phase passed its + /// deadline: the abort begun, route restoration done, the lease + /// withdrawal and candidate cleanup still owed, and the deadline stale. + #[cfg(unix)] + fn wedge_abort(routed: &RoutedWorld, error: &str) -> Result<()> { + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("{error}"))?; + routed.resume()?; + let aborting = routed.transaction()?.post_fencing_abort.context("no abort")?; + assert_eq!(aborting.route_restoration, CleanupEvidence::Complete); + assert_eq!(aborting.candidate_cleanup, CleanupEvidence::Pending); + pass_deadline(&routed.world) + } + + #[cfg(unix)] + fn lease_of(routed: &RoutedWorld) -> Result<(IdunnProcessWriteLeaseRecord, String)> { + let leasing = routed.transaction()?.leasing.context("no leasing evidence")?; + Ok(( + leasing.issued_lease().context("no issued lease")?.clone(), + leasing.lease_sha256().context("no granted lease")?.to_owned(), + )) + } + + #[cfg(unix)] + fn lease_driver(routed: &RoutedWorld) -> (CultCacheWriteLeaseDriver, PathBuf) { + let path = routed.world.root.join("lease/process-write-lease.cc"); + (CultCacheWriteLeaseDriver::new("service", &path), path) + } + + /// The live shape of tx-4eaed669 (Odin): Routing, lease granted and + /// adopted, the proof failing, the abort begun at the Routing deadline. + /// A pre-c2c Idunn went on to stop the candidate and the target lost + /// its only running copy. The boot lift returns the record to forward + /// retry; nothing is stopped, and the proof, once it passes, commits. + #[cfg(unix)] + #[test] + fn a_wedged_odin_shaped_continuity_abort_is_returned_to_forward_retry_at_boot() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + assert!(routed.transaction()?.lease_adoption.is_some()); + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + wedge_abort( + &routed, + "the Routing phase passed its deadline at unix ms 1792324133638", + )?; + let stops = routed.workload.stops.load(Ordering::SeqCst); + + let lifted_at = now_millis()?; + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + for _ in 0..10 { + routed.resume()?; + } + assert_eq!( + routed.workload.stops.load(Ordering::SeqCst), + stops, + "the candidate was stopped" + ); + let retrying = routed.transaction()?; + assert_eq!(returned, [retrying.transaction_id.clone()]); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + assert!(retrying.post_fencing_abort.is_none(), "the abort came back"); + assert!(retrying.completion.is_none()); + assert_eq!(retrying.phase, DeploymentPhase::Routing); + let deadline = retrying.phase_deadline.context("no deadline")?; + assert!(deadline.entered_at_unix_millis >= lifted_at, "the stale deadline stayed"); + assert!(routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty()); + + routed.stub.hang_up.store(false, Ordering::SeqCst); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert!(routed.fragment_path().exists(), "the route membership was not re-installed"); + Ok(()) + } + + /// The live shape of tx-039b91e9 (ghostlight): AwaitingReady with the + /// lease granted and not adopted, aborted at the readiness deadline. + /// Returned to forward retry, the candidate that later answers is + /// adopted and the record enters Routing. + #[cfg(unix)] + #[test] + fn a_wedged_ghostlight_shaped_continuity_abort_is_returned_to_forward_retry_at_boot() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + wedge_abort( + &routed, + "lease-not-adopted: the AwaitingReady phase passed its deadline at unix ms 1792324133638", + )?; + let stops = routed.workload.stops.load(Ordering::SeqCst); + + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + for _ in 0..5 { + routed.resume()?; + } + let waiting = routed.transaction()?; + assert_eq!(returned, [waiting.transaction_id.clone()]); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + assert!(waiting.post_fencing_abort.is_none() && waiting.completion.is_none()); + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.lease_adoption.is_none()); + + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Routing)?; + let adopted = routed.transaction()?; + assert_eq!(adopted.lease_adoption.context("no adoption recorded")?.write_lease_sha256, lease_sha256); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + Ok(()) + } + + /// The lift is a function of its inputs: each case lacks one thing the + /// lift requires, and its abort stays and runs to its end. + #[cfg(unix)] + #[test] + fn the_boot_lift_leaves_every_abort_it_cannot_prove_harmless() -> Result<()> { + #[derive(Clone, Copy, Debug)] + enum Case { + CandidateNotRunning, + LeaseWithdrawn, + CandidateCleaned, + LeaseRevokedUnderIt, + DeployWithRunningCandidate, + CompletedRecord, + } + for case in [ + Case::CandidateNotRunning, + Case::LeaseWithdrawn, + Case::CandidateCleaned, + Case::LeaseRevokedUnderIt, + Case::DeployWithRunningCandidate, + Case::CompletedRecord, + ] { + let deploy = matches!(case, Case::DeployWithRunningCandidate); + // The cleanup comes after the lease withdrawal, so a case with + // cleanup done and the withdrawal not is a stateless target's. + let stateful = !matches!(case, Case::CandidateCleaned); + let routed = build_routed_world( + Odin::Unreachable, + 1, + DeploymentPhase::Fencing, + false, + stateful, + if deploy { CommandKind::Deploy } else { CommandKind::Continuity }, + )?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + routed.world.engine.begin_post_fencing_abort( + &resident(&routed.world)?, + anyhow!("the AwaitingReady phase passed its deadline"), + )?; + // Route restoration; then each case takes the abort one real + // step, or one input, away from the lift's predicate. + routed.resume()?; + match case { + Case::CandidateNotRunning => routed.workload.kill(), + Case::LeaseWithdrawn | Case::CandidateCleaned => routed.resume()?, + Case::LeaseRevokedUnderIt => { + let (lease, _) = lease_of(&routed)?; + lease_driver(&routed).0.revoke_exact(Some(&lease))?; + } + Case::DeployWithRunningCandidate => {} + Case::CompletedRecord => { + routed.resume_until_terminal()?; + } + } + let abort = routed.transaction().ok().and_then(|t| t.post_fencing_abort); + match case { + Case::LeaseWithdrawn => assert_eq!( + abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), + Some((CleanupEvidence::Complete, CleanupEvidence::Pending)) + ), + Case::CandidateCleaned => assert_eq!( + abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), + Some((CleanupEvidence::Skipped, CleanupEvidence::Complete)) + ), + _ => {} + } + let before = ControlSnapshot::read(&routed.world.state_store)?; + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty(), "lifted {returned:?}"); + let after = ControlSnapshot::read(&routed.world.state_store)?; + assert_eq!( + before.transactions.iter().map(|t| &t.envelope).collect::>(), + after.transactions.iter().map(|t| &t.envelope).collect::>(), + "the lift wrote a record it left" + ); + let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.completion + ); + } + Ok(()) + } + /// B5 Soul F. A stateful candidate Odin reports on: its Ready word /// carries the granted lease, and that Ready records the adoption, /// sourced from Odin's topology and naming the presence Odin observed. From 063409489e95682117ad5afb29e4441d74bde2e8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:33:29 +0200 Subject: [PATCH 27/47] Cut c2c: a continuity past Fencing retries forward and pages, never aborts PhaseEnd::RetryForward: phase_end answers it for every continuity at or past Fencing, because aborting restores the body its candidate already runs (ruling c2c-retry-decided-by-record). The resolver leaves it to the operator (operator-required incident, report_once) and the phase step keeps running. leaves_to_operator covers it. DeploymentTransaction::expirable is the one spelling the expire CLI, the spent-request retirement and the resolver's requested flag read: a continuity past Fencing cannot be expired. Deploy keeps abort-to-incumbent and the stateful operator-required arm. Tests: the PhaseEnd table, the Routing deadline forward retry and its incident episode, the stateful deadline, expiry refusal and spent-request retirement, and a deployment still aborting to its incumbent. Abort-mechanics tests begin their aborts directly. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 318 ++++++++++++++++++++++++++++++++----------- 1 file changed, 236 insertions(+), 82 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 092324d..3e36787 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -981,6 +981,10 @@ enum PhaseEnd { /// A stateful deployment whose candidate holds an issued lease: the phase /// is reported and not resolved. OperatorRequired, + /// A continuity at or past Fencing: its candidate is the only running copy + /// of the admitted body, so aborting it restores what it already runs. The + /// resolver leaves it to the operator and its phase step keeps running. + RetryForward, } /// How a post-fencing failure recovers. `RestoreIncumbent` is what a @@ -1844,17 +1848,29 @@ impl DeploymentTransaction { && (self.pre_fencing_abort.is_some() || self.post_fencing_abort.is_some() || self.drains_incumbent() - || (self.resolves_by_deadline() && self.phase_end() == PhaseEnd::OperatorRequired)) + || (self.resolves_by_deadline() + && matches!(self.phase_end(), PhaseEnd::OperatorRequired | PhaseEnd::RetryForward))) } - /// What an ended post-fencing phase resolves to (Q3). A candidate that was - /// never issued a write lease cannot have written, so the transaction - /// aborts to the incumbent through the post-fencing path, stateless or not. - /// A continuity always aborts: its candidate is the admitted release, and - /// supervision restarts that release under the continuity backoff. A - /// deployment whose candidate was issued a lease may have written under a - /// new state contract, and restoring the incumbent over that is not - /// Idunn's call: it is left for the operator. + /// Whether an operator's expiry can end this record's phase: it resolves by + /// deadline, and is not a continuity past Fencing, whose expiry abort would + /// stop the only running copy of the admitted body (ruling + /// c2c-expire-refuses-continuity-past-fencing). + fn expirable(&self) -> bool { + self.resolves_by_deadline() + && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) + } + + /// What an ended post-fencing phase resolves to (Q3). A continuity at or + /// past Fencing retries forward: its incumbent is the admitted generation, + /// which supervision mints a continuity only after seeing dead, and from + /// Fencing the continuity owns the target, so aborting restores the body + /// its candidate already runs (ruling c2c-retry-decided-by-record). A + /// deployment whose candidate was never issued a write lease cannot have + /// written, so it aborts to the incumbent through the post-fencing path, + /// stateless or not. One whose candidate was issued a lease may have + /// written under a new state contract, and restoring the incumbent over + /// that is not Idunn's call: it is left for the operator. fn phase_end(&self) -> PhaseEnd { if self.phase < DeploymentPhase::Fencing { return PhaseEnd::AbortBeforeFencing; @@ -1863,8 +1879,9 @@ impl DeploymentTransaction { let granted = leasing.and_then(LeasingEvidence::lease).is_some(); let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); match self.command_kind { + CommandKind::Continuity => PhaseEnd::RetryForward, CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, - _ => PhaseEnd::Abort { + CommandKind::Deploy => PhaseEnd::Abort { lease_not_adopted: granted && self.lease_adoption.is_none(), }, } @@ -3119,8 +3136,13 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> "command has multiple live transactions" ); ensure!( - current.value.resolves_by_deadline(), - "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + current.value.expirable(), + "{}", + if current.value.resolves_by_deadline() { + "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" + } else { + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + } ); let now = now_millis()?; let request = ExpiryRequest { @@ -9460,8 +9482,11 @@ impl Engine { /// Every live record is held to a deadline. Before a transaction runs its /// step, a deadline phase whose deadline has passed, or whose command an - /// operator asked to expire, is resolved as `phase_end` says; an abort or - /// a drain past its deadline is left to the operator and its step retries. + /// operator asked to expire, is resolved as `phase_end` says: aborted before + /// the fence, aborted after it, or left to the operator with its step + /// retrying (a stateful deploy, or a continuity, whose candidate is the only + /// running copy of its body). An abort or a drain past its deadline is left + /// to the operator the same way. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the @@ -9513,10 +9538,11 @@ impl Engine { } return Ok(None); } - let requested = ControlSnapshot::read(&self.options.state_store)? - .expiry_requests - .iter() - .any(|request| request.value.command_id == transaction.command_id); + let requested = transaction.expirable() + && ControlSnapshot::read(&self.options.state_store)? + .expiry_requests + .iter() + .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { format!( "the {:?} phase passed its deadline at unix ms {}", @@ -9548,6 +9574,15 @@ impl Engine { ); Ok(None) } + PhaseEnd::RetryForward => { + self.leave_to_operator( + transaction, + &format!( + "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, stop its unit by hand" + ), + ); + Ok(None) + } } } @@ -9578,7 +9613,7 @@ impl Engine { .filter(|request| { !snapshot.transactions.iter().any(|stored| { stored.value.command_id == request.value.command_id - && stored.value.resolves_by_deadline() + && stored.value.expirable() }) }) .map(|request| request.envelope.clone()) @@ -14061,22 +14096,24 @@ mod tests { (Deploy, Some(prepared.clone()), None, PhaseEnd::OperatorRequired), (Deploy, Some(granted.clone()), None, PhaseEnd::OperatorRequired), (Deploy, Some(granted.clone()), adoption.clone(), PhaseEnd::OperatorRequired), - (Continuity, Some(LeasingEvidence::SkippedStateless), None, abort(false)), - (Continuity, Some(prepared), None, abort(false)), - (Continuity, Some(granted.clone()), None, abort(true)), - (Continuity, Some(granted), adoption.clone(), abort(false)), + (Continuity, None, None, PhaseEnd::RetryForward), + (Continuity, Some(LeasingEvidence::SkippedStateless), None, PhaseEnd::RetryForward), + (Continuity, Some(prepared), None, PhaseEnd::RetryForward), + (Continuity, Some(granted.clone()), None, PhaseEnd::RetryForward), + (Continuity, Some(granted), adoption.clone(), PhaseEnd::RetryForward), ]; for (kind, leasing, lease_adoption, expected) in cases { transaction.command_kind = kind; transaction.leasing = leasing.clone(); transaction.lease_adoption = lease_adoption.clone(); assert_eq!(transaction.phase_end(), expected, "{kind:?} {leasing:?} {lease_adoption:?}"); - // Only a stateful deploy is left to the operator when its phase ends. + // Only a stateful deploy and a continuity are left to the operator + // when their phase ends; the other deploys abort. transaction.retime(1_000); let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; assert_eq!( transaction.leaves_to_operator(due), - expected == PhaseEnd::OperatorRequired, + matches!(expected, PhaseEnd::OperatorRequired | PhaseEnd::RetryForward), "{kind:?} {leasing:?} {lease_adoption:?}" ); assert!(!transaction.leaves_to_operator(due - 1)); @@ -22482,11 +22519,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// Soul's F1 probe, committed. A continuity's route changes are /// Survival, so no ceiling refuses its install and rollback while its - /// route proof keeps failing. The Routing deadline ends it: no attempt - /// runs after the deadline. + /// route proof keeps failing. Its Routing deadline does not end it: the + /// continuity pages the operator and retries forward, never stopping its + /// candidate (ruling post-fence-forward-retry). #[cfg(unix)] #[test] - fn a_continuity_whose_route_proof_fails_ends_at_its_routing_deadline() -> Result<()> { + fn a_continuity_whose_route_proof_fails_retries_forward_past_its_routing_deadline_and_pages() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; @@ -22504,36 +22542,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(waiting.last_error.is_some()); assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); + let stops = routed.workload.stops.load(Ordering::SeqCst); pass_deadline(&routed.world)?; - routed.resume()?; - let abort = routed - .transaction()? - .post_fencing_abort - .context("the passed deadline did not end the phase")?; - assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); - let finished = routed.resume_until_terminal()?; - assert!( - matches!( - finished.completion, - Some(TransactionCompletion::FailedAfterFencing { - recovery: TerminalRecovery::RestoreIncumbent, - .. - }) - ), - "{:?}", - finished.completion + for _ in 0..5 { + routed.resume()?; + } + // Past the deadline the continuity is not aborted: aborting it + // would stop the only running copy of the admitted body. It is left + // to the operator, and its Routing step keeps running. + let left = routed.transaction()?; + assert!(left.post_fencing_abort.is_none(), "the passed deadline aborted a continuity"); + assert_eq!(left.phase, DeploymentPhase::Routing); + assert!(left.completion.is_none()); + assert_eq!( + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, + [left.transaction_id.clone()] ); - // After the deadline no attempt runs; the withdrawal writes and - // reloads once, whatever the rolled-back disk shows. - assert_eq!(routed.reloads(), 7, "an attempt ran after the deadline"); + assert_eq!(routed.world.engine.incidents().read()?.len(), 1, "one page per episode"); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert!(routed.reloads() > 6, "no attempt ran after the deadline"); + + // The proof passes: the phase moves on, and the first pass after + // closes the incident. + routed.stub.hang_up.store(false, Ordering::SeqCst); + routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; + routed.resume()?; + assert!( + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?.is_empty(), + "the incident outlived the episode" + ); + let records = routed.world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); Ok(()) } /// The live shape of F1. A continuity reuses the admitted Expected, so /// its membership is the admitted fragment already on disk: while its /// proof fails, the install and the rollback write nothing and reload - /// nothing, and neither does the withdrawal after its deadline. + /// nothing, and neither does its forward retry after its deadline. #[cfg(unix)] #[test] fn a_continuity_over_its_admitted_route_reloads_nothing_while_its_proof_fails() @@ -22562,8 +22610,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(routed.world.route_stubs.count(""), 0, "a failing continuity ran a host program"); pass_deadline(&routed.world)?; - routed.resume_until_terminal()?; - assert_eq!(routed.world.route_stubs.count(""), 0, "its withdrawal ran a host program"); + for _ in 0..5 { + routed.resume()?; + } + assert!(routed.transaction()?.post_fencing_abort.is_none(), "a passed deadline aborted it"); + assert_eq!(routed.world.route_stubs.count(""), 0, "its retry ran a host program"); assert_eq!(std::fs::read(routed.fragment_path())?, admitted, "the admitted route moved"); Ok(()) } @@ -22689,10 +22740,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .expect_err("an aborting transaction was expired"); assert!(format!("{refused:#}").contains("idunn cancel"), "{refused:#}"); - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let untouched = resident(&routed.world)?.envelope; - expire(&routed.world.state_store, "continuity-service", "operator")?; - assert!(expire(&routed.world.state_store, "continuity-service", "operator").is_err()); + expire(&routed.world.state_store, "up-service", "operator")?; + assert!(expire(&routed.world.state_store, "up-service", "operator").is_err()); let snapshot = ControlSnapshot::read(&routed.world.state_store)?; assert_eq!(snapshot.expiry_requests.len(), 1); assert_eq!(resident(&routed.world)?.envelope, untouched, "the CLI wrote the transaction"); @@ -22708,6 +22759,36 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ControlSnapshot::read(&routed.world.state_store)?.expiry_requests.is_empty(), "the spent request stayed" ); + + // A continuity past Fencing retries forward: expiring it would + // abort the only running copy of its body. The CLI refuses, and a + // request a pre-c2c binary stored for one is retired as spent. + let continuity = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let refused = expire(&continuity.world.state_store, "continuity-service", "operator") + .expect_err("a continuity past Fencing was expired"); + assert!(format!("{refused:#}").contains("retries forward"), "{refused:#}"); + assert!(ControlSnapshot::read(&continuity.world.state_store)?.expiry_requests.is_empty()); + let now = now_millis()?; + let stored = ExpiryRequest { + schema_version: EXPIRY_REQUEST_SCHEMA.into(), + command_id: "continuity-service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now, + }; + assert!(SingleFileMessagePackBackingStore::new(&continuity.world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: ExpiryRequest::TYPE.into(), + key: stored.command_id.clone(), + current: None, + }], + &[expiry_request_envelope(&stored, now)?], + )?); + continuity.resume()?; + assert!( + ControlSnapshot::read(&continuity.world.state_store)?.expiry_requests.is_empty(), + "the request for a continuity past Fencing stayed" + ); + assert!(continuity.transaction()?.post_fencing_abort.is_none(), "the request aborted a continuity"); Ok(()) } @@ -22739,7 +22820,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(format!("{refused:#}").contains("no live transaction"), "{refused:#}"); // Another target's deployment in flight is not this command's. - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let now = now_millis()?; let other = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -22765,10 +22846,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ], &[command_envelope(&other, now)?, transaction_envelope(&sealing, now)?], )?); - expire(&routed.world.state_store, "continuity-service", "operator")?; + expire(&routed.world.state_store, "up-service", "operator")?; let requests = ControlSnapshot::read(&routed.world.state_store)?.expiry_requests; assert_eq!(requests.len(), 1); - assert_eq!(requests[0].value.command_id, "continuity-service"); + assert_eq!(requests[0].value.command_id, "up-service"); Ok(()) } @@ -22776,7 +22857,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// live transactions, a request for the other leaves this one running. #[test] fn an_expiry_request_ends_only_its_own_commands_phase() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let now = now_millis()?; let other = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), @@ -22789,8 +22870,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let sealing = DeploymentTransaction::new(&other, "other".into(), 0, None, now)?; // The request names a command whose id has this transaction's // length and first bytes, and differs only in its last. - let near = "continuity-servicx"; - assert_eq!(near.len(), "continuity-service".len()); + let near = "up-servicx"; + assert_eq!(near.len(), "up-service".len()); let request = ExpiryRequest { schema_version: EXPIRY_REQUEST_SCHEMA.into(), command_id: near.into(), @@ -22800,11 +22881,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide // And commands whose ids are a strict prefix of this one's, and // extend it: a match is the whole id, in neither direction a part. let prefix = ExpiryRequest { - command_id: "continuity-servic".into(), + command_id: "up-servic".into(), ..request.clone() }; let extension = ExpiryRequest { - command_id: "continuity-service-2".into(), + command_id: "up-service-2".into(), ..request.clone() }; let store = SingleFileMessagePackBackingStore::new(&routed.world.state_store); @@ -22835,24 +22916,26 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let mine = snapshot .transactions .iter() - .find(|t| t.value.command_id == "continuity-service") + .find(|t| t.value.command_id == "up-service") .context("no continuity")?; assert_eq!(routed.world.engine.resolve_phase_deadline(mine)?, None); let after = ControlSnapshot::read(&routed.world.state_store)?; let unchanged = after .transactions .iter() - .find(|t| t.value.command_id == "continuity-service") + .find(|t| t.value.command_id == "up-service") .context("no continuity")?; assert_eq!(unchanged.envelope, mine.envelope, "another command's request ended this phase"); Ok(()) } - /// Q3 at the deadline, through the Engine. A continuity whose candidate - /// was granted a lease and never proved it held it fails as - /// `lease-not-adopted`; one whose Ready proved it restarts the admitted - /// release. A deployment whose candidate holds a lease is left to the - /// operator: nothing is aborted and its step still runs. + /// The deadline of a stateful target, through the Engine. A deployment + /// whose candidate holds a lease is left to the operator: nothing is + /// aborted and its step still runs. So is a continuity, adopted or not, + /// whose candidate is the only running copy of the admitted body. An + /// abort begun on a continuity directly still names its recovery: it + /// restarts the admitted release when its Ready proved the lease, and + /// restores the incumbent when it never did. #[test] fn a_stateful_deadline_resolves_by_lease_adoption() -> Result<()> { let granted = |routed: &RoutedWorld| -> Result { @@ -22866,13 +22949,29 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .to_owned()) }; + // Past its deadline a continuity is not aborted, adopted or not: + // its candidate is the only running copy of the admitted body. It + // is left to the operator and its step runs. + let open = |routed: &RoutedWorld| -> Result> { + routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired) + }; let unadopted = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; granted(&unadopted)?; pass_deadline(&unadopted.world)?; - unadopted.resume()?; - let abort = unadopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(abort.error.starts_with("lease-not-adopted: "), "{}", abort.error); - assert!(abort.error.contains("AwaitingReady phase passed its deadline"), "{}", abort.error); + for _ in 0..3 { + unadopted.resume()?; + } + let held = unadopted.transaction()?; + assert!(held.post_fencing_abort.is_none(), "an unadopted continuity was aborted"); + assert_eq!(held.phase, DeploymentPhase::AwaitingReady); + assert_eq!(held.last_error.as_deref(), Some("candidate is still warming"), "its step did not run"); + assert_eq!(open(&unadopted)?, [held.transaction_id.clone()]); + // An abort begun on it directly (a candidate found dead) still + // restores the incumbent when the lease was never adopted. + unadopted + .world + .engine + .begin_post_fencing_abort(&resident(&unadopted.world)?, anyhow!("test abort"))?; let finished = unadopted.resume_until_terminal()?; assert!(matches!( finished.completion, @@ -22896,10 +22995,19 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(adoption.signed_presence_sha256, evidence.canonical_sha256); assert_eq!(adoption.source, AdoptionSource::Direct); adopted.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + adopted.stub.hang_up.store(true, Ordering::SeqCst); pass_deadline(&adopted.world)?; - adopted.resume()?; - let abort = adopted.transaction()?.post_fencing_abort.context("not ended")?; - assert!(!abort.error.contains("lease-not-adopted"), "{}", abort.error); + for _ in 0..3 { + adopted.resume()?; + } + let held = adopted.transaction()?; + assert!(held.post_fencing_abort.is_none(), "an adopted continuity was aborted"); + assert_eq!(held.phase, DeploymentPhase::Routing); + assert_eq!(open(&adopted)?, [held.transaction_id.clone()]); + adopted + .world + .engine + .begin_post_fencing_abort(&resident(&adopted.world)?, anyhow!("test abort"))?; let finished = adopted.resume_until_terminal()?; assert!( matches!( @@ -22991,7 +23099,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } /// B5 Soul A. The physical grant lands before `Granted` is durable. A - /// continuity that dies in that window and passes its deadline is + /// continuity that dies in that window and whose abort begins is /// aborted from `Prepared`; the abort still withdraws the lease on /// disk, so the next fencing of the target is not refused. #[test] @@ -23011,9 +23119,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ); assert!(driver.observe_exact(&lease)?, "the grant is not on disk"); - pass_deadline(&routed.world)?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); routed.resume_until_terminal()?; assert!(driver.observe_empty()?, "the abort left the granted lease on disk"); // What the next fencing does to a candidate's lease path. @@ -23034,9 +23145,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.workload.hold_lease_lock(&lease_path); let driver = CultCacheWriteLeaseDriver::new("service", &lease_path); - pass_deadline(&routed.world)?; + routed + .world + .engine + .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; - assert!(routed.transaction()?.post_fencing_abort.is_some(), "the deadline did not end it"); + assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); let stops = routed.workload.stops.load(Ordering::SeqCst); let finished = routed.resume_until_terminal()?; assert!( @@ -23049,6 +23163,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// The deadline's abort is a continuity's no longer, and a deployment's + /// still: a deployment's candidate is a new release and its incumbent + /// the admitted body, which an abort restores. + #[cfg(unix)] + #[test] + fn a_deploy_with_a_live_incumbent_still_aborts_to_it_at_its_deadline() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_none(), "aborted before its deadline"); + + pass_deadline(&routed.world)?; + routed.resume()?; + let abort = routed + .transaction()? + .post_fencing_abort + .context("the passed deadline did not end a deployment")?; + assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!( + routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), + "the boot lift took a deployment's abort" + ); + assert!(routed.transaction()?.post_fencing_abort.is_some()); + let finished = routed.resume_until_terminal()?; + assert!( + matches!( + finished.completion, + Some(TransactionCompletion::FailedAfterFencing { + recovery: TerminalRecovery::RestoreIncumbent, + .. + }) + ), + "{:?}", + finished.completion + ); + Ok(()) + } + /// What a pre-c2c Idunn left when a continuity's phase passed its /// deadline: the abort begun, route restoration done, the lease /// withdrawal and candidate cleanup still owed, and the deadline stale. From cc4a72ad43bf89f1e34c18be959fd5bbab8b6a57 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:37:03 +0200 Subject: [PATCH 28/47] Cut c2c: a record left to the operator backs off to the continuity cap ResumeBackoff stores the ceiling it was armed with and the skip check reads it. resume_ceiling is the continuity cap (300 s) for a record that leaves_to_operator and the short cap otherwise, for every such record: a forward-retrying continuity, a stateful deploy, a wedged abort, a drain. A step that returns Ok without moving a record left to the operator is backed off like a failed attempt, so a silent candidate is challenged once per backoff, not every poll. Nothing in TargetSupervision is touched. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 136 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 122 insertions(+), 14 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3e36787..f750257 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5112,11 +5112,26 @@ impl RouteActuationGate for EngineRouteGate<'_> { struct ResumeBackoff { failures: u32, not_before_unix_millis: u64, + /// The longest wait this backoff was armed with: the skip check reads it, + /// so the check and the wait cannot disagree. + ceiling_millis: u64, } /// Longest wait between attempts at a failing transaction step. const RESUME_BACKOFF_CEILING_MILLIS: u64 = 60_000; +/// The longest wait between attempts at one transaction's step: the continuity +/// cap for a record left to the operator, which retries on the slow cadence its +/// page promises (ruling c2c-forward-retry-cadence), and the shorter one for +/// every other. +fn resume_ceiling(transaction: &DeploymentTransaction, now: u64) -> u64 { + if transaction.leaves_to_operator(now) { + CONTINUITY_BACKOFF_CEILING_MILLIS + } else { + RESUME_BACKOFF_CEILING_MILLIS + } +} + impl Engine { fn open(options: RuntimeOptions) -> Result { Self::open_with_systemd_workload( @@ -6054,7 +6069,7 @@ impl Engine { .get(id) .is_some_and(|backoff| { now_millis().is_ok_and(|now| { - is_waiting(now, backoff.not_before_unix_millis, RESUME_BACKOFF_CEILING_MILLIS) + is_waiting(now, backoff.not_before_unix_millis, backoff.ceiling_millis) }) }) { @@ -6064,15 +6079,29 @@ impl Engine { self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); return Ok(false); } + let left = now_millis().is_ok_and(|now| current.value.leaves_to_operator(now)); let advanced = self.advance_transaction(current); if advanced.is_ok() { - // The step ran and succeeded: the fault, if any, has recovered, so - // a later failure is news. A skipped or failed attempt clears nothing. - self.resume_backoff - .lock() - .expect("resume backoff mutex") - .remove(id); - self.clear_fault(id); + // A step that returns Ok without moving a record left to the + // operator (a gate-wait on a silent candidate) is an attempt like + // a failed one: it backs off, and the fault it recorded stands. + let unmoved = left + && ControlSnapshot::read(&self.options.state_store)? + .transactions + .iter() + .find(|stored| stored.value.transaction_id == id) + .is_some_and(|live| !state_advanced(¤t.value, &live.value)); + if unmoved { + self.back_off(¤t.value)?; + } else { + // The step ran and succeeded: the fault, if any, has recovered, so + // a later failure is news. A skipped or failed attempt clears nothing. + self.resume_backoff + .lock() + .expect("resume backoff mutex") + .remove(id); + self.clear_fault(id); + } } if let Err(error) = advanced { let latest_snapshot = ControlSnapshot::read(&self.options.state_store)?; @@ -6088,7 +6117,7 @@ impl Engine { .remove(id); return Ok(true); } - self.back_off(id)?; + self.back_off(&latest.value)?; if latest.value.phase < DeploymentPhase::Fencing { // Before the fence an abort that cannot finish a step is // resumable, never a post-fence abort: that path refuses a @@ -6142,16 +6171,20 @@ impl Engine { /// Wait twice as long after each consecutive failure of a transaction's /// step, from one poll interval up to a ceiling. - fn back_off(&self, transaction_id: &str) -> Result<()> { + fn back_off(&self, transaction: &DeploymentTransaction) -> Result<()> { let now = now_millis()?; + let ceiling_millis = resume_ceiling(transaction, now); let mut all = self.resume_backoff.lock().expect("resume backoff mutex"); - let failures = all.get(transaction_id).map_or(0, |backoff| backoff.failures); - let wait = backoff_wait(self.options.poll_millis, failures, RESUME_BACKOFF_CEILING_MILLIS); + let failures = all + .get(&transaction.transaction_id) + .map_or(0, |backoff| backoff.failures); + let wait = backoff_wait(self.options.poll_millis, failures, ceiling_millis); all.insert( - transaction_id.to_string(), + transaction.transaction_id.clone(), ResumeBackoff { failures: failures.saturating_add(1), not_before_unix_millis: now.saturating_add(wait), + ceiling_millis, }, ); Ok(()) @@ -14061,11 +14094,45 @@ mod tests { Ok(()) } + /// The slow cadence belongs to a record that is left to the operator, + /// whichever way it came to be: it waits up to the continuity cap between + /// attempts from its deadline on, and up to the short cap before. + #[test] + fn a_record_left_to_the_operator_backs_off_to_the_continuity_cap() -> Result<()> { + let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; + let lease = write_lease(); + let leased = LeasingEvidence::Granted { + lease_sha256: lease.canonical_sha256()?, + lease, + }; + transaction.retime(1_000); + let due = transaction.phase_deadline.context("no deadline")?.deadline_at_unix_millis; + let mut aborting = transaction.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "wedged")); + let mut deploy = transaction.clone(); + deploy.command_kind = CommandKind::Deploy; + let mut stateful = deploy.clone(); + stateful.leasing = Some(leased); + let mut continuity = transaction.clone(); + continuity.command_kind = CommandKind::Continuity; + for (what, record) in [ + ("a forward-retrying continuity", &continuity), + ("a stateful deploy", &stateful), + ("a wedged abort", &aborting), + ] { + assert_eq!(resume_ceiling(record, due - 1), RESUME_BACKOFF_CEILING_MILLIS, "{what} before its deadline"); + assert_eq!(resume_ceiling(record, due), CONTINUITY_BACKOFF_CEILING_MILLIS, "{what} at its deadline"); + } + // A deploy that aborts at its deadline is not left to anyone. + assert_eq!(resume_ceiling(&deploy, due), RESUME_BACKOFF_CEILING_MILLIS); + Ok(()) + } + /// Q3 as a table: which ended phases abort, which name the unadopted /// lease, which are left to the operator, and which recovery the terminal /// record names. #[test] - fn an_ended_phase_aborts_unless_a_deployment_candidate_was_issued_a_lease() -> Result<()> { + fn an_ended_phase_is_resolved_by_command_kind_and_lease() -> Result<()> { let (_, mut transaction) = fixture_transaction(FIXTURE_TRANSACTIONS[2].1)?; let lease = write_lease(); let sha256 = lease.canonical_sha256()?; @@ -15794,6 +15861,7 @@ mod tests { ResumeBackoff { failures: 3, not_before_unix_millis, + ceiling_millis: RESUME_BACKOFF_CEILING_MILLIS, }, ); }; @@ -23203,6 +23271,46 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// A record left to the operator whose step returns Ok without moving it + /// (a gate-wait on a silent candidate) is backed off like a failed + /// attempt, to the continuity cap: its candidate is challenged once per + /// backoff, not once per pass. Before its deadline every pass challenges. + #[test] + fn a_continuity_left_to_the_operator_challenges_a_silent_candidate_once_per_backoff() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let engine = &routed.world.engine; + let id = routed.transaction()?.transaction_id; + let hits = || routed.stub.candidate_hits.load(Ordering::SeqCst); + + let before = hits(); + for _ in 0..3 { + engine.resume_one_transaction()?; + } + assert!(hits() >= before + 3, "a pass inside the deadline did not challenge"); + assert!(engine.resume_backoff.lock().unwrap().get(&id).is_none()); + + pass_deadline(&routed.world)?; + engine.resume_one_transaction()?; + let first = hits(); + assert!(first > before + 3, "the pass past the deadline did not challenge"); + let armed = *engine.resume_backoff.lock().unwrap().get(&id).context("the unmoved step set no backoff")?; + assert_eq!(armed.ceiling_millis, CONTINUITY_BACKOFF_CEILING_MILLIS); + for _ in 0..5 { + engine.resume_one_transaction()?; + } + assert_eq!(hits(), first, "a wedged candidate was challenged inside its backoff"); + assert!(routed.transaction()?.post_fencing_abort.is_none()); + + // The skip check reads the ceiling the backoff was armed with: a + // wait longer than the short cap is still a wait. + engine.resume_backoff.lock().unwrap().get_mut(&id).unwrap().not_before_unix_millis = + now_millis()? + 200_000; + engine.resume_one_transaction()?; + assert_eq!(hits(), first, "the skip check read the short cap"); + Ok(()) + } + /// What a pre-c2c Idunn left when a continuity's phase passed its /// deadline: the abort begun, route restoration done, the lease /// withdrawal and candidate cleanup still owed, and the deadline stale. From bff308aea781817e44a935c38ce2d26e0ddbf47e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:39:32 +0200 Subject: [PATCH 29/47] Cut c2c: expire's refusal names its reason without reading the deadline predicate Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index f750257..ad06ea9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -3138,7 +3138,7 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> ensure!( current.value.expirable(), "{}", - if current.value.resolves_by_deadline() { + if current.value.command_kind == CommandKind::Continuity && current.value.phase >= DeploymentPhase::Fencing && current.value.post_fencing_abort.is_none() { "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" } else { "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" From d9846e8847a675008f2877fa26842991fdc45f6d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:39:33 +0200 Subject: [PATCH 30/47] Cut c2c: route-continuity invariant 5 states the continuity forward retry A continuity past its post-fencing deadline is left to the operator and retries forward on a capped backoff; a stateless deployment aborts to its incumbent; a stateful deployment is left to the operator (ruling post-fence-forward-retry). Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index 77c201b..c4d3bc8 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -52,12 +52,13 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: the admitted incarnation. The proof answers an Idunn challenge and is signed with the credential Idunn issued to that launch. Invariant 3 changes where the proof travels, not what it proves. -5. **Every post-fencing phase ends.** Each phase after fencing has an admitted - deadline. When a deadline expires, Idunn resolves it: stateless targets abort - to the incumbent. Stateful targets get an explicit terminal record with a - named recovery. A granted lease that the process does not pick up within its - window is a named failure with an Idunn-owned recovery. It is never a silent - wait. Resumable-forever is gone. +5. **Every post-fencing phase has a deadline and is never a silent wait.** A + deployment's stateless phase that passes its deadline aborts to its + incumbent. A continuity past its deadline is left to the operator + (operator-required incident) and retries its phase forward on a capped + backoff, because aborting it would stop the only running copy of the admitted + body (ruling post-fence-forward-retry). A stateful deployment is left to the + operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never meets drift between the receipt and the projection. Rollback demotes the exact From a7ad3de5593a086e2522eb8bafda1b9f72cd9af4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 19:58:09 +0200 Subject: [PATCH 31/47] c2d negative: a waiting Deploy must land over a forward-retrying continuity past its deadline Red at e529858: the waiter ends FailedBeforeFencing owner-left-to-operator. The yield (ruling continuity-yields-to-deploy) lands in the next commit. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 96 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 11 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index de88449..28ea709 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17643,6 +17643,80 @@ mod tests { } // END c2b-forever-wait harness + // BEGIN c2d continuity-yields + // A deployment waits at Warming beside a continuity that retries forward + // past its deadline (ruling continuity-yields-to-deploy). The pass order is + // the hostile one: the waiter always looks at the owner first. + + /// One pass over the waiter and then the owner, never clearing the resume + /// backoff. A record that left the live set is skipped. + fn waiter_then_owner(world: &EngineFixture, waiter: &str, owner: &str) -> Result<()> { + for id in [waiter, owner] { + if let Ok(stored) = resident_id(world, id) { + world.engine.resume_candidate(&stored)?; + } + } + Ok(()) + } + + /// A continuity at Fencing driven alone to AwaitingReady, where it + /// gate-waits for an Odin word that never comes, beside a Deploy waiting in + /// Warming. Returns the world, the workload, and the two ids. + fn waiter_beside_gate_waiting_continuity() + -> Result<(EngineFixture, Arc, DeploymentTransaction, String, String, AdmittedGeneration)> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let (deploy, continuity, incumbent) = + deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; + let (deploy_id, owner_id) = (deploy.transaction_id.clone(), continuity.transaction_id.clone()); + drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::AwaitingReady)?; + pass_deadline_of(&world, &owner_id)?; + Ok((world, workload, deploy, deploy_id, owner_id, incumbent)) + } + + /// NEGATIVE. The continuity's deadline has passed and it retries forward; + /// the deployment waiting on it must not end, the continuity must yield. + #[test] + fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { + let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); + for _ in 0..2 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + let waiter = latest_id(&world, &deploy_id)?; + assert!(waiter.completion.is_none(), "the waiter ended: {:?}", waiter.completion); + assert!(waiter.pre_fencing_abort.is_none()); + let owner = resident_id(&world, &owner_id)?.value; + let abort = owner.post_fencing_abort.as_ref().context("the continuity did not yield")?; + assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); + // Nothing stopped the candidate before the yield began. + assert_eq!(workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + + for _ in 0..8 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + world.engine.resume_backoff.lock().unwrap().clear(); + } + // The yield's abort retimed the record: its incident, if one opened, closed. + assert!(!operator_required(&world)?.contains(&owner_id)); + let ended = latest_id(&world, &owner_id)?; + assert!(matches!(ended.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), "{:?}", ended.completion); + assert_eq!(workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops + 1); + + odin_reports_ready(&world, &deploy, seeded_sequence(Some(&incumbent)) + 10)?; + drive_id(&world, &deploy_id, |t| t.is_terminal())?; + let landed = latest_id(&world, &deploy_id)?; + assert!(matches!(landed.completion, Some(TransactionCompletion::Admitted { .. })), "{:?}", landed.completion); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("nothing admitted")? + .value + .clone(); + assert_eq!(admitted.transaction_id, deploy_id); + assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); + Ok(()) + } + // END c2d continuity-yields + #[test] fn an_operator_required_incident_closes_when_its_transaction_ends() -> Result<()> { let workload = SwitchWorkload::new(); @@ -22726,7 +22800,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(waiting.last_error.is_some()); assert_eq!(waiting.phase_deadline, Some(deadline), "a resume moved the deadline"); - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); pass_deadline(&routed.world)?; for _ in 0..5 { routed.resume()?; @@ -22743,7 +22817,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide [left.transaction_id.clone()] ); assert_eq!(routed.world.engine.incidents().read()?.len(), 1, "one page per episode"); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped"); assert!(routed.reloads() > 6, "no attempt ran after the deadline"); // The proof passes: the phase moves on, and the first pass after @@ -22758,7 +22832,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let records = routed.world.engine.incidents().read()?; assert_eq!(records.len(), 1); assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); Ok(()) } @@ -23335,7 +23409,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .begin_post_fencing_abort(&resident(&routed.world)?, anyhow!("test abort"))?; routed.resume()?; assert!(routed.transaction()?.post_fencing_abort.is_some(), "the abort did not begin"); - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let finished = routed.resume_until_terminal()?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), @@ -23343,7 +23417,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide finished.last_error ); assert!(driver.observe_empty()?, "the abort left the lease on disk"); - assert!(routed.workload.stops.load(Ordering::SeqCst) > stops, "the candidate was never stopped"); + assert!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst) > stops, "the candidate was never stopped"); Ok(()) } @@ -23480,7 +23554,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide &routed, "the Routing phase passed its deadline at unix ms 1792324133638", )?; - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let lifted_at = now_millis()?; let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; @@ -23488,7 +23562,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.resume()?; } assert_eq!( - routed.workload.stops.load(Ordering::SeqCst), + routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped" ); @@ -23504,7 +23578,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.stub.hang_up.store(false, Ordering::SeqCst); routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Committing)?; - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); assert!(routed.fragment_path().exists(), "the route membership was not re-installed"); Ok(()) } @@ -23525,7 +23599,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide &routed, "lease-not-adopted: the AwaitingReady phase passed its deadline at unix ms 1792324133638", )?; - let stops = routed.workload.stops.load(Ordering::SeqCst); + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; for _ in 0..5 { @@ -23533,7 +23607,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } let waiting = routed.transaction()?; assert_eq!(returned, [waiting.transaction_id.clone()]); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops, "the candidate was stopped"); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops, "the candidate was stopped"); assert!(driver.observe_exact(&lease)?, "the lease left the disk"); assert!(waiting.post_fencing_abort.is_none() && waiting.completion.is_none()); assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); @@ -23544,7 +23618,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.drive_until_resuming(|t| t.phase == DeploymentPhase::Routing)?; let adopted = routed.transaction()?; assert_eq!(adopted.lease_adoption.context("no adoption recorded")?.write_lease_sha256, lease_sha256); - assert_eq!(routed.workload.stops.load(Ordering::SeqCst), stops); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); Ok(()) } From 4f68809f0b300c5a8830f51f18c5b2d061e41172 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 19:59:37 +0200 Subject: [PATCH 32/47] Continuity yields to a Deploy awaiting the handover (ruling continuity-yields-to-deploy) A continuity past its deadline at or past Fencing begins its own post-fencing abort when a Deploy on its target awaits the handover; the waiter keeps waiting on an owner that retries forward. Adds retries_forward and awaits_handover, both derived. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 53 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 28ea709..34cf549 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -983,7 +983,9 @@ enum PhaseEnd { OperatorRequired, /// A continuity at or past Fencing: its candidate is the only running copy /// of the admitted body, so aborting it restores what it already runs. The - /// resolver leaves it to the operator and its phase step keeps running. + /// resolver leaves it to the operator and its phase step keeps running, + /// until a deployment awaits the handover, when the continuity yields + /// through its own post-fencing abort (ruling continuity-yields-to-deploy). RetryForward, } @@ -1861,6 +1863,14 @@ impl DeploymentTransaction { && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) } + /// A continuity at or past Fencing that its deadline leaves to the operator + /// while its step runs. It yields to a deployment awaiting the handover + /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. + /// Derived, never stored. + fn retries_forward(&self) -> bool { + self.resolves_by_deadline() && self.phase_end() == PhaseEnd::RetryForward + } + /// What an ended post-fencing phase resolves to (Q3). A continuity at or /// past Fencing retries forward: its incumbent is the admitted generation, /// which supervision mints a continuity only after seeing dead, and from @@ -2064,6 +2074,16 @@ impl DeploymentTransaction { self.phase != DeploymentPhase::Complete } + /// A live deployment whose next move is the handover: not ended, not + /// aborting, one phase short of `takes_target_at`. Derived, never stored. + /// Read by the continuity's yield and the boot lift. + fn awaits_handover(&self) -> bool { + self.command_kind == CommandKind::Deploy + && self.completion.is_none() + && self.pre_fencing_abort.is_none() + && self.phase as u8 + 1 == self.takes_target_at() as u8 + } + /// From the handover until Complete the transaction owns the target's /// incumbent slot: it fences, leases, routes and commits. Derived from the /// phase, never stored. Read by validation (one owner), supervision (which @@ -8535,9 +8555,11 @@ impl Engine { /// A deployment waits on the transaction that owns or drains its target. /// The owner bounds the wait, so the waiter's own Warming deadline is /// disarmed (the resolver re-arms it when the target is free). When the - /// owner's deadline has passed in a state Idunn will not end by itself, - /// the waiter ends through its own pre-fencing abort (the caller's Err - /// path): it never aborts the owner. + /// owner's deadline has passed in a state Idunn will not end by itself and + /// will not yield (a wedged abort or drain), the waiter ends through its own + /// pre-fencing abort (the caller's Err path): it never aborts the owner. A + /// waiter keeps waiting on an owner that retries forward, because that + /// owner yields to it on its next pass. fn wait_for_owner( &self, current: &Stored, @@ -8547,7 +8569,9 @@ impl Engine { let now = now_millis()?; let snapshot = ControlSnapshot::read(&self.options.state_store)?; if snapshot.transactions.iter().any(|stored| { - stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) + stored.value.transaction_id == owner_id + && stored.value.leaves_to_operator(now) + && !stored.value.retries_forward() }) { bail!( "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", @@ -9522,7 +9546,9 @@ impl Engine { /// the fence, aborted after it, or left to the operator with its step /// retrying (a stateful deploy, or a continuity, whose candidate is the only /// running copy of its body). An abort or a drain past its deadline is left - /// to the operator the same way. + /// to the operator the same way. A continuity left to the operator yields, + /// through its own post-fencing abort, to a deployment awaiting the + /// handover; it never writes the deployment. /// Returns `Some(moved)` when it decided the tick, `None` to run the step. /// /// Only `retime` sets a deadline (at phase entry, an abort's beginning, the @@ -9611,6 +9637,21 @@ impl Engine { Ok(None) } PhaseEnd::RetryForward => { + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + if let Some(deploy) = snapshot + .on_target(&transaction.target) + .find(|other| other.value.awaits_handover()) + { + self.begin_post_fencing_abort( + current, + anyhow!( + "continuity-yields: {cause}; deployment {} awaits the handover of {}", + deploy.value.transaction_id, + transaction.target + ), + )?; + return Ok(Some(true)); + } self.leave_to_operator( transaction, &format!( From 196e0dc884f1a573a37c545f80dbb52f14f80127 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:03:48 +0200 Subject: [PATCH 33/47] c2d: a resolver write spends the resume backoff; the boot lift leaves a yield resume_candidate removes the transaction's backoff when the resolver moved its record, so a yield abort is not held behind the forward retry's backoff (ruling c2d-yield-mechanics). return_wedged_continuity_aborts_forward skips an abort whose target holds a Deploy awaiting the handover. Tests: the yield table, the unit rows, the c2b waiter-still-ends row, the backoff spend, and lift case (vii). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 197 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 189 insertions(+), 8 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 34cf549..e2494f9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5690,10 +5690,12 @@ impl Engine { /// c2c-boot-returns-wedged-aborts-forward). The result is the exact inverse /// of `begin_post_fencing_abort`. /// - /// A continuity abort now begins only over a permanently stopped - /// candidate, so this predicate cannot match a new record: it is a - /// migration with a deletion line (follow-up c2c-delete-boot-lift). A - /// record it cannot decide is reported and left aborting. + /// A continuity abort now begins only over a permanently stopped candidate + /// or to yield to a deployment awaiting the handover, so this predicate + /// matches only aborts over a running candidate with nothing awaiting the + /// target, which Idunn no longer creates: it is a migration with a deletion + /// line (follow-up c2c-delete-boot-lift). A record it cannot decide is + /// reported and left aborting. fn return_wedged_continuity_aborts_forward(&self) -> Result> { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut returned = Vec::new(); @@ -5706,6 +5708,9 @@ impl Engine { || transaction.completion.is_some() || abort.candidate_cleanup != CleanupEvidence::Pending || abort.lease_withdrawal == CleanupEvidence::Complete + || snapshot + .on_target(&transaction.target) + .any(|other| other.value.awaits_handover()) { continue; } @@ -6082,6 +6087,11 @@ impl Engine { fn resume_candidate(&self, current: &Stored) -> Result { let id = current.value.transaction_id.as_str(); if let Some(moved) = self.resolve_phase_deadline(current)? { + if moved { + // A phase-ending write is movement: the next step is not held + // behind the backoff a forward retry armed. + self.resume_backoff.lock().expect("resume backoff mutex").remove(id); + } return Ok(moved); } if self @@ -17721,9 +17731,7 @@ mod tests { fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); - for _ in 0..2 { - waiter_then_owner(&world, &deploy_id, &owner_id)?; - } + waiter_then_owner(&world, &deploy_id, &owner_id)?; let waiter = latest_id(&world, &deploy_id)?; assert!(waiter.completion.is_none(), "the waiter ended: {:?}", waiter.completion); assert!(waiter.pre_fencing_abort.is_none()); @@ -17756,6 +17764,167 @@ mod tests { assert_eq!(landed.fencing, Some(FencingEvidence::SkippedStateless)); Ok(()) } + + fn stops_of(workload: &SwitchWorkload) -> usize { + workload.stops.load(std::sync::atomic::Ordering::SeqCst) + } + + /// The yield is a function of its inputs: the deadline, the continuity's + /// own record, and a deployment that awaits the handover. + #[test] + fn a_continuity_that_retries_forward_yields_only_past_its_deadline_and_only_to_a_deploy_awaiting_the_handover( + ) -> Result<()> { + // (i) Before the deadline nothing yields and the Deploy waits. + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let stops = stops_of(&workload); + for _ in 0..5 { + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + let waiting = resident_id(&world, &deploy_id)?.value; + assert!(waiting.pre_fencing_abort.is_none() && waiting.completion.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(&owner_id))); + assert_eq!(stops_of(&workload), stops); + + // (ii) Past the deadline, a Deploy still building awaits nothing. + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let stops = stops_of(&workload); + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + pass_deadline_of(&world, &owner_id)?; + for _ in 0..4 { + waiter_then_owner(&world, "", &owner_id)?; + } + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + + // (iii) A Deploy already ending before the fence awaits nothing. + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + + // (iv) Nor does a Deploy that has ended. + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; + for _ in 0..12 { + if latest_id(&world, &deploy_id)?.completion.is_some() { + break; + } + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, &deploy_id, "")?; + } + assert!(latest_id(&world, &deploy_id)?.completion.is_some()); + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + Ok(()) + } + + /// `awaits_handover` and `retries_forward` row by row. + #[test] + fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + let waiting = resident_id(&world, &deploy_id)?.value; + assert_eq!(waiting.phase, DeploymentPhase::Warming); + assert!(waiting.awaits_handover()); + for phase in [ + DeploymentPhase::Sealing, + DeploymentPhase::Starting, + DeploymentPhase::Fencing, + DeploymentPhase::Complete, + ] { + let mut moved = waiting.clone(); + moved.phase = phase; + assert!(!moved.awaits_handover(), "{phase:?}"); + } + let mut ended = waiting.clone(); + ended.completion = Some(TransactionCompletion::FailedBeforeFencing { error: "x".into() }); + assert!(!ended.awaits_handover()); + let mut continuity = resident_id(&world, &owner_id)?.value; + let mut as_continuity = waiting.clone(); + as_continuity.command_kind = CommandKind::Continuity; + assert!(!as_continuity.awaits_handover()); + world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("x"))?; + assert!(!resident_id(&world, &deploy_id)?.value.awaits_handover()); + + // The stop-then-start handover is Starting: its Sealing tail awaits it. + let (stop_first, stop_deploy, _) = stop_first_world(Arc::new(StillWorkload))?; + let mut sealing = resident_id(&stop_first, &stop_deploy)?.value; + sealing.phase = DeploymentPhase::Sealing; + assert!(sealing.awaits_handover()); + sealing.phase = DeploymentPhase::Warming; + assert!(!sealing.awaits_handover()); + + // A continuity retries forward from Fencing to Committing, until aborting. + for phase in [ + DeploymentPhase::Fencing, + DeploymentPhase::Leasing, + DeploymentPhase::AwaitingReady, + DeploymentPhase::Routing, + DeploymentPhase::Committing, + ] { + continuity.phase = phase; + assert!(continuity.retries_forward(), "{phase:?}"); + } + let mut deploy_record = waiting.clone(); + deploy_record.phase = DeploymentPhase::Fencing; + assert!(!deploy_record.retries_forward()); + continuity.post_fencing_abort = Some(post_fencing_abort_intent(&continuity, "x")); + assert!(!continuity.retries_forward()); + Ok(()) + } + + /// The c2b waiter still ends on an owner that will not yield: the yield's + /// own abort wedged past its deadline. + #[test] + fn the_c2b_waiter_still_ends_on_an_owner_that_will_not_yield() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + workload.fail_stop_of(&runtime_instance_of(&world, &owner_id)?); + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, &deploy_id, &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + pass_deadline_of(&world, &owner_id)?; + for _ in 0..20 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, &deploy_id, &owner_id)?; + } + assert!(!resident_id(&world, &owner_id)?.value.is_terminal()); + assert_ended_for_owner(&world, &deploy_id, &owner_id) + } + + /// The yield's abort is not held behind the backoff the forward retry + /// armed: movement spends it. + #[test] + fn a_yield_abort_is_not_held_behind_the_forward_retry_backoff() -> Result<()> { + let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + // The Deploy is still building, so the pass past the deadline retries + // forward and arms the continuity backoff. + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + pass_deadline_of(&world, &owner_id)?; + waiter_then_owner(&world, "", &owner_id)?; + let armed = *world + .engine + .resume_backoff + .lock() + .unwrap() + .get(&owner_id) + .context("the unmoved step set no backoff")?; + assert_eq!(armed.ceiling_millis, CONTINUITY_BACKOFF_CEILING_MILLIS); + // It reaches the handover: the next pass yields, and the abort runs + // with no clearing and no fake time. + rephase(&world, &deploy_id, DeploymentPhase::Warming)?; + waiter_then_owner(&world, "", &owner_id)?; + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + for _ in 0..3 { + waiter_then_owner(&world, "", &owner_id)?; + } + let done = latest_id(&world, &owner_id)?; + let cleaned = done.post_fencing_abort.as_ref().map(|abort| abort.candidate_cleanup); + assert_eq!(cleaned, Some(CleanupEvidence::Complete), "{:?}", done.last_error); + Ok(()) + } // END c2d continuity-yields #[test] @@ -23676,6 +23845,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide LeaseRevokedUnderIt, DeployWithRunningCandidate, CompletedRecord, + DeployAwaitsTheTarget, } for case in [ Case::CandidateNotRunning, @@ -23684,6 +23854,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Case::LeaseRevokedUnderIt, Case::DeployWithRunningCandidate, Case::CompletedRecord, + Case::DeployAwaitsTheTarget, ] { let deploy = matches!(case, Case::DeployWithRunningCandidate); // The cleanup comes after the lease withdrawal, so a case with @@ -23713,11 +23884,17 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide lease_driver(&routed).0.revoke_exact(Some(&lease))?; } Case::DeployWithRunningCandidate => {} + // The abort of the Odin-shaped negative, begun to yield: a + // Deploy awaits the handover of its target. + Case::DeployAwaitsTheTarget => { + seeded_transaction(&routed.world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + } Case::CompletedRecord => { routed.resume_until_terminal()?; } } - let abort = routed.transaction().ok().and_then(|t| t.post_fencing_abort); + let owner_id = the_continuity(&routed.world).ok().map(|c| c.value.transaction_id); + let abort = the_continuity(&routed.world).ok().and_then(|c| c.value.post_fencing_abort); match case { Case::LeaseWithdrawn => assert_eq!( abort.as_ref().map(|a| (a.lease_withdrawal, a.candidate_cleanup)), @@ -23738,6 +23915,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide after.transactions.iter().map(|t| &t.envelope).collect::>(), "the lift wrote a record it left" ); + if let (Case::DeployAwaitsTheTarget, Some(owner)) = (case, &owner_id) { + assert!(abort.is_some(), "the lift withdrew the yield"); + clear_other_live_transactions(&routed.world, owner)?; + } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), From a8683212598053a2bbce82ca3a454e056de82e6f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:06:00 +0200 Subject: [PATCH 34/47] c2d yield table: name the row an error comes from Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e2494f9..dc57b9d 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17780,7 +17780,7 @@ mod tests { for _ in 0..5 { waiter_then_owner(&world, &deploy_id, &owner_id)?; } - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(i)")?.value.post_fencing_abort.is_none()); let waiting = resident_id(&world, &deploy_id)?.value; assert!(waiting.pre_fencing_abort.is_none() && waiting.completion.is_none()); assert!(waiting.last_error.as_deref().is_some_and(|note| note.contains(&owner_id))); @@ -17794,7 +17794,7 @@ mod tests { for _ in 0..4 { waiter_then_owner(&world, "", &owner_id)?; } - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(ii)")?.value.post_fencing_abort.is_none()); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); @@ -17803,7 +17803,7 @@ mod tests { world.engine.begin_pre_fencing_abort(&resident_id(&world, &deploy_id)?, anyhow!("refused"))?; pass_deadline_of(&world, &owner_id)?; waiter_then_owner(&world, "", &owner_id)?; - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(iii)")?.value.post_fencing_abort.is_none()); // (iv) Nor does a Deploy that has ended. let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; @@ -17818,7 +17818,7 @@ mod tests { assert!(latest_id(&world, &deploy_id)?.completion.is_some()); pass_deadline_of(&world, &owner_id)?; waiter_then_owner(&world, "", &owner_id)?; - assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(resident_id(&world, &owner_id).with_context(|| "(iv)")?.value.post_fencing_abort.is_none()); Ok(()) } From fd6e3e65af2aa781c39b638aeb42ea773a421e62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:06:48 +0200 Subject: [PATCH 35/47] c2d yield table: show how row (i) ends --- src/control_plane.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index dc57b9d..5db88d4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17779,6 +17779,8 @@ mod tests { let stops = stops_of(&workload); for _ in 0..5 { waiter_then_owner(&world, &deploy_id, &owner_id)?; + let owner = latest_id(&world, &owner_id)?; + assert!(owner.completion.is_none(), "(i) {:?} {:?} {:?}", owner.completion, owner.phase, owner.last_error); } assert!(resident_id(&world, &owner_id).with_context(|| "(i)")?.value.post_fencing_abort.is_none()); let waiting = resident_id(&world, &deploy_id)?.value; From f79deecdb70ce6a5b845ca178850e961c01c247a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:07:30 +0200 Subject: [PATCH 36/47] c2d tests: the helper stops short of the deadline; each test passes it --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5db88d4..3b2d51e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17721,7 +17721,6 @@ mod tests { deploy_beside_continuity(&world, DeploymentPhase::Warming, DeploymentPhase::Fencing)?; let (deploy_id, owner_id) = (deploy.transaction_id.clone(), continuity.transaction_id.clone()); drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::AwaitingReady)?; - pass_deadline_of(&world, &owner_id)?; Ok((world, workload, deploy, deploy_id, owner_id, incumbent)) } @@ -17730,6 +17729,7 @@ mod tests { #[test] fn a_waiting_deployment_lands_over_a_continuity_that_retries_forward_past_its_deadline() -> Result<()> { let (world, workload, deploy, deploy_id, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + pass_deadline_of(&world, &owner_id)?; let stops = workload.stops.load(std::sync::atomic::Ordering::SeqCst); waiter_then_owner(&world, &deploy_id, &owner_id)?; let waiter = latest_id(&world, &deploy_id)?; From 09bd12764323b3dc4d023041ca16bd7c845c164f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:09:28 +0200 Subject: [PATCH 37/47] c2d: stateful yield test, an Odin-shaped continuity yields its lease to a waiting Deploy Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3b2d51e..49fe7f9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -23834,6 +23834,76 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// NEGATIVE. The live shape of tx-4eaed669 (Odin) left to the operator + /// at its Routing deadline, with a Deploy now awaiting the handover of + /// its target. The continuity yields through its own abort: the lease + /// leaves the disk only after the stop, the target is free, and the + /// Deploy, unended, fences a path that is already empty. + #[cfg(unix)] + #[test] + fn an_odin_shaped_continuity_left_to_the_operator_yields_its_lease_to_a_waiting_deploy() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + assert!(routed.transaction()?.lease_adoption.is_some()); + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + let owner_id = routed.transaction()?.transaction_id; + pass_deadline(&routed.world)?; + routed.resume()?; + let left = routed.transaction()?; + assert!(left.post_fencing_abort.is_none(), "the continuity aborted with nothing awaiting the target"); + assert_eq!(routed.world.engine.incidents().open_subjects(IncidentCondition::OperatorRequired)?, [owner_id.as_str()]); + assert!(driver.observe_exact(&lease)?); + + let deploy = seeded_transaction(&routed.world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + for _ in 0..12 { + routed.resume()?; + let owner = latest_id(&routed.world, &owner_id)?; + if owner.post_fencing_abort.is_none() { + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(driver.observe_exact(&lease)?, "the lease left before the yield"); + } + if owner.completion.is_some() { + break; + } + } + let ended = latest_id(&routed.world, &owner_id)?; + assert!( + matches!( + ended.completion, + Some(TransactionCompletion::FailedAfterFencing { recovery: TerminalRecovery::RestartAdmitted, .. }) + ), + "{:?}", + ended.completion + ); + assert!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst) > stops); + assert!(!driver.observe_exact(&lease)?, "the lease stayed on disk"); + assert!(ControlSnapshot::read(&routed.world.state_store)?.target_owner("service", None).is_none()); + let waiting = resident_id(&routed.world, &deploy.transaction_id)?.value; + assert!(waiting.completion.is_none() && waiting.pre_fencing_abort.is_none()); + + // Its next move is the fence, over a lease path that is empty. + rephase(&routed.world, &deploy.transaction_id, DeploymentPhase::Fencing)?; + routed.world.engine.advance_transaction(&resident_id(&routed.world, &deploy.transaction_id)?)?; + let fenced = latest_id(&routed.world, &deploy.transaction_id)?; + assert!( + matches!( + fenced.fencing, + Some(FencingEvidence::Revoked { candidate_lease_path_verified_empty: true, .. }) + ), + "{:?} {:?}", + fenced.fencing, + fenced.last_error + ); + Ok(()) + } + /// The lift is a function of its inputs: each case lacks one thing the /// lift requires, and its abort stays and runs to its end. #[cfg(unix)] From 4e66a65758873eab4d3f41ef395a0096dc0ff9bd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:10:18 +0200 Subject: [PATCH 38/47] c2d stateful yield test: drop the Fencing half the seeded Deploy cannot honestly assert Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 49fe7f9..d872c4e 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -23838,7 +23838,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// at its Routing deadline, with a Deploy now awaiting the handover of /// its target. The continuity yields through its own abort: the lease /// leaves the disk only after the stop, the target is free, and the - /// Deploy, unended, fences a path that is already empty. + /// Deploy is unended. (The Fencing half is not asserted here: the seeded + /// Deploy plans a stateless target, so its fence skips the lease path.) #[cfg(unix)] #[test] fn an_odin_shaped_continuity_left_to_the_operator_yields_its_lease_to_a_waiting_deploy() -> Result<()> { @@ -23887,20 +23888,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(ControlSnapshot::read(&routed.world.state_store)?.target_owner("service", None).is_none()); let waiting = resident_id(&routed.world, &deploy.transaction_id)?.value; assert!(waiting.completion.is_none() && waiting.pre_fencing_abort.is_none()); - - // Its next move is the fence, over a lease path that is empty. - rephase(&routed.world, &deploy.transaction_id, DeploymentPhase::Fencing)?; - routed.world.engine.advance_transaction(&resident_id(&routed.world, &deploy.transaction_id)?)?; - let fenced = latest_id(&routed.world, &deploy.transaction_id)?; - assert!( - matches!( - fenced.fencing, - Some(FencingEvidence::Revoked { candidate_lease_path_verified_empty: true, .. }) - ), - "{:?} {:?}", - fenced.fencing, - fenced.last_error - ); Ok(()) } From 81bfb037123d97287c024fdab407404253233a1a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 20:11:10 +0200 Subject: [PATCH 39/47] route-continuity-target invariant 5: the continuity yields to a deployment awaiting the handover Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index c4d3bc8..c014a2a 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -57,7 +57,10 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: incumbent. A continuity past its deadline is left to the operator (operator-required incident) and retries its phase forward on a capped backoff, because aborting it would stop the only running copy of the admitted - body (ruling post-fence-forward-retry). A stateful deployment is left to the + body (ruling post-fence-forward-retry), until a deployment awaits the + handover of its target: then it yields through its own post-fencing abort, + and the deployment fences what is admitted (ruling + continuity-yields-to-deploy). A stateful deployment is left to the operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never From 11f7fdbf09dc4c6447eacb86f83c525f03d828c7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:28:06 +0200 Subject: [PATCH 40/47] c2c fix 1/3: negative test, a lift that cannot observe must not let the abort stop the candidate Fails at 81bfb03: an observe error leaves the abort running and its first step stops the candidate. Also pins the lease_withdrawal==Complete conjunct (case LeaseWithdrawalRecordedComplete) and makes the refusal table's dead-candidate case provably dead (SwitchWorkload::bury). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 106 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d872c4e..e7297b0 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14522,6 +14522,10 @@ mod tests { stops: std::sync::atomic::AtomicUsize, /// Runtime instances whose stop fails, whatever is asked. failing_stops: std::sync::Mutex>, + /// The unit is absent or failed, which the driver reports as + /// permanently stopped: positive evidence of death, unlike an + /// observation that merely errs. + buried: std::sync::atomic::AtomicBool, } impl SwitchWorkload { @@ -14535,9 +14539,16 @@ mod tests { holds: std::sync::Mutex::new(Vec::new()), stops: std::sync::atomic::AtomicUsize::new(0), failing_stops: std::sync::Mutex::new(Vec::new()), + buried: std::sync::atomic::AtomicBool::new(false), }) } + /// The process is dead and the driver can prove it. + fn bury(&self) { + self.kill(); + self.buried.store(true, std::sync::atomic::Ordering::SeqCst); + } + /// From now on a start launches a process distinct from every recorded one. fn launching(&self) { self.launches.store(true, std::sync::atomic::Ordering::SeqCst); @@ -14584,6 +14595,8 @@ mod tests { } fn revive(&self) { + self.buried + .store(false, std::sync::atomic::Ordering::SeqCst); self.alive .store(true, std::sync::atomic::Ordering::SeqCst); self.rebooted @@ -14695,7 +14708,7 @@ mod tests { Ok(()) } fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { - Ok(false) + Ok(self.buried.load(std::sync::atomic::Ordering::SeqCst)) } fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { Ok(self.rebooted.load(std::sync::atomic::Ordering::SeqCst)) @@ -23834,6 +23847,81 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// NEGATIVE. The boot lift runs before anything it needs may answer: a + /// systemctl error, a host-actuator hub no actuator has reached. The + /// abort of a candidate it could not observe is held, not run (its first + /// step stops the candidate), and the lift is retried on later passes. + /// Once observation answers Ok the record is lifted; once it answers + /// dead the abort proceeds. + #[cfg(unix)] + #[test] + fn a_boot_lift_that_cannot_observe_holds_the_abort_and_retries() -> Result<()> { + for hub_unattached in [false, true] { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let (lease, lease_sha256) = lease_of(&routed)?; + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(lease_sha256); + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + let (driver, path) = lease_driver(&routed); + routed.workload.hold_lease_lock(&path); + routed.stub.hang_up.store(true, Ordering::SeqCst); + wedge_abort(&routed, "the Routing phase passed its deadline at unix ms 1792324133638")?; + let stops = routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst); + let observation_fails = || { + if hub_unattached { + routed.workload.silence(true); + } else { + routed.workload.kill(); + } + }; + + // Boot: observation errs. Nothing is lifted and nothing stops. + observation_fails(); + let returned = routed.world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty(), "lifted {returned:?} it could not observe"); + for _ in 0..5 { + routed.resume()?; + assert_eq!( + routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), + stops, + "an undecided abort stopped the candidate (hub_unattached {hub_unattached})" + ); + } + let held = routed.transaction()?; + let abort = held.post_fencing_abort.context("the abort was dropped")?; + assert_eq!(abort.lease_withdrawal, CleanupEvidence::Pending); + assert!(driver.observe_exact(&lease)?, "the lease left the disk"); + + // Observation answers Ok: the next pass lifts the record. + routed.workload.silence(false); + routed.workload.revive(); + routed.resume()?; + let lifted = routed.transaction()?; + assert!(lifted.post_fencing_abort.is_none(), "a later pass did not lift the record"); + assert_eq!(lifted.phase, DeploymentPhase::Routing); + assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + } + + // Observation answers dead: the abort proceeds to its end. + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + wedge_abort(&routed, "the AwaitingReady phase passed its deadline")?; + routed.workload.silence(true); + assert!(routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty()); + routed.resume()?; + assert!(routed.transaction()?.post_fencing_abort.is_some_and(|a| a.candidate_cleanup == CleanupEvidence::Pending)); + routed.workload.silence(false); + routed.workload.bury(); + let finished = routed.resume_until_terminal()?; + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), + "{:?}", + finished.completion + ); + Ok(()) + } + /// NEGATIVE. The live shape of tx-4eaed669 (Odin) left to the operator /// at its Routing deadline, with a Deploy now awaiting the handover of /// its target. The continuity yields through its own abort: the lease @@ -23899,6 +23987,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[derive(Clone, Copy, Debug)] enum Case { CandidateNotRunning, + LeaseWithdrawalRecordedComplete, LeaseWithdrawn, CandidateCleaned, LeaseRevokedUnderIt, @@ -23908,6 +23997,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } for case in [ Case::CandidateNotRunning, + Case::LeaseWithdrawalRecordedComplete, Case::LeaseWithdrawn, Case::CandidateCleaned, Case::LeaseRevokedUnderIt, @@ -23936,13 +24026,25 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide // step, or one input, away from the lift's predicate. routed.resume()?; match case { - Case::CandidateNotRunning => routed.workload.kill(), + Case::CandidateNotRunning => routed.workload.bury(), Case::LeaseWithdrawn | Case::CandidateCleaned => routed.resume()?, Case::LeaseRevokedUnderIt => { let (lease, _) = lease_of(&routed)?; lease_driver(&routed).0.revoke_exact(Some(&lease))?; } Case::DeployWithRunningCandidate => {} + // The record says the lease was withdrawn while the candidate + // still runs and the lease is still exact on disk: only the + // record's own evidence keeps the lift off it. + Case::LeaseWithdrawalRecordedComplete => { + let stored = resident(&routed.world)?; + let mut next = stored.value.clone(); + next.post_fencing_abort + .as_mut() + .context("no abort")? + .lease_withdrawal = CleanupEvidence::Complete; + replace_transaction(&routed.world.state_store, &stored, &next)?; + } // The abort of the Odin-shaped negative, begun to yield: a // Deploy awaits the handover of its target. Case::DeployAwaitsTheTarget => { From 35d9c7fcf8b3f11be03f0ef0878714e8f70d8807 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:30:18 +0200 Subject: [PATCH 41/47] c2c fix 2/3: the lift defers on an observe error; the retry exit is worded from the driver An abort over a candidate the boot lift could not observe is held in process memory: resume_candidate runs no step of it and retries the lift each pass (retry_held_lift) until observation answers Ok (lifted) or the driver proves the candidate dead (the abort proceeds). The predicate and write move into consider_lifting_continuity_abort, which boot and the retry share. Ruling c2c-lift-defers-on-observe-error. The page and the expire refusal name the manual exit from the plan's workload binding: a unit to stop for systemd, a process to kill for a host actuator. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 233 ++++++++++++++++++++++++++++++++----------- 1 file changed, 173 insertions(+), 60 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index e7297b0..786486f 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1863,6 +1863,23 @@ impl DeploymentTransaction { && !(self.command_kind == CommandKind::Continuity && self.phase >= DeploymentPhase::Fencing) } + /// The operator's only way to restart a forward-retrying continuity, worded + /// from the driver its plan binds: a systemd target has a unit to stop (a + /// stopped transient unit is forgotten, which is permanently stopped); a + /// host-actuator target has no unit, only a process on the managed host + /// that the actuator then reports not running. + fn manual_exit(&self) -> &'static str { + match self + .plan + .as_ref() + .and_then(|plan| plan.parsed_inputs().ok()) + .map(|(_, binding)| binding.workload) + { + Some(WorkloadBinding::HostActuator(_)) => "kill its process on the managed host by hand", + _ => "stop its unit by hand", + } + } + /// A continuity at or past Fencing that its deadline leaves to the operator /// while its step runs. It yields to a deployment awaiting the handover /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. @@ -3159,9 +3176,12 @@ fn expire(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> current.value.expirable(), "{}", if current.value.command_kind == CommandKind::Continuity && current.value.phase >= DeploymentPhase::Fencing && current.value.post_fencing_abort.is_none() { - "a continuity past Fencing retries forward and cannot be expired; to restart it, stop its candidate unit and Idunn aborts it as dead" + format!( + "a continuity past Fencing retries forward and cannot be expired; to restart it, {} and Idunn aborts it as dead", + current.value.manual_exit() + ) } else { - "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel" + "only a live transaction in Warming through Committing, and not already aborting, can be expired; before Warming, use idunn cancel".to_owned() } ); let now = now_millis()?; @@ -5097,6 +5117,10 @@ struct Engine { /// Process-local on purpose: a restart retries at once, and no schema /// carries a clock that only pacing reads. resume_backoff: Mutex>, + /// Continuity aborts the boot lift could not decide (their candidate could + /// not be observed): no step of them runs until `retry_held_lift` decides. + /// Process-local: a restart asks again at boot. + lift_held: Mutex>, /// When each routed target's next challenge is due, held only when the /// stored `RouteSupervisionState` could not record the last one. The stored /// value owns the schedule and survives restarts; this spaces attempts @@ -5209,6 +5233,7 @@ impl Engine { history_report: ReportOnce::default(), fault_reports: Mutex::default(), resume_backoff: Mutex::default(), + lift_held: Mutex::default(), challenge_wait: Mutex::default(), }) } @@ -5682,6 +5707,17 @@ struct ProjectionReconciliation { unexplained: Vec, } +/// What the lift decided about one continuity abort. +enum LiftDecision { + /// The abort was withdrawn and the record retimed. + Lifted, + /// The abort stands and runs: not a lift candidate, or its candidate is + /// provably dead. + KeepAborting, + /// The candidate could not be observed and is not provably dead. + Undecided(anyhow::Error), +} + impl Engine { /// Once at boot: withdraw the abort of a continuity that has not stopped /// its running candidate, and retime the record so its phase retries @@ -5694,73 +5730,126 @@ impl Engine { /// or to yield to a deployment awaiting the handover, so this predicate /// matches only aborts over a running candidate with nothing awaiting the /// target, which Idunn no longer creates: it is a migration with a deletion - /// line (follow-up c2c-delete-boot-lift). A record it cannot decide is - /// reported and left aborting. + /// line (follow-up c2c-delete-boot-lift). A record whose candidate cannot + /// be observed is held, not aborted: `resume_candidate` runs no step of it + /// and asks `retry_held_lift` each pass, until observation answers (ruling + /// c2c-lift-defers-on-observe-error). fn return_wedged_continuity_aborts_forward(&self) -> Result> { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let mut returned = Vec::new(); for stored in &snapshot.transactions { - let transaction = &stored.value; - let Some(abort) = &transaction.post_fencing_abort else { - continue; - }; - if transaction.command_kind != CommandKind::Continuity - || transaction.completion.is_some() - || abort.candidate_cleanup != CleanupEvidence::Pending - || abort.lease_withdrawal == CleanupEvidence::Complete - || snapshot - .on_target(&transaction.target) - .any(|other| other.value.awaits_handover()) - { - continue; - } - let id = &transaction.transaction_id; - let lifted = (|| -> Result { - let plan = required(&transaction.plan, "transaction plan")?; - let expected = required(&transaction.expected, "Expected projection")?; - let activation = required(&transaction.activation, "activation")?; - let Some(workload) = &transaction.workload else { - return Ok(false); - }; - self.workload_for(plan)? - .observe(expected, activation, workload)?; - if let Some(lease) = transaction - .leasing - .as_ref() - .and_then(LeasingEvidence::issued_lease) - { - let path = plan - .parsed_inputs()? - .1 - .process_write_lease - .context("leased target has no write-lease binding")? - .record_path; - if !CultCacheWriteLeaseDriver::new(&transaction.target, path) - .observe_exact(lease)? - { - return Ok(false); - } + let id = &stored.value.transaction_id; + match self.consider_lifting_continuity_abort(&snapshot, stored) { + LiftDecision::Lifted => returned.push(id.clone()), + LiftDecision::KeepAborting => {} + LiftDecision::Undecided(error) => { + eprintln!( + "Idunn could not decide whether continuity {id} may retry forward; its abort is held, not run: {error:#}" + ); + self.lift_held.lock().expect("lift held mutex").insert(id.clone()); } - let now = now_millis()?; - let mut next = transaction.clone(); - next.post_fencing_abort = None; - next.last_error = Some(abort.error.clone()); - next.updated_at_unix_millis = now; - next.retime(now); - replace_transaction(&self.options.state_store, stored, &next)?; - Ok(true) - })(); - match lifted { - Ok(true) => returned.push(id.clone()), - Ok(false) => {} - Err(error) => eprintln!( - "Idunn could not decide whether continuity {id} may retry forward; its abort is left running: {error:#}" - ), } } Ok(returned) } + /// A held record's later pass: lift it, release it to its abort over + /// positive evidence of death, or keep holding. `Some(moved)` means the + /// pass is spent on the record; `None` lets the pass go on. + fn retry_held_lift(&self, current: &Stored) -> Result> { + let id = current.value.transaction_id.as_str(); + if !self.lift_held.lock().expect("lift held mutex").contains(id) { + return Ok(None); + } + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + match self.consider_lifting_continuity_abort(&snapshot, current) { + LiftDecision::Undecided(error) => { + self.note_fault("holds continuity abort", id, &error); + Ok(Some(false)) + } + LiftDecision::Lifted => { + self.lift_held.lock().expect("lift held mutex").remove(id); + self.clear_fault(id); + println!( + "returned continuity {id} to forward retry: its abort had not stopped its running candidate" + ); + Ok(Some(true)) + } + LiftDecision::KeepAborting => { + self.lift_held.lock().expect("lift held mutex").remove(id); + self.clear_fault(id); + Ok(None) + } + } + } + + /// The lift's predicate and write for one record. An abort proceeds only + /// over positive evidence that its candidate is dead; an observation that + /// merely errs is undecided. + fn consider_lifting_continuity_abort( + &self, + snapshot: &ControlSnapshot, + stored: &Stored, + ) -> LiftDecision { + let transaction = &stored.value; + let Some(abort) = &transaction.post_fencing_abort else { + return LiftDecision::KeepAborting; + }; + if transaction.command_kind != CommandKind::Continuity + || transaction.completion.is_some() + || abort.candidate_cleanup != CleanupEvidence::Pending + || abort.lease_withdrawal == CleanupEvidence::Complete + || snapshot + .on_target(&transaction.target) + .any(|other| other.value.awaits_handover()) + { + return LiftDecision::KeepAborting; + } + let lifted = (|| -> Result { + let plan = required(&transaction.plan, "transaction plan")?; + let expected = required(&transaction.expected, "Expected projection")?; + let activation = required(&transaction.activation, "activation")?; + let Some(workload) = &transaction.workload else { + return Ok(false); + }; + self.workload_for(plan)? + .observe(expected, activation, workload)?; + if let Some(lease) = transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::issued_lease) + { + let path = plan + .parsed_inputs()? + .1 + .process_write_lease + .context("leased target has no write-lease binding")? + .record_path; + if !CultCacheWriteLeaseDriver::new(&transaction.target, path) + .observe_exact(lease)? + { + return Ok(false); + } + } + let now = now_millis()?; + let mut next = transaction.clone(); + next.post_fencing_abort = None; + next.last_error = Some(abort.error.clone()); + next.updated_at_unix_millis = now; + next.retime(now); + replace_transaction(&self.options.state_store, stored, &next)?; + Ok(true) + })(); + match lifted { + Ok(true) => LiftDecision::Lifted, + Ok(false) => LiftDecision::KeepAborting, + Err(error) => match self.candidate_is_permanently_stopped(transaction) { + Ok(true) => LiftDecision::KeepAborting, + _ => LiftDecision::Undecided(error), + }, + } + } + /// Once at boot: demote the activation a failed continuity left standing, /// where the projection still names exactly the activation that failed /// transaction issued. A continuity abort before the single resolution @@ -6086,6 +6175,9 @@ impl Engine { /// transaction would report progress every tick. fn resume_candidate(&self, current: &Stored) -> Result { let id = current.value.transaction_id.as_str(); + if let Some(moved) = self.retry_held_lift(current)? { + return Ok(moved); + } if let Some(moved) = self.resolve_phase_deadline(current)? { if moved { // A phase-ending write is movement: the next step is not held @@ -9665,7 +9757,8 @@ impl Engine { self.leave_to_operator( transaction, &format!( - "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, stop its unit by hand" + "{cause}; its candidate is the only running copy of the admitted body, so Idunn retries the phase forward and never stops it; to restart it, {}", + transaction.manual_exit() ), ); Ok(None) @@ -16173,6 +16266,21 @@ mod tests { } } + /// The operator's exit from a forward-retrying continuity is worded from + /// the driver the plan binds: a host-actuator target has no unit to stop. + #[test] + fn the_manual_exit_is_worded_from_the_targets_driver() -> Result<()> { + let (world, _deploy, host_continuity) = stop_first_world(SwitchWorkload::new())?; + let host = resident_id(&world, &host_continuity)?.value; + assert!(host.manual_exit().contains("managed host"), "{}", host.manual_exit()); + assert!(!host.manual_exit().contains("unit"), "{}", host.manual_exit()); + let systemd = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&systemd)?; + let unit = seeded_transaction(&systemd, DeploymentPhase::Fencing, CommandKind::Continuity, Some(&incumbent))?; + assert!(unit.manual_exit().contains("unit"), "{}", unit.manual_exit()); + Ok(()) + } + /// The step that ends a transaction archives it in its own write, so a /// record gone from the live set is finished only if history holds it. One /// that vanished from both is a fault, never progress. @@ -24080,6 +24188,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(abort.is_some(), "the lift withdrew the yield"); clear_other_live_transactions(&routed.world, owner)?; } + if matches!(case, Case::LeaseWithdrawalRecordedComplete) { + // The record's claim is made true, so the abort can finish. + let (lease, _) = lease_of(&routed)?; + lease_driver(&routed).0.revoke_exact(Some(&lease))?; + } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( matches!(finished.completion, Some(TransactionCompletion::FailedAfterFencing { .. })), From 1beca61ac458081fa171fbc1884c9d5e65ecaa0f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:31:36 +0200 Subject: [PATCH 42/47] c2c fix 3/3: the refusal table's recorded-withdrawal case ends at the lift Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 786486f..3f9ef43 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -24189,9 +24189,9 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide clear_other_live_transactions(&routed.world, owner)?; } if matches!(case, Case::LeaseWithdrawalRecordedComplete) { - // The record's claim is made true, so the abort can finish. - let (lease, _) = lease_of(&routed)?; - lease_driver(&routed).0.revoke_exact(Some(&lease))?; + // An edited record, not a reachable one: the lift's refusal + // is the whole claim, and its abort is not run to the end. + continue; } let finished = routed.resume_until_terminal().with_context(|| format!("{case:?}"))?; assert!( From 04389e6afc2696d895dca271a208e2625484f8ee Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:38:10 +0200 Subject: [PATCH 43/47] c2d fix 1/2: negative tests, no yield to a serving continuity or a braked Deploy Fail at 1beca61 (the c2d logic is 81bfb03's): a Committing continuity past its deadline yields to a waiting Deploy and stops serving; a stop-then-start Deploy at its Sealing tail with the deployment brake unreleased draws the yield. The second test releases the brake and pins that the yield then happens and closes the continuity's operator-required incident Recovered. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 95 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 3f9ef43..e34c1c4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17945,7 +17945,7 @@ mod tests { Ok(()) } - /// `awaits_handover` and `retries_forward` row by row. + /// `awaits_handover` and `yields_to_a_deploy` row by row. #[test] fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; @@ -17999,6 +17999,93 @@ mod tests { Ok(()) } + /// NEGATIVE. A continuity past its route proof is serving. Past its deadline + /// it keeps retrying and pages; a waiting Deploy does not stop it. + #[test] + fn a_committing_continuity_past_its_deadline_keeps_serving_and_pages() -> Result<()> { + let (world, workload, _, _, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + rephase(&world, &owner_id, DeploymentPhase::Committing)?; + pass_deadline_of(&world, &owner_id)?; + let stops = stops_of(&workload); + for _ in 0..4 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + let owner = resident_id(&world, &owner_id)?.value; + assert!(owner.post_fencing_abort.is_none(), "a serving continuity yielded"); + assert_eq!(owner.phase, DeploymentPhase::Committing); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + Ok(()) + } + + /// NEGATIVE. A Deploy held at its Sealing tail by the deployment brake + /// cannot take the handover, so a continuity does not stop its candidate + /// for it; once the brake releases, the continuity yields. The yield closes + /// the continuity's operator-required incident Recovered. + #[test] + fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + make_stop_first(&world, &deploy_id)?; + let sealing = resident_id(&world, &deploy_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + assert!(resident_id(&world, &deploy_id)?.value.awaits_handover()); + pass_deadline_of(&world, &owner_id)?; + let stops = stops_of(&workload); + for _ in 0..4 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + let held = resident_id(&world, &owner_id)?.value; + assert!(held.post_fencing_abort.is_none(), "the continuity yielded to a braked Deploy"); + assert_eq!(operator_required(&world)?, [owner_id.as_str()]); + assert_eq!(stops_of(&workload), stops); + + // The operator releases the brake for this exact Deploy. + let deploy = resident_id(&world, &deploy_id)?.value; + let authorization = deployment_authorization( + &world, + deploy.expected.as_ref().context("no Expected")?, + &deploy_id, + now_millis()?, + )?; + let store = deploy.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + assert!(SingleFileMessagePackBackingStore::new(&store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + key: record.brake_id.clone(), + current: None, + }], + &[CultCacheEnvelope { + key: record.brake_id.clone(), + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + payload: authorization.canonical_brake_bytes.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), + }], + )?); + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + let yielded = resident_id(&world, &owner_id)?.value; + let abort = yielded.post_fencing_abort.as_ref().context("the continuity did not yield")?; + assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); + // The yield retimed the record: the next pass closes its incident + // Recovered, and it stays closed. + for _ in 0..3 { + world.engine.resume_backoff.lock().unwrap().clear(); + world.engine.resume_one_transaction()?; + } + assert!(operator_required(&world)?.is_empty()); + let records = world.engine.incidents().read()?; + assert_eq!(records.len(), 1); + assert_eq!(records[0].subject, owner_id); + assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); + Ok(()) + } + /// The c2b waiter still ends on an owner that will not yield: the yield's /// own abort wedged past its deadline. #[test] @@ -18311,7 +18398,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide bind_provider( world, &format!("{head}{HOST}[runtime_identity]{identity}[brakes]{brakes}") - .replace("candidate-then-promote", "stop-then-start"), + .replace("candidate-then-promote", "stop-then-start") + .replace( + "/var/lib/gamecult/idunn/service-deployment-brake.cc", + &world.root.join("service-deployment-brake.cc").display().to_string(), + ), ) } From b119e500adb0d318de56a48a1739947c58f34532 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 21:43:34 +0200 Subject: [PATCH 44/47] c2d fix 2/2: a continuity yields only while unproven, and only to a Deploy that can move serving_proven (Committing, or routing evidence recorded) ends the yield: yields_to_a_deploy replaces retries_forward, and wait_for_owner reads it, so a Deploy no longer waits on an owner that will keep serving. deploy_can_move asks the deployment brake live for a Deploy at its Sealing tail, so a braked Deploy draws no yield and the yield happens once the brake releases. The tests of 04389e6 are repaired to reach their worlds (the failures they showed at 1beca61 were the right ones). Rulings c2d-yield-only-unproven-to-movable. Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-target.md | 12 ++-- src/control_plane.rs | 109 ++++++++++++++++++++++++-------- 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/docs/route-continuity-target.md b/docs/route-continuity-target.md index c014a2a..7c25d0c 100644 --- a/docs/route-continuity-target.md +++ b/docs/route-continuity-target.md @@ -57,11 +57,13 @@ web app and service through Idunn v2. The live evidence is on Yggdrasil: incumbent. A continuity past its deadline is left to the operator (operator-required incident) and retries its phase forward on a capped backoff, because aborting it would stop the only running copy of the admitted - body (ruling post-fence-forward-retry), until a deployment awaits the - handover of its target: then it yields through its own post-fencing abort, - and the deployment fences what is admitted (ruling - continuity-yields-to-deploy). A stateful deployment is left to the - operator. + body (ruling post-fence-forward-retry), until a deployment that the + deployment brake has released awaits the handover of its target and the + continuity's route proof has not succeeded: then it yields through its own + post-fencing abort, and the deployment fences what is admitted (rulings + continuity-yields-to-deploy, c2d-yield-only-unproven-to-movable). A + continuity that has proven its route is serving and never yields. A + stateful deployment is left to the operator. 6. **The admitted receipt describes the running incarnation.** Continuity commits update the admitted generation atomically, so the rollback path never meets drift between the receipt and the projection. Rollback demotes the exact diff --git a/src/control_plane.rs b/src/control_plane.rs index e34c1c4..5d1389a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1880,12 +1880,23 @@ impl DeploymentTransaction { } } + /// The candidate's route proof has succeeded: it is Committing, or its + /// routing evidence is recorded and only the transition remains. It serves, + /// so no waiting deployment stops it (ruling c2d-yield-only-unproven-to- + /// movable). Derived, never stored. + fn serving_proven(&self) -> bool { + self.phase >= DeploymentPhase::Committing || self.routing.is_some() + } + /// A continuity at or past Fencing that its deadline leaves to the operator - /// while its step runs. It yields to a deployment awaiting the handover - /// (ruling continuity-yields-to-deploy), so a waiter never ends on it. - /// Derived, never stored. - fn retries_forward(&self) -> bool { - self.resolves_by_deadline() && self.phase_end() == PhaseEnd::RetryForward + /// while its step runs, and whose candidate is still unproven: it yields to + /// a deployment that can take the handover (ruling continuity-yields-to- + /// deploy), so a waiter never ends on it. A proven continuity keeps serving + /// and paging and does not yield. Derived, never stored. + fn yields_to_a_deploy(&self) -> bool { + self.resolves_by_deadline() + && self.phase_end() == PhaseEnd::RetryForward + && !self.serving_proven() } /// What an ended post-fencing phase resolves to (Q3). A continuity at or @@ -8546,6 +8557,17 @@ impl Engine { })) } + /// Whether a deployment awaiting the handover is free to take it: the + /// deployment brake has released its exact transaction, so the stop a yield + /// makes is followed by the deployment's own landing. A Deploy at its + /// Sealing tail has not passed the brake yet and is read live; one that is + /// still building, or whose brake cannot be read, is held. An operator's + /// hold is a reason to leave a serving or retrying body alone. + fn deploy_can_move(&self, deploy: &Stored, now: u64) -> bool { + deploy.value.deployment_authorization.is_some() + || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) + } + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { let plan = required(¤t.value.plan, "continuity plan")?; let binding = plan.parsed_inputs()?.1; @@ -8673,7 +8695,7 @@ impl Engine { if snapshot.transactions.iter().any(|stored| { stored.value.transaction_id == owner_id && stored.value.leaves_to_operator(now) - && !stored.value.retries_forward() + && !stored.value.yields_to_a_deploy() }) { bail!( "owner-left-to-operator: target {} is owned by {owner_id}, which Idunn has left to the operator; this deployment ends before fencing", @@ -9740,10 +9762,12 @@ impl Engine { } PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - if let Some(deploy) = snapshot - .on_target(&transaction.target) - .find(|other| other.value.awaits_handover()) - { + let now = now_millis()?; + if let Some(deploy) = snapshot.on_target(&transaction.target).find(|other| { + transaction.yields_to_a_deploy() + && other.value.awaits_handover() + && self.deploy_can_move(other, now) + }) { self.begin_post_fencing_abort( current, anyhow!( @@ -17947,7 +17971,7 @@ mod tests { /// `awaits_handover` and `yields_to_a_deploy` row by row. #[test] - fn awaits_handover_and_retries_forward_are_exact_functions_of_the_record() -> Result<()> { + fn awaits_handover_and_yields_to_a_deploy_are_exact_functions_of_the_record() -> Result<()> { let (world, _, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; let waiting = resident_id(&world, &deploy_id)?.value; assert_eq!(waiting.phase, DeploymentPhase::Warming); @@ -17980,22 +18004,29 @@ mod tests { sealing.phase = DeploymentPhase::Warming; assert!(!sealing.awaits_handover()); - // A continuity retries forward from Fencing to Committing, until aborting. + // A continuity yields from Fencing until its route proof succeeds, and + // until aborting; a proven one keeps serving. for phase in [ DeploymentPhase::Fencing, DeploymentPhase::Leasing, DeploymentPhase::AwaitingReady, DeploymentPhase::Routing, - DeploymentPhase::Committing, ] { continuity.phase = phase; - assert!(continuity.retries_forward(), "{phase:?}"); + assert!(continuity.yields_to_a_deploy(), "{phase:?}"); + assert!(!continuity.serving_proven(), "{phase:?}"); } + continuity.phase = DeploymentPhase::Committing; + assert!(continuity.serving_proven() && !continuity.yields_to_a_deploy()); + continuity.phase = DeploymentPhase::Routing; + continuity.routing = Some(RoutingEvidence::SkippedUnrouted); + assert!(continuity.serving_proven() && !continuity.yields_to_a_deploy()); + continuity.routing = None; let mut deploy_record = waiting.clone(); deploy_record.phase = DeploymentPhase::Fencing; - assert!(!deploy_record.retries_forward()); + assert!(!deploy_record.yields_to_a_deploy()); continuity.post_fencing_abort = Some(post_fencing_abort_intent(&continuity, "x")); - assert!(!continuity.retries_forward()); + assert!(!continuity.yields_to_a_deploy()); Ok(()) } @@ -18003,15 +18034,43 @@ mod tests { /// it keeps retrying and pages; a waiting Deploy does not stop it. #[test] fn a_committing_continuity_past_its_deadline_keeps_serving_and_pages() -> Result<()> { - let (world, workload, _, _, owner_id, _) = waiter_beside_gate_waiting_continuity()?; - rephase(&world, &owner_id, DeploymentPhase::Committing)?; + let (world, workload, _, _, owner_id, incumbent) = waiter_beside_gate_waiting_continuity()?; + let owner = resident_id(&world, &owner_id)?.value; + // The continuity's incarnation already has a correlation (the + // incumbent's, same release): Odin re-stamps it Ready for this activation. + let expected = owner.expected.as_ref().context("no Expected")?; + let backing = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); + let key = crate::drivers::incarnation_key_of(&expected.target, &expected.canonical_sha256()?); + let current = backing + .pull_all_read_only_snapshot()? + .into_iter() + .find(|entry| entry.key == key); + assert!(backing.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + key: key.clone(), + current, + }], + &[CultCacheEnvelope { + key, + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: signed_correlation(&world, &owner, seeded_sequence(Some(&incumbent)) + 10, true)?, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into()), + }], + )?); + let phases = drive_id(&world, &owner_id, |t| t.phase == DeploymentPhase::Committing)?; + let at = latest_id(&world, &owner_id)?; + assert_eq!(at.phase, DeploymentPhase::Committing, "{phases:?} {:?} {:?}", at.completion, at.last_error); + // Its Committing step now fails on every attempt: the body is gone. + workload.kill(); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); for _ in 0..4 { world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; } - let owner = resident_id(&world, &owner_id)?.value; + let owner = latest_id(&world, &owner_id)?; assert!(owner.post_fencing_abort.is_none(), "a serving continuity yielded"); assert_eq!(owner.phase, DeploymentPhase::Committing); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); @@ -18026,25 +18085,25 @@ mod tests { #[test] fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; - make_stop_first(&world, &deploy_id)?; - let sealing = resident_id(&world, &deploy_id)?; + make_stop_first(&world, &deploy_id).context("A make_stop_first")?; + let sealing = resident_id(&world, &deploy_id).context("B sealing")?; let mut tail = sealing.value.clone(); tail.deployment_authorization = None; replace_transaction(&world.state_store, &sealing, &tail)?; - assert!(resident_id(&world, &deploy_id)?.value.awaits_handover()); + assert!(resident_id(&world, &deploy_id).context("C awaits")?.value.awaits_handover()); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); for _ in 0..4 { world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; } - let held = resident_id(&world, &owner_id)?.value; + let held = latest_id(&world, &owner_id)?; assert!(held.post_fencing_abort.is_none(), "the continuity yielded to a braked Deploy"); assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); // The operator releases the brake for this exact Deploy. - let deploy = resident_id(&world, &deploy_id)?.value; + let deploy = resident_id(&world, &deploy_id).context("E deploy")?.value; let authorization = deployment_authorization( &world, deploy.expected.as_ref().context("no Expected")?, @@ -18069,7 +18128,7 @@ mod tests { )?); world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; - let yielded = resident_id(&world, &owner_id)?.value; + let yielded = resident_id(&world, &owner_id).context("F yielded")?.value; let abort = yielded.post_fencing_abort.as_ref().context("the continuity did not yield")?; assert!(abort.error.starts_with("continuity-yields") && abort.error.contains(&deploy_id), "{}", abort.error); // The yield retimed the record: the next pass closes its incident From b993615c69eb34dc8d07e1eb47af8260b100f952 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:47:48 +0200 Subject: [PATCH 45/47] Negative tests: the lift must not yield to a braked waiting Deploy Two tests, from Soul's h2 attacks: a running candidate at boot beside a braked stop-then-start Deploy at its Sealing tail, and a held (unobservable) candidate when such a Deploy appears. Both must leave the candidate unstopped. They fail here: the lift treats any awaits_handover Deploy as a reason to keep aborting. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 68 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5d1389a..bf08f52 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -17914,6 +17914,74 @@ mod tests { workload.stops.load(std::sync::atomic::Ordering::SeqCst) } + /// A Deploy at its Sealing tail, braked, beside a continuity: it awaits the + /// handover but cannot take it (the brake has not released it). + fn braked_sealing_deploy(world: &EngineFixture, deploy_id: &str) -> Result<()> { + make_stop_first(world, deploy_id)?; + let sealing = resident_id(world, deploy_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + let tail = resident_id(world, deploy_id)?.value; + assert!(tail.awaits_handover()); + assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?, now_millis()?)); + Ok(()) + } + + /// NEGATIVE. The boot lift yields to a Deploy only by the continuity's own + /// yield test: a braked Deploy awaiting the handover is no reason to keep a + /// pre-c2c abort running over the only copy of the body. + #[test] + fn the_boot_lift_returns_a_running_candidate_forward_beside_a_braked_waiting_deploy() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + braked_sealing_deploy(&world, &deploy_id)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + let returned = world.engine.return_wedged_continuity_aborts_forward()?; + assert_eq!(returned, vec![owner_id.clone()]); + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the running candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + Ok(()) + } + + /// NEGATIVE. A held record (its candidate unobservable, not provably dead) + /// is not released to its abort because a braked Deploy starts awaiting the + /// handover. + #[test] + fn a_held_continuity_abort_is_not_released_by_a_braked_waiting_deploy() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + rephase(&world, &deploy_id, DeploymentPhase::Sealing)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + workload.silence(true); + let returned = world.engine.return_wedged_continuity_aborts_forward()?; + assert!(returned.is_empty()); + assert!(world.engine.lift_held.lock().unwrap().contains(&owner_id)); + for _ in 0..3 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + braked_sealing_deploy(&world, &deploy_id)?; + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the unobservable candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_some()); + assert!(world.engine.lift_held.lock().unwrap().contains(&owner_id)); + Ok(()) + } + /// The yield is a function of its inputs: the deadline, the continuity's /// own record, and a deployment that awaits the handover. #[test] From c4edc66c48ae80ea8a72ac37a450c16de4bd5ba2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:50:46 +0200 Subject: [PATCH 46/47] The boot lift yields to a Deploy by the continuity's own yield test The lift kept an abort running for any awaits_handover Deploy, so a braked Deploy released a held record to an abort that stopped the only copy of the body. One predicate, deploy_the_continuity_yields_to (candidate unproven, a waiting Deploy that deploy_can_move), is now what both the deadline's yield and the lift ask (rulings c2d-yield-only-unproven-to-movable, c2c-lift-defers-on-observe-error). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 46 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index bf08f52..c681051 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -2104,7 +2104,8 @@ impl DeploymentTransaction { /// A live deployment whose next move is the handover: not ended, not /// aborting, one phase short of `takes_target_at`. Derived, never stored. - /// Read by the continuity's yield and the boot lift. + /// Read by `deploy_the_continuity_yields_to`: the continuity's yield and the + /// boot lift. fn awaits_handover(&self) -> bool { self.command_kind == CommandKind::Deploy && self.completion.is_none() @@ -5810,12 +5811,20 @@ impl Engine { || transaction.completion.is_some() || abort.candidate_cleanup != CleanupEvidence::Pending || abort.lease_withdrawal == CleanupEvidence::Complete - || snapshot - .on_target(&transaction.target) - .any(|other| other.value.awaits_handover()) { return LiftDecision::KeepAborting; } + match now_millis() { + Ok(now) => { + if self + .deploy_the_continuity_yields_to(snapshot, transaction, now) + .is_some() + { + return LiftDecision::KeepAborting; + } + } + Err(error) => return LiftDecision::Undecided(error), + } let lifted = (|| -> Result { let plan = required(&transaction.plan, "transaction plan")?; let expected = required(&transaction.expected, "Expected projection")?; @@ -8568,6 +8577,25 @@ impl Engine { || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) } + /// The deployment a continuity gives way to: one on its target that awaits + /// the handover and can take it, while the continuity's own candidate is + /// unproven (ruling c2d-yield-only-unproven-to-movable). The one test the + /// deadline's yield and the boot lift's abort both ask, so neither stops a + /// serving candidate, or one a braked Deploy cannot replace. + fn deploy_the_continuity_yields_to<'a>( + &self, + snapshot: &'a ControlSnapshot, + continuity: &DeploymentTransaction, + now: u64, + ) -> Option<&'a Stored> { + if continuity.serving_proven() { + return None; + } + snapshot.on_target(&continuity.target).find(|other| { + other.value.awaits_handover() && self.deploy_can_move(other, now) + }) + } + fn lifecycle_allows(&self, current: &Stored, now: u64) -> Result { let plan = required(¤t.value.plan, "continuity plan")?; let binding = plan.parsed_inputs()?.1; @@ -9763,11 +9791,11 @@ impl Engine { PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let now = now_millis()?; - if let Some(deploy) = snapshot.on_target(&transaction.target).find(|other| { - transaction.yields_to_a_deploy() - && other.value.awaits_handover() - && self.deploy_can_move(other, now) - }) { + if let Some(deploy) = transaction + .yields_to_a_deploy() + .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction, now)) + .flatten() + { self.begin_post_fencing_abort( current, anyhow!( From 8760a4fab377d3153a5621a64336e82a4e509ee7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 20 Oct 2026 01:51:57 +0200 Subject: [PATCH 47/47] A boot lift that errs fails closed If the boot lift returned Err, boot logged and served with nothing held, so every pending continuity abort ran and stopped a running candidate. Boot now sets lift_unsettled; retry_held_lift runs the lift for each continuity abort until one succeeds, and no abort step runs before. Boot's call moves into Engine::boot_lift so the failure is testable. Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 97 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index c681051..d2a7635 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5133,6 +5133,10 @@ struct Engine { /// not be observed): no step of them runs until `retry_held_lift` decides. /// Process-local: a restart asks again at boot. lift_held: Mutex>, + /// Set when the boot lift itself failed, so no record's decision exists: + /// every continuity with a post-fencing abort is held until a later lift + /// succeeds. Process-local, like `lift_held`. + lift_unsettled: std::sync::atomic::AtomicBool, /// When each routed target's next challenge is due, held only when the /// stored `RouteSupervisionState` could not record the last one. The stored /// value owns the schedule and survives restarts; this spaces attempts @@ -5246,6 +5250,7 @@ impl Engine { fault_reports: Mutex::default(), resume_backoff: Mutex::default(), lift_held: Mutex::default(), + lift_unsettled: std::sync::atomic::AtomicBool::new(false), challenge_wait: Mutex::default(), }) } @@ -5731,6 +5736,28 @@ enum LiftDecision { } impl Engine { + /// Boot's call of the lift. A lift that errs decided nothing, so boot fails + /// closed: every continuity abort is held, none of its steps run, until + /// `retry_held_lift` completes a lift. + fn boot_lift(&self) { + match self.return_wedged_continuity_aborts_forward() { + Ok(returned) => { + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + } + Err(error) => { + self.lift_unsettled + .store(true, std::sync::atomic::Ordering::SeqCst); + eprintln!( + "Idunn boot could not lift wedged continuity aborts; every continuity abort is held until a lift succeeds: {error:#}" + ); + } + } + } + /// Once at boot: withdraw the abort of a continuity that has not stopped /// its running candidate, and retime the record so its phase retries /// forward. A pre-c2c Idunn aborted a continuity at its deadline; that @@ -5770,6 +5797,29 @@ impl Engine { /// pass is spent on the record; `None` lets the pass go on. fn retry_held_lift(&self, current: &Stored) -> Result> { let id = current.value.transaction_id.as_str(); + if current.value.command_kind == CommandKind::Continuity + && current.value.post_fencing_abort.is_some() + && self.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst) + { + match self.return_wedged_continuity_aborts_forward() { + Err(error) => { + self.note_fault("holds continuity abort", id, &error); + return Ok(Some(false)); + } + Ok(returned) => { + self.lift_unsettled + .store(false, std::sync::atomic::Ordering::SeqCst); + for transaction_id in &returned { + println!( + "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" + ); + } + if returned.iter().any(|lifted| lifted == id) { + return Ok(Some(true)); + } + } + } + } if !self.lift_held.lock().expect("lift held mutex").contains(id) { return Ok(None); } @@ -5983,16 +6033,7 @@ fn boot(options: RuntimeOptions) -> Result<(ProcessLock, Engine)> { ControlSnapshot::read(&options.state_store).context("validating all Idunn records")?; let engine = Engine::open(options)?; engine.validate_durable_authority(&ControlSnapshot::read(&engine.options.state_store)?)?; - match engine.return_wedged_continuity_aborts_forward() { - Ok(returned) => { - for transaction_id in &returned { - println!( - "returned continuity {transaction_id} to forward retry: its abort had not stopped its running candidate" - ); - } - } - Err(error) => eprintln!("Idunn boot could not lift wedged continuity aborts: {error:#}"), - } + engine.boot_lift(); match engine.reconcile_failed_continuity_projections() { Ok(outcome) => { for transaction_id in &outcome.demoted { @@ -17979,6 +18020,42 @@ mod tests { Ok(()) } + /// NEGATIVE. A boot lift that errs decided nothing: no continuity abort + /// runs a step until a later lift succeeds, and then the running candidate + /// is returned forward instead of stopped. + #[test] + fn a_boot_lift_that_errs_holds_every_continuity_abort_until_a_lift_succeeds() -> Result<()> { + let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; + braked_sealing_deploy(&world, &deploy_id)?; + let owner = resident_id(&world, &owner_id)?; + world.engine.begin_post_fencing_abort( + &owner, + anyhow!("the AwaitingReady phase passed its deadline at unix ms 2"), + )?; + // The lift's only fallible read fails: the store is unreadable at boot. + let intact = std::fs::read(&world.state_store)?; + std::fs::write(&world.state_store, b"not a control store")?; + world.engine.boot_lift(); + assert!(world.engine.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)); + std::fs::write(&world.state_store, &intact)?; + let stored = resident_id(&world, &owner_id)?; + assert!(stored.value.post_fencing_abort.is_some()); + // Still failing: the abort is held, no step runs. + std::fs::write(&world.state_store, b"not a control store")?; + assert!(!world.engine.resume_candidate(&stored)?, "a held abort moved"); + std::fs::write(&world.state_store, &intact)?; + assert_eq!(stops_of(&workload), 0); + // A later lift succeeds: the running candidate is returned forward. + for _ in 0..5 { + world.engine.resume_backoff.lock().unwrap().clear(); + waiter_then_owner(&world, "", &owner_id)?; + } + assert_eq!(stops_of(&workload), 0, "the running candidate was stopped"); + assert!(resident_id(&world, &owner_id)?.value.post_fencing_abort.is_none()); + assert!(!world.engine.lift_unsettled.load(std::sync::atomic::Ordering::SeqCst)); + Ok(()) + } + /// NEGATIVE. A held record (its candidate unobservable, not provably dead) /// is not released to its abort because a braked Deploy starts awaiting the /// handover.