From e9c718e0732477a7f09f3685c8a84f76feeff9c8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:21:58 +0200 Subject: [PATCH] B2-3d: the route actuation ceiling counts only Forward Ruling 3d A (2026-09-30): the 12/h per-target route actuation ceiling counts only Forward actuations (deploys). Survival actuations (continuity restarts, rollbacks, withdrawals, admitted-route repairs) are recorded for visibility in a separate TargetSupervision.route_survivals log (key 6, newest 12 kept) and are never counted against the ceiling or refused by it. A target whose continuity spent its six restarts can therefore always be rescued by a deploy. Each Survival path keeps its own bound: a continuity's actuations the restart log and its doubling wait, an admitted-route repair the challenge backoff, a deployment's rollback or withdrawal the Forward change it undoes. A candidate whose route proof keeps failing now reloads at most twice the ceiling per window (12 installs, each with its rollback) instead of 12. status prints the survival record beside the ceiling line. TargetSupervision stays at v1 with the added slot: no installed Idunn has written the record (B2 is merged, not installed). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 231 ++++++++++++++++++++++++++++++++----------- src/drivers.rs | 8 +- 2 files changed, 175 insertions(+), 64 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index da57968..f2e78e4 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -73,9 +73,17 @@ const CONTINUITY_RESTART_BACKOFF_MILLIS: u64 = 5_000; /// Forward route actuations (fragment write, firewall, `nginx -t`, reload, /// private-mount validation) one target may perform per sliding window. A /// healthy target performs none; a legitimate deployment performs one or two. -/// Survival actuations are counted against it but never refused. +/// Only deployments count: a Survival actuation is recorded beside this +/// ceiling, never counted against it and never refused by it, so a target +/// whose continuity spent its restarts can still be rescued by a deploy. Each +/// Survival path has its own bound instead: a continuity's actuations the +/// restart log, an admitted-route repair the challenge backoff, and a +/// deployment's rollback or withdrawal the Forward change it undoes. const ROUTE_ACTUATION_CEILING: usize = 12; const ROUTE_ACTUATION_WINDOW_MILLIS: u64 = 3_600_000; +/// How many Survival route actuation times a target keeps, for `status` +/// only: they are a record, not a meter, and no decision reads them. +const ROUTE_SURVIVAL_RECORD: usize = 12; /// The longest wait between route challenges while proofs keep failing. The /// shortest is the observation max age. const ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS: u64 = 600_000; @@ -934,8 +942,9 @@ impl RouteSupervisionState { } /// What one target has been allowed to do, kept across every generation and -/// present before the first: route actuations and continuity restarts, each a -/// sliding log of the times they happened. Owned by the meter primitives below; +/// present before the first: Forward route actuations and continuity restarts, +/// each a sliding log of the times they happened, and a record of Survival +/// route actuations. Owned by the meter primitives below; /// the route driver only asks, and no generation write carries it. #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( @@ -947,7 +956,8 @@ struct TargetSupervision { schema_version: String, #[cultcache(key = 1)] target: String, - /// The newest `ROUTE_ACTUATION_CEILING` actuation times, ascending. + /// The newest `ROUTE_ACTUATION_CEILING` Forward actuation times, + /// ascending: the only entries the ceiling counts. #[cultcache(key = 2)] route_actuations: Vec, /// The newest `CONTINUITY_RESTART_ATTEMPTS` restart times, ascending. @@ -958,6 +968,10 @@ struct TargetSupervision { continuity_deferred_until: Option, #[cultcache(key = 5)] continuity_deferral_reason: Option, + /// The newest `ROUTE_SURVIVAL_RECORD` Survival actuation times, ascending. + /// Visibility only: never counted, never refused. + #[cultcache(key = 6)] + route_survivals: Vec, } /// The entries of a sliding log still inside `window` at `now`. An entry from @@ -970,8 +984,8 @@ fn live_entries(log: &[u64], now: u64, window: u64) -> Vec { .collect() } -/// Record `now`, keeping the newest `limit` entries. That is exact for "at most -/// `limit` in any window", even after a forced entry pushes past the limit. +/// Record `now`, keeping the newest `limit` entries: all a meter needs to +/// decide "at most `limit` in any window". fn record_entry(log: &mut Vec, now: u64, limit: usize) { clamp_to(log, now); log.push(now); @@ -999,14 +1013,16 @@ impl TargetSupervision { continuity_restarts: Vec::new(), continuity_deferred_until: None, continuity_deferral_reason: None, + route_survivals: Vec::new(), } } - /// Clamp both logs to `now`. Callers that decide from a log write the + /// Clamp every log to `now`. Callers that decide from a log write the /// result back when it changed. fn settle(&mut self, now: u64) { clamp_to(&mut self.route_actuations, now); clamp_to(&mut self.continuity_restarts, now); + clamp_to(&mut self.route_survivals, now); } fn route_used(&self, now: u64) -> usize { @@ -1020,23 +1036,33 @@ impl TargetSupervision { .then(|| live[live.len() - ROUTE_ACTUATION_CEILING] + ROUTE_ACTUATION_WINDOW_MILLIS) } - /// Count one route actuation. Only a `Forward` change can be refused; a - /// `Survival` is always admitted, and counted whenever the ledger can record it. + /// Charge one route actuation. A `Forward` change is counted against the + /// ceiling and refused at it; a `Survival` is only recorded, never + /// counted and never refused. fn charge_route(&mut self, now: u64, kind: RouteActuation) -> Result<(), RouteActuationRefused> { self.settle(now); - if kind == RouteActuation::Forward - && let Some(reopens_at) = self.route_reopens_at(now) - { - return Err(RouteActuationRefused { - target: self.target.clone(), - used: self.route_used(now), - reopens_at_unix_millis: reopens_at, - }); + match kind { + RouteActuation::Forward => { + if let Some(reopens_at) = self.route_reopens_at(now) { + return Err(RouteActuationRefused { + target: self.target.clone(), + used: self.route_used(now), + reopens_at_unix_millis: reopens_at, + }); + } + record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING); + } + RouteActuation::Survival => { + record_entry(&mut self.route_survivals, now, ROUTE_SURVIVAL_RECORD); + } } - record_entry(&mut self.route_actuations, now, ROUTE_ACTUATION_CEILING); Ok(()) } + fn survivals_in_window(&self, now: u64) -> usize { + live_entries(&self.route_survivals, now, ROUTE_ACTUATION_WINDOW_MILLIS).len() + } + fn restarts_used(&self, now: u64) -> usize { live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS).len() } @@ -1091,6 +1117,7 @@ impl TargetSupervision { for (log, limit, what) in [ (&self.route_actuations, ROUTE_ACTUATION_CEILING, "route actuation"), (&self.continuity_restarts, CONTINUITY_RESTART_ATTEMPTS, "continuity restart"), + (&self.route_survivals, ROUTE_SURVIVAL_RECORD, "survival route actuation"), ] { ensure!(log.len() <= limit, "{what} log exceeds its bound"); ensure!( @@ -4226,6 +4253,7 @@ fn render_supervision( } if generation.is_none_or(|value| value.route_supervision.is_some()) || !meters.route_actuations.is_empty() + || !meters.route_survivals.is_empty() { lines.push(format!( " route actuations {}/{} in window, reopens-at {}", @@ -4233,6 +4261,15 @@ fn render_supervision( ROUTE_ACTUATION_CEILING, at(meters.route_reopens_at(now)) )); + // A record, not a meter: only the newest are kept, so a saturated + // count says "at least". + let survivals = meters.survivals_in_window(now); + lines.push(format!( + " route survival actuations {}{} in window (uncounted), last-at {}", + if survivals == ROUTE_SURVIVAL_RECORD { "at least " } else { "" }, + survivals, + at(meters.route_survivals.last().copied()) + )); } lines } @@ -4368,7 +4405,7 @@ struct Engine { } /// The route driver's door to the actuation ceiling: every group of host -/// mutations a driver makes for `target` is counted through here. The gate +/// mutations a driver makes for `target` is charged through here. The gate /// owns which charge a request is: a driver asks for the change it is making, /// and the command that owns the transaction decides whether that change is /// deployment (`Forward`, refusable) or the continuity of the admitted @@ -4475,20 +4512,20 @@ impl Engine { } } - /// Count one route actuation against the target's sliding ceiling, in its - /// `TargetSupervision`, before the driver acts. The record is created by - /// the first charge, so a first deployment is metered like any other. A - /// refused `Forward` is the typed `RouteActuationRefused`, and a `Forward` - /// that cannot be recorded is refused too: no charge, no actuation. A - /// `Survival` is never refused, so it is counted when it can be and runs - /// regardless when it cannot: an unwritable store must not stop a rollback, - /// a withdrawal or a repair. + /// Charge one route actuation to the target's `TargetSupervision` before + /// the driver acts. The record is created by the first charge, so a first + /// deployment is metered like any other. A `Forward` is counted against + /// the sliding ceiling; a refused one is the typed `RouteActuationRefused`, + /// and one that cannot be recorded is refused too: no charge, no + /// actuation. A `Survival` is never counted or refused, only recorded when + /// it can be, and runs regardless when it cannot: an unwritable store must + /// not stop a rollback, a withdrawal or a repair. fn charge_route_actuation(&self, target: &str, kind: RouteActuation) -> Result<()> { match self.record_route_actuation(target, kind) { Ok(charged) => charged.map_err(anyhow::Error::new), Err(error) if kind == RouteActuation::Survival => { eprintln!( - "Idunn could not count a survival route actuation of {target}; it proceeds uncounted: {error:#}" + "Idunn could not record a survival route actuation of {target}; it proceeds unrecorded: {error:#}" ); Ok(()) } @@ -12317,6 +12354,8 @@ mod tests { meters(|m| m.route_actuations = (1..=13).collect()).contains("exceeds its bound") ); assert!(meters(|m| m.continuity_restarts = (1..=7).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.route_survivals = (1..=13).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.route_survivals = vec![9, 5]).contains("ascending")); assert!(meters(|m| m.continuity_deferred_until = Some(5)).contains("time or a reason")); assert!(meters(|m| m.continuity_deferral_reason = Some("x".into())).contains("time or a reason")); assert!(meters(|m| m.schema_version = "idunn.target_supervision.v0".into()) @@ -15017,22 +15056,32 @@ mod tests { } #[test] - fn a_survival_actuation_is_counted_past_the_limit_and_never_refused() -> Result<()> { + fn a_survival_actuation_is_recorded_but_never_counted_or_refused() -> Result<()> { let t = 1_000_000_000_u64; let mut meters = TargetSupervision::new("service"); + // Survivals first, past the ceiling: every one is admitted and + // recorded, and none takes a Forward slot. + for offset in 0..20 { + meters.charge_route(t + offset, RouteActuation::Survival)?; + } + assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); + assert_eq!(meters.route_survivals.last(), Some(&(t + 19))); + assert!(meters.route_actuations.is_empty()); + // Twelve Forward changes interleaved with more Survivals: the twelve + // are admitted, and the thirteenth is refused with a reopen time that + // only the oldest Forward entry decides. for offset in 0..12 { - meters.charge_route(t + offset, RouteActuation::Forward)?; - } - meters.charge_route(t + 20, RouteActuation::Survival)?; - assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); - assert_eq!(meters.route_actuations.last(), Some(&(t + 20))); - // The survival took a slot: the forward ceiling is still exact, and - // reopens when the oldest surviving entry leaves. - let refused = meters.charge_route(t + 21, RouteActuation::Forward).unwrap_err(); - assert_eq!(refused.reopens_at_unix_millis, t + 1 + ROUTE_ACTUATION_WINDOW_MILLIS); - for _ in 0..20 { - meters.charge_route(t + 22, RouteActuation::Survival)?; + meters.charge_route(t + 100 + 2 * offset, RouteActuation::Forward)?; + meters.charge_route(t + 101 + 2 * offset, RouteActuation::Survival)?; } + let refused = meters.charge_route(t + 200, RouteActuation::Forward).unwrap_err(); + assert_eq!( + (refused.used, refused.reopens_at_unix_millis), + (ROUTE_ACTUATION_CEILING, t + 100 + ROUTE_ACTUATION_WINDOW_MILLIS) + ); + // A full Forward ceiling still admits and records a Survival. + meters.charge_route(t + 201, RouteActuation::Survival)?; + assert_eq!(meters.route_survivals.last(), Some(&(t + 201))); meters.validate()?; Ok(()) } @@ -15260,6 +15309,21 @@ mod tests { now - 5_000 + ROUTE_ACTUATION_WINDOW_MILLIS )), "{lines}"); assert!(lines.contains("route healthy consecutive-failures 0"), "{lines}"); + assert!( + lines.contains("route survival actuations 0 in window (uncounted), last-at none"), + "{lines}" + ); + meters.route_survivals = vec![now - 7_000, now - 3_000]; + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains(&format!( + "route survival actuations 2 in window (uncounted), last-at {}", + now - 3_000 + )), "{lines}"); + meters.route_survivals = (0..ROUTE_SURVIVAL_RECORD as u64).map(|n| now - 100 + n).collect(); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains("route survival actuations at least 12 in window"), "{lines}"); + // Survivals never move the ceiling's count. + assert!(lines.contains("route actuations 12/12 in window"), "{lines}"); // A first deployment has meters and no generation: it shows them alone. let mut first = TargetSupervision::new("fresh"); @@ -17380,10 +17444,11 @@ mod tests { #[test] fn a_full_ledger_still_restores_the_admitted_and_incumbent_route() -> Result<()> { let full = ROUTE_ACTUATION_CEILING; + // The newest recorded Survival, or 0; the Forward ledger stays full. let newest = |world: &EngineFixture| -> Result { let meters = meters_of(world, "service")?; assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); - Ok(*meters.route_actuations.last().unwrap()) + Ok(meters.route_survivals.last().copied().unwrap_or(0)) }; // (i) A post-fence abort withdraws the candidate's membership. @@ -17396,7 +17461,7 @@ mod tests { .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; aborting.step()?; assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); - assert!(newest(&aborting.world)? > before, "the withdrawal was not counted"); + assert!(newest(&aborting.world)? > before, "the withdrawal was not recorded"); // (ii) Supervision repairs a drifted admitted fragment. let supervised = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; @@ -17406,11 +17471,12 @@ mod tests { supervised.drift()?; assert!(supervised.tick()?); assert_eq!(supervised.reloads(), reloads + 1, "the repair did not run"); - assert!(newest(&supervised.world)? > before, "the repair was not counted"); + assert!(newest(&supervised.world)? > before, "the repair was not recorded"); assert!(admitted(&supervised)?.route_supervision.unwrap().degraded_since_unix_millis.is_none()); // (iii) A failed proof at Routing rolls the route back, though the - // install that preceded it took the last free slot. + // install that preceded it took the last free slot. The rollback + // is recorded and leaves the Forward ledger exactly full. let failing = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; failing.stub.set_state("active"); failing.run_to_routing()?; @@ -17419,7 +17485,7 @@ mod tests { let before = now_millis()?; assert!(failing.step().is_err(), "an unanswered stable route was admitted"); assert_eq!(failing.reloads(), 2, "install then rollback"); - assert!(newest(&failing.world)? >= before, "the rollback was not counted"); + assert!(newest(&failing.world)? >= before, "the rollback was not recorded"); assert!(!failing.fragment_path().exists(), "the rollback left the candidate's route"); Ok(()) } @@ -17427,32 +17493,37 @@ mod tests { #[cfg(unix)] #[test] fn a_continuity_restart_is_never_refused_by_a_full_ledger() -> Result<()> { - // Warming: the preflight of a continuity runs and is counted. + // Warming: the preflight of a continuity runs and is recorded, + // and the Forward ledger is left as it was. let warming = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; warming.step()?; assert!(warming.transaction()?.warming.is_some()); warming.ledger(ROUTE_ACTUATION_CEILING)?; - let before = *meters_of(&warming.world, "service")?.route_actuations.last().unwrap(); + let ledger = meters_of(&warming.world, "service")?.route_actuations; warming.step()?; assert!(warming.transaction()?.route_preflight.is_some(), "the preflight was refused"); - assert!(*meters_of(&warming.world, "service")?.route_actuations.last().unwrap() > before); + let after = meters_of(&warming.world, "service")?; + assert_eq!(after.route_survivals.len(), 1, "the preflight was not recorded"); + assert_eq!(after.route_actuations, ledger, "a survival was counted"); - // Routing: its install runs and is counted. + // Routing: its install runs and is recorded, and not counted. let routing = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routing.stub.set_state("active"); routing.run_to_routing()?; routing.ledger(ROUTE_ACTUATION_CEILING)?; - let before = *meters_of(&routing.world, "service")?.route_actuations.last().unwrap(); + let ledger = meters_of(&routing.world, "service")?.route_actuations; routing.step()?; assert!(routing.reloads() >= 1, "the install did not run"); assert!(routing.fragment_path().exists()); - assert!(*meters_of(&routing.world, "service")?.route_actuations.last().unwrap() > before); + let after = meters_of(&routing.world, "service")?; + assert_eq!(after.route_survivals.len(), 1, "the install was not recorded"); + assert_eq!(after.route_actuations, ledger, "a survival was counted"); Ok(()) } #[cfg(unix)] #[test] - fn a_survival_that_cannot_be_counted_runs_and_a_forward_is_refused() -> Result<()> { + fn a_survival_that_cannot_be_recorded_runs_and_a_forward_is_refused() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; let transaction = routed.transaction()?; let preflight = transaction.route_preflight.clone().context("no preflight")?; @@ -17466,7 +17537,7 @@ mod tests { .route .context("no route binding")?, ); - // The ledger cannot be read, so nothing can be counted. + // The ledger cannot be read, so nothing can be recorded. std::fs::write(&routed.world.state_store, b"unreadable")?; let engine = &routed.world.engine; @@ -17482,7 +17553,7 @@ mod tests { &preflight, &engine.route_gate("service", CommandKind::Deploy), )?; - assert_eq!(routed.reloads(), 1, "the uncounted withdrawal did not run"); + assert_eq!(routed.reloads(), 1, "the unrecorded withdrawal did not run"); Ok(()) } @@ -17669,7 +17740,9 @@ mod tests { assert_eq!(state.consecutive_failures, 1); assert!(state.degraded_since_unix_millis.is_some()); assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - assert_eq!(meters_of(&routed.world, "service")?.route_actuations.len(), 2); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_survivals.len(), 2); + assert!(meters.route_actuations.is_empty()); // Tick 2: the route waits, and the dead unit is restarted from a // fresh read, counted in the same write that mints it. @@ -17724,21 +17797,59 @@ mod tests { #[cfg(unix)] #[test] - fn a_candidate_whose_route_proof_fails_reloads_at_most_the_ceiling() -> Result<()> { + fn a_candidate_whose_route_proof_fails_reloads_at_most_twice_the_ceiling() -> Result<()> { let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.stub.hang_up.store(true, Ordering::SeqCst); let mut last = None; - for _ in 0..30 { + for _ in 0..40 { last = Some(routed.step().unwrap_err()); } - // Every install is followed by its rollback and both are counted, so - // twelve reloads spend the window; the rest are refused. - assert_eq!(routed.reloads(), ROUTE_ACTUATION_CEILING); + // Every install is a counted Forward and is followed by one + // recorded rollback, so the rollbacks are bounded by the installs: + // twelve installs spend the window, each reloads twice, and every + // later attempt is refused before it runs anything. + assert_eq!(routed.reloads(), 2 * ROUTE_ACTUATION_CEILING); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_used(now_millis()?), ROUTE_ACTUATION_CEILING); + assert_eq!(meters.route_survivals.len(), ROUTE_SURVIVAL_RECORD); let last = last.unwrap(); assert!(last.downcast_ref::().is_some(), "{last:#}"); Ok(()) } + + /// The rescue (ruling 3d A): a target whose continuity spent every + /// restart, each charging its preflight and install as Survival, is + /// still deployed: its route changes are the ceiling's first Forward. + #[cfg(unix)] + #[test] + fn a_target_that_spent_its_restarts_can_still_be_deployed() -> Result<()> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(routed.transaction()?.warming.is_some()); + set_meters( + &routed.world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + for _ in 0..2 * CONTINUITY_RESTART_ATTEMPTS { + routed.world.engine.charge_route_actuation("service", RouteActuation::Survival)?; + } + let spent = meters_of(&routed.world, "service")?; + assert!(spent.restarts_exhausted(now_millis()?)); + assert_eq!(spent.route_survivals.len(), 2 * CONTINUITY_RESTART_ATTEMPTS); + + // Warming's preflight and Routing's install are both admitted. + routed.step()?; + assert!(routed.transaction()?.route_preflight.is_some(), "the preflight was refused"); + routed.run_to_routing()?; + routed.promote()?; + assert!(routed.fragment_path().exists(), "the install was refused"); + assert!(routed.reloads() >= 1); + let after = meters_of(&routed.world, "service")?; + assert_eq!(after.route_actuations.len(), 2, "preflight and install are the Forward"); + Ok(()) + } } } diff --git a/src/drivers.rs b/src/drivers.rs index af18a80..189779b 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5079,10 +5079,10 @@ impl Default for RouteActuators { } } -/// Why a route actuation is being asked for. Only a `Forward` change can be -/// refused: restoring the admitted or incumbent route, and every actuation of a -/// continuity, is survival, not deployment, so a `Survival` is always admitted -/// and counted whenever the ledger can record it. +/// Why a route actuation is being asked for. Only a `Forward` change is +/// counted against the ceiling and can be refused: restoring the admitted or +/// incumbent route, and every actuation of a continuity, is survival, not +/// deployment, so a `Survival` is always admitted and only recorded. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RouteActuation { Forward,