diff --git a/src/control_plane.rs b/src/control_plane.rs index 22ad024..08c72b0 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1033,7 +1033,10 @@ impl TargetSupervision { /// number of restarts still inside the window. fn next_restart_at(&self, now: u64) -> Option { let live = live_entries(&self.continuity_restarts, now, CONTINUITY_RESTART_WINDOW_MILLIS); - let last = *live.last()?; + if live.is_empty() { + return None; + } + let last = *self.continuity_restarts.last()?; Some(last.saturating_add(CONTINUITY_RESTART_BACKOFF_MILLIS << (live.len() - 1))) } @@ -8973,6 +8976,70 @@ mod tests { state_store: PathBuf, odin_signer: ServiceIdentitySigner, root: PathBuf, + /// The host programs every route driver of this Engine actuates + /// through: logged stubs, so nothing touches the workstation's nginx. + #[cfg(unix)] + route_stubs: RouteStubs, + } + + /// Stub nginx, systemd-run, systemctl and ufw that append each call to one + /// log. `reload-fails` makes `systemctl reload` exit non-zero. + #[cfg(unix)] + struct RouteStubs { + calls: PathBuf, + reload_fails: PathBuf, + } + + #[cfg(unix)] + impl RouteStubs { + fn new(root: &Path) -> Result<(RouteActuators, Self)> { + use std::os::unix::fs::PermissionsExt; + + let dir = root.join("route-stubs"); + std::fs::create_dir_all(&dir)?; + let calls = dir.join("calls"); + let reload_fails = dir.join("reload-fails"); + let program = |name: &str, refuse: &str| -> Result { + let path = dir.join(name); + std::fs::write( + &path, + format!( + "#!/bin/sh\necho \"{name} $*\" >> '{}'\n{refuse}exit 0\n", + calls.display() + ), + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))?; + Ok(path) + }; + let actuators = RouteActuators { + nginx: program("nginx", "")?, + systemd_run: program("systemd-run", "")?, + systemctl: program( + "systemctl", + &format!( + "if [ \"$1\" = reload ] && [ -e '{}' ]; then exit 1; fi\n", + reload_fails.display() + ), + )?, + ufw: program("ufw", "")?, + preflight_root: root.join("route-preflight"), + }; + Ok((actuators, Self { calls, reload_fails })) + } + + fn refuse_reloads(&self) -> Result<()> { + std::fs::write(&self.reload_fails, b"x")?; + Ok(()) + } + + /// How many logged calls begin with `prefix`, e.g. `systemctl reload`. + fn count(&self, prefix: &str) -> usize { + std::fs::read_to_string(&self.calls) + .unwrap_or_default() + .lines() + .filter(|line| line.starts_with(prefix)) + .count() + } } impl EngineFixture { @@ -9001,6 +9068,8 @@ mod tests { export_service_identity_trust_anchor(&odin_signer, &odin_anchor)?; let state_store = root.join("control.cc"); + #[cfg(unix)] + let (route_actuators, route_stubs) = RouteStubs::new(root)?; let options = RuntimeOptions { state_store: state_store.clone(), bindings_dir: root.join("bindings"), @@ -9011,6 +9080,8 @@ mod tests { odin_trust_anchor: odin_anchor, idunn_identity_store: idunn, deployment_brake_operator_anchor: root.join("brake-anchor.cc"), + #[cfg(unix)] + route_actuators, ..RuntimeOptions::default() }; let engine = match workload { @@ -9023,6 +9094,8 @@ mod tests { engine, state_store, odin_signer, + #[cfg(unix)] + route_stubs, }) } } @@ -10118,6 +10191,7 @@ mod tests { }, ], admitted: Vec::new(), + targets: Vec::new(), }; assert_eq!(snapshot.max_odin_sequence("ghostlight", "odin-signer"), 7); @@ -10270,56 +10344,6 @@ mod tests { Ok(()) } - #[test] - fn continuity_gives_up_on_a_release_that_will_not_start() -> Result<()> { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "continuity-1".into(), - kind: CommandKind::Continuity, - selector: "odin".into(), - requested_by: "idunn-continuity".into(), - requested_at_unix_millis: 100, - }; - let failed_restart = |generation: &str, ordinal: u32| -> Result { - let mut transaction = - DeploymentTransaction::new(&command, "odin".into(), ordinal, None, 100)?; - transaction.incumbent_generation_id = Some(generation.into()); - transaction.completion = Some(TransactionCompletion::FailedBeforeFencing { - error: "systemd unit is not running".into(), - }); - Ok(transaction) - }; - - let refused = |transactions: &[DeploymentTransaction], generation: &str| -> usize { - transactions - .iter() - .filter(|value| { - value.target == "odin" - && value.command_kind == CommandKind::Continuity - && value.incumbent_generation_id.as_deref() == Some(generation) - && matches!( - value.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - | Some(TransactionCompletion::FailedAfterFencing { .. }) - ) - }) - .count() - }; - - let attempts = vec![ - failed_restart("generation-1", 0)?, - failed_restart("generation-1", 1)?, - failed_restart("generation-1", 2)?, - ]; - assert!(refused(&attempts, "generation-1") >= CONTINUITY_RESTART_ATTEMPTS); - - // A restart that succeeds admits a new generation, and the count is - // kept against the generation id, so the next release is not condemned - // by the failures of the one it replaced. - assert_eq!(refused(&attempts, "generation-2"), 0); - Ok(()) - } - #[test] fn activation_without_workload_still_requires_durable_candidate_cleanup() { assert_eq!( @@ -10506,6 +10530,7 @@ mod tests { }, ], admitted: Vec::new(), + targets: Vec::new(), }; assert!(snapshot.validate_relations().is_err()); Ok(()) @@ -10530,6 +10555,7 @@ mod tests { }) .collect(), admitted: Vec::new(), + targets: Vec::new(), }; assert!(!snapshot.has_earlier_authority_sibling(&first)); assert!(snapshot.has_earlier_authority_sibling(&second)); @@ -10659,6 +10685,10 @@ mod tests { const FIXTURE_GENERATION_REPAIRING: &str = include_str!( "../tests/fixtures/idunn-control-legacy/generation-route-repair-started.hex" ); + const FIXTURE_GENERATION_V3_ODIN: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-odin.hex"); + const FIXTURE_GENERATION_V3_ROUTE_PROOF: &str = + include_str!("../tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex"); const FIXTURE_TRANSACTIONS: [(&str, &str); 6] = [ ( "fencing", @@ -10698,7 +10728,9 @@ mod tests { .step_by(2) .map(|index| u8::from_str_radix(&hex[index..index + 2], 16)) .collect::, _>>()?; - let key = if record_type == AdmittedGeneration::TYPE { + let key = if record_type == AdmittedGeneration::TYPE && schema == ADMITTED_GENERATION_SCHEMA_V3 { + rmp_serde::from_slice::(&payload)?.target + } else if record_type == AdmittedGeneration::TYPE { rmp_serde::from_slice::(&payload)?.target } else { rmp_serde::from_slice::(&payload)?.transaction_id @@ -10741,11 +10773,6 @@ mod tests { legacy.odin_publisher_sequence_cursor ); assert_eq!(lifted.ready.voucher(), Voucher::Odin); - // The repair intent still decides whatever it decided. - assert_eq!( - lifted.route_repair_started_at_unix_millis, - legacy.route_repair_started_at_unix_millis - ); assert_eq!(lifted.routing, legacy.routing); assert_eq!(lifted.leasing, legacy.leasing); assert_eq!(lifted.plan, legacy.plan); @@ -10755,17 +10782,13 @@ mod tests { assert_eq!(lifted.generation_id, legacy.generation_id); // The new state starts empty and the plan is still a v2 plan. assert_eq!(lifted.route_supervision, Some(RouteSupervisionState::default())); - assert_eq!(lifted.continuity_backoff, ContinuityBackoff::default()); + assert_eq!(lifted.last_error, None); assert_eq!(lifted.plan.schema, crate::deployment_plan::COMPILED_DEPLOYMENT_PLAN_SCHEMA_V2); assert_eq!( lifted.plan.phase_deadlines(), crate::deployment_plan::PhaseDeadlines::IDUNN_DEFAULTS ); } - let (_, repairing) = fixture_generation(FIXTURE_GENERATION_REPAIRING)?; - assert!(repairing.route_repair_started_at_unix_millis.is_some()); - let (_, plain) = fixture_generation(FIXTURE_GENERATION)?; - assert!(plain.route_repair_started_at_unix_millis.is_none()); Ok(()) } @@ -11048,51 +11071,20 @@ mod tests { } #[test] - fn a_new_generation_inherits_actuation_and_backoff_but_not_observation() -> Result<()> { - let (_, mut incumbent) = fixture_generation(FIXTURE_GENERATION)?; - let mut state = RouteSupervisionState { - last_challenge_at_unix_millis: Some(5), - consecutive_failures: 4, - next_challenge_at_unix_millis: Some(9), - degraded_since_unix_millis: Some(6), - actuations: ActuationWindow { - window_started_at_unix_millis: 3, - count: 2, - }, - }; - incumbent.route_supervision = Some(state.clone()); - incumbent.continuity_backoff = ContinuityBackoff { - window_started_at_unix_millis: Some(1), - attempts: 2, - next_restart_at_unix_millis: Some(8), - }; + fn a_new_generation_starts_unobserved_and_undegraded() -> Result<()> { let committing = fixture_transaction(FIXTURE_TRANSACTIONS[4].1)?.1; - incumbent.odin_receipts()?; + let (_, incumbent) = fixture_generation(FIXTURE_GENERATION)?; let authority = incumbent .odin_authority .clone() .context("fixture generation has no Odin authority")?; let next = AdmittedGeneration::from_transaction( - &committing, - Some(authority.clone()), - Some(&incumbent), - 1_700_000_200_000, - )?; - state = RouteSupervisionState { - actuations: state.actuations, - ..RouteSupervisionState::default() - }; - assert_eq!(next.route_supervision, Some(state)); - assert_eq!(next.continuity_backoff, incumbent.continuity_backoff); - - let first = AdmittedGeneration::from_transaction( &committing, Some(authority), - None, 1_700_000_200_000, )?; - assert_eq!(first.route_supervision, Some(RouteSupervisionState::default())); - assert_eq!(first.continuity_backoff, ContinuityBackoff::default()); + assert_eq!(next.route_supervision, Some(RouteSupervisionState::default())); + assert_eq!(next.last_error, None); Ok(()) } @@ -12033,7 +12025,7 @@ mod tests { } #[test] - fn route_supervision_and_continuity_backoff_are_checked_on_the_generation() -> Result<()> { + fn route_supervision_is_checked_on_the_generation_and_the_meters_on_their_own_record() -> Result<()> { let (_, generation) = fixture_generation(FIXTURE_GENERATION)?; generation.validate()?; let supervised = |change: fn(&mut RouteSupervisionState)| { @@ -12043,22 +12035,23 @@ mod tests { }; assert!(supervised(|state| state.degraded_since_unix_millis = Some(0)) .contains("no start time")); - assert!(supervised(|state| state.actuations.count = 1) - .contains("count but no start")); - let mut degraded = generation.clone(); - let state = degraded.route_supervision.as_mut().unwrap(); - state.degraded_since_unix_millis = Some(5); - state.actuations = ActuationWindow { - window_started_at_unix_millis: 5, - count: 2, - }; - degraded.validate()?; - let mut backoff = generation.clone(); - backoff.continuity_backoff.attempts = 2; - assert!(error_text(backoff.validate()).contains("attempts but no window")); - backoff.continuity_backoff.window_started_at_unix_millis = Some(5); - backoff.validate()?; + let meters = |change: fn(&mut TargetSupervision)| { + let mut meters = TargetSupervision::new("service"); + change(&mut meters); + error_text(meters.validate()) + }; + TargetSupervision::new("service").validate()?; + assert!(meters(|m| m.route_actuations = vec![0]).contains("ascending")); + assert!(meters(|m| m.route_actuations = vec![9, 5]).contains("ascending")); + assert!( + meters(|m| m.route_actuations = (1..=13).collect()).contains("exceeds its bound") + ); + assert!(meters(|m| m.continuity_restarts = (1..=7).collect()).contains("exceeds its bound")); + assert!(meters(|m| m.continuity_deferred_until = Some(5)).contains("time or a reason")); + assert!(meters(|m| m.continuity_deferral_reason = Some("x".into())).contains("time or a reason")); + assert!(meters(|m| m.schema_version = "idunn.target_supervision.v0".into()) + .contains("unsupported")); Ok(()) } @@ -12081,6 +12074,7 @@ mod tests { }, value: generation, }], + targets: Vec::new(), }; assert_eq!(snapshot.max_odin_sequence(&target, &authority.signer_identity_id), 88); assert_eq!(snapshot.max_odin_sequence(&target, "another-signer"), 0); @@ -12603,6 +12597,45 @@ mod tests { Ok(next) } + /// Replace or create a target's meters in the store. + fn set_meters(world: &EngineFixture, meters: &TargetSupervision) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + world.engine.write_target_supervision( + snapshot.supervision_for(&meters.target), + meters, + now_millis()?, + ) + } + + fn meters_of(world: &EngineFixture, target: &str) -> Result { + Ok(ControlSnapshot::read(&world.state_store)?.supervision_or_new(target)) + } + + /// Meters for "service" whose restarts happened these many milliseconds ago. + fn restarted_ago(ages: &[u64]) -> Result { + let now = now_millis()?; + let mut meters = TargetSupervision::new("service"); + meters.continuity_restarts = ages.iter().map(|age| now - age).collect(); + meters.continuity_restarts.sort_unstable(); + Ok(meters) + } + + /// Drop every live transaction and command: what a finished restart leaves. + fn clear_live_transactions(world: &EngineFixture) -> Result<()> { + let snapshot = ControlSnapshot::read(&world.state_store)?; + let envelopes = snapshot + .transactions + .iter() + .map(|stored| stored.envelope.clone()) + .chain(snapshot.commands.iter().map(|stored| stored.envelope.clone())) + .collect::>(); + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store) + .delete_batch_if_unchanged(&envelopes)? + ); + Ok(()) + } + fn incumbent_envelope(world: &EngineFixture) -> Result { Ok(ControlSnapshot::read(&world.state_store)? .admitted_for("service") @@ -12699,14 +12732,9 @@ mod tests { fn a_deploy_that_fences_commits_and_retires_the_incumbent_it_replaced() -> Result<()> { let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; admit_incumbent(&world)?; - let backoff = ContinuityBackoff { - window_started_at_unix_millis: Some(1_000), - attempts: 2, - next_restart_at_unix_millis: Some(9_000), - }; - let incumbent = edit_incumbent(&world, |generation| { - generation.continuity_backoff = backoff.clone(); - })?; + let meters = restarted_ago(&[600_000, 60_000])?; + set_meters(&world, &meters)?; + let incumbent = edit_incumbent(&world, |_| {})?; let candidate = seeded_transaction( &world, @@ -12726,8 +12754,8 @@ mod tests { .clone(); assert_eq!(admitted.transaction_id, candidate.transaction_id); assert_ne!(admitted.generation_id, incumbent.generation_id); - // The restart ceiling belongs to the target: a commit must not reset it. - assert_eq!(admitted.continuity_backoff, backoff); + // The restart log belongs to the target: a commit must not reset it. + assert_eq!(meters_of(&world, "service")?, meters); assert_eq!(admitted.route_supervision, incumbent.route_supervision); // The commit did not stop the incumbent: it is retired after. @@ -13614,9 +13642,13 @@ mod tests { assert_eq!(backoff_wait(500, 3), 4000); assert_eq!(backoff_wait(500, 30), RESUME_BACKOFF_CEILING_MILLIS); assert_eq!(backoff_wait(u64::MAX, 5), RESUME_BACKOFF_CEILING_MILLIS); - assert!(is_waiting(9, 10)); - assert!(!is_waiting(10, 10)); - assert!(!is_waiting(11, 10)); + assert!(is_waiting(9, 10, 5)); + assert!(!is_waiting(10, 10, 5)); + assert!(!is_waiting(11, 10, 5)); + // Due further ahead than the longest wait the owner can set: the clock + // stepped back, so the attempt is due, not stalled for the step. + assert!(is_waiting(5, 10, 5)); + assert!(!is_waiting(4, 10, 5)); } #[test] @@ -13759,7 +13791,6 @@ mod tests { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; let incumbent = admit_incumbent(&world)?; - assert_eq!(incumbent.continuity_backoff, ContinuityBackoff::default()); // The projection names an activation that is not the incumbent's. let anchor = world.engine.provider_anchor_for_plan(&incumbent.plan)?; @@ -13800,13 +13831,14 @@ mod tests { snapshot.transactions.is_empty(), "continuity minted a restart over a projection it could not demote" ); - let deferred = snapshot - .admitted_for("service") - .context("no generation")? + let recorded = snapshot + .supervision_for("service") + .context("the failed demotion was not recorded")? .value - .continuity_backoff - .next_restart_at_unix_millis - .context("the failed demotion was not recorded")?; + .clone(); + let deferred = recorded.continuity_deferred_until.context("no deferral time")?; + assert!(recorded.continuity_deferral_reason.is_some()); + assert!(recorded.continuity_restarts.is_empty(), "a deferral spent an attempt"); let after = now_millis()?; assert!( (before + CONTINUITY_DEFERRAL_MILLIS..=after + CONTINUITY_DEFERRAL_MILLIS) @@ -13814,21 +13846,24 @@ mod tests { ); // Deferred: the next tick neither retries nor writes. - let envelope = incumbent_envelope(&world)?; assert!(!world.engine.supervise_one_admitted_generation()?); - assert_eq!(incumbent_envelope(&world)?, envelope); + assert_eq!(meters_of(&world, "service")?, recorded); assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); // The projection is repaired and the deferral has run out: the restart // proceeds, from an Expected-only projection. topology.withdraw_stale_incarnation(&incumbent.expected, &anchor)?; project_admitted(&world, &incumbent)?; - edit_incumbent(&world, |generation| { - generation.continuity_backoff.next_restart_at_unix_millis = Some(1); - })?; + let mut ran_out = recorded.clone(); + ran_out.continuity_deferred_until = Some(1); + set_meters(&world, &ran_out)?; assert!(world.engine.supervise_one_admitted_generation()?); let snapshot = ControlSnapshot::read(&world.state_store)?; assert_eq!(snapshot.transactions.len(), 1); + let minted = snapshot.supervision_for("service").context("no meters")?; + assert_eq!(minted.value.continuity_restarts.len(), 1); + assert_eq!(minted.value.continuity_deferred_until, None); + assert_eq!(minted.value.continuity_deferral_reason, None); assert_eq!(snapshot.transactions[0].value.command_kind, CommandKind::Continuity); assert_eq!(projected_under(&world, &incumbent.expected)?, expected_only()); Ok(()) @@ -14031,24 +14066,18 @@ mod tests { } #[test] - fn unreadable_history_stops_continuity_instead_of_counting_no_failures() -> Result<()> { + fn unreadable_history_does_not_stop_continuity() -> Result<()> { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; admit_incumbent(&world)?; workload.kill(); corrupt_history(&world)?; - let envelope = incumbent_envelope(&world)?; - // A dead workload would be restarted at once, with the ceiling counting - // zero refusals. With history unreadable nothing is actuated. - assert!(!world.engine.supervise_one_admitted_generation()?); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); - assert_eq!(incumbent_envelope(&world)?, envelope); - - // Readable again, the same tick restarts it. - std::fs::remove_file(history_store_path(&world.state_store))?; + // Crash recovery reads the target's own log, never history: the dead + // workload is restarted, and the restart is counted there. assert!(world.engine.supervise_one_admitted_generation()?); assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); Ok(()) } @@ -14121,27 +14150,93 @@ mod tests { } #[test] - fn continuity_stops_after_three_refused_restarts_counted_from_history() -> Result<()> { + fn continuity_stops_when_the_targets_own_log_has_spent_the_window() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + // Six restarts inside the window, the last long enough ago that its + // doubling wait (160 s) is over: only the ceiling holds. + set_meters( + &world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + assert!(!world.engine.supervise_one_admitted_generation()?); + assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + assert!( + world + .engine + .fault_reports + .lock() + .unwrap() + .contains_key("continuity:service"), + "exhaustion was not reported" + ); + + // The oldest leaves the window: five remain, so one more is allowed. + set_meters( + &world, + &restarted_ago(&[3_700_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert_eq!(ControlSnapshot::read(&world.state_store)?.transactions.len(), 1); + Ok(()) + } + + #[test] + fn continuity_restarts_are_spaced_by_a_doubling_wait_and_counted_in_the_mint() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + workload.kill(); + let restarts = |world: &EngineFixture| -> Result { + let progressed = world.engine.supervise_one_admitted_generation()?; + let minted = !ControlSnapshot::read(&world.state_store)?.transactions.is_empty(); + assert_eq!(progressed, minted); + clear_live_transactions(world)?; + Ok(minted) + }; + + // The first restart is minted, and the same write counts it. + assert!(restarts(&world)?); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 1); + // At once again: the 5 s wait has not passed. + assert!(!restarts(&world)?); + // The second wait is 10 s and the third 20 s: each is the last one doubled. + set_meters(&world, &restarted_ago(&[100_000, 6_000])?)?; + assert!(!restarts(&world)?); + set_meters(&world, &restarted_ago(&[100_000, 11_000])?)?; + assert!(restarts(&world)?); + assert_eq!(meters_of(&world, "service")?.continuity_restarts.len(), 3); + set_meters(&world, &restarted_ago(&[200_000, 100_000, 15_000])?)?; + assert!(!restarts(&world)?); + set_meters(&world, &restarted_ago(&[200_000, 100_000, 21_000])?)?; + assert!(restarts(&world)?); + Ok(()) + } + + #[test] + fn a_deployment_yields_to_a_restart_only_while_the_target_has_restarts_left() -> Result<()> { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; let incumbent = admit_incumbent(&world)?; - for round in 0..CONTINUITY_RESTART_ATTEMPTS { - let failed = seeded_transaction( - &world, - DeploymentPhase::Warming, - CommandKind::Continuity, - Some(&incumbent), - )?; - world - .engine - .begin_pre_fencing_abort(&resident(&world)?, anyhow!("refused {round}"))?; - drive(&world, |transaction| transaction.completion.is_some())?; - assert!(record_of(&world, &failed.transaction_id)?.is_terminal()); - } + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; workload.kill(); - // Three refusals are in history and none is live: the ceiling holds. - assert!(!world.engine.supervise_one_admitted_generation()?); - assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); + // The target's restarts are spent: nothing is left to yield to, and + // aborting the deployment would close the last door. + set_meters( + &world, + &restarted_ago(&[3_000_000, 2_400_000, 1_800_000, 1_200_000, 600_000, 200_000])?, + )?; + world.engine.supervise_one_admitted_generation()?; + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + assert!(after.pre_fencing_abort.is_none(), "yielded with no restarts left"); + + set_meters(&world, &restarted_ago(&[200_000])?)?; + world.engine.supervise_one_admitted_generation()?; + assert!(resident(&world)?.value.pre_fencing_abort.is_some()); Ok(()) } @@ -14554,6 +14649,244 @@ mod tests { Ok(()) } + // --------------------------------------------------------------------- + // B2: the meters, the v4 lift, and what `status` shows. + // --------------------------------------------------------------------- + + #[test] + fn the_actuation_log_slides() -> Result<()> { + let window = ROUTE_ACTUATION_WINDOW_MILLIS; + let t = 1_000_000_000_u64; + let mut meters = TargetSupervision::new("service"); + for offset in 0..12 { + meters.charge_route(t + offset, RouteActuation::Forward)?; + } + let refused = meters.charge_route(t + window - 1, RouteActuation::Forward).unwrap_err(); + assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + window)); + // Only the first entry has left the window: exactly one slot reopens. + meters.charge_route(t + window, RouteActuation::Forward)?; + let refused = meters.charge_route(t + window, RouteActuation::Forward).unwrap_err(); + assert_eq!((refused.used, refused.reopens_at_unix_millis), (12, t + 1 + window)); + meters.validate()?; + Ok(()) + } + + #[test] + fn a_survival_actuation_is_counted_past_the_limit_and_never_refused() -> Result<()> { + let t = 1_000_000_000_u64; + let mut meters = TargetSupervision::new("service"); + for offset in 0..12 { + meters.charge_route(t + offset, RouteActuation::Forward)?; + } + meters.charge_route(t + 20, RouteActuation::Survival)?; + assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); + assert_eq!(meters.route_actuations.last(), Some(&(t + 20))); + // The survival took a slot: the forward ceiling is still exact, and + // reopens when the oldest surviving entry leaves. + let refused = meters.charge_route(t + 21, RouteActuation::Forward).unwrap_err(); + assert_eq!(refused.reopens_at_unix_millis, t + 1 + ROUTE_ACTUATION_WINDOW_MILLIS); + for _ in 0..20 { + meters.charge_route(t + 22, RouteActuation::Survival)?; + } + meters.validate()?; + Ok(()) + } + + #[test] + fn a_clock_stepped_back_stalls_nothing_past_one_window() -> Result<()> { + let t = 5_000_000_000_u64; + let now = t - 86_400_000; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = vec![t; ROUTE_ACTUATION_CEILING]; + meters.continuity_restarts = vec![t; CONTINUITY_RESTART_ATTEMPTS]; + meters.continuity_deferred_until = Some(t); + meters.continuity_deferral_reason = Some("demotion failed".into()); + meters.validate()?; + + // Entries from the future count as now: the meters hold for one window. + let refused = meters.charge_route(now, RouteActuation::Forward).unwrap_err(); + assert!(refused.reopens_at_unix_millis <= now + ROUTE_ACTUATION_WINDOW_MILLIS); + assert!(meters.restarts_exhausted(now)); + assert!(!meters.restarts_exhausted(now + CONTINUITY_RESTART_WINDOW_MILLIS)); + meters.charge_route(now + ROUTE_ACTUATION_WINDOW_MILLIS, RouteActuation::Forward)?; + + // Waits set for the future are due, not stalled for the length of the step. + assert!(!meters.continuity_is_waiting(now)); + let route = RouteSupervisionState { + next_challenge_at_unix_millis: Some(t), + ..RouteSupervisionState::default() + }; + assert!(!route.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); + assert!(!is_waiting(now, t, RESUME_BACKOFF_CEILING_MILLIS)); + // The same waits, set for a moment ahead, still wait. + let ahead = RouteSupervisionState { + next_challenge_at_unix_millis: Some(now + 5_000), + ..RouteSupervisionState::default() + }; + assert!(ahead.is_waiting(now, DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS)); + Ok(()) + } + + #[test] + fn a_route_challenge_wait_widens_to_a_cap_and_a_proof_clears_it() { + let age = DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS; + let mut state = RouteSupervisionState::default(); + let mut waits = Vec::new(); + for _ in 0..8 { + state.record_failed_challenge(1_000, age); + waits.push(state.next_challenge_at_unix_millis.unwrap() - 1_000); + } + assert_eq!(&waits[..3], &[age, age * 2, age * 4]); + assert_eq!(waits[7], ROUTE_CHALLENGE_BACKOFF_CAP_MILLIS); + assert_eq!(state.degraded_since_unix_millis, Some(1_000)); + state.record_proved_challenge(9_000); + assert_eq!(state.consecutive_failures, 0); + assert_eq!(state.next_challenge_at_unix_millis, None); + assert_eq!(state.degraded_since_unix_millis, None); + } + + #[test] + fn a_v3_generation_lifts_to_v4_once_and_reencodes_canonically() -> Result<()> { + for text in [FIXTURE_GENERATION_V3_ODIN, FIXTURE_GENERATION_V3_ROUTE_PROOF] { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let envelope = fixture_envelope(text, AdmittedGeneration::TYPE)?; + let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; + SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(envelope.clone())?; + // The v3 bytes are not v4 bytes: only the typed lift may read them. + assert!(decode_record::(&envelope).is_err()); + + assert_eq!(migrate_control_store_to_current_schema(&store)?, 1); + assert_eq!(migrate_control_store_to_current_schema(&store)?, 0); + let snapshot = ControlSnapshot::read(&store)?; + let migrated = snapshot.admitted.first().context("the lift lost the record")?; + assert_eq!(migrated.envelope.schema_id.as_deref(), Some(ADMITTED_GENERATION_SCHEMA)); + // After the lift the canonical re-encode accepts its own bytes. + let lifted: AdmittedGeneration = decode_record(&migrated.envelope)?; + assert_eq!(lifted, migrated.value); + + // The decisions are the ones the v3 record encoded; only the retired + // fields are gone, and no meters were invented. + assert_eq!(lifted.target, legacy.target); + assert_eq!(lifted.generation_id, legacy.generation_id); + assert_eq!(lifted.plan, legacy.plan); + assert_eq!(lifted.expected, legacy.expected); + assert_eq!(lifted.activation, legacy.activation); + assert_eq!(lifted.workload, legacy.workload); + assert_eq!(lifted.leasing, legacy.leasing); + assert_eq!(lifted.ready, legacy.ready); + assert_eq!(lifted.latest_odin_observation, legacy.latest_odin_observation); + assert_eq!(lifted.routing, legacy.routing); + assert_eq!(lifted.odin_authority, legacy.odin_authority); + assert_eq!( + lifted.odin_publisher_sequence_cursor, + legacy.odin_publisher_sequence_cursor + ); + assert_eq!( + lifted.route_supervision, + legacy.route_supervision.map(|state| RouteSupervisionState { + last_challenge_at_unix_millis: state.last_challenge_at_unix_millis, + consecutive_failures: state.consecutive_failures, + next_challenge_at_unix_millis: state.next_challenge_at_unix_millis, + degraded_since_unix_millis: state.degraded_since_unix_millis, + }) + ); + assert_eq!(lifted.last_error, None); + assert!(snapshot.targets.is_empty()); + } + Ok(()) + } + + #[test] + fn a_v3_generation_that_shows_written_meters_is_refused_not_dropped() -> Result<()> { + let envelope = fixture_envelope(FIXTURE_GENERATION_V3_ROUTE_PROOF, AdmittedGeneration::TYPE)?; + let legacy: LegacyAdmittedGenerationV3 = rmp_serde::from_slice(&envelope.payload)?; + assert!(legacy.route_supervision.is_some(), "the fixture is a routed generation"); + let refused = |change: fn(&mut LegacyAdmittedGenerationV3)| -> Result { + let mut edited = legacy.clone(); + change(&mut edited); + let mut tampered = envelope.clone(); + tampered.payload = rmp_serde::to_vec(&edited)?; + Ok(error_text(read_generation_record(&tampered).map(|_| ()))) + }; + let restarts = refused(|edited| edited.continuity_backoff.attempts = 2)?; + assert!(restarts.contains("refusing to drop"), "{restarts}"); + assert!(restarts.contains(&legacy.target), "the error names the record: {restarts}"); + let actuations = + refused(|edited| edited.route_supervision.as_mut().unwrap().actuations.count = 1)?; + assert!(actuations.contains("refusing to drop"), "{actuations}"); + + // A whole store carrying one refuses to boot, naming the record. + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + let mut edited = legacy.clone(); + edited.continuity_backoff.attempts = 2; + let mut tampered = envelope.clone(); + tampered.payload = rmp_serde::to_vec(&edited)?; + SingleFileMessagePackBackingStore::new(&store).insert_entry_if_absent(tampered)?; + assert!(migrate_control_store_to_current_schema(&store).is_err()); + Ok(()) + } + + #[test] + fn target_supervision_reads_back_and_its_key_must_be_its_target() -> Result<()> { + let world = EngineFixture::new()?; + let meters = restarted_ago(&[5_000])?; + set_meters(&world, &meters)?; + assert_eq!(meters_of(&world, "service")?, meters); + assert_eq!(ControlSnapshot::read(&world.state_store)?.targets.len(), 1); + + let mut elsewhere = target_supervision_envelope(&meters, 5)?; + elsewhere.key = "other".into(); + SingleFileMessagePackBackingStore::new(&world.state_store).insert_entry_if_absent(elsewhere)?; + let error = error_text(ControlSnapshot::read(&world.state_store).map(|_| ())); + assert!(error.contains("not its target"), "{error}"); + Ok(()) + } + + #[test] + fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { + let now = 10_000_000_u64; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = (0..12).map(|offset| now - 5_000 + offset).collect(); + meters.continuity_restarts = vec![now - 9_000, now - 4_000]; + meters.continuity_deferred_until = Some(now + 500); + meters.continuity_deferral_reason = Some("projection would not demote".into()); + + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.last_error = Some("held".into()); + let lines = render_supervision("service", Some(&generation), Some(&meters), now).join("\n"); + assert!(lines.contains(&format!("target service {}", generation.generation_id)), "{lines}"); + assert!(lines.contains("held: held"), "{lines}"); + assert!(lines.contains(&format!( + "continuity restarts 2/6 next-restart-at {}", + now - 4_000 + 10_000 + )), "{lines}"); + assert!(lines.contains("continuity deferred until 10000500: projection would not demote"), "{lines}"); + assert!(lines.contains(&format!( + "route actuations 12/12 in window, reopens-at {}", + now - 5_000 + ROUTE_ACTUATION_WINDOW_MILLIS + )), "{lines}"); + assert!(lines.contains("route healthy consecutive-failures 0"), "{lines}"); + + // A first deployment has meters and no generation: it shows them alone. + let mut first = TargetSupervision::new("fresh"); + first.route_actuations = vec![now - 1]; + let lines = render_supervision("fresh", None, Some(&first), now).join("\n"); + assert!(lines.contains("target fresh no-admitted-generation"), "{lines}"); + assert!(lines.contains("route actuations 1/12 in window, reopens-at none"), "{lines}"); + // An unrouted, unmetered target shows no route line at all. + let lines = render_supervision("plain", Some(&unrouted_generation()?), None, now).join("\n"); + assert!(!lines.contains("route actuations"), "{lines}"); + Ok(()) + } + + fn unrouted_generation() -> Result { + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.route_supervision = None; + Ok(generation) + } + // --------------------------------------------------------------------- // Route-proof readiness: a routed target that declares no Odin dependency // is admitted by Idunn's own challenges. Odin is never read. @@ -14772,6 +15105,7 @@ mod tests { struct RoutedWorld { world: EngineFixture, + workload: Arc, stub: Arc, odin: Odin, poisoned: Vec, @@ -14807,7 +15141,8 @@ mod tests { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, }; - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; let candidate_listener = TcpListener::bind("127.0.0.1:0")?; let stable_listener = TcpListener::bind("127.0.0.1:0")?; let candidate_port = candidate_listener.local_addr()?.port(); @@ -15065,6 +15400,7 @@ mod tests { }; Ok(RoutedWorld { world, + workload, stub, odin, poisoned, @@ -15134,30 +15470,19 @@ mod tests { Ok(()) } - /// Routing installs an nginx fragment and needs the host's programs, - /// so the world writes the fragment itself and records the promotion - /// the stable challenge proves, then enters Committing. - fn promote_by_hand(&self) -> Result<()> { - let current = resident(&self.world)?; - let transaction = ¤t.value; - assert_eq!(transaction.phase, DeploymentPhase::Routing); - let expected = transaction.expected.as_ref().unwrap(); - let binding = transaction.plan.as_ref().unwrap().parsed_inputs()?.1; - let driver = NginxRouteDriver::new(binding.route.unwrap()); - let rendered = driver.render(expected)?; - std::fs::write(&driver.binding.config_path, &rendered)?; - let observation = self.world.engine.prove_stable_route( - transaction, - &driver, - sha256_id(&rendered), - )?; - let mut next = transaction.clone(); - next.routing = Some(RoutingEvidence::Promoted { - promoted_at_unix_millis: observation.observed_at_unix_millis, - observation, - }); - next.enter_phase(DeploymentPhase::Committing, now_millis()?); - replace_transaction(&self.world.state_store, ¤t, &next) + /// Routing runs for real: it installs the fragment through the + /// world's stub host programs and proves the stable route against + /// the stub runtime. The world only drives the steps until the + /// transaction enters Committing. + fn promote(&self) -> Result<()> { + assert_eq!(self.transaction()?.phase, DeploymentPhase::Routing); + for _ in 0..4 { + if self.transaction()?.phase == DeploymentPhase::Committing { + return Ok(()); + } + self.step()?; + } + bail!("the transaction never reached Committing") } } @@ -15166,7 +15491,7 @@ mod tests { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; - routed.promote_by_hand()?; + routed.promote()?; routed.step()?; admitted(&routed) } @@ -15206,7 +15531,7 @@ mod tests { )); routed.step()?; assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); - routed.promote_by_hand()?; + routed.promote()?; routed.step()?; let generation = admitted(&routed)?; @@ -15237,7 +15562,7 @@ mod tests { let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; - routed.promote_by_hand()?; + routed.promote()?; routed.step()?; let generation = admitted(&routed)?; assert_eq!(generation.ready.voucher(), Voucher::Candidate); @@ -15282,7 +15607,7 @@ mod tests { let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; - routed.promote_by_hand()?; + routed.promote()?; routed.stub.capacity.store(1, Ordering::SeqCst); let failure = routed.step().expect_err("a shortfall must be refused"); // The shortfall is a typed value, not a sentence to search. @@ -15325,23 +15650,21 @@ mod tests { Ok(()) } + #[cfg(unix)] #[test] - fn route_admission_gets_past_its_gate_without_reading_odin() -> Result<()> { - // The step below would run the host's real nginx, so it only runs - // where there is none. - if Path::new("/usr/sbin/nginx").exists() { - return Ok(()); - } + fn route_admission_installs_through_the_host_programs_without_reading_odin() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; - // The step fails at installing the route: past the admission gate, - // which would have failed on Odin. - let failure = format!("{:#}", routed.step().expect_err("no nginx in the test world")); - assert!( - failure.contains("starting route actuator /usr/sbin/nginx"), - "{failure}" - ); + // One real Routing step: the fragment goes in through the world's + // stub nginx, ufw and systemctl, and the stable listener proves it. + routed.step()?; + assert!(matches!( + routed.transaction()?.routing, + Some(RoutingEvidence::Promoted { .. }) + )); + assert_eq!(routed.world.route_stubs.count("systemctl reload"), 1); + assert_eq!(routed.world.route_stubs.count("ufw allow"), 1); routed.assert_odin_untouched()?; Ok(()) } @@ -15442,7 +15765,7 @@ mod tests { routed.run_to_routing()?; let ready = routed.transaction()?; assert!(ready.ready.as_ref().and_then(ReadinessEvidence::odin).is_some()); - routed.promote_by_hand()?; + routed.promote()?; routed.step()?; let generation = admitted(&routed)?; generation.validate()?; @@ -15464,7 +15787,7 @@ mod tests { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; - routed.promote_by_hand()?; + routed.promote()?; routed.step()?; let snapshot = ControlSnapshot::read(&routed.world.state_store)?; routed.world.engine.validate_durable_authority(&snapshot)?; @@ -15556,7 +15879,7 @@ mod tests { routed.assert_boots()?; routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; - routed.promote_by_hand()?; + routed.promote()?; routed.assert_boots()?; routed.step()?; @@ -15814,7 +16137,7 @@ mod tests { let mut bare = routing.clone(); bare.ready = None; assert!(error_text(bare.validate()).contains("lacks Ready evidence")); - routed.promote_by_hand()?; + routed.promote()?; let mut bare = routed.transaction()?; assert_eq!(bare.phase, DeploymentPhase::Committing); bare.ready = None; @@ -15948,7 +16271,7 @@ mod tests { routed.stub.set_state("active"); routed.run_to_routing()?; let at_routing = routed.transaction()?; - routed.promote_by_hand()?; + routed.promote()?; routed.step()?; let snapshot = ControlSnapshot::read(&routed.world.state_store)?; @@ -16263,5 +16586,250 @@ mod tests { )); Ok(()) } + + // ----------------------------------------------------------------- + // B2: the route gate, the meters and the supervision pass, through + // the Engine, over the world's stub host programs. + // ----------------------------------------------------------------- + + impl RoutedWorld { + /// Run a fresh world to an admitted generation whose transaction + /// is retired: what admitted-route supervision looks at. + fn admit(&self) -> Result<()> { + self.stub.set_state("active"); + self.run_to_routing()?; + self.promote()?; + self.drive_until(|transaction| transaction.is_terminal())?; + assert!(self.world.engine.retire_one_terminal_transaction()?); + Ok(()) + } + + fn fragment_path(&self) -> PathBuf { + self.world.root.join("service.conf") + } + + /// The target's route ledger holds `entries` recent actuations. + fn ledger(&self, entries: usize) -> Result<()> { + let now = now_millis()?; + let mut meters = TargetSupervision::new("service"); + meters.route_actuations = (0..entries as u64).map(|offset| now - 1_000 + offset).collect(); + set_meters(&self.world, &meters) + } + + fn reloads(&self) -> usize { + self.world.route_stubs.count("systemctl reload") + } + + /// Something else rewrote the fragment: the admitted membership drifted. + fn drift(&self) -> Result<()> { + std::fs::write(self.fragment_path(), b"drifted bytes\n")?; + Ok(()) + } + + fn tick(&self) -> Result { + self.world.engine.supervise_one_admitted_generation() + } + } + + #[cfg(unix)] + #[test] + fn a_full_ledger_still_restores_the_admitted_and_incumbent_route() -> Result<()> { + let full = ROUTE_ACTUATION_CEILING; + let newest = |world: &EngineFixture| -> Result { + let meters = meters_of(world, "service")?; + assert_eq!(meters.route_actuations.len(), ROUTE_ACTUATION_CEILING); + Ok(*meters.route_actuations.last().unwrap()) + }; + + // (i) A post-fence abort withdraws the candidate's membership. + let aborting = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + aborting.ledger(full)?; + let before = newest(&aborting.world)?; + aborting + .world + .engine + .begin_post_fencing_abort(&resident(&aborting.world)?, anyhow!("test"))?; + aborting.step()?; + assert_eq!(aborting.reloads(), 1, "the withdrawal did not run"); + assert!(newest(&aborting.world)? > before, "the withdrawal was not counted"); + + // (ii) Supervision repairs a drifted admitted fragment. + let supervised = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + supervised.admit()?; + supervised.ledger(full)?; + let (before, reloads) = (newest(&supervised.world)?, supervised.reloads()); + supervised.drift()?; + assert!(supervised.tick()?); + assert_eq!(supervised.reloads(), reloads + 1, "the repair did not run"); + assert!(newest(&supervised.world)? > before, "the repair was not counted"); + assert!(admitted(&supervised)?.route_supervision.unwrap().degraded_since_unix_millis.is_none()); + + // (iii) A failed proof at Routing rolls the route back, though the + // install that preceded it took the last free slot. + let failing = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + failing.stub.set_state("active"); + failing.run_to_routing()?; + failing.ledger(full - 1)?; + failing.stub.hang_up.store(true, Ordering::SeqCst); + let before = now_millis()?; + assert!(failing.step().is_err(), "an unanswered stable route was admitted"); + assert_eq!(failing.reloads(), 2, "install then rollback"); + assert!(newest(&failing.world)? >= before, "the rollback was not counted"); + assert!(!failing.fragment_path().exists(), "the rollback left the candidate's route"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_first_routed_deploy_is_metered_and_refused_at_the_ceiling() -> Result<()> { + let metered = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + metered.stub.set_state("active"); + metered.run_to_routing()?; + assert!(ControlSnapshot::read(&metered.world.state_store)?.targets.is_empty()); + metered.step()?; + assert_eq!(meters_of(&metered.world, "service")?.route_actuations.len(), 1); + + let refused = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + refused.stub.set_state("active"); + refused.run_to_routing()?; + refused.ledger(ROUTE_ACTUATION_CEILING)?; + let error = refused.step().unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert_eq!(refused.world.route_stubs.count(""), 0, "a refused install ran a program"); + assert!(!refused.fragment_path().exists()); + + // Past the fence the refusal is resumable, not an abort: the resume + // backoff spaces it and no actuation runs while it waits. + assert!(!refused.world.engine.resume_one_transaction()?); + let waiting = refused.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::Routing); + assert!(waiting.post_fencing_abort.is_none()); + assert!(waiting.last_error.as_deref().is_some_and(|text| text.contains("ceiling"))); + assert_eq!(refused.world.route_stubs.count(""), 0); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_refused_preflight_starts_no_private_mount_unit_and_fails_at_once_with_the_reopen_time() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + assert!(routed.transaction()?.warming.is_some()); + routed.ledger(ROUTE_ACTUATION_CEILING)?; + let reopens_at = meters_of(&routed.world, "service")? + .route_reopens_at(now_millis()?) + .context("the ledger is not full")?; + + // The refusal is an error of its own phase, before the fence: the + // deployment fails now, and its error carries the reopen time. + routed.world.engine.resume_one_transaction()?; + assert_eq!(routed.world.route_stubs.count("systemd-run"), 0); + let aborting = routed.transaction()?; + let abort = aborting.pre_fencing_abort.context("the refusal did not abort the deployment")?; + assert!( + abort.error.contains(&format!("reopens at unix ms {reopens_at}")), + "{}", + abort.error + ); + routed.drive_until(|transaction| transaction.completion.is_some())?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = + routed.transaction()?.completion + else { + bail!("the deployment did not fail before the fence"); + }; + assert!(error.contains(&reopens_at.to_string()), "{error}"); + assert_eq!(routed.world.route_stubs.count(""), 0); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.workload.kill(); + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + + // Tick 1: the repair charges the target's meters and the proof + // fails. Both are writes, so they end the pass: nothing else is + // decided from the snapshot they made stale. + assert!(routed.tick()?); + let state = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(state.consecutive_failures, 1); + assert!(state.degraded_since_unix_millis.is_some()); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + assert_eq!(meters_of(&routed.world, "service")?.route_actuations.len(), 2); + + // Tick 2: the route waits, and the dead unit is restarted from a + // fresh read, counted in the same write that mints it. + assert!(routed.tick()?); + assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); + assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_proved_challenge_clears_the_degradation() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + assert!(routed.tick()?); + let degraded = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(degraded.consecutive_failures, 1); + assert!(degraded.degraded_since_unix_millis.is_some()); + assert!(degraded.next_challenge_at_unix_millis.is_some()); + + routed.stub.hang_up.store(false, Ordering::SeqCst); + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); + })?; + routed.drift()?; + assert!(routed.tick()?); + let healed = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(healed.consecutive_failures, 0); + assert_eq!(healed.next_challenge_at_unix_millis, None); + assert_eq!(healed.degraded_since_unix_millis, None); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failing_reload_that_deletes_the_fragment_backs_the_route_off() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.world.route_stubs.refuse_reloads()?; + let reloads = routed.reloads(); + routed.drift()?; + for _ in 0..200 { + routed.tick()?; + } + assert_eq!(routed.reloads() - reloads, 1, "the route was reloaded again and again"); + assert!(!routed.fragment_path().exists()); + assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_candidate_whose_route_proof_fails_reloads_at_most_the_ceiling() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + let mut last = None; + for _ in 0..30 { + last = Some(routed.step().unwrap_err()); + } + // Every install is followed by its rollback and both are counted, so + // twelve reloads spend the window; the rest are refused. + assert_eq!(routed.reloads(), ROUTE_ACTUATION_CEILING); + let last = last.unwrap(); + assert!(last.downcast_ref::().is_some(), "{last:#}"); + Ok(()) + } } } diff --git a/tests/fixtures/idunn-control-legacy/README.md b/tests/fixtures/idunn-control-legacy/README.md index d1b00ec..3d420f3 100644 --- a/tests/fixtures/idunn-control-legacy/README.md +++ b/tests/fixtures/idunn-control-legacy/README.md @@ -42,3 +42,21 @@ borrowed from `transaction-fencing`. | `transaction-pre-fence-abort-terminal` | the same abort finished: Complete, `FailedBeforeFencing` | | `transaction-post-fence-abort` | Fencing continuity, post-fencing abort in flight | | `provider-anchor` | one line: the provider trust-anchor file bytes, hex. A test writes it to the fixed path so the Engine can read the plan's anchor | + +## Generation v3 (the B2 lift) + +`generation-v3-odin` and `generation-v3-route-proof` are the exact +`idunn.admitted_generation.v3` payloads the encoder at Idunn `583b2a7` wrote +(`admitted_envelope` over `rmp_serde::to_vec`), printed by a throwaway test that +is not kept. The first is the v2 `generation-with-receipts` record lifted by that +revision's own `LegacyAdmittedGeneration::into_current`, the second is the +generation `route_proof::committed_generation` admits through a real route-proof +deployment. Both are synthetic. + +| File | Schema | What it is | +|---|---|---| +| `generation-v3-odin` | `idunn.admitted_generation.v3` | routed, Odin-correlated generation | +| `generation-v3-route-proof` | `idunn.admitted_generation.v3` | routed, route-proof generation, no Odin receipts | + +`control_plane.rs` decodes them through `LegacyAdmittedGenerationV3`; the +fixtures die with that struct. diff --git a/tests/fixtures/idunn-control-legacy/generation-v3-odin.hex b/tests/fixtures/idunn-control-legacy/generation-v3-odin.hex new file mode 100644 index 0000000..23e87fa --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/generation-v3-odin.hex @@ -0,0 +1,2 @@ +idunn.admitted_generation.v3 +dc0015bc6964756e6e2e61646d69747465645f67656e65726174696f6e2e7633a773657276696365d93267656e65726174696f6e2d74782d37636236616537392d323635302d346261322d623664392d643137333338643566393031af636f6e74696e756974792d74657374d92774782d37636236616537392d323635302d346261322d623664392d643137333338643566393031cf0000018bcfe6eea099d9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7632d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d353962636334323766373939343035636134613531326232386138303637343435366661333134393732333631323830656130626332316363356532663635338091a87265662d6865616464dc058e0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0a5b5b646570656e64656e636965735d5d0a6b696e64203d20227368617265642d696e667261737472756374757265220a6361706162696c697479203d20226f64696e2e76657273652d72656e64657a766f7573220a736368656d61203d20226f64696e2e76657273652d746f706f6c6f67792e7631220a636f6d7061746962696c697479203d20227631220adc076e0a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d2022736572766963652d72756e74696d652d7369676e6572220a74727573745f616e63686f725f73746f7265203d20222f6574632f67616d6563756c742f74727573742f736572766963652e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3137393939220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2031383030300a707269766174655f706f72745f656e64203d2031383030390a636f6e6669675f70617468203d20222f6574632f6e67696e782f6964756e6e2d73747265616d2d726f757465732f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a919296b57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6e97a46f64696e98ad6d616e616765642d7265616479a46f64696eb26f64696e2d696e6361726e6174696f6e2d31d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161d9477368613235362d65656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565d9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238d9477368613235362d6137313730323730313265383236396235666636363961313135386466636236313764346463636230663963333633656631373738656561633230353332633907b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b57463703a2f2f31302e37372e302e313a3137383731cd465196b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237d9477368613235362d643963646464656263666363623263663162613961643662393235663366666162376465616363323666616536316461386632363538613361393430396562329195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bd2f7372762f736572766963652f72656c65617365732f63757272656e74dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d64396364646465626366636362326366316261396164366239323566336666616237646561636332366661653631646138663236353861336139343039656232b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d35663738353165373634333366383261376562666431626232646234323065366263333263646263346536666538373563356633326461363434643535656237bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d35396263633432376637393934303563613461353132623238613830363734343536666133313439373233363132383065613062633231636335653266363533b1736572766963652d79676764726173696cb6736572766963652d72756e74696d652d7369676e6572ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3137393939b6687474703a2f2f3132372e302e302e313a31383030319194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101919ab57368617265642d696e667261737472756374757265b56f64696e2e76657273652d72656e64657a766f7573b66f64696e2e76657273652d746f706f6c6f67792e7631a2763101b06265666f72652d70726f6d6f74696f6ea46f64696eb36d616e616765642d696e6361726e6174696f6ed9477368613235362d63343263333331373863366539666534633435313162623563663962626461376431633138356536646330373064323234653463653431303538626338633238b57463703a2f2f31302e37372e302e313a31373837319abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d39613662663530353466616430306232343363323331386330333436303931333230343665643436356231356534636561666532613736623966663231313039b1736572766963652d79676764726173696cd9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d94032363535323862353235386435353563356434646533383931303830363164383965663439616336306631373337626464656337316535333335643163623935c420c06afef8f66e6ec88c96c81ad6d74dbd9933738032f4d8069a46412380e2272dcf0000018bcfe567ecd94066346536393735326334333432363965323262306130346137623430613065336235386364343561383238376364303936383263313864326565366539376239a765643235353139c44046048cf77cc9f524da966e16a131edb34180f4ea8420f5f24a64fd53178e44988bef994d6df103a5f117470283d533e9d7e8245c4f9965f9a4c0a0114ed1c201dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d34626635313232663334343535346335336264653265626238636432623765336431363030616436333163333835613564376363653233633737383534353961d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc002001010101010101010101010101010101010101010101010101010101010101019091b1736b69707065642d73746174656c65737392af6f64696e2d636f7272656c61746564959140d9477368613235362d63333634316638353434643763303266333538306230376330663938383766306336613237666635616231643461336532396361663139376366633239396165ab6f64696e2d7369676e65722864959141d9477368613235362d35353961656164303832363464353739356433393039373138636464303561626434393537326538346665353535393065656633316138386130386664666664ab6f64696e2d7369676e6572296493a870726f6d6f74656495a773657276696365d9477368613235362d37316361633366396164336534346139333861343762623035653738306566366264396365303266396139636631333464643765363865396162323563663262d9477368613235362d62663563663539653335363635323235333236386336303463626638646638636664623033613461306433326232376164313538653538313730396338306534d9477368613235362d34313266303131626462396566646138616137323036636364336333613438633637613131386164646161363266616436303463326236346664396163643063cf0000018bcfe6ee96cf0000018bcfe6ee9692d94038663736666430326265623133353233373733373632306361666530383234636138663937326438623332643134373762346336333430656139666634643762dc002048ccd7cc85cc8c14ccb8384dccbccc8fcc83ccbacca50f25ccb072ccc126cc9fccee0cccd660316f5bcccf5e6f172529c095c000c0c092000093c000c0 diff --git a/tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex b/tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex new file mode 100644 index 0000000..fbcc390 --- /dev/null +++ b/tests/fixtures/idunn-control-legacy/generation-v3-route-proof.hex @@ -0,0 +1,2 @@ +idunn.admitted_generation.v3 +dc0015bc6964756e6e2e61646d69747465645f67656e65726174696f6e2e7633a773657276696365d93267656e65726174696f6e2d74782d35653364386238382d646632352d346465662d386237632d633736636636376564643933b2636f6e74696e756974792d73657276696365d92774782d35653364386238382d646632352d346465662d386237632d633736636636376564643933cf000001a0ef449fec9ad9216964756e6e2e636f6d70696c65645f6465706c6f796d656e745f706c616e2e7633d9477368613235362d65373533396236376136393534613039373834363234653763643832623335653365386663363833306664646565656330636438646366343262633436333066b5736572766963652d696e6361726e6174696f6e2d316e9bbf6964756e6e2e736f757263655f73656c656374696f6e5f66616374732e7631d92768747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974af726566732f68656164732f6d61696ed92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d92833333333333333333333333333333333333333333333333333333333333333333333333333333333bc6465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6cd9477368613235362d393135366563643230356266343439336166636532363961623565613631336133663535663935303766393161666537633633303162613566333735623063378091a87265662d6865616464dc050f0a736368656d61203d202267616d6563756c742e6964756e6e2e7461726765745f6465636c61726174696f6e2e7631220a746172676574203d202273657276696365220a736f757263655f7374616d705f656e7669726f6e6d656e74203d2022534552564943455f4255494c445f434f4d4d4954220a0a5b5b73746570735d5d0a6964203d20226275696c64220a7068617365203d20226275696c64220a72756e6e6572203d202272757374220a61726776203d205b22636172676f222c20226275696c64222c20222d2d6c6f636b6564225d0a0a5b5b65787465726e616c5f696e707574735d5d0a6964203d2022746f6f6c636861696e2d696e646578220a75726c203d202268747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578220a736861323536203d202264646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464220a72756e6e6572203d202272757374220a64657374696e6174696f6e203d2022696e707574732f746f6f6c636861696e2d696e646578220a0a5b5b6172746966616374735d5d0a6964203d20226461656d6f6e220a736f757263655f6b696e64203d202272756e6e65722d6f7574707574220a72756e6e6572203d202272757374220a736f75726365203d20227461726765742f72656c656173652f73657276696365220a64657374696e6174696f6e203d202273657276696365220a65787065637465645f736861323536203d202263636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363220a65786563757461626c65203d20747275650a0a5b736572766963655d0a65786563757461626c655f6172746966616374203d20226461656d6f6e220a617267756d656e7473203d205b0a20207b206b696e64203d202262696e64696e67222c206e616d65203d202273746174655f726f6f7422207d2c0a5d0a7472616e73706f7274203d202268747470220a726f7574655f7265717569726564203d20747275650a72657175697265645f656e7669726f6e6d656e74203d205b2247414d4543554c545f4944554e4e5f43414e4449444154455f42494e44222c202247414d4543554c545f4944554e4e5f52554e54494d455f42554e444c45225d0a0a5b736572766963652e6865616c74685d0a636f6e7472616374203d2022736572766963652e6865616c7468220a0a5b73746174655d0a736368656d615f67656e65726174696f6e203d20227631220a0a5b5b73746174652e736c6f74735d5d0a6964203d202272756e74696d652d6361636865220a72656c61746976655f70617468203d202272756e74696d652d63616368652e6363220a6b696e64203d202263756c7463616368652d66696c65220a736368656d61203d2022736572766963652e72756e74696d652d63616368652e7631220a777269746572203d20226e6f6e65220a7265636f76657279203d202272656275696c6461626c65220a73746172747570203d20226f70656e2d61742d7374617274220a0a5b5b70726f76696465735d5d0a6361706162696c697479203d2022736572766963652e72756e74696d65220a6361706163697479203d20310a736368656d61203d2022736572766963652e72756e74696d652e7631220a636f6d7061746962696c697479203d20227631220a0adc07980a736368656d61203d202267616d6563756c742e6964756e6e2e6f70657261746f725f62696e64696e672e7632220a746172676574203d202273657276696365220a0a5b7265706f7369746f72795d0a6f726967696e203d202268747470733a2f2f6769746875622e636f6d2f47616d6543756c742f536572766963652e676974220a61646d69747465645f726566203d2022726566732f68656164732f6d61696e220a6d696e696d756d5f7265766973696f6e203d202231313131313131313131313131313131313131313131313131313131313131313131313131313131220a73656c656374696f6e203d20227265662d68656164220a636865636b6f7574203d20222f7372762f6275696c642f53657276696365220a7265636970655f70617468203d20226465706c6f796d656e742f6964756e6e2f7265636970652e746f6d6c220a0a5b72756e6e6572732e727573745d0a647269766572203d2022646f636b6572220a696d616765203d202272757374407368613235363a61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161220a75736572203d2022313030303a31303030220a6166666f7264616e636573203d205b22736f757263652d72656164222c202261727469666163742d7772697465225d0a616c6c6f7765645f70726f6772616d73203d205b22636172676f225d0a6e6574776f726b5f70726f66696c65203d20226275696c642d646570656e64656e63792d656772657373220a6d656d6f72795f6d6562696279746573203d20323034380a6370755f71756f74615f70657263656e74203d203230300a706964735f6c696d6974203d203531320a746d7066735f6d6562696279746573203d203531320a0a5b776f726b6c6f61645d0a647269766572203d202273797374656d642d7472616e7369656e74220a73746174655f67726f7570203d202273657276696365220a756e69745f707265666978203d20226964756e6e2d73657276696365220a72656c656173655f726f6f74203d20222f7372762f736572766963652f72656c6561736573220a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a72756e74696d655f726f6f74203d20222f72756e2f67616d6563756c742f6964756e6e2f73657276696365220a6e6574776f726b203d2022686f73742d70726976617465220a68617264656e696e67203d2022737472696374220a6d656d6f72795f6d6562696279746573203d20313032340a6370755f71756f74615f70657263656e74203d203130300a0a5b776f726b6c6f61642e617267756d656e745f62696e64696e67735d0a73746174655f726f6f74203d20222f7661722f6c69622f67616d6563756c742f73657276696365220a0a5b776f726b6c6f61642e7365637265745f66696c65735d0a47414d4543554c545f52554e54494d455f50524553454e43455f4944454e54495459203d20222f6574632f67616d6563756c742f736572766963652f72756e74696d652d70726573656e63652d6964656e746974792e6363220a0a5b72756e74696d655f6964656e746974795d0a72756e74696d655f6964203d2022736572766963652d79676764726173696c220a65787065637465645f7369676e65725f6964656e746974795f6964203d202235353064336235303862343832366564623532356666386339313636393530306230396563306134303639373133626666633137363465663463636363336637220a74727573745f616e63686f725f73746f7265203d20222f746d702f2e746d706f4d424135692f6964656e7469746965732f70726f76696465722d616e63686f722e6363220a0a5b726f7574655d0a647269766572203d20226e67696e782d73747265616d2d746370220a726f7574655f6964203d202273657276696365220a737461626c655f656e64706f696e74203d2022687474703a2f2f3132372e302e302e313a3337373433220a707269766174655f686f7374203d20223132372e302e302e31220a707269766174655f706f72745f7374617274203d2034303537370a707269766174655f706f72745f656e64203d2034303537380a636f6e6669675f70617468203d20222f746d702f2e746d706f4d424135692f736572766963652e636f6e66220a72656c6f61645f756e6974203d20226e67696e782e73657276696365220a0a5b6272616b65735d0a6465706c6f796d656e745f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6465706c6f796d656e742d6272616b652e6363220a6c6966656379636c655f73746f7265203d20222f7661722f6c69622f67616d6563756c742f6964756e6e2f736572766963652d6c6966656379636c652d6272616b652e6363220a0a5b726f6c6c6f75745d0a7374726174656779203d202263616e6469646174652d7468656e2d70726f6d6f7465220a647261696e5f7365636f6e6473203d2033300a72657461696e5f72656c6561736573203d20320a0a5b706c6163656d656e745d0a646573697265645f7265706c69636173203d20310a6e6f646573203d205b2279676764726173696c225d0a90cd9e81957878cd012c787896b76964756e6e2e7365616c65645f72656c656173652e7631d9477368613235362d34613533323635653265393464656434663131623936383434366566346137306265613363666635313634383238333736663739623063356436666330653539d9477368613235362d653735333962363761363935346130393738343632346537636438326233356533653866633638333066646465656563306364386463663432626334363330669195a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac39196af746f6f6c636861696e2d696e646578d92768747470733a2f2f6578616d706c652e696e76616c69642f746f6f6c636861696e2d696e646578d9477368613235362d64646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464646464a472757374b6696e707574732f746f6f6c636861696e2d696e646578117892d9477368613235362d34613533323635653265393464656434663131623936383434366566346137306265613363666635313634383238333736663739623063356436666330653539ba2f7372762f736572766963652f72656c65617365732f74657374dc0012bd6964756e6e2e65787065637465645f696e6361726e6174696f6e2e7632a773657276696365d9477368613235362d65373533396236376136393534613039373834363234653763643832623335653365386663363833306664646565656330636438646366343262633436333066b5736572766963652d696e6361726e6174696f6e2d31d9477368613235362d34613533323635653265393464656434663131623936383434366566346137306265613363666635313634383238333736663739623063356436666330653539bb6769746875622e636f6d2f47616d6543756c742f53657276696365d92832323232323232323232323232323232323232323232323232323232323232323232323232323232d9477368613235362d39313536656364323035626634343933616663653236396162356561363133613366353566393530376639316166653763363330316261356633373562306337b1736572766963652d79676764726173696cd94035353064336235303862343832366564623532356666386339313636393530306230396563306134303639373133626666633137363465663463636363336637ae736572766963652e6865616c7468d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363a27631d9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731c294a773657276696365a468747470b6687474703a2f2f3132372e302e302e313a3337373433b6687474703a2f2f3132372e302e302e313a34303537379194af736572766963652e72756e74696d65b2736572766963652e72756e74696d652e7631a2763101909abb6964756e6e2e72756e74696d655f61637469766174696f6e2e7632d9477368613235362d38646437343836646535373263333732316234383965663531373232633931353334396366376365383632366662313737636261316639386137613061663063b1736572766963652d79676764726173696cd9477368613235362d30636363383737383930313339323639383066383865373661653537663236666238363131643735373435346631656661366133653536313762366636643032d94031653534396337363539396133323564343435626561343135393233613563656638303833353439353262396266383237646533663134366466306465666530c4202f287f2191a013ed6c10ba58e6b52a20be204c0590b415164efae071c5aca54fcf000001a0ef449ce9d94033373962393163363738363561346139626266366334333662653633363965333433646464643235613437316538656638613635316539663863353362323864a765643235353139c4406267597c3a042ebaa5a89355da21e37f59bf72fa816db670bc749c25c6a6150d98ac21475c598f943477402028e3b17bb09f7a98353294e3e7c6ee538b2a1206dc0033b26964756e6e2d343234322e73657276696365af4964756e6e20746573742034323432af696e766f636174696f6e2d3432343201a465786563a6616c77617973ad636f6e74726f6c2d67726f7570c3a57534323432a57534323432a0a0a0c3c3a9696e76697369626c65a3706964c3a430303737a0a0cd10920194cd1092cd1092cd1092cd109294cd1092cd1092cd1092cd109291cd10920000000000c391010807b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d63636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363d9477368613235362d30636363383737383930313339323639383066383865373661653537663236666238363131643735373435346631656661366133653536313762366636643032a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d6462633162346339303066666534386435373562356461356336333830343031323566363564623066653365323434393462373665613938363435376439383690d9477368613235362d30383466656430386239373861663464376431393661373434366138366235383030396536333662363131646231363231316236356139616164666632396335d9202f73797374656d2e736c6963652f6964756e6e2d343234322e73657276696365c090aa61637469766174696f6edc002001010101010101010101010101010101010101010101010101010101010101019091b1736b69707065642d73746174656c65737392ab726f7574652d70726f6f6695dc03c0ccdc0018ccd92367616d6563756c742e72756e74696d655f70726573656e63655f6865616c74682e7632cca773657276696365ccd9477368613235362d38646437343836646535373263333732316234383965663531373232633931353334396366376365383632366662313737636261316639386137613061663063ccd9477368613235362d65373533396236376136393534613039373834363234653763643832623335653365386663363833306664646565656330636438646366343262633436333066ccb5736572766963652d696e6361726e6174696f6e2d31ccd9477368613235362d34613533323635653265393464656434663131623936383434366566346137306265613363666635313634383238333736663739623063356436666330653539ccd9477368613235362d66663136383733633362666530333730613962336332386466353632663361663464356265313736343630333738633931653032396566396634396266323938cca27631ccd9477368613235362d32663837303134316366353638316566656332653234316364636435663731303831616539326162316535633534633839636438393666386335656236323731ccb1736572766963652d79676764726173696cccd9477368613235362d30636363383737383930313339323639383066383865373661653537663236666238363131643735373435346631656661366133653536313762366636643032ccb6687474703a2f2f3132372e302e302e313a3430353737cc91cc94ccaf736572766963652e72756e74696d65ccb2736572766963652e72756e74696d652e7631cca2763101ccae736572766963652e6865616c7468cca6616374697665ccd93c726f7574652d6f62736572766174696f6e3a63616e6469646174652d3934373439373738356531333466633639306133316461616430626631363136ccc0ccd9403535306433623530386234383236656462353235666638633931363639353030623039656330613430363937313362666663313736346566346363636333663702cccf000001cca0ccef44cc9e5acca765643235353139ccc440ccaa3009610eccf72acccccc8a643633ccf92d3bcc80ccde2dccefccc0ccd911cc8ccca1ccd3ccb053704d041027ccb5ccfdccf01fccd5ccb4cce628ccadcc91cca9ccb6651e27cca778cc912930ccbdcc9e1c1f12cc9724ccedccf4ccd33803ccd94031653534396337363539396133323564343435626561343135393233613563656638303833353439353262396266383237646533663134366466306465666530ccc4405c6dccdecc86cc83cc8f38ccac6b357ccce5ccc2ccd80ecc8709cca822ccbf5106ccc1cce701ccf77bcce211686a744b592ccc93491bcca415cc9f4c75cccbcc860fccf23bcc9acc84ccd6ccf867cce8cc9bcc92ccf6ccf4ccd5ccb84ecc9dcc9c0bd9477368613235362d64626664333862636466343133343862363335653630663865313835356339323836363637653639303534656632353063623530653935303935346363353131d92a63616e6469646174652d3934373439373738356531333466633639306133316461616430626631363136cf000001a0ef449e4fcf000001a0ef449e5bc093a870726f6d6f74656495a773657276696365d9477368613235362d30636363383737383930313339323639383066383865373661653537663236666238363131643735373435346631656661366133653536313762366636643032d9477368613235362d33396133356465613735303130363633646162386133613436366338366530343630383538363036306531666366363662656331366163633837366630366335d9477368613235362d35366536323264306634633834383138303738356162613366356262356533353333336233303661666135656664633539643739373030393563663430353965cf000001a0ef449f3acf000001a0ef449f3ac000c095c000c0c092000093c000c0