From b230703a5c2024871e2b6af77f32b8b933fcb840 Mon Sep 17 00:00:00 2001 From: Eureka Date: Fri, 13 Nov 2026 17:36:57 +0000 Subject: [PATCH] Idunn: a stop halts before irreversible steps; fail-closed reader; break-glass clear - Re-read the stop at the static flip intent and at entry into Fencing for a deploy (stop_note, the same path as Sealing). After the intent or fencing the transaction runs to its end; documented. - Absent means NotFound in an existing directory; every other lookup error, a zero-length file and a missing parent fail closed. Unreadable names the idunn-provision deployment-brake-clear command. - replace_stop is the operator's break-glass: it replaces v1, foreign, ambiguous, zero-length and caller-owned undecodable stores, never a symlink or another user's file. - Mutation-proven tests: foreign-owner Clear, Unreadable admitting at Sealing, author from USER. - docs: stop semantics, v6 downgrade hazard and backup; legacy script no longer requires the store and drops the signed-grant wording; two stale doc lines. Co-Authored-By: Claude Sonnet 5.5 --- deploy/legacy/idunn-yggdrasil | 18 ++-- docs/deployment-authority.md | 25 +++++ docs/route-continuity-cut.md | 2 +- docs/verify-transaction-cut.md | 2 +- src/brake.rs | 146 ++++++++++++++++++++++++++--- src/control_plane.rs | 162 +++++++++++++++++++++++++++++---- 6 files changed, 313 insertions(+), 42 deletions(-) diff --git a/deploy/legacy/idunn-yggdrasil b/deploy/legacy/idunn-yggdrasil index d299ec8..62a18eb 100644 --- a/deploy/legacy/idunn-yggdrasil +++ b/deploy/legacy/idunn-yggdrasil @@ -153,7 +153,7 @@ deployment_target_policy() { esac } -require_signed_deployment_grant() { +require_deployment_admission() { source_commit="$3" deployment_request_id="$4" release_authority_id="$5" @@ -179,15 +179,13 @@ require_signed_deployment_grant() { [ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; } require_root_owned_resolved_executable "$idunn_provision" - require_root_owned_regular_path "$brake_store" - require_root_owned_regular_path "$brake_lock" - # This validation is the deployment-admission point. The signed expiry is a - # start-before deadline: it denies a later transaction, but does not become a - # timer-shaped veto over one already admitted. The root actuator remains the - # sole shared-lock holder through the synchronous consequence, so an explicit - # engage/release transition cannot race or rewrite that transaction. - exec 9<"$brake_lock" + # This validation is the deployment-admission point. An absent or clear stop + # admits; an engaged or unreadable one makes the status verb exit nonzero. + # The root actuator remains the sole shared-lock holder through the + # synchronous consequence, so an engage or clear cannot race or rewrite it. + # The lock file is created here when no store has ever been written. + exec 9>>"$brake_lock" /usr/bin/flock --shared 9 "$idunn_provision" deployment-brake-status \ --store "$brake_store" \ @@ -312,7 +310,7 @@ case "$action:$target" in manifest="/srv/odin/deploy-manifests/$target" require_root_owned_regular_path "$manifest" [ -x "$manifest" ] || { echo "root actuator manifest is not executable: $manifest" >&2; exit 1; } - require_signed_deployment_grant "$@" + require_deployment_admission "$@" export IDUNN_ACTUATOR=1 export IDUNN_COMMAND_AUTHORITY=idunn-daemon export IDUNN_SOURCE_COMMIT="$source_commit" diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index f454248..4722734 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -299,6 +299,31 @@ names both content addresses and the executable artifact digest. Present and Ready remain Odin-owned correlations over independently verified runtime evidence; the deployment-plan module deliberately does not model them. +## The deployment stop + +`src/brake.rs` is the only reader and writer of the stop. A stop halts work +that has not crossed an irreversible step, and it is read again at each such +step: Sealing (`advance_sealing`), the static flip intent (`advance_static`) and +the entry into Fencing for a deploy (`advance_warming`). An engaged stop there +makes the transaction wait with a note naming who engaged it and why; nothing +aborts and nothing moves. After the flip intent is recorded, or once a deploy +has fenced, the transaction runs to its end: a stop halts before the +irreversible step, never mid-fence or mid-flip. Continuity is never gated. +The sequence of a stop record is positive and advances by one per verb; nothing +reads it for authority, so it is not validated further. + +Absent means the store file is missing in an existing directory. Anything else +Idunn cannot judge (a symlink, a zero-length or undecodable file, another +runtime's record, another owner, an unsearchable or non-directory parent) reads +as unreadable and fails closed; the wait note carries the +`idunn-provision deployment-brake-clear` command that replaces it. + +Record v6 (`deployment_admitted_at_unix_millis`, key 38) is forward-only: an +Idunn that predates v6 cannot read a migrated `control.cc`, so, as for v5 (see +`route-continuity-cut.md`, "An older Idunn refuses the v5 store"), take a +byte-compared pre-install backup of `control.cc` and `history.cc` with Idunn +stopped, and roll back by restoring the backup, never by swapping the binary. + ## Initial drivers and replaceable boundary The first Yggdrasil implementation uses: diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index a278165..d1f52d2 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -1160,7 +1160,7 @@ target's generation or to its `TargetSupervision` ends that target's iteration: - **Why no existing owner can serve:** - The generation does not exist for a first deploy (the ruling). - The transaction dies with each attempt. - - Brakes are operator-signed and are not Idunn's to write. + - The deployment brake is a stop that a steward or the operator writes (`src/brake.rs`); Idunn writes no brake. - **What it replaces:** two carry compensators, `for_new_incarnation` and the backoff copy in `from_transaction`, plus a history scan. diff --git a/docs/verify-transaction-cut.md b/docs/verify-transaction-cut.md index 87387dd..5bc21db 100644 --- a/docs/verify-transaction-cut.md +++ b/docs/verify-transaction-cut.md @@ -155,7 +155,7 @@ cleaned up. `idunn redeploy --daemon …`. The installed binary has no `redeploy` command (probe of its usage text), so the script is dead and is **not** the current pattern. The current pattern is `idunn up` followed by - `idunn-provision deployment-brake-release` + `idunn-provision deployment-brake-clear` when a stop is engaged (an absent or clear store admits) (`runbooks/idunn-host-raven.md`, "Deploy Muninn to Raven"). - F24. Build placement (`gamecult-ops/inventory.md:559-566`): build on or for the target's host. Yggdrasil has 16 vCPU and 62 GiB and carries live services, so diff --git a/src/brake.rs b/src/brake.rs index 570cf94..d44cd96 100644 --- a/src/brake.rs +++ b/src/brake.rs @@ -189,14 +189,40 @@ fn check_file(path: &Path, _: u32) -> Result<()> { Ok(()) } +/// Absent means exactly this: the final component does not exist and its +/// parent is an existing directory. Every other failure to look (permission, +/// a parent that is a file, a symlink loop) is a store this runtime cannot +/// judge, and fails closed. +fn is_absent(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let parent = path.parent().filter(|p| !p.as_os_str().is_empty()); + let parent = parent.unwrap_or_else(|| Path::new(".")); + ensure!( + std::fs::metadata(parent).is_ok_and(|m| m.is_dir()), + "deployment stop store directory {} does not exist", + parent.display() + ); + Ok(true) + } + Err(error) => Err(error) + .with_context(|| format!("looking for deployment stop store {}", path.display())), + } +} + fn read_checked(path: &Path, runtime_id: &str, owner: u32) -> Result> { - if std::fs::symlink_metadata(path).is_err() { + if is_absent(path)? { return Ok(None); } check_file(path, owner)?; + ensure!( + std::fs::metadata(path)?.len() > 0, + "deployment stop store is zero length" + ); let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; match entries.as_slice() { - [] => Ok(None), + [] => bail!("deployment stop store holds no record"), [entry] => { let record = DeploymentStop::decode(entry)?; ensure!( @@ -215,15 +241,26 @@ pub(crate) fn read_stop(path: &Path, runtime_id: &str, expected_owner_uid: u32) Ok(None) => Stop::Absent, Ok(Some(record)) if record.state == StopState::Clear => Stop::Clear(record), Ok(Some(record)) => Stop::Engaged(record), - Err(error) => Stop::Unreadable(format!("{error:#}")), + Err(error) => Stop::Unreadable(format!( + "{error:#}; to replace it run: idunn-provision deployment-brake-clear \ + --store {} --runtime-id {runtime_id} --reason (or deployment-brake-engage)", + path.display() + )), } } /// Write the next record under a compare-exchange of the whole store, so a -/// concurrent verb or a store that moved since it was read is refused, and a -/// foreign or corrupt predecessor is replaced only by an explicit verb. The +/// concurrent verb or a store that moved since it was read is refused. The /// author is the login name of the process uid and the sequence is the -/// predecessor's plus one; neither is an argument. +/// predecessor's plus one (1 when there is none or it cannot be decoded); +/// neither is an argument. +/// +/// This is also the operator's break-glass. A predecessor that is a retired v1 +/// brake, another runtime's record, ambiguous, or zero length is replaced. A +/// file that is not a CultCache store at all (junk bytes) is replaced only +/// when it is a regular file owned by the caller, by removing it first. A +/// symlink or another user's file is refused. The sequence is not validated +/// beyond being positive: nothing reads it for authority. pub(crate) fn replace_stop( path: &Path, runtime_id: &str, @@ -232,14 +269,22 @@ pub(crate) fn replace_stop( now_unix_millis: u64, ) -> Result { let store = SingleFileMessagePackBackingStore::new(path); - let before = store.pull_all_read_only_snapshot()?; + let before = match store.pull_all_read_only_snapshot() { + Ok(entries) => entries, + Err(undecodable) => { + check_file(path, process_uid()).with_context(|| { + format!("deployment stop store is undecodable ({undecodable:#}) and not the caller's to replace") + })?; + std::fs::remove_file(path)?; + Vec::new() + } + }; let sequence = match before.as_slice() { - [] => 1, [entry] => match DeploymentStop::decode(entry) { Ok(previous) => previous.sequence.saturating_add(1), Err(_) => 1, }, - _ => bail!("deployment stop store is ambiguous"), + _ => 1, }; let record = DeploymentStop { schema_version: DEPLOYMENT_STOP_SCHEMA.into(), @@ -296,8 +341,12 @@ mod tests { engage(&path, "held", 100)?; // Another runtime's record. assert!(!read_stop(&path, "other", me).admits()); - // Owned by someone other than the expected uid. - assert!(!read_stop(&path, "runtime", me + 1).admits()); + // Owned by someone other than the expected uid: even a CLEAR record + // from a foreign owner must not admit. + let cleared = dir.path().join("cleared.cc"); + replace_stop(&cleared, "runtime", StopState::Clear, "thaw", 100)?; + assert!(read_stop(&cleared, "runtime", me).admits()); + assert!(matches!(read_stop(&cleared, "runtime", me + 1), Stop::Unreadable(_))); // A symlink to a valid, clear store. let real = dir.path().join("real.cc"); replace_stop(&real, "runtime", StopState::Clear, "clear", 100)?; @@ -386,4 +435,79 @@ mod tests { assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "moved"); Ok(()) } + + #[test] + fn only_a_missing_file_in_an_existing_directory_is_absent() -> Result<()> { + let dir = tempfile::tempdir()?; + let me = process_uid(); + // Parent is a regular file: ENOTDIR, not absence. + let file = dir.path().join("plain"); + std::fs::write(&file, b"x")?; + assert!(matches!(read_stop(&file.join("stop.cc"), "runtime", me), Stop::Unreadable(_))); + // Parent does not exist. + assert!(matches!( + read_stop(&dir.path().join("nope/stop.cc"), "runtime", me), + Stop::Unreadable(_) + )); + // Zero-length file. + let empty = dir.path().join("empty.cc"); + std::fs::write(&empty, b"")?; + assert!(matches!(read_stop(&empty, "runtime", me), Stop::Unreadable(_))); + // An existing directory with no store is absent. + assert_eq!(read_stop(&dir.path().join("stop.cc"), "runtime", me), Stop::Absent); + // A parent that cannot be searched (skipped as root, which can). + #[cfg(unix)] + if me != 0 { + use std::os::unix::fs::PermissionsExt; + let locked = dir.path().join("locked"); + std::fs::create_dir(&locked)?; + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000))?; + let reading = read_stop(&locked.join("stop.cc"), "runtime", me); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700))?; + assert!(matches!(reading, Stop::Unreadable(_)), "{reading:?}"); + } + Ok(()) + } + + #[test] + fn the_author_is_the_uid_login_never_the_environment() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + // SAFETY: no other test reads USER, LOGNAME or LNAME. + unsafe { + std::env::set_var("USER", "spoof"); + std::env::set_var("LOGNAME", "spoof"); + } + let written = engage(&path, "held", 10)?; + assert_eq!(written.by, login_name(process_uid())?); + assert_ne!(written.by, "spoof"); + Ok(()) + } + + #[test] + fn unreadable_names_the_command_and_a_break_glass_clear_replaces_junk_and_v1() -> Result<()> { + let dir = tempfile::tempdir()?; + let me = process_uid(); + let path = dir.path().join("stop.cc"); + std::fs::write(&path, b"junk that is no store")?; + let Stop::Unreadable(why) = read_stop(&path, "runtime", me) else { + bail!("junk admitted") + }; + assert!(why.contains("idunn-provision deployment-brake-clear --store"), "{why}"); + assert!(why.contains("--runtime-id runtime"), "{why}"); + let cleared = replace_stop(&path, "runtime", StopState::Clear, "operator break-glass", 50)?; + assert_eq!(cleared.sequence, 1); + assert!(read_stop(&path, "runtime", me).admits()); + // A symlink is not the caller's to replace. + let real = dir.path().join("real.cc"); + std::fs::write(&real, b"junk")?; + let link = dir.path().join("link.cc"); + #[cfg(unix)] + { + std::os::unix::fs::symlink(&real, &link)?; + assert!(replace_stop(&link, "runtime", StopState::Clear, "x", 60).is_err()); + assert_eq!(std::fs::read(&real)?, b"junk"); + } + Ok(()) + } } diff --git a/src/control_plane.rs b/src/control_plane.rs index dc71d7a..5b06543 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -8028,26 +8028,10 @@ impl Engine { let mut next = current.value.clone(); match current.value.command_kind { CommandKind::Deploy | CommandKind::Rollback => { - match self.deployment_stop(current)? { - Stop::Absent | Stop::Clear(_) => { - next.deployment_admitted_at_unix_millis = Some(now) - } - Stop::Engaged(held) => { - return self.record_gate_wait( - current, - &format!( - "deployment brake engaged by {}: {}", - held.by, held.reason - ), - ); - } - Stop::Unreadable(why) => { - return self.record_gate_wait( - current, - &format!("deployment brake is unreadable and fails closed: {why}"), - ); - } + if let Some(note) = self.stop_note(current)? { + return self.record_gate_wait(current, ¬e); } + next.deployment_admitted_at_unix_millis = Some(now); } CommandKind::Continuity => { if !self.lifecycle_allows(current, now)? { @@ -8176,6 +8160,11 @@ impl Engine { let Some(flip) = ¤t.value.flip else { // Nothing has moved yet, so a refusal here ends the transaction // honestly: the release is still what was sealed, or it is not. + // The flip intent is the last point a stop can halt a static + // release: once it is recorded the flip runs to its end. + if let Some(note) = self.stop_note(current)? { + return self.record_gate_wait(current, ¬e); + } self.static_release.verify_installed(workload, release)?; let previous_release_id = self.static_release.observe_current(workload)?; let release_id = release.sealed_release_id.clone(); @@ -8188,6 +8177,7 @@ impl Engine { flipped_at_unix_millis: None, }); next.updated_at_unix_millis = now; + next.last_error = None; Ok(()) }); }; @@ -8418,6 +8408,14 @@ impl Engine { }); } + // Fencing is the last pre-fencing abort-able point of a deploy: an + // engaged stop halts here, before the incumbent is fenced. A fenced + // deploy runs to its end. Continuity is never gated by this stop. + if current.value.command_kind == CommandKind::Deploy + && let Some(note) = self.stop_note(current)? + { + return self.record_gate_wait(current, ¬e); + } self.transition(current, DeploymentPhase::Fencing) } @@ -9343,6 +9341,22 @@ impl Engine { .context("no private candidate port remains in the operator range") } + /// Why the deployment stop holds this transaction, or None when it admits. + /// Read live at each point that is still reversible: Sealing, the static + /// flip intent and the entry into Fencing. + fn stop_note(&self, current: &Stored) -> Result> { + Ok(match self.deployment_stop(current)? { + Stop::Absent | Stop::Clear(_) => None, + Stop::Engaged(held) => Some(format!( + "deployment brake engaged by {}: {}", + held.by, held.reason + )), + Stop::Unreadable(why) => Some(format!( + "deployment brake is unreadable and fails closed: {why}" + )), + }) + } + /// What the binding's deployment stop says about this transaction's /// runtime. Read live; never cached, never consumed. fn deployment_stop(&self, current: &Stored) -> Result { @@ -14111,6 +14125,10 @@ mod tests { .split_once("[brakes]") .context("binding has no brakes table")? .1 + ) + .replace( + "/var/lib/gamecult/idunn/service-deployment-brake.cc", + &world.root.join("service-deployment-brake.cc").display().to_string(), ), )?; let recipe = RECIPE @@ -19999,6 +20017,66 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(deploy) } + /// An engaged stop halts a deploy before it fences, the last pre-fencing + /// abort-able point; clearing lets the same transaction fence. + #[test] + fn an_engaged_stop_halts_entry_into_fencing_and_a_clear_lets_it_through() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + let plan = deploy.plan.as_ref().context("no plan")?; + set_stop(plan, StopState::Engaged, "freeze before fencing")?; + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + let held = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(held.phase, DeploymentPhase::Warming); + assert!(held.pre_fencing_abort.is_none() && held.completion.is_none()); + assert!( + held.last_error.as_deref().is_some_and(|note| note.contains("deployment brake engaged by") + && note.contains("freeze before fencing")), + "{:?}", + held.last_error + ); + set_stop(plan, StopState::Clear, "thaw")?; + assert!(world.engine.run_scheduler_tick()?); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + Ok(()) + } + + /// An Unreadable stop holds a Sealing deploy with a message that names the + /// remedy; it never admits. + #[test] + fn an_unreadable_stop_keeps_a_sealing_deploy_held_with_the_remedy() -> Result<()> { + let world = EngineFixture::new()?; + let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + make_stop_first(&world, &deploy.transaction_id)?; + let sealing = resident_id(&world, &deploy.transaction_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_admitted_at_unix_millis = None; + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + let store = tail.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; + std::fs::create_dir_all(store.parent().context("no parent")?)?; + for bytes in [&b"junk that is no store"[..], &b""[..]] { + std::fs::write(&store, bytes)?; + let held = resident_id(&world, &deploy.transaction_id)?; + rephase(&world, &deploy.transaction_id, DeploymentPhase::Sealing)?; + let held_now = resident_id(&world, &deploy.transaction_id)?; + world.engine.advance_transaction(&held_now)?; + let after = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(after.phase, DeploymentPhase::Sealing, "{:?}", held.value.last_error); + assert!(!after.deployment_admitted()); + assert!( + after.last_error.as_deref().is_some_and(|note| note.contains("fails closed") + && note.contains("idunn-provision deployment-brake-clear")), + "{:?}", + after.last_error + ); + } + Ok(()) + } + #[test] fn a_continuity_minted_as_a_deployment_takes_the_target_is_resolved_next_pass() -> Result<()> { // (a) The deployment fences first in the tick, so supervision reads its @@ -26239,6 +26317,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide )?; std::fs::create_dir_all(site.join("www"))?; std::fs::create_dir_all(site.join("config"))?; + std::fs::create_dir_all(site.join("authority"))?; std::fs::write(site.join("config/site.json"), SITE_CONFIG_ONE)?; *world.engine.runner_stand_in.lock().unwrap() = Some(Arc::new(SealingRunner { tree: tree.clone() })); @@ -26404,6 +26483,51 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + #[cfg(unix)] + #[test] + fn a_stop_engaged_after_admission_halts_the_static_flip_until_it_clears() -> Result<()> { + let w = static_world()?; + let first = w.deploy("up-service")?; + let incumbent = w.current_release().context("no incumbent")?; + assert!(first.is_terminal()); + enqueue_deploy(&w.world, "up-service-2")?; + let id = w.transaction_of("up-service-2")?; + // Admitted by the absent-or-clear stop, flip intent not yet recorded. + w.tick_until(&id, "admission", |t| t.phase == DeploymentPhase::Starting)?; + assert!(latest_id(&w.world, &id)?.flip.is_none()); + w.hold()?; + for _ in 0..4 { + w.world.engine.resume_backoff.lock().unwrap().clear(); + let _ = w.world.engine.run_scheduler_tick(); + } + let held = latest_id(&w.world, &id)?; + assert_eq!(held.phase, DeploymentPhase::Starting); + assert!(held.flip.is_none(), "a halted flip recorded an intent"); + assert!(held.pre_fencing_abort.is_none() && held.completion.is_none()); + assert!(held.last_error.as_deref().is_some_and(|n| n.contains("deployment brake engaged by")), "{:?}", held.last_error); + assert_eq!(w.current_release().as_deref(), Some(incumbent.as_str()), "the link moved under a stop"); + w.release_brake(&held)?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert_ne!(w.current_release().as_deref(), Some(incumbent.as_str())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_stop_engaged_after_the_flip_intent_does_not_halt_the_flip() -> Result<()> { + let w = static_world()?; + let id = w.transaction_of("up-service")?; + w.tick_until(&id, "the flip intent", |t| t.flip.is_some())?; + w.hold()?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert!(done.flip.as_ref().is_some_and(FlipEvidence::landed)); + Ok(()) + } + #[cfg(unix)] #[test] fn static_release_failed_flip_leaves_current_and_ends_before_fencing() -> Result<()> {