From a20568ef70ff2f14d0fac8313536c2e3a71d1295 Mon Sep 17 00:00:00 2001 From: Eureka Date: Fri, 13 Nov 2026 17:24:34 +0000 Subject: [PATCH 1/2] Idunn: the deployment brake is a stop, not a per-deploy gate Delete the signed release path (deployment_brake_release, the operator enroll/export verbs, the anchor option and flag, the consumed-once scan, the durable re-verification, DeploymentAuthorization::validate_shape). src/brake.rs is the one reader and writer of the stop (idunn.deployment_stop.v1: engaged|clear, by from the process uid, reason, at, sequence). Absent or clear admits; a foreign, corrupt, symlinked, other-runtime or foreign-owned store fails closed. advance_sealing stamps deployment_admitted_at_unix_millis (key 38) for Deploy and Rollback alike, so static releases and rollbacks use the same stop. Control record v6: key 38 changes canonical bytes, so v5 gets a legacy layout, a lift, boot migration and pinned bytes tests. Key 16 stays a never-written legacy field so old terminal records decode. idunn-provision: deployment-brake-engage|clear|status take --store, --runtime-id, --reason only; status exits nonzero unless deployments admit. Co-Authored-By: Claude Sonnet 5.5 --- deploy/idunn-yggdrasil.service | 1 - deploy/legacy/idunn-yggdrasil | 12 +- docs/authority-map.md | 13 +- docs/deploy-provenance-target.md | 9 + docs/deployment-authority.md | 23 +- src/brake.rs | 389 ++++++++++++++ src/control_plane.rs | 843 +++++++++++++++++-------------- src/lib.rs | 1 + src/provisioning.rs | 479 ++++-------------- 9 files changed, 981 insertions(+), 789 deletions(-) create mode 100644 src/brake.rs diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index 1c45594..c7407ca 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -45,7 +45,6 @@ ExecStart=/usr/local/bin/idunn serve \ --idunn-identity-store /var/lib/gamecult/idunn/idunn-service-identity.cc \ --odin-correlation-store /var/lib/gamecult/odin/topology.cc \ --odin-trust-anchor /etc/gamecult/idunn/odin-topology-anchor.cc \ - --deployment-brake-operator-anchor /etc/gamecult/idunn/deployment-brake-operator-anchor.cc \ --host-actuator-bind 10.77.0.1:17890 \ --source-uid 986 \ --source-gid 978 diff --git a/deploy/legacy/idunn-yggdrasil b/deploy/legacy/idunn-yggdrasil index 26970c0..d299ec8 100644 --- a/deploy/legacy/idunn-yggdrasil +++ b/deploy/legacy/idunn-yggdrasil @@ -16,7 +16,6 @@ target="${2:-}" compose=/srv/compose/yggdrasil-apps.yaml brake_store=/var/lib/gamecult/idunn-authority/deployment-brake.cc brake_lock=/var/lib/gamecult/idunn-authority/deployment-brake.cc.lock -brake_anchor=/etc/gamecult/idunn/deployment-brake-operator-anchor.cc generation_lock=/var/lib/gamecult/idunn-authority/idunn-generation.lock idunn_provision=/usr/local/bin/idunn-provision @@ -182,7 +181,6 @@ require_signed_deployment_grant() { require_root_owned_resolved_executable "$idunn_provision" require_root_owned_regular_path "$brake_store" require_root_owned_regular_path "$brake_lock" - require_root_owned_regular_path "$brake_anchor" # This validation is the deployment-admission point. The signed expiry is a # start-before deadline: it denies a later transaction, but does not become a @@ -191,17 +189,9 @@ require_signed_deployment_grant() { # engage/release transition cannot race or rewrite that transaction. exec 9<"$brake_lock" /usr/bin/flock --shared 9 - now_unix_millis="$(/usr/bin/date +%s%3N)" - case "$now_unix_millis" in - ''|*[!0-9]*) echo "root actuator clock did not produce Unix milliseconds" >&2; exit 1 ;; - esac "$idunn_provision" deployment-brake-status \ --store "$brake_store" \ - --operator-anchor "$brake_anchor" \ - --runtime-id yggdrasil \ - --release-id "$source_commit" \ - --deployment-id "$deployment_request_id" \ - --now-unix-millis "$now_unix_millis" + --runtime-id yggdrasil if [ "$minimum_source_revision_required" = true ]; then require_root_owned_resolved_executable /usr/local/bin/idunn diff --git a/docs/authority-map.md b/docs/authority-map.md index c8a8546..001b644 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -240,7 +240,7 @@ Fencing retries its phase forward on a capped backoff and never stops its live candidate, since that is the only running copy of the admitted body; the deadline opens the same incident. The one exception is yielding: a continuity whose route proof has not succeeded gives way, through its own post-fencing -abort, to a deployment the deployment brake has released and that awaits the +abort, to a deployment the deployment stop admits and that awaits the handover; a continuity that has proven its route never yields. The `ContinuityExhausted` incident (six charged attempts in an hour, `restarts_page`) pages and gates nothing. When continuity next acts is derived @@ -257,9 +257,14 @@ held until a lift succeeds. It is a migration with a deletion line (follow-up still owe dependency failures not growing the streak and the dead-generation route refresh. -The deployment brake gates changes to source, release, configuration, schema, -unit, or authority binding. The lifecycle brake separately gates restart of an -already-admitted incarnation. Neither target brake may gate Idunn itself or an +The deployment brake is a stop, not a per-deploy gate: it gates changes to +source, release, configuration, schema, unit, or authority binding only while +engaged. Its one reader and writer is `src/brake.rs` (`idunn.deployment_stop.v1`: +engaged or clear, who, why, when, sequence). An absent or clear store admits; +a store that is present but unreadable, foreign, for another runtime, a symlink +or owned by another uid fails closed. Nothing is signed, bound to a release or +consumed, and one clear admits every deploy until it is engaged again. The +lifecycle brake separately gates restart of an already-admitted incarnation. Neither target brake may gate Idunn itself or an unrelated service. If Odin is unavailable, Idunn preserves existing routes and continuity for diff --git a/docs/deploy-provenance-target.md b/docs/deploy-provenance-target.md index 6b6293d..e1086e8 100644 --- a/docs/deploy-provenance-target.md +++ b/docs/deploy-provenance-target.md @@ -4,6 +4,15 @@ Status: target, Self, 2026-10-19 (campaign `idunn-deploy-provenance` in Eureka's mind). It states the ends. The map will live in `deploy-provenance-map.md`. +Live as of cut `brake-is-a-stop` (2026-11-13): the deployment brake is the +stop of invariant 3 and the signed per-deploy release is gone. `src/brake.rs` +reads and writes it; `idunn-provision deployment-brake-engage|clear|status` +take only `--store`, `--runtime-id` and `--reason`; the writer's login name and +the verb's clock are recorded; an absent or clear store admits. The +transaction stamps `deployment_admitted_at_unix_millis` (record v6). Still +planned: the typed authority provenance on the transaction (invariant 2), and a +steward-scoped store. + ## Why Operator, 2026-10-19: diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index 1c5a7f4..f454248 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -48,8 +48,8 @@ and health. CultMesh carries typed projections between them. scaling, restart, host reboot recovery, and crash recovery use the same source-to-recipe-to-artifact provenance and the same distinct write-lease and route-membership primitives. - Deployment may change that chain only while the deployment brake admits the - exact transaction. Continuity may restart only the already-admitted chain and + Deployment may change that chain only while the deployment stop admits it + (absent or clear; an engaged stop halts). Continuity may restart only the already-admitted chain and is governed by a separate lifecycle brake. - **Deletion line:** hard-coded target copies, raw `deploy_command` and `restart_command` state, `sh -c` actuation, root Git inspection, staged @@ -209,8 +209,8 @@ non-regular carry source fails the deployment before anything moves. What `current_link` names is observed from the symlink and confirmed by the latest Admitted static transaction's `flip.release_id`; it is stored nowhere else. A rollback is a flip back to `flip.previous_release_id`: release -directories stay, and Idunn prunes none. The brake is read exactly as for a -service, naming the binding's runtime id and the sealed release id. Release ids are per transaction, so +directories stay, and Idunn prunes none. The deployment stop is read exactly as for a +service, by the binding's runtime id. Release ids are per transaction, so redeploying an unchanged tarball with a changed site file is a new release that carries the new file, never an in-place re-carry of an existing one: the carry is part of a release. If `current_link` is replaced by a directory or by a link @@ -230,9 +230,8 @@ transaction (`CommandKind::Rollback`) that reuses the previous release exactly: the plan, sealed release and installed release are copied from the transaction that flipped it, so there is no source, no build and no source cleanup, and it creates no admitted generation. It starts at Sealing with that evidence and -reaches the same deployment brake as a deploy, which the operator releases for -the rollback transaction's id and the previous sealed release id; then -`advance_static` flips, with the same three durable steps, the same +reaches the same deployment stop as a deploy, which admits it unless engaged; +then `advance_static` flips, with the same three durable steps, the same nothing-aborts-after-the-rename rule and the same stuck-flip behavior. The previous release is derived, never stored: the newest Admitted static @@ -338,7 +337,7 @@ For a singleton incarnation: declaration, binding, launch contract, and input digests. 5. Derive the release-bound Expected incarnation from the validated sealed release and its exact compiled plan. Publish it only once the deployment - brake has admitted this exact transaction, under the candidate's own + stop has admitted this transaction (stamping `deployment_admitted_at_unix_millis`), under the candidate's own incarnation key, beside whatever the target's admitted incarnation projects. 6. Issue one opaque runtime-instance activation bound to the Expected digest, pass the exact Expected and activation documents through the actuator's @@ -361,8 +360,8 @@ For a singleton incarnation: and stop the incumbent. Sealing resolves the source, freezes it, runs every recipe step, and installs -the release before the deployment brake is consulted at all (`advance_sealing`). -The brake gates step 5, the first Verse-visible change: publishing the +the release before the deployment stop is consulted at all (`advance_sealing`). +The stop gates step 5, the first Verse-visible change: publishing the candidate's Expected incarnation on the move into `Starting`. Build and test already run unbraked, inside the sealing phase. @@ -393,7 +392,7 @@ identity, and route membership. Every restart receives a new runtime-instance activation and, for a state writer, a new process-bound lease only after the new warming presence is observed. A lease from the dead process is never reused. A continuity restart does not fetch a branch, rebuild, rerun a migration, alter -the launch contract, or consume the deployment brake. An explicit lifecycle +the launch contract, or pass the deployment stop. An explicit lifecycle brake may suspend this actuation without changing deployment authority. Continuity never stops. Six charged attempts in an hour only open the @@ -426,7 +425,7 @@ the admitted body. Both operator cases open an `operator-required` incident on. The one yield: a continuity whose route proof has not succeeded gives way to a -deployment the deployment brake has released that awaits the handover of the +deployment the deployment stop admits that awaits the handover of the target. It begins its own post-fencing abort and the deployment fences what is admitted. A continuity that has proven its route is serving never yields. diff --git a/src/brake.rs b/src/brake.rs new file mode 100644 index 0000000..570cf94 --- /dev/null +++ b/src/brake.rs @@ -0,0 +1,389 @@ +//! The deployment stop: the one reader and the one writer of the record that +//! halts deployments of a runtime. Engaged halts; absent or clear admits. Who +//! engaged it and why are on the record, and nothing is signed, issued, bound +//! to a release or consumed: the brake is a stop, not a per-deploy gate (ruling +//! brake-is-a-stop). The lifecycle brake is a different record and a different +//! store; nothing here touches it. +//! +//! The reader fails closed. A store that is present but foreign, ambiguous, +//! noncanonical, for another runtime, a symlink, or owned by anyone but the +//! expected uid reads as `Unreadable`, which does not admit. + +use std::path::Path; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore}; +use serde::{Deserialize, Serialize}; + +pub(crate) const DEPLOYMENT_STOP_SCHEMA: &str = "idunn.deployment_stop.v1"; +pub(crate) const DEPLOYMENT_STOP_KEY: &str = "deployment-stop"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum StopState { + Engaged, + Clear, +} + +impl StopState { + pub(crate) fn name(self) -> &'static str { + match self { + Self::Engaged => "engaged", + Self::Clear => "clear", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.deployment_stop", schema = "idunn.deployment_stop.v1")] +pub(crate) struct DeploymentStop { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) runtime_id: String, + #[cultcache(key = 2)] + pub(crate) state: StopState, + /// The login name of the user who ran the verb, read from the process uid. + #[cultcache(key = 3)] + pub(crate) by: String, + #[cultcache(key = 4)] + pub(crate) reason: String, + /// The verb's clock. + #[cultcache(key = 5)] + pub(crate) at_unix_millis: u64, + /// The previous record's sequence plus one; 1 for the first. + #[cultcache(key = 6)] + pub(crate) sequence: u64, +} + +fn bounded(value: &str, label: &str, max: usize) -> Result<()> { + ensure!( + !value.trim().is_empty() && value.len() <= max && !value.chars().any(char::is_control), + "deployment stop {label} is empty, oversized, or contains control characters" + ); + Ok(()) +} + +impl DeploymentStop { + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == DEPLOYMENT_STOP_SCHEMA, + "deployment stop schema is unsupported" + ); + bounded(&self.runtime_id, "runtime id", 256)?; + bounded(&self.by, "author", 256)?; + bounded(&self.reason, "reason", 1024)?; + ensure!( + self.at_unix_millis > 0 && self.sequence > 0, + "deployment stop time or sequence is zero" + ); + Ok(()) + } + + fn envelope(&self) -> Result { + self.validate()?; + let stored_at = chrono::DateTime::from_timestamp_millis(i64::try_from(self.at_unix_millis)?) + .ok_or_else(|| anyhow!("deployment stop time is out of range"))? + .to_rfc3339(); + Ok(CultCacheEnvelope { + key: DEPLOYMENT_STOP_KEY.into(), + r#type: Self::TYPE.into(), + payload: rmp_serde::to_vec(self)?, + stored_at, + schema_id: Some(DEPLOYMENT_STOP_SCHEMA.into()), + }) + } + + fn decode(envelope: &CultCacheEnvelope) -> Result { + ensure!( + envelope.r#type == Self::TYPE + && envelope.key == DEPLOYMENT_STOP_KEY + && envelope.schema_id.as_deref() == Some(DEPLOYMENT_STOP_SCHEMA), + "deployment stop store holds a foreign record" + ); + let record: Self = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&record)? == envelope.payload, + "deployment stop is noncanonical" + ); + record.validate()?; + Ok(record) + } +} + +/// What one store says about a runtime's deployments. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Stop { + /// No store, or an empty one: nothing halts. + Absent, + Clear(DeploymentStop), + Engaged(DeploymentStop), + /// Present but not a record this runtime can trust. Does not admit. + Unreadable(String), +} + +impl Stop { + pub(crate) fn admits(&self) -> bool { + matches!(self, Self::Absent | Self::Clear(_)) + } +} + +#[cfg(unix)] +pub(crate) fn process_uid() -> u32 { + // SAFETY: geteuid has no failure mode and no arguments. + unsafe { libc::geteuid() } +} + +#[cfg(not(unix))] +pub(crate) fn process_uid() -> u32 { + 0 +} + +#[cfg(unix)] +fn login_name(uid: u32) -> Result { + // SAFETY: getpwuid returns null or a pointer into libc's static buffer, + // copied before any other call; the verbs are single threaded. + unsafe { + let entry = libc::getpwuid(uid); + ensure!(!entry.is_null(), "uid {uid} has no login name"); + Ok(std::ffi::CStr::from_ptr((*entry).pw_name) + .to_string_lossy() + .into_owned()) + } +} + +#[cfg(not(unix))] +fn login_name(_: u32) -> Result { + std::env::var("USERNAME").context("USERNAME is unset") +} + +/// Open without following a final symlink and judge the file by fstat before +/// anything is read from it. +#[cfg(unix)] +fn check_file(path: &Path, expected_owner_uid: u32) -> Result<()> { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + .with_context(|| format!("opening deployment stop store {}", path.display()))?; + let metadata = file.metadata()?; + ensure!( + metadata.is_file(), + "deployment stop store is not a regular file" + ); + ensure!( + metadata.uid() == expected_owner_uid, + "deployment stop store is owned by uid {}, not {expected_owner_uid}", + metadata.uid() + ); + Ok(()) +} + +#[cfg(not(unix))] +fn check_file(path: &Path, _: u32) -> Result<()> { + ensure!( + std::fs::metadata(path)?.is_file(), + "deployment stop store is not a regular file" + ); + Ok(()) +} + +fn read_checked(path: &Path, runtime_id: &str, owner: u32) -> Result> { + if std::fs::symlink_metadata(path).is_err() { + return Ok(None); + } + check_file(path, owner)?; + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + match entries.as_slice() { + [] => Ok(None), + [entry] => { + let record = DeploymentStop::decode(entry)?; + ensure!( + record.runtime_id == runtime_id, + "deployment stop belongs to runtime {}", + record.runtime_id + ); + Ok(Some(record)) + } + _ => bail!("deployment stop store is ambiguous"), + } +} + +pub(crate) fn read_stop(path: &Path, runtime_id: &str, expected_owner_uid: u32) -> Stop { + match read_checked(path, runtime_id, expected_owner_uid) { + Ok(None) => Stop::Absent, + Ok(Some(record)) if record.state == StopState::Clear => Stop::Clear(record), + Ok(Some(record)) => Stop::Engaged(record), + Err(error) => Stop::Unreadable(format!("{error:#}")), + } +} + +/// Write the next record under a compare-exchange of the whole store, so a +/// concurrent verb or a store that moved since it was read is refused, and a +/// foreign or corrupt predecessor is replaced only by an explicit verb. The +/// author is the login name of the process uid and the sequence is the +/// predecessor's plus one; neither is an argument. +pub(crate) fn replace_stop( + path: &Path, + runtime_id: &str, + state: StopState, + reason: &str, + now_unix_millis: u64, +) -> Result { + let store = SingleFileMessagePackBackingStore::new(path); + let before = store.pull_all_read_only_snapshot()?; + let sequence = match before.as_slice() { + [] => 1, + [entry] => match DeploymentStop::decode(entry) { + Ok(previous) => previous.sequence.saturating_add(1), + Err(_) => 1, + }, + _ => bail!("deployment stop store is ambiguous"), + }; + let record = DeploymentStop { + schema_version: DEPLOYMENT_STOP_SCHEMA.into(), + runtime_id: runtime_id.into(), + state, + by: login_name(process_uid())?, + reason: reason.into(), + at_unix_millis: now_unix_millis, + sequence, + }; + let next = record.envelope()?; + ensure!( + store.compare_exchange_snapshot(&before, &[next])?, + "deployment stop changed during transition" + ); + Ok(record) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn engage(path: &Path, reason: &str, at: u64) -> Result { + replace_stop(path, "runtime", StopState::Engaged, reason, at) + } + + #[test] + fn absent_and_empty_stores_admit_a_clear_one_admits_an_engaged_one_does_not() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + let me = process_uid(); + assert_eq!(read_stop(&path, "runtime", me), Stop::Absent); + engage(&path, "freeze for the audit", 100)?; + let Stop::Engaged(held) = read_stop(&path, "runtime", me) else { + bail!("an engaged stop did not read as engaged") + }; + assert_eq!(held.reason, "freeze for the audit"); + assert_eq!(held.by, login_name(me)?); + assert!(!Stop::Engaged(held).admits()); + replace_stop(&path, "runtime", StopState::Clear, "audit done", 200)?; + let cleared = read_stop(&path, "runtime", me); + assert!(matches!(&cleared, Stop::Clear(record) if record.sequence == 2)); + assert!(cleared.admits()); + // Clear is not consumed: the same record admits again. + assert!(read_stop(&path, "runtime", me).admits()); + Ok(()) + } + + #[test] + fn corrupt_foreign_other_runtime_symlinked_or_foreign_owned_stores_stop() -> Result<()> { + let dir = tempfile::tempdir()?; + let me = process_uid(); + let path = dir.path().join("stop.cc"); + engage(&path, "held", 100)?; + // Another runtime's record. + assert!(!read_stop(&path, "other", me).admits()); + // Owned by someone other than the expected uid. + assert!(!read_stop(&path, "runtime", me + 1).admits()); + // A symlink to a valid, clear store. + let real = dir.path().join("real.cc"); + replace_stop(&real, "runtime", StopState::Clear, "clear", 100)?; + let link = dir.path().join("link.cc"); + #[cfg(unix)] + std::os::unix::fs::symlink(&real, &link)?; + #[cfg(unix)] + assert!(matches!(read_stop(&link, "runtime", me), Stop::Unreadable(_))); + // Bytes that are no store at all. + let junk = dir.path().join("junk.cc"); + std::fs::write(&junk, b"not msgpack at all")?; + assert!(matches!(read_stop(&junk, "runtime", me), Stop::Unreadable(_))); + // A record of another type under another key (the retired v1 brake). + let foreign = dir.path().join("foreign.cc"); + let store = SingleFileMessagePackBackingStore::new(&foreign); + let envelope = CultCacheEnvelope { + key: "deployment".into(), + r#type: "idunn.deployment_brake".into(), + payload: vec![0x90], + stored_at: "1970-01-01T00:00:00+00:00".into(), + schema_id: Some("idunn.deployment_brake.v1".into()), + }; + assert!(store.compare_exchange_snapshot(&[], &[envelope])?); + assert!(matches!(read_stop(&foreign, "runtime", me), Stop::Unreadable(_))); + // A noncanonical payload under the right identity. + let mut stop = DeploymentStop::decode(&store_entry(&path)?)?; + stop.sequence = 9; + let mut loose = stop.envelope()?; + loose.payload.push(0xc0); + let noncanonical = dir.path().join("noncanonical.cc"); + assert!(SingleFileMessagePackBackingStore::new(&noncanonical) + .compare_exchange_snapshot(&[], &[loose])?); + assert!(matches!(read_stop(&noncanonical, "runtime", me), Stop::Unreadable(_))); + Ok(()) + } + + fn store_entry(path: &Path) -> Result { + let mut entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + entries.pop().context("no entry") + } + + #[test] + fn a_verb_replaces_a_retired_brake_record_and_the_sequence_advances_by_one() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + let retired = CultCacheEnvelope { + key: "deployment".into(), + r#type: "idunn.deployment_brake".into(), + payload: vec![0x90], + stored_at: "1970-01-01T00:00:00+00:00".into(), + schema_id: Some("idunn.deployment_brake.v1".into()), + }; + assert!(store.compare_exchange_snapshot(&[], &[retired])?); + assert_eq!(engage(&path, "first", 10)?.sequence, 1); + assert_eq!(store.pull_all_read_only_snapshot()?.len(), 1); + assert_eq!(engage(&path, "second", 11)?.sequence, 2); + assert_eq!( + replace_stop(&path, "runtime", StopState::Clear, "third", 12)?.sequence, + 3 + ); + assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "third"); + Ok(()) + } + + #[test] + fn a_stale_exchange_is_refused_and_a_record_without_a_reason_is_not_written() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + engage(&path, "held", 10)?; + let store = SingleFileMessagePackBackingStore::new(&path); + let stale = store.pull_all_read_only_snapshot()?; + engage(&path, "moved", 11)?; + let next = DeploymentStop { + schema_version: DEPLOYMENT_STOP_SCHEMA.into(), + runtime_id: "runtime".into(), + state: StopState::Clear, + by: "someone".into(), + reason: "late".into(), + at_unix_millis: 12, + sequence: 2, + } + .envelope()?; + assert!(!store.compare_exchange_snapshot(&stale, &[next])?); + assert!(replace_stop(&path, "runtime", StopState::Clear, " ", 13).is_err()); + assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "moved"); + Ok(()) + } +} diff --git a/src/control_plane.rs b/src/control_plane.rs index 8e96a93..dc71d7a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14,22 +14,23 @@ use cultcache_rs::{ }; use cultnet_rs::{ AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, - IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, - IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, - IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, + IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, correlate_runtime_presence_claim, derive_service_identity_id, - evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, - verify_idunn_deployment_brake_authorization, verify_runtime_authority, + evaluate_idunn_continuity_restart, open_service_identity_at, verify_runtime_authority, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; +use crate::brake::{Stop, process_uid, read_stop}; +#[cfg(test)] +use crate::brake::StopState; use crate::deployment::{ DependencyKind, OperatorBinding, RolloutStrategy, RouteBinding, WorkloadBinding, capability_compatible, @@ -56,7 +57,8 @@ use crate::host_actuator::{ }; const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2"; -const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v5"; +const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v6"; +const DEPLOYMENT_TRANSACTION_SCHEMA_V5: &str = "idunn.deployment_transaction.v5"; const DEPLOYMENT_TRANSACTION_SCHEMA_V4: &str = "idunn.deployment_transaction.v4"; const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3"; const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2"; @@ -243,8 +245,12 @@ enum DeploymentPhase { Complete, } +/// The signed brake release a Deploy once consumed. Never written and never +/// verified now: the deployment brake is a stop (`brake.rs`). It stays so the +/// terminal records that hold one in key 16 still decode. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] +#[allow(dead_code)] struct DeploymentAuthorization { authorization_id: String, brake_sha256: String, @@ -252,27 +258,6 @@ struct DeploymentAuthorization { authorized_at_unix_millis: u64, } -impl DeploymentAuthorization { - fn validate_shape(&self) -> Result<()> { - require_id(&self.authorization_id, "deployment authorization id")?; - ensure!( - !self.canonical_brake_bytes.is_empty() - && self.brake_sha256 == sha256_id(&self.canonical_brake_bytes) - && self.authorized_at_unix_millis > 0, - "deployment authorization receipt is incomplete" - ); - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&self.canonical_brake_bytes)?; - record.validate()?; - ensure!( - rmp_serde::to_vec(&record)? == self.canonical_brake_bytes - && record.authorization_id.as_deref() == Some(&self.authorization_id), - "deployment authorization receipt is noncanonical or mismatched" - ); - Ok(()) - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct TopologyEvidence { @@ -1517,7 +1502,7 @@ mod cleanup_evidence { } } - /// The schema migration's one entry point: read a stored v2, v3 or v4 + /// The schema migration's one entry point: read a stored v2, v3, v4 or v5 /// transaction (lifting older ones) and return its key and current-shape /// envelope. The record may carry the marker the lift just set. pub(super) fn migrate_transaction_record( @@ -1678,7 +1663,7 @@ impl PostCommitCleanup { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v5" + schema = "idunn.deployment_transaction.v6" )] struct DeploymentTransaction { #[cultcache(key = 0)] @@ -1764,6 +1749,155 @@ struct DeploymentTransaction { /// predates it, which decodes as None: no schema bump (precedent key 34). #[cultcache(key = 37, default)] flip: Option, + /// When the deployment stop admitted this Deploy or Rollback at Sealing: + /// the stamp that replaced the signed release in key 16. Absent on every + /// record that predates it. + #[cultcache(key = 38, default)] + deployment_admitted_at_unix_millis: Option, +} + +/// The v5 transaction layout: the one before `deployment_admitted_at_unix_millis` +/// (key 38), which replaced the signed brake release. Records are decoded by it +/// and lifted, because the control store refuses a record whose bytes differ +/// from its re-encoding. Migration to v6 is forward-only: an Idunn that predates +/// v6 cannot read a migrated store. `history.cc` is never migrated, so this +/// struct lives until history is pruned. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_transaction", + schema = "idunn.deployment_transaction.v5" +)] +struct LegacyDeploymentTransactionV5 { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + transaction_id: String, + #[cultcache(key = 2)] + command_id: String, + #[cultcache(key = 3)] + command_kind: CommandKind, + #[cultcache(key = 4)] + target: String, + #[cultcache(key = 5)] + ordinal: u32, + #[cultcache(key = 6)] + phase: DeploymentPhase, + #[cultcache(key = 7)] + created_at_unix_millis: u64, + #[cultcache(key = 8)] + updated_at_unix_millis: u64, + #[cultcache(key = 9)] + incumbent_generation_id: Option, + #[cultcache(key = 10)] + plan: Option, + #[cultcache(key = 11)] + frozen_source: Option, + #[cultcache(key = 12)] + sealed_release: Option, + #[cultcache(key = 13)] + installed_release: Option, + #[cultcache(key = 14)] + expected: Option, + #[cultcache(key = 15)] + expected_publication_sha256: Option, + #[cultcache(key = 16)] + deployment_authorization: Option, + #[cultcache(key = 17)] + lifecycle_authorized_at_unix_millis: Option, + #[cultcache(key = 18)] + activation: Option, + #[cultcache(key = 19)] + workload: Option, + #[cultcache(key = 20)] + activation_publication_sha256: Option, + #[cultcache(key = 21)] + latest_odin_observation: Option, + #[cultcache(key = 22)] + warming: Option, + #[cultcache(key = 23)] + route_preflight: Option, + #[cultcache(key = 24)] + isolation: Option, + #[cultcache(key = 25)] + fencing: Option, + #[cultcache(key = 26)] + leasing: Option, + #[cultcache(key = 27)] + ready: Option, + #[cultcache(key = 28)] + routing: Option, + #[cultcache(key = 29)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 30)] + last_error: Option, + #[cultcache(key = 31)] + completion: Option, + #[cultcache(key = 32)] + pre_fencing_abort: Option, + #[cultcache(key = 33)] + post_commit_cleanup: Option, + // because every transaction already durable predates this field: + // an absent key 34 decodes as None, which is exactly "this transaction has + // no post-fencing abort". Without it the daemon refuses to read its own + // control store and crashloops. + #[cultcache(key = 34, default)] + post_fencing_abort: Option, + /// Set by `transition` on entry to each post-fencing phase. + #[cultcache(key = 35)] + phase_deadline: Option, + #[cultcache(key = 36)] + lease_adoption: Option, + /// A static release's carry and flip. Absent on every record that + /// predates it, which decodes as None: no schema bump (precedent key 34). + #[cultcache(key = 37, default)] + flip: Option, +} + + +impl LegacyDeploymentTransactionV5 { + fn into_current(self) -> DeploymentTransaction { + DeploymentTransaction { + schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), + transaction_id: self.transaction_id, + command_id: self.command_id, + command_kind: self.command_kind, + target: self.target, + ordinal: self.ordinal, + phase: self.phase, + created_at_unix_millis: self.created_at_unix_millis, + updated_at_unix_millis: self.updated_at_unix_millis, + incumbent_generation_id: self.incumbent_generation_id, + plan: self.plan, + frozen_source: self.frozen_source, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + expected_publication_sha256: self.expected_publication_sha256, + deployment_authorization: self.deployment_authorization, + lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, + activation: self.activation, + workload: self.workload, + activation_publication_sha256: self.activation_publication_sha256, + latest_odin_observation: self.latest_odin_observation, + warming: self.warming, + route_preflight: self.route_preflight, + isolation: self.isolation, + fencing: self.fencing, + leasing: self.leasing, + ready: self.ready, + routing: self.routing, + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + last_error: self.last_error, + completion: self.completion, + pre_fencing_abort: self.pre_fencing_abort, + post_commit_cleanup: self.post_commit_cleanup, + post_fencing_abort: self.post_fencing_abort, + phase_deadline: self.phase_deadline, + lease_adoption: self.lease_adoption, + flip: self.flip, + deployment_admitted_at_unix_millis: None, + } + } } /// The v4 transaction layout: the current one before a static release's @@ -1900,6 +2034,7 @@ impl LegacyDeploymentTransactionV4 { phase_deadline: self.phase_deadline, lease_adoption: self.lease_adoption, flip: None, + deployment_admitted_at_unix_millis: None, } } } @@ -2151,9 +2286,9 @@ impl DeploymentTransaction { self.flip.is_some() && error.downcast_ref::().is_none() } - /// The runtime and the sealed release a deployment brake release must name. - /// One reading for both kinds of target: the binding's runtime identity and - /// the sealed release, which a service's Expected projection copies. + /// The runtime and the sealed release this transaction names, for the + /// status listing. One reading for both kinds of target: the binding's + /// runtime identity and the sealed release. fn brake_subject(&self) -> Result<(String, String)> { let plan = required(&self.plan, "transaction plan")?; let release = required(&self.sealed_release, "sealed release")?; @@ -2219,6 +2354,7 @@ impl DeploymentTransaction { phase_deadline: None, lease_adoption: None, flip: None, + deployment_admitted_at_unix_millis: None, }; transaction.validate()?; Ok(transaction) @@ -2400,6 +2536,12 @@ impl DeploymentTransaction { /// What a static release transaction may hold. It never runs a workload, so /// every piece of evidence of one is absent, and it uses three phases: /// Sealing, Starting (the flip, behind the brake) and Complete. + /// Whether the deployment stop admitted this transaction: stamped at + /// Sealing now, or a legacy record that holds a consumed signed release. + fn deployment_admitted(&self) -> bool { + self.deployment_admitted_at_unix_millis.is_some() || self.deployment_authorization.is_some() + } + fn validate_static(&self) -> Result<()> { ensure!( self.command_kind != CommandKind::Continuity, @@ -2443,13 +2585,13 @@ impl DeploymentTransaction { && self.sealed_release.is_some() && self.installed_release.is_some() && (self.command_kind == CommandKind::Rollback || self.frozen_source.is_some()) - && self.deployment_authorization.is_some(), - "static release entered Starting without sealed release evidence and brake consumption" + && self.deployment_admitted(), + "static release entered Starting without sealed release evidence and the stop's admission" ); } else { ensure!( - self.flip.is_none() && self.deployment_authorization.is_none(), - "static release flipped or consumed the brake before Starting" + self.flip.is_none() && !self.deployment_admitted(), + "static release flipped or was admitted by the stop before Starting" ); } if self.phase == DeploymentPhase::Complete { @@ -2661,9 +2803,6 @@ impl DeploymentTransaction { "lease adoption does not name the granted lease" ); } - if let Some(authorization) = &self.deployment_authorization { - authorization.validate_shape()?; - } let is_static = self.is_static(); if let Some(flip) = &self.flip { ensure!(is_static, "a flip belongs to a static release transaction"); @@ -2818,13 +2957,13 @@ impl DeploymentTransaction { ); match self.command_kind { CommandKind::Deploy => ensure!( - self.frozen_source.is_some() && self.deployment_authorization.is_some(), - "deployment entered Starting without frozen source and brake consumption" + self.frozen_source.is_some() && self.deployment_admitted(), + "deployment entered Starting without frozen source and the stop's admission" ), CommandKind::Rollback => bail!("a rollback is a static release transaction"), CommandKind::Continuity => ensure!( self.lifecycle_authorized_at_unix_millis.is_some() - && self.deployment_authorization.is_none(), + && !self.deployment_admitted(), "continuity entered Starting without its lifecycle gate" ), } @@ -3202,7 +3341,6 @@ struct RuntimeOptions { odin_correlation_store: PathBuf, odin_trust_anchor: PathBuf, idunn_identity_store: PathBuf, - deployment_brake_operator_anchor: PathBuf, source_identity: Option, topology_maximum_age_millis: u64, topology_maximum_future_skew_millis: u64, @@ -3232,9 +3370,6 @@ impl Default for RuntimeOptions { idunn_identity_store: PathBuf::from( "/var/lib/gamecult/idunn/idunn-service-identity.cc", ), - deployment_brake_operator_anchor: PathBuf::from( - "/etc/gamecult/idunn/deployment-brake-operator-anchor.cc", - ), source_identity: None, topology_maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, topology_maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, @@ -3405,9 +3540,6 @@ fn parse_serve(mut args: impl Iterator) -> Result { "--idunn-identity-store" => { options.idunn_identity_store = path_value(&mut args, &argument)? } - "--deployment-brake-operator-anchor" => { - options.deployment_brake_operator_anchor = path_value(&mut args, &argument)? - } "--topology-maximum-age-millis" => { options.topology_maximum_age_millis = u64_value(&mut args, &argument)? } @@ -3906,17 +4038,6 @@ impl ControlSnapshot { } } } - let mut authorizations = BTreeSet::new(); - for authorization in self - .transactions - .iter() - .filter_map(|stored| stored.value.deployment_authorization.as_ref()) - { - ensure!( - authorizations.insert(authorization.authorization_id.as_str()), - "deployment authorization was consumed more than once" - ); - } Ok(()) } @@ -4234,6 +4355,7 @@ impl LegacyDeploymentTransaction { phase_deadline: None, lease_adoption: None, flip: None, + deployment_admitted_at_unix_millis: None, } } } @@ -4442,10 +4564,19 @@ impl LegacyAdmittedGenerationV3 { } } -/// Decode a v2, v3 or v4 transaction and lift it to the current shape, without +/// Decode a v2, v3, v4 or v5 transaction and lift it to the current shape, without /// validating it. History reads stop here; the control store validates. fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result { let schema = match envelope.schema_id.as_deref() { + Some(schema @ DEPLOYMENT_TRANSACTION_SCHEMA_V5) => { + let value: LegacyDeploymentTransactionV5 = rmp_serde::from_slice(&envelope.payload) + .with_context(|| format!("decoding a {schema} deployment transaction"))?; + ensure!( + value.schema_version == schema, + "stored transaction schema differs from its envelope" + ); + return Ok(value.into_current()); + } Some(schema @ DEPLOYMENT_TRANSACTION_SCHEMA_V4) => { let value: LegacyDeploymentTransactionV4 = rmp_serde::from_slice(&envelope.payload) .with_context(|| format!("decoding a {schema} deployment transaction"))?; @@ -4473,7 +4604,7 @@ fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result Result { DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3 | DEPLOYMENT_TRANSACTION_SCHEMA_V4 + | DEPLOYMENT_TRANSACTION_SCHEMA_V5 ) )) || (envelope.r#type == AdmittedGeneration::TYPE @@ -4914,6 +5046,7 @@ fn decode_history_transactions( DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3 | DEPLOYMENT_TRANSACTION_SCHEMA_V4 + | DEPLOYMENT_TRANSACTION_SCHEMA_V5 ) => { lift_legacy_transaction(&envelope) } @@ -6030,41 +6163,15 @@ impl Engine { /// and authorize nothing further, so re-proving them only creates ways for /// history to refuse a boot. On yggdrasil that is 259 of 260 records. /// - /// The operator anchor is read on demand rather than up front. Reading it - /// unconditionally meant an absent brake artifact refused startup against - /// an *empty* store -- a deployment brake gating Idunn itself, which - /// `F:\Projects\CLAUDE.md` forbids outright: a target brake "may never gate - /// Idunn itself or unrelated service lifecycle". + /// The deployment stop is not durable authority: it is read live at + /// Sealing and gates nothing already admitted, so no brake artifact is + /// read here and none can refuse a boot. fn validate_durable_authority(&self, snapshot: &ControlSnapshot) -> Result<()> { - let mut operator_anchor = None; for stored in &snapshot.transactions { let transaction = &stored.value; if transaction.is_terminal() { continue; } - if let Some(authorization) = &transaction.deployment_authorization { - authorization.validate_shape()?; - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; - if operator_anchor.is_none() { - operator_anchor = - Some(read_trust_anchor::( - &self.options.deployment_brake_operator_anchor, - )?); - } - verify_idunn_deployment_brake_authorization( - &record, - operator_anchor.as_ref().expect("read directly above"), - )?; - let (runtime_id, sealed_release_id) = transaction.brake_subject()?; - ensure!( - record.authorized_release_id.as_deref() == Some(sealed_release_id.as_str()) - && record.authorized_deployment_id.as_deref() - == Some(transaction.transaction_id.as_str()) - && record.runtime_id == runtime_id, - "durable deployment authorization names another release or transaction" - ); - } let lease = transaction .leasing .as_ref() @@ -6341,16 +6448,11 @@ impl Engine { { return LiftDecision::KeepAborting; } - match now_millis() { - Ok(now) => { - if self - .deploy_the_continuity_yields_to(snapshot, transaction, now) - .is_some() - { - return LiftDecision::KeepAborting; - } - } - Err(error) => return LiftDecision::Undecided(error), + if self + .deploy_the_continuity_yields_to(snapshot, transaction) + .is_some() + { + return LiftDecision::KeepAborting; } let lifted = (|| -> Result { let plan = required(&transaction.plan, "transaction plan")?; @@ -7926,13 +8028,26 @@ impl Engine { let mut next = current.value.clone(); match current.value.command_kind { CommandKind::Deploy | CommandKind::Rollback => { - let Some(authorization) = self.deployment_authorization(current, now)? else { - return self.record_gate_wait( - current, - "deployment brake has not released this exact transaction", - ); - }; - next.deployment_authorization = Some(authorization); + match self.deployment_stop(current)? { + Stop::Absent | Stop::Clear(_) => { + next.deployment_admitted_at_unix_millis = Some(now) + } + Stop::Engaged(held) => { + return self.record_gate_wait( + current, + &format!( + "deployment brake engaged by {}: {}", + held.by, held.reason + ), + ); + } + Stop::Unreadable(why) => { + return self.record_gate_wait( + current, + &format!("deployment brake is unreadable and fails closed: {why}"), + ); + } + } } CommandKind::Continuity => { if !self.lifecycle_allows(current, now)? { @@ -9228,66 +9343,27 @@ impl Engine { .context("no private candidate port remains in the operator range") } - fn deployment_authorization( - &self, - current: &Stored, - now: u64, - ) -> Result> { - let (runtime_id, sealed_release_id) = current.value.brake_subject()?; + /// What the binding's deployment stop says about this transaction's + /// runtime. Read live; never cached, never consumed. + fn deployment_stop(&self, current: &Stored) -> Result { let plan = required(¤t.value.plan, "transaction plan")?; let binding = plan.parsed_inputs()?.1; - let Some((record, canonical_bytes)) = - read_deployment_brake(&binding.brakes.deployment_store)? - else { - return Ok(None); - }; - let anchor = read_trust_anchor::( - &self.options.deployment_brake_operator_anchor, - )?; - if evaluate_idunn_deployment_brake( - IdunnDeploymentBrakeObservation::Present(&record), - &anchor, - &runtime_id, - &sealed_release_id, - ¤t.value.transaction_id, - now, - ) - .is_err() - { - return Ok(None); - } - let authorization_id = record - .authorization_id - .clone() - .context("released brake has no authorization id")?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - ensure!( - !snapshot.transactions.iter().any(|stored| { - stored - .value - .deployment_authorization - .as_ref() - .is_some_and(|used| used.authorization_id == authorization_id) - }), - "deployment authorization was already consumed" - ); - Ok(Some(DeploymentAuthorization { - authorization_id, - brake_sha256: sha256_id(&canonical_bytes), - canonical_brake_bytes: canonical_bytes, - authorized_at_unix_millis: now, - })) + Ok(read_stop( + &binding.brakes.deployment_store, + &binding.runtime_identity.runtime_id, + process_uid(), + )) } /// Whether a deployment awaiting the handover is free to take it: the - /// deployment brake has released its exact transaction, so the stop a yield - /// makes is followed by the deployment's own landing. A Deploy at its - /// Sealing tail has not passed the brake yet and is read live; one that is - /// still building, or whose brake cannot be read, is held. An operator's + /// deployment stop admits it, so the stop a yield makes is followed by the + /// deployment's own landing. A Deploy at its Sealing tail has not passed + /// the stop yet and is read live; one that is still building, or whose + /// stop is engaged or cannot be read, is held. An operator's /// hold is a reason to leave a serving or retrying body alone. - fn deploy_can_move(&self, deploy: &Stored, now: u64) -> bool { - deploy.value.deployment_authorization.is_some() - || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) + fn deploy_can_move(&self, deploy: &Stored) -> bool { + deploy.value.deployment_admitted() + || self.deployment_stop(deploy).is_ok_and(|stop| stop.admits()) } /// The deployment a continuity gives way to: one on its target that awaits @@ -9299,13 +9375,12 @@ impl Engine { &self, snapshot: &'a ControlSnapshot, continuity: &DeploymentTransaction, - now: u64, ) -> Option<&'a Stored> { if continuity.serving_proven() { return None; } snapshot.on_target(&continuity.target).find(|other| { - other.value.awaits_handover() && self.deploy_can_move(other, now) + other.value.awaits_handover() && self.deploy_can_move(other) }) } @@ -10503,10 +10578,9 @@ impl Engine { } PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let now = now_millis()?; if let Some(deploy) = transaction .yields_to_a_deploy() - .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction, now)) + .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction)) .flatten() { self.begin_post_fencing_abort( @@ -11159,24 +11233,6 @@ fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation && record.disagreements.is_empty() } -fn read_deployment_brake(path: &Path) -> Result)>> { - let Some(envelope) = read_single_envelope(path)? else { - return Ok(None); - }; - ensure!( - envelope.r#type == IdunnDeploymentBrakeRecord::TYPE - && envelope.schema_id.as_deref() == Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA), - "deployment brake store contains a foreign record" - ); - let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&envelope.payload)?; - record.validate()?; - ensure!( - rmp_serde::to_vec(&record)? == envelope.payload && envelope.key == record.brake_id, - "deployment brake is noncanonical or keyed by another authority" - ); - Ok(Some((record, envelope.payload))) -} - fn read_lifecycle_brake(path: &Path) -> Result> { let Some(envelope) = read_single_envelope(path)? else { return Ok(None); @@ -11659,7 +11715,6 @@ mod tests { odin_correlation_store: root.join("odin-correlation.cc"), odin_trust_anchor: odin_anchor, idunn_identity_store: idunn, - deployment_brake_operator_anchor: root.join("brake-anchor.cc"), #[cfg(unix)] route_actuators, ..RuntimeOptions::default() @@ -11883,27 +11938,6 @@ mod tests { Ok(()) } - #[test] - fn an_absent_brake_anchor_does_not_gate_an_empty_store() -> Result<()> { - // `F:\Projects\CLAUDE.md`: a target brake "may never gate Idunn itself - // or unrelated service lifecycle". Reading the operator anchor up front - // meant a missing brake artifact refused startup against a store with - // nothing in it -- the deployment authority held hostage by an artifact - // describing one target's consent. - let world = EngineFixture::new()?; - assert!( - !world - .engine - .options - .deployment_brake_operator_anchor - .exists(), - "fixture must not have written a brake anchor" - ); - let snapshot = ControlSnapshot::read(&world.state_store)?; - world.engine.validate_durable_authority(&snapshot)?; - Ok(()) - } - #[test] fn one_unreadable_record_faults_its_tick_without_touching_the_store() -> Result<()> { let world = EngineFixture::new()?; @@ -13979,77 +14013,18 @@ mod tests { if incumbent.is_some() { 14 } else { 4 } } - /// A released deployment brake receipt, signed by an operator identity - /// enrolled in the world: what a Deploy's `deployment_authorization` holds. - fn deployment_authorization( - world: &EngineFixture, - expected: &IdunnExpectedIncarnationRecord, - transaction_id: &str, - now: u64, - ) -> Result { - released_brake(world, &expected.runtime_id, &expected.sealed_release_id, transaction_id, now) - } - - /// The release receipt for one runtime, release and transaction: all a - /// brake release names, so a target with no Expected can be released too. - fn released_brake( - world: &EngineFixture, - runtime_id: &str, - sealed_release_id: &str, - transaction_id: &str, - now: u64, - ) -> Result { - use cultnet_rs::{ - IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, - IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, - IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, - }; - // One operator per world, however many deployments it releases. - let signer_path = world.root.join("identities/brake-operator.cc"); - let signer = if signer_path.exists() { - open_service_identity_at::(&signer_path)? - } else { - let signer = - enroll_service_identity_at::(&signer_path)?; - export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; - signer - }; - let authorization_id = format!("authorization-{transaction_id}"); - let mut record = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: runtime_id.to_owned(), - status: "released".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: "test rollout".into(), - observed_at_unix_millis: now, - expires_at_unix_millis: Some(now + 600_000), - authorization_id: Some(authorization_id.clone()), - authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), - authorized_release_id: Some(sealed_release_id.to_owned()), - authorized_deployment_id: Some(transaction_id.to_owned()), - authorized_by: Some(signer.trust_anchor()?.identity_id), - authorization_issued_at_unix_millis: Some(now), - authorization_expires_at_unix_millis: Some(now + 600_000), - signature_algorithm: Some("ed25519".into()), - signature: None, - private_state_exposed: false, - updated_by: "operator/test".into(), - }; - record.signature = Some( - signer - .sign::(&rmp_serde::to_vec(&record)?) - .signature, - ); - record.validate()?; - let canonical_brake_bytes = rmp_serde::to_vec(&record)?; - Ok(DeploymentAuthorization { - authorization_id, - brake_sha256: sha256_id(&canonical_brake_bytes), - canonical_brake_bytes, - authorized_at_unix_millis: now, - }) + /// Write the deployment stop of the binding `plan` names, as the verb + /// would: engaged halts that runtime's deployments, clear admits them. + fn set_stop(plan: &CompiledDeploymentPlan, state: StopState, reason: &str) -> Result<()> { + let binding = plan.parsed_inputs()?.1; + crate::brake::replace_stop( + &binding.brakes.deployment_store, + &binding.runtime_identity.runtime_id, + state, + reason, + now_millis()?, + )?; + Ok(()) } /// `binding` with its provider trust anchor and signer pointed at this @@ -14233,12 +14208,7 @@ mod tests { plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - transaction.deployment_authorization = Some(deployment_authorization( - world, - &expected, - &transaction.transaction_id, - now, - )?); + transaction.deployment_admitted_at_unix_millis = Some(now); } } transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); @@ -18760,11 +18730,12 @@ mod tests { make_stop_first(world, deploy_id)?; let sealing = resident_id(world, deploy_id)?; let mut tail = sealing.value.clone(); - tail.deployment_authorization = None; + tail.deployment_admitted_at_unix_millis = None; replace_transaction(&world.state_store, &sealing, &tail)?; + set_stop(tail.plan.as_ref().context("no plan")?, StopState::Engaged, "test hold")?; let tail = resident_id(world, deploy_id)?.value; assert!(tail.awaits_handover()); - assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?, now_millis()?)); + assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?)); Ok(()) } @@ -19027,13 +18998,14 @@ mod tests { /// for it; once the brake releases, the continuity yields. The yield closes /// the continuity's operator-required incident Recovered. #[test] - fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { + fn a_continuity_does_not_yield_to_a_stopped_deploy_and_yields_once_the_stop_clears() -> Result<()> { let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; make_stop_first(&world, &deploy_id).context("A make_stop_first")?; let sealing = resident_id(&world, &deploy_id).context("B sealing")?; let mut tail = sealing.value.clone(); - tail.deployment_authorization = None; + tail.deployment_admitted_at_unix_millis = None; replace_transaction(&world.state_store, &sealing, &tail)?; + set_stop(tail.plan.as_ref().context("no plan")?, StopState::Engaged, "test hold")?; assert!(resident_id(&world, &deploy_id).context("C awaits")?.value.awaits_handover()); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); @@ -19046,30 +19018,9 @@ mod tests { assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); - // The operator releases the brake for this exact Deploy. + // The steward clears the stop; no per-deploy release exists. let deploy = resident_id(&world, &deploy_id).context("E deploy")?.value; - let authorization = deployment_authorization( - &world, - deploy.expected.as_ref().context("no Expected")?, - &deploy_id, - now_millis()?, - )?; - let store = deploy.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; - let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; - assert!(SingleFileMessagePackBackingStore::new(&store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - key: record.brake_id.clone(), - current: None, - }], - &[CultCacheEnvelope { - key: record.brake_id.clone(), - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - payload: authorization.canonical_brake_bytes.clone(), - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), - }], - )?); + set_stop(deploy.plan.as_ref().context("no plan")?, StopState::Clear, "test thaw")?; world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; let yielded = resident_id(&world, &owner_id).context("F yielded")?.value; @@ -19460,8 +19411,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - next.deployment_authorization = - Some(deployment_authorization(world, &expected, id, now)?); + next.deployment_admitted_at_unix_millis = Some(now); } next.installed_release = Some(crate::drivers::InstalledReleaseObservation { sealed_release_id: release.sealed_release_id.clone(), @@ -19791,8 +19741,9 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let sealing = resident_id(&world, deploy_id)?; world.engine.advance_transaction(&sealing)?; let after = resident_id(&world, deploy_id)?.value; + // The target has no stop, so Sealing passes it and is not held. assert!( - after.last_error.as_deref().is_some_and(|note| !note.contains(&continuity.transaction_id)), + after.last_error.as_deref().is_none_or(|note| !note.contains(&continuity.transaction_id)), "Sealing was held by the owner: {:?}", after.last_error ); @@ -22019,12 +21970,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - transaction.deployment_authorization = Some(deployment_authorization( - &world, - &expected, - &transaction.transaction_id, - now, - )?); + transaction.deployment_admitted_at_unix_millis = Some(now); } } let launch = IdunnRuntimeActivationLaunch::issue( @@ -25835,7 +25781,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); if envelope.r#type == DeploymentTransaction::TYPE { transactions += 1; - assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v5")); + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v6")); } } } @@ -25874,9 +25820,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; /// The v4 bytes of the same record, captured at base 804ed75: a legacy - /// fixture. v5 adds key 37 (`flip`), so it is one element longer and names v5. + /// fixture. v5 added key 37 (`flip`) and v6 key 38, so v4 is two elements shorter. const PINNED_TRANSACTION_HEX_V4: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; - const PINNED_TRANSACTION_HEX: &str = "dc0026bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7635a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0c0"; + const PINNED_TRANSACTION_HEX_V5: &str = "dc0026bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7635a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0c0"; + /// The v6 bytes: v5 with key 38 (`deployment_admitted_at_unix_millis`) appended, so + /// one element longer and naming v6. + const PINNED_TRANSACTION_HEX: &str = "dc0027bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7636a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0c0c0"; fn pinned_v4_envelope() -> Result { let payload = (0..PINNED_TRANSACTION_HEX_V4.len()) @@ -25893,14 +25842,15 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } #[test] - fn a_pinned_v4_transaction_lifts_and_reencodes_as_the_pinned_v5() -> Result<()> { + fn a_pinned_v4_transaction_lifts_and_reencodes_as_the_pinned_v6() -> Result<()> { let lifted = read_transaction_record(&pinned_v4_envelope()?)?; assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); assert_eq!(lifted.flip, None); + assert_eq!(lifted.deployment_admitted_at_unix_millis, None); let bytes = encode_transaction(&lifted, 100)?.payload; let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); assert_eq!(hex, PINNED_TRANSACTION_HEX); - // Read as v5 the v4 bytes are refused, never silently accepted. + // Read as v6 the v4 bytes are refused, never silently accepted. let mut mislabeled = pinned_v4_envelope()?; mislabeled.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()); assert!(read_transaction_record(&mislabeled).is_err()); @@ -25909,7 +25859,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// Always a copy in a temp dir: never a live control.cc. #[test] - fn a_v4_control_store_boots_into_v5_and_a_v5_store_is_left_alone() -> Result<()> { + fn a_v4_control_store_boots_into_v6_and_a_v6_store_is_left_alone() -> Result<()> { let temporary = tempfile::tempdir()?; let path = temporary.path().join("control.cc"); let store = SingleFileMessagePackBackingStore::new(&path); @@ -25929,7 +25879,68 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(after[0].stored_at, v4.stored_at); let migrated: String = after[0].payload.iter().map(|byte| format!("{byte:02x}")).collect(); assert_eq!(migrated, PINNED_TRANSACTION_HEX); - // Negative: an already-v5 store is untouched, byte for byte. + // Negative: an already-v6 store is untouched, byte for byte. + let file_before = std::fs::read(&path)?; + assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); + assert_eq!(std::fs::read(&path)?, file_before); + read_transaction_record(&after[0])?; + Ok(()) + } + fn pinned_v5_envelope() -> Result { + let payload = (0..PINNED_TRANSACTION_HEX_V5.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&PINNED_TRANSACTION_HEX_V5[i..i + 2], 16)) + .collect::, _>>()?; + Ok(CultCacheEnvelope { + key: "tx-pinned".into(), + r#type: DeploymentTransaction::TYPE.into(), + payload, + stored_at: rfc3339_millis(100)?, + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA_V5.into()), + }) + } + + #[test] + fn a_pinned_v5_transaction_lifts_and_reencodes_as_the_pinned_v6() -> Result<()> { + let lifted = read_transaction_record(&pinned_v5_envelope()?)?; + assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + assert_eq!(lifted.deployment_admitted_at_unix_millis, None); + let bytes = encode_transaction(&lifted, 100)?.payload; + let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + assert_eq!(hex, PINNED_TRANSACTION_HEX); + // Read as v6 the v5 bytes are refused, never silently accepted. + let mut mislabeled = pinned_v5_envelope()?; + mislabeled.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()); + assert!(read_transaction_record(&mislabeled).is_err()); + // History is never migrated: it lifts a v5 record the same way. + let (history, undecodable) = decode_history_transactions(vec![pinned_v5_envelope()?]); + assert!(undecodable.is_empty(), "{undecodable:?}"); + assert_eq!(history, vec![lifted]); + Ok(()) + } + + /// Always a copy in a temp dir: never a live control.cc. + #[test] + fn a_v5_control_store_copy_boots_into_v6_and_a_v6_store_is_left_alone() -> Result<()> { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + let v5 = pinned_v5_envelope()?; + assert!(store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: v5.r#type.clone(), + key: v5.key.clone(), + current: None, + }], + std::slice::from_ref(&v5), + )?); + assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); + let after = store.pull_all_read_only_snapshot()?; + assert_eq!(after.len(), 1); + assert_eq!(after[0].schema_id.as_deref(), Some(DEPLOYMENT_TRANSACTION_SCHEMA)); + assert_eq!(after[0].stored_at, v5.stored_at); + let migrated: String = after[0].payload.iter().map(|byte| format!("{byte:02x}")).collect(); + assert_eq!(migrated, PINNED_TRANSACTION_HEX); let file_before = std::fs::read(&path)?; assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); assert_eq!(std::fs::read(&path)?, file_before); @@ -26107,8 +26118,23 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide latest_id(&self.world, &id) } + /// Engage the stop for this world's runtime, as a steward would. + fn hold(&self) -> Result<()> { + crate::brake::replace_stop( + &self.site.join("authority/deployment-brake.cc"), + "site-yggdrasil", + StopState::Engaged, + "test hold", + now_millis()?, + )?; + Ok(()) + } + /// Submit a rollback command and tick until the daemon freezes it. + /// A rollback is born at Sealing with its evidence, so a test that + /// wants it held at the stop engages the stop first. fn submit_rollback(&self, command_id: &str) -> Result { + self.hold()?; commit_deployment_command( &self.world.state_store, CommandKind::Rollback, @@ -26160,51 +26186,32 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(error) } - /// The operator releases the brake for this exact transaction. + /// The steward clears the stop. It is not bound to this transaction and + /// not consumed by it. fn release_brake(&self, transaction: &DeploymentTransaction) -> Result<()> { - let (runtime_id, release_id) = transaction.brake_subject()?; - let authorization = released_brake( - &self.world, - &runtime_id, - &release_id, - &transaction.transaction_id, - now_millis()?, - )?; - let store = self.site.join("authority/deployment-brake.cc"); - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; - let backing = SingleFileMessagePackBackingStore::new(&store); - let current = backing - .pull_all_read_only_snapshot() - .ok() - .and_then(|entries| entries.into_iter().next()); - ensure!(backing.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - key: record.brake_id.clone(), - current, - }], - &[CultCacheEnvelope { - key: record.brake_id.clone(), - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - payload: authorization.canonical_brake_bytes.clone(), - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), - }], - )?); - Ok(()) + set_stop( + transaction.plan.as_ref().context("no plan")?, + StopState::Clear, + "test thaw", + ) } - /// Run a Deploy up to the brake and say where it waits. + /// Run a Deploy or a Rollback up to the stop and say where it waits. fn run_to_brake(&self, id: &str) -> Result { - self.tick_until(id, "the brake", |t| t.installed_release.is_some())?; + self.hold()?; + self.tick_until(id, "the sealed release", |t| t.installed_release.is_some())?; for _ in 0..2 { self.world.engine.resume_backoff.lock().unwrap().clear(); let _ = self.world.engine.run_scheduler_tick(); } let waiting = latest_id(&self.world, id)?; assert_eq!(waiting.phase, DeploymentPhase::Sealing, "{waiting:?}"); - assert!(waiting.deployment_authorization.is_none()); + assert!(!waiting.deployment_admitted()); + assert!( + waiting.last_error.as_deref().is_some_and(|why| why.contains("deployment brake engaged by")), + "{:?}", + waiting.last_error + ); Ok(waiting) } } @@ -26286,7 +26293,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(flip.previous_release_id, None); assert_eq!(flip.carried_sha256.as_deref(), Some(sha256_of(SITE_CONFIG_TWO).as_str())); assert_eq!(flip.carry_source, w.site_config()); - assert!(done.deployment_authorization.is_some(), "the brake was not consumed"); + assert!(done.deployment_admitted_at_unix_millis.is_some(), "the stop did not stamp its admission"); assert_eq!(w.current_release().as_deref(), Some(sealed.sealed_release_id.as_str())); let carried = w.current_link().join("config.json"); assert_eq!(std::fs::read(&carried)?, SITE_CONFIG_TWO); @@ -26351,6 +26358,52 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + #[cfg(unix)] + #[test] + fn absent_store_admits_and_stamps_admission() -> Result<()> { + let w = static_world()?; + let id = w.transaction_of("up-service")?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + assert!(!w.site.join("authority/deployment-brake.cc").exists()); + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert!(done.deployment_admitted_at_unix_millis.is_some()); + assert!(done.deployment_authorization.is_none()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn one_clear_admits_many_deploys_and_an_engaged_stop_names_who() -> Result<()> { + let w = static_world()?; + let first_id = w.transaction_of("up-service")?; + let first = w.run_to_brake(&first_id)?; + let held = first.last_error.clone().context("no wait note")?; + let me = crate::brake::process_uid(); + let store = w.site.join("authority/deployment-brake.cc"); + let crate::brake::Stop::Engaged(engaged) = crate::brake::read_stop(&store, "site-yggdrasil", me) else { + bail!("the stop is not engaged") + }; + assert_eq!(held, format!("deployment brake engaged by {}: {}", engaged.by, engaged.reason)); + w.release_brake(&first)?; + w.tick_until(&first_id, "the first end", |t| t.is_terminal())?; + // No engage in between: one Clear admits the second deploy too, with + // no release id or transaction id in sight. + enqueue_deploy(&w.world, "up-service-2")?; + let second_id = w.transaction_of("up-service-2")?; + w.tick_until(&second_id, "the second end", |t| t.is_terminal())?; + let crate::brake::Stop::Clear(record) = crate::brake::read_stop(&store, "site-yggdrasil", me) else { + bail!("the stop did not stay clear") + }; + assert_eq!(record.sequence, 2); + for id in [&first_id, &second_id] { + let done = latest_id(&w.world, id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert!(done.deployment_admitted_at_unix_millis.is_some()); + } + Ok(()) + } + #[cfg(unix)] #[test] fn static_release_failed_flip_leaves_current_and_ends_before_fencing() -> Result<()> { @@ -26500,15 +26553,26 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + /// A populated record an older Idunn could have written: a static deploy + /// that consumed a signed brake release (key 16, now legacy) and ended + /// before fencing. `drop_keys` is how many trailing nil keys the older + /// layout lacks: v5 lacks key 38, v4 also lacks key 37. #[cfg(unix)] - #[test] - fn a_populated_v4_transaction_lifts_to_the_exact_v5_field_by_field() -> Result<()> { + fn populated_legacy_lifts_to_the_exact_current_field_by_field( + schema: &str, + drop_keys: usize, + ) -> Result<()> { let w = static_world()?; let done = drive_static_to_end(&w)?; - // A populated record a v4 Idunn could have written: a deploy that - // consumed its brake and ended before fencing. let mut expected = done.clone(); expected.flip = None; + expected.deployment_admitted_at_unix_millis = None; + expected.deployment_authorization = Some(DeploymentAuthorization { + authorization_id: "authorization-legacy".into(), + brake_sha256: sha256_id(b"legacy brake bytes"), + canonical_brake_bytes: b"legacy brake bytes".to_vec(), + authorized_at_unix_millis: 99, + }); expected.completion = Some(TransactionCompletion::FailedBeforeFencing { error: "refused".into() }); expected.pre_fencing_abort = Some(PreFencingAbort { error: "refused".into(), @@ -26519,42 +26583,47 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide expected.post_commit_cleanup = None; expected.phase_deadline = None; expected.last_error = Some("refused".into()); + // Starting accepts the legacy receipt as the admission. expected.validate()?; assert_ne!(expected.created_at_unix_millis, expected.updated_at_unix_millis); - assert!(expected.completion.is_some() && expected.deployment_authorization.is_some()); - // The v4 bytes of this record: the v5 encoding with the old schema name, - // one element shorter (key 37 is the last, a nil here). - let mut as_v4 = expected.clone(); - as_v4.schema_version = DEPLOYMENT_TRANSACTION_SCHEMA_V4.into(); - let mut payload = rmp_serde::to_vec(&as_v4)?; - assert_eq!(&payload[..3], &[0xdc, 0x00, 0x26]); - payload[2] = 0x25; - assert_eq!(payload.pop(), Some(0xc0)); + // The older bytes: the current encoding with the old schema name and + // `drop_keys` fewer elements, every one a nil. + let mut older = expected.clone(); + older.schema_version = schema.into(); + let mut payload = rmp_serde::to_vec(&older)?; + assert_eq!(&payload[..3], &[0xdc, 0x00, 0x27]); + payload[2] -= u8::try_from(drop_keys)?; + for _ in 0..drop_keys { + assert_eq!(payload.pop(), Some(0xc0)); + } let envelope = CultCacheEnvelope { key: expected.transaction_id.clone(), r#type: DeploymentTransaction::TYPE.into(), payload, stored_at: rfc3339_millis(100)?, - schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA_V4.into()), + schema_id: Some(schema.into()), }; let lifted = read_transaction_record(&envelope)?; assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); - assert_eq!(lifted.transaction_id, expected.transaction_id); - assert_eq!(lifted.created_at_unix_millis, expected.created_at_unix_millis); - assert_eq!(lifted.updated_at_unix_millis, expected.updated_at_unix_millis); - assert_eq!(lifted.completion, expected.completion); - assert_eq!(lifted.plan, expected.plan); - assert_eq!(lifted.sealed_release, expected.sealed_release); - assert_eq!(lifted.installed_release, expected.installed_release); - assert_eq!(lifted.frozen_source, expected.frozen_source); assert_eq!(lifted.deployment_authorization, expected.deployment_authorization); - assert_eq!(lifted.post_commit_cleanup, expected.post_commit_cleanup); - assert_eq!(lifted.phase_deadline, expected.phase_deadline); + assert_eq!(lifted.deployment_admitted_at_unix_millis, None); assert_eq!(lifted.flip, None); assert_eq!(lifted, expected, "some field changed in the lift"); Ok(()) } + #[cfg(unix)] + #[test] + fn a_populated_v4_transaction_lifts_to_the_exact_v6_field_by_field() -> Result<()> { + populated_legacy_lifts_to_the_exact_current_field_by_field(DEPLOYMENT_TRANSACTION_SCHEMA_V4, 2) + } + + #[cfg(unix)] + #[test] + fn a_populated_v5_transaction_with_a_consumed_release_lifts_to_the_exact_v6_field_by_field() -> Result<()> { + populated_legacy_lifts_to_the_exact_current_field_by_field(DEPLOYMENT_TRANSACTION_SCHEMA_V5, 1) + } + #[cfg(unix)] #[test] fn static_release_with_a_foreign_current_retries_visibly_and_cannot_be_cancelled() -> Result<()> { @@ -26616,7 +26685,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(flip.carried_sha256.as_deref(), Some(sha256_of(SITE_CONFIG_THREE).as_str())); assert_eq!(w.current_release().as_deref(), Some(first.as_str())); assert_eq!(std::fs::read(w.current_link().join("config.json"))?, SITE_CONFIG_THREE); - assert!(done.deployment_authorization.is_some(), "the brake was not consumed"); + assert!(done.deployment_admitted_at_unix_millis.is_some(), "the stop did not stamp its admission"); // Negatives: no source, no source cleanup, no generation, no pruning. assert!(done.frozen_source.is_none()); assert_eq!( diff --git a/src/lib.rs b/src/lib.rs index b5499a6..6b034d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +mod brake; pub mod control_plane; pub mod deployment; pub mod deployment_plan; diff --git a/src/provisioning.rs b/src/provisioning.rs index 1c74a83..56e104f 100644 --- a/src/provisioning.rs +++ b/src/provisioning.rs @@ -1,4 +1,5 @@ -use anyhow::{Context, Result, anyhow, bail}; +use crate::brake::{Stop, StopState, process_uid, read_stop, replace_stop}; +use anyhow::{Context, Result, anyhow, bail, ensure}; use cultcache_rs::{ CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, }; @@ -6,15 +7,13 @@ use cultnet_rs::{ GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, GameCultServiceTrustAnchorRecord, IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA, IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SIGNING_PURPOSE, - IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, - IDUNN_DEPLOYMENT_BRAKE_ID, IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_SCOPE, - IDUNN_DEPLOYMENT_RELEASE_PURPOSE, IDUNN_LIFECYCLE_BRAKE_AUTHORITY, + IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA, + IDUNN_LIFECYCLE_BRAKE_AUTHORITY, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCOPE, IdunnDaemonHealthTrustBindingRecord, - IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, - IdunnDeploymentBrakeRecord, IdunnDeploymentBrakeReleasePurpose, IdunnLifecycleBrakeObservation, + IdunnLifecycleBrakeObservation, IdunnLifecycleBrakeRecord, IdunnServiceIdentity, OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentityTrustAnchor, derive_service_identity_id, enroll_service_identity_at, - evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, + evaluate_idunn_continuity_restart, export_service_identity_trust_anchor, open_service_identity_at, }; use std::collections::{BTreeMap, BTreeSet}; @@ -80,17 +79,8 @@ pub fn run(args: impl IntoIterator) -> Result<()> { "export-odin-topology-public-anchor" => { export_identity_anchor::(&options, "Odin topology public anchor")? } - "enroll-deployment-brake-operator" => { - enroll_identity::(&options)? - } - "export-deployment-brake-operator-anchor" => { - export_identity_anchor::( - &options, - "deployment brake operator anchor", - )? - } - "deployment-brake-engage" => deployment_brake_engage(&options)?, - "deployment-brake-release" => deployment_brake_release(&options)?, + "deployment-brake-engage" => deployment_brake_write(&options, StopState::Engaged)?, + "deployment-brake-clear" => deployment_brake_write(&options, StopState::Clear)?, "deployment-brake-status" => deployment_brake_status(&options)?, "lifecycle-brake-engage" => lifecycle_brake_engage(&options)?, "lifecycle-brake-release" => lifecycle_brake_release(&options)?, @@ -157,153 +147,58 @@ fn provider_health_public_key(options: &BTreeMap) -> Result) -> Result<()> { - require_only( - options, - &[ - "store", - "runtime-id", - "owner", - "reason", - "observed-at-unix-millis", - ], +/// Engage or clear the deployment stop. The author is the process uid's login +/// name and the time is this verb's clock: neither is an argument. +fn deployment_brake_write(options: &BTreeMap, state: StopState) -> Result<()> { + require_only(options, &["store", "runtime-id", "reason"])?; + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), )?; - let observed = parse_u64(options, "observed-at-unix-millis")?; - let record = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: required(options, "runtime-id")?.into(), - status: "engaged".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: required(options, "reason")?.into(), - observed_at_unix_millis: observed, - expires_at_unix_millis: None, - authorization_id: None, - authorization_purpose: None, - authorized_release_id: None, - authorized_deployment_id: None, - authorized_by: None, - authorization_issued_at_unix_millis: None, - authorization_expires_at_unix_millis: None, - signature_algorithm: None, - signature: None, - private_state_exposed: false, - updated_by: required(options, "owner")?.into(), - }; - record.validate()?; - replace_brake(&path(options, "store")?, record, observed) -} - -fn deployment_brake_release(options: &BTreeMap) -> Result<()> { - require_only( - options, - &[ - "store", - "private-store", - "runtime-id", - "owner", - "reason", - "authorization-id", - "release-id", - "deployment-id", - "issued-at-unix-millis", - "expires-at-unix-millis", - ], - )?; - let issued = parse_u64(options, "issued-at-unix-millis")?; - let expires = parse_u64(options, "expires-at-unix-millis")?; - let signer = open_service_identity_at::(&path( - options, - "private-store", - )?)?; - let mut record = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: required(options, "runtime-id")?.into(), - status: "released".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: required(options, "reason")?.into(), - observed_at_unix_millis: issued, - expires_at_unix_millis: Some(expires), - authorization_id: Some(required(options, "authorization-id")?.into()), - authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), - authorized_release_id: Some(required(options, "release-id")?.into()), - authorized_deployment_id: Some(required(options, "deployment-id")?.into()), - authorized_by: Some(signer.trust_anchor()?.identity_id), - authorization_issued_at_unix_millis: Some(issued), - authorization_expires_at_unix_millis: Some(expires), - signature_algorithm: Some("ed25519".into()), - signature: None, - private_state_exposed: false, - updated_by: required(options, "owner")?.into(), - }; - record.signature = Some( - signer - .sign::(&rmp_serde::to_vec(&record)?) - .signature, - ); - record.validate()?; - replace_brake(&path(options, "store")?, record, issued) -} - -fn deployment_brake_status(options: &BTreeMap) -> Result<()> { - allow_only( - options, - &[ - "store", - "operator-anchor", - "runtime-id", - "release-id", - "deployment-id", - "now-unix-millis", - ], - )?; - let exact_actuation_fields = ["release-id", "deployment-id", "now-unix-millis"]; - let exact_actuation_field_count = exact_actuation_fields - .into_iter() - .filter(|field| options.contains_key(*field)) - .count(); - if exact_actuation_field_count != 0 - && exact_actuation_field_count != exact_actuation_fields.len() - { - bail!("exact deployment-brake actuation validation requires release, deployment, and time"); - } - let record = read_brake(&path(options, "store")?)?; - if record.runtime_id != required(options, "runtime-id")? { - bail!("deployment brake belongs to another runtime") - } - if record.status == "engaged" { - if exact_actuation_field_count != 0 { - bail!("engaged deployment brake denies exact actuation validation"); - } - println!( - "engaged runtime={} scope={} owner={} reason={}", - record.runtime_id, record.scope, record.updated_by, record.reason - ); - return Ok(()); - } - let anchor = read_operator_anchor(&path(options, "operator-anchor")?)?; - evaluate_idunn_deployment_brake( - IdunnDeploymentBrakeObservation::Present(&record), - &anchor, + let record = replace_stop( + &path(options, "store")?, required(options, "runtime-id")?, - required(options, "release-id")?, - required(options, "deployment-id")?, - parse_u64(options, "now-unix-millis")?, - ) - .map_err(|d| anyhow!("deployment brake denied actuation: {d:?}"))?; + state, + required(options, "reason")?, + now, + )?; println!( - "released runtime={} release={} deployment={} owner={}", + "{} runtime={} by={} sequence={}", + record.state.name(), record.runtime_id, - record.authorized_release_id.unwrap(), - record.authorized_deployment_id.unwrap(), - record.updated_by + record.by, + record.sequence ); Ok(()) } +/// Print the stop and exit nonzero unless deployments admit, so a script can +/// gate on it as it could on the retired exact-actuation validation. +fn deployment_brake_status(options: &BTreeMap) -> Result<()> { + require_only(options, &["store", "runtime-id"])?; + let stop = read_stop( + &path(options, "store")?, + required(options, "runtime-id")?, + process_uid(), + ); + match &stop { + Stop::Absent => println!("absent: deployments admit"), + Stop::Unreadable(why) => println!("unreadable: deployments wait ({why})"), + Stop::Clear(record) | Stop::Engaged(record) => println!( + "{} runtime={} by={} at={} sequence={} reason={}", + record.state.name(), + record.runtime_id, + record.by, + record.at_unix_millis, + record.sequence, + record.reason + ), + } + ensure!(stop.admits(), "the deployment brake does not admit deployments"); + Ok(()) +} + fn lifecycle_brake_engage(options: &BTreeMap) -> Result<()> { require_only( options, @@ -482,70 +377,6 @@ fn lifecycle_identifier<'a>(options: &'a BTreeMap, name: &str) - Ok(value) } -fn replace_brake(path: &Path, record: IdunnDeploymentBrakeRecord, millis: u64) -> Result<()> { - let store = SingleFileMessagePackBackingStore::new(path); - let entries = store.pull_all_read_only_snapshot()?; - let current = match entries.as_slice() { - [] => None, - [entry] - if entry.r#type == IdunnDeploymentBrakeRecord::TYPE - && entry.key == IDUNN_DEPLOYMENT_BRAKE_ID => - { - Some(entry.clone()) - } - _ => bail!("deployment brake store is corrupt or ambiguous"), - }; - let next = typed_envelope( - IDUNN_DEPLOYMENT_BRAKE_ID, - &record, - IDUNN_DEPLOYMENT_BRAKE_SCHEMA, - millis, - )?; - if !store.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - key: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - current, - }], - &[next], - )? { - bail!("deployment brake changed during transition"); - } - Ok(()) -} - -fn read_brake(path: &Path) -> Result { - let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - let [entry] = entries.as_slice() else { - bail!("deployment brake is missing or ambiguous") - }; - if entry.r#type != IdunnDeploymentBrakeRecord::TYPE - || entry.key != IDUNN_DEPLOYMENT_BRAKE_ID - || entry.schema_id.as_deref() != Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA) - { - bail!("deployment brake is foreign") - } - let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&entry.payload)?; - if rmp_serde::to_vec(&record)? != entry.payload { - bail!("deployment brake is noncanonical") - } - record.validate()?; - Ok(record) -} - -fn read_operator_anchor(path: &Path) -> Result { - let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - let [entry] = entries.as_slice() else { - bail!("operator anchor is missing or ambiguous") - }; - if entry.r#type != IdunnDeploymentBrakeOperatorIdentity::TRUST_ANCHOR_TYPE - || entry.key != IdunnDeploymentBrakeOperatorIdentity::TRUST_ANCHOR_KEY - { - bail!("operator anchor is foreign") - } - Ok(rmp_serde::from_slice(&entry.payload)?) -} - fn validate_health_binding_store(path: &Path) -> Result<()> { let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; validated_health_bindings(&entries)?; @@ -1127,7 +958,7 @@ fn normalized(path: &Path) -> Result { } fn usage() -> &'static str { - "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision export-provider-health-public-anchor --private-store --public-anchor \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-odin-topology-identity --private-store \n idunn-provision export-odin-topology-public-anchor --private-store --public-anchor \n idunn-provision enroll-deployment-brake-operator --private-store \n idunn-provision export-deployment-brake-operator-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --owner --reason --observed-at-unix-millis \n idunn-provision deployment-brake-release --store --private-store --runtime-id --owner --reason --authorization-id --release-id --deployment-id --issued-at-unix-millis --expires-at-unix-millis \n idunn-provision deployment-brake-status --store --operator-anchor --runtime-id [--release-id --deployment-id --now-unix-millis ]\n idunn-provision lifecycle-brake-engage --store --runtime-id --target --reason --updated-at-unix-millis \n idunn-provision lifecycle-brake-release --store --runtime-id --target --reason --updated-at-unix-millis [--released-until-unix-millis ]\n idunn-provision lifecycle-brake-status --store --runtime-id --target --now-unix-millis \n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " + "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision export-provider-health-public-anchor --private-store --public-anchor \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-odin-topology-identity --private-store \n idunn-provision export-odin-topology-public-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --reason \n idunn-provision deployment-brake-clear --store --runtime-id --reason \n idunn-provision deployment-brake-status --store --runtime-id \n idunn-provision lifecycle-brake-engage --store --runtime-id --target --reason --updated-at-unix-millis \n idunn-provision lifecycle-brake-release --store --runtime-id --target --reason --updated-at-unix-millis [--released-until-unix-millis ]\n idunn-provision lifecycle-brake-status --store --runtime-id --target --now-unix-millis \n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " } #[cfg(test)] @@ -2157,120 +1988,74 @@ mod tests { } #[test] - fn brake_operator_can_issue_one_exact_bounded_rollout_grant() -> Result<()> { + fn provision_engage_clear_sequence() -> Result<()> { let temp = TempDir::new()?; - let private = temp.path().join("operator-private.cc"); - let anchor = temp.path().join("operator-public.cc"); let store = temp.path().join("brake.cc"); - invoke(&[ - "enroll-deployment-brake-operator", - "--private-store", - private.to_str().unwrap(), - ])?; - invoke(&[ - "export-deployment-brake-operator-anchor", - "--private-store", - private.to_str().unwrap(), - "--public-anchor", - anchor.to_str().unwrap(), - ])?; - invoke(&[ - "deployment-brake-engage", - "--store", - store.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--owner", - "operator/metacrat", - "--reason", - "sleep mode", - "--observed-at-unix-millis", - "100", - ])?; - assert_eq!(read_brake(&store)?.status, "engaged"); + let me = process_uid(); + let engage = |verb: &str, reason: &str| { + invoke(&[ + verb, + "--store", + store.to_str().unwrap(), + "--runtime-id", + "yggdrasil", + "--reason", + reason, + ]) + }; + assert_eq!(read_stop(&store, "yggdrasil", me), Stop::Absent); invoke(&[ "deployment-brake-status", "--store", store.to_str().unwrap(), - "--operator-anchor", - anchor.to_str().unwrap(), "--runtime-id", "yggdrasil", ])?; - assert!( + engage("deployment-brake-engage", "freeze")?; + let status = || { invoke(&[ "deployment-brake-status", "--store", store.to_str().unwrap(), - "--operator-anchor", - anchor.to_str().unwrap(), "--runtime-id", "yggdrasil", - "--release-id", - "commit-4", - "--deployment-id", - "request-4", - "--now-unix-millis", - "150", ]) - .is_err() - ); - invoke(&[ - "deployment-brake-release", - "--store", - store.to_str().unwrap(), - "--private-store", - private.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--owner", - "operator/metacrat", - "--reason", - "one rollout", - "--authorization-id", - "auth/r4", - "--release-id", - "commit-4", - "--deployment-id", - "request-4", - "--issued-at-unix-millis", - "200", - "--expires-at-unix-millis", - "800", - ])?; - invoke(&[ - "deployment-brake-status", - "--store", - store.to_str().unwrap(), - "--operator-anchor", - anchor.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--release-id", - "commit-4", - "--deployment-id", - "request-4", - "--now-unix-millis", - "500", - ])?; - assert!( - invoke(&[ - "deployment-brake-status", - "--store", - store.to_str().unwrap(), - "--operator-anchor", - anchor.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--release-id", - "substituted", - "--deployment-id", - "request-4", - "--now-unix-millis", - "500" - ]) - .is_err() - ); + }; + assert!(status().is_err(), "an engaged stop must not pass status"); + let Stop::Engaged(first) = read_stop(&store, "yggdrasil", me) else { + bail!("engage did not engage") + }; + assert_eq!((first.sequence, first.reason.as_str()), (1, "freeze")); + assert!(!first.by.is_empty() && first.at_unix_millis > 0); + engage("deployment-brake-clear", "thawed")?; + status()?; + let Stop::Clear(second) = read_stop(&store, "yggdrasil", me) else { + bail!("clear did not clear") + }; + assert_eq!((second.sequence, second.by), (2, first.by)); + assert!(second.at_unix_millis >= first.at_unix_millis); + // The author and the time are not arguments, and the retired + // signing arguments are refused. + for stray in ["--owner", "--observed-at-unix-millis", "--private-store"] { + assert!( + invoke(&[ + "deployment-brake-engage", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "yggdrasil", + "--reason", + "x", + stray, + "y", + ]) + .is_err(), + "{stray}" + ); + } + for gone in ["deployment-brake-release", "enroll-deployment-brake-operator"] { + assert!(invoke(&[gone]).is_err(), "{gone}"); + } Ok(()) } @@ -2309,7 +2094,6 @@ mod tests { assert_eq!(engaged.status, "engaged"); assert_eq!(engaged.runtime_id, "idunn-yggdrasil"); assert_eq!(engaged.target, "ghostlight"); - assert!(read_brake(&store).is_err()); assert!( invoke(&[ "lifecycle-brake-status", @@ -2426,57 +2210,4 @@ mod tests { assert_eq!(std::fs::read(&store)?, released_bytes); Ok(()) } - - #[test] - fn brake_compare_exchange_rejects_stale_transition_snapshot() -> Result<()> { - let temp = TempDir::new()?; - let store_path = temp.path().join("brake.cc"); - let base = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: "yggdrasil".into(), - status: "engaged".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: "first".into(), - observed_at_unix_millis: 100, - expires_at_unix_millis: None, - authorization_id: None, - authorization_purpose: None, - authorized_release_id: None, - authorized_deployment_id: None, - authorized_by: None, - authorization_issued_at_unix_millis: None, - authorization_expires_at_unix_millis: None, - signature_algorithm: None, - signature: None, - private_state_exposed: false, - updated_by: "operator/a".into(), - }; - replace_brake(&store_path, base.clone(), 100)?; - let stale = SingleFileMessagePackBackingStore::new(&store_path) - .pull_all_read_only_snapshot()? - .remove(0); - let mut changed = base.clone(); - changed.reason = "racing writer".into(); - changed.observed_at_unix_millis = 101; - replace_brake(&store_path, changed, 101)?; - let candidate = typed_envelope( - IDUNN_DEPLOYMENT_BRAKE_ID, - &base, - IDUNN_DEPLOYMENT_BRAKE_SCHEMA, - 102, - )?; - assert!( - !SingleFileMessagePackBackingStore::new(&store_path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - key: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - current: Some(stale) - }], - &[candidate] - )? - ); - Ok(()) - } } -- 2.47.3 From b230703a5c2024871e2b6af77f32b8b933fcb840 Mon Sep 17 00:00:00 2001 From: Eureka Date: Fri, 13 Nov 2026 17:36:57 +0000 Subject: [PATCH 2/2] Idunn: a stop halts before irreversible steps; fail-closed reader; break-glass clear - Re-read the stop at the static flip intent and at entry into Fencing for a deploy (stop_note, the same path as Sealing). After the intent or fencing the transaction runs to its end; documented. - Absent means NotFound in an existing directory; every other lookup error, a zero-length file and a missing parent fail closed. Unreadable names the idunn-provision deployment-brake-clear command. - replace_stop is the operator's break-glass: it replaces v1, foreign, ambiguous, zero-length and caller-owned undecodable stores, never a symlink or another user's file. - Mutation-proven tests: foreign-owner Clear, Unreadable admitting at Sealing, author from USER. - docs: stop semantics, v6 downgrade hazard and backup; legacy script no longer requires the store and drops the signed-grant wording; two stale doc lines. Co-Authored-By: Claude Sonnet 5.5 --- deploy/legacy/idunn-yggdrasil | 18 ++-- docs/deployment-authority.md | 25 +++++ docs/route-continuity-cut.md | 2 +- docs/verify-transaction-cut.md | 2 +- src/brake.rs | 146 ++++++++++++++++++++++++++--- src/control_plane.rs | 162 +++++++++++++++++++++++++++++---- 6 files changed, 313 insertions(+), 42 deletions(-) diff --git a/deploy/legacy/idunn-yggdrasil b/deploy/legacy/idunn-yggdrasil index d299ec8..62a18eb 100644 --- a/deploy/legacy/idunn-yggdrasil +++ b/deploy/legacy/idunn-yggdrasil @@ -153,7 +153,7 @@ deployment_target_policy() { esac } -require_signed_deployment_grant() { +require_deployment_admission() { source_commit="$3" deployment_request_id="$4" release_authority_id="$5" @@ -179,15 +179,13 @@ require_signed_deployment_grant() { [ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; } require_root_owned_resolved_executable "$idunn_provision" - require_root_owned_regular_path "$brake_store" - require_root_owned_regular_path "$brake_lock" - # This validation is the deployment-admission point. The signed expiry is a - # start-before deadline: it denies a later transaction, but does not become a - # timer-shaped veto over one already admitted. The root actuator remains the - # sole shared-lock holder through the synchronous consequence, so an explicit - # engage/release transition cannot race or rewrite that transaction. - exec 9<"$brake_lock" + # This validation is the deployment-admission point. An absent or clear stop + # admits; an engaged or unreadable one makes the status verb exit nonzero. + # The root actuator remains the sole shared-lock holder through the + # synchronous consequence, so an engage or clear cannot race or rewrite it. + # The lock file is created here when no store has ever been written. + exec 9>>"$brake_lock" /usr/bin/flock --shared 9 "$idunn_provision" deployment-brake-status \ --store "$brake_store" \ @@ -312,7 +310,7 @@ case "$action:$target" in manifest="/srv/odin/deploy-manifests/$target" require_root_owned_regular_path "$manifest" [ -x "$manifest" ] || { echo "root actuator manifest is not executable: $manifest" >&2; exit 1; } - require_signed_deployment_grant "$@" + require_deployment_admission "$@" export IDUNN_ACTUATOR=1 export IDUNN_COMMAND_AUTHORITY=idunn-daemon export IDUNN_SOURCE_COMMIT="$source_commit" diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index f454248..4722734 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -299,6 +299,31 @@ names both content addresses and the executable artifact digest. Present and Ready remain Odin-owned correlations over independently verified runtime evidence; the deployment-plan module deliberately does not model them. +## The deployment stop + +`src/brake.rs` is the only reader and writer of the stop. A stop halts work +that has not crossed an irreversible step, and it is read again at each such +step: Sealing (`advance_sealing`), the static flip intent (`advance_static`) and +the entry into Fencing for a deploy (`advance_warming`). An engaged stop there +makes the transaction wait with a note naming who engaged it and why; nothing +aborts and nothing moves. After the flip intent is recorded, or once a deploy +has fenced, the transaction runs to its end: a stop halts before the +irreversible step, never mid-fence or mid-flip. Continuity is never gated. +The sequence of a stop record is positive and advances by one per verb; nothing +reads it for authority, so it is not validated further. + +Absent means the store file is missing in an existing directory. Anything else +Idunn cannot judge (a symlink, a zero-length or undecodable file, another +runtime's record, another owner, an unsearchable or non-directory parent) reads +as unreadable and fails closed; the wait note carries the +`idunn-provision deployment-brake-clear` command that replaces it. + +Record v6 (`deployment_admitted_at_unix_millis`, key 38) is forward-only: an +Idunn that predates v6 cannot read a migrated `control.cc`, so, as for v5 (see +`route-continuity-cut.md`, "An older Idunn refuses the v5 store"), take a +byte-compared pre-install backup of `control.cc` and `history.cc` with Idunn +stopped, and roll back by restoring the backup, never by swapping the binary. + ## Initial drivers and replaceable boundary The first Yggdrasil implementation uses: diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index a278165..d1f52d2 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -1160,7 +1160,7 @@ target's generation or to its `TargetSupervision` ends that target's iteration: - **Why no existing owner can serve:** - The generation does not exist for a first deploy (the ruling). - The transaction dies with each attempt. - - Brakes are operator-signed and are not Idunn's to write. + - The deployment brake is a stop that a steward or the operator writes (`src/brake.rs`); Idunn writes no brake. - **What it replaces:** two carry compensators, `for_new_incarnation` and the backoff copy in `from_transaction`, plus a history scan. diff --git a/docs/verify-transaction-cut.md b/docs/verify-transaction-cut.md index 87387dd..5bc21db 100644 --- a/docs/verify-transaction-cut.md +++ b/docs/verify-transaction-cut.md @@ -155,7 +155,7 @@ cleaned up. `idunn redeploy --daemon …`. The installed binary has no `redeploy` command (probe of its usage text), so the script is dead and is **not** the current pattern. The current pattern is `idunn up` followed by - `idunn-provision deployment-brake-release` + `idunn-provision deployment-brake-clear` when a stop is engaged (an absent or clear store admits) (`runbooks/idunn-host-raven.md`, "Deploy Muninn to Raven"). - F24. Build placement (`gamecult-ops/inventory.md:559-566`): build on or for the target's host. Yggdrasil has 16 vCPU and 62 GiB and carries live services, so diff --git a/src/brake.rs b/src/brake.rs index 570cf94..d44cd96 100644 --- a/src/brake.rs +++ b/src/brake.rs @@ -189,14 +189,40 @@ fn check_file(path: &Path, _: u32) -> Result<()> { Ok(()) } +/// Absent means exactly this: the final component does not exist and its +/// parent is an existing directory. Every other failure to look (permission, +/// a parent that is a file, a symlink loop) is a store this runtime cannot +/// judge, and fails closed. +fn is_absent(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let parent = path.parent().filter(|p| !p.as_os_str().is_empty()); + let parent = parent.unwrap_or_else(|| Path::new(".")); + ensure!( + std::fs::metadata(parent).is_ok_and(|m| m.is_dir()), + "deployment stop store directory {} does not exist", + parent.display() + ); + Ok(true) + } + Err(error) => Err(error) + .with_context(|| format!("looking for deployment stop store {}", path.display())), + } +} + fn read_checked(path: &Path, runtime_id: &str, owner: u32) -> Result> { - if std::fs::symlink_metadata(path).is_err() { + if is_absent(path)? { return Ok(None); } check_file(path, owner)?; + ensure!( + std::fs::metadata(path)?.len() > 0, + "deployment stop store is zero length" + ); let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; match entries.as_slice() { - [] => Ok(None), + [] => bail!("deployment stop store holds no record"), [entry] => { let record = DeploymentStop::decode(entry)?; ensure!( @@ -215,15 +241,26 @@ pub(crate) fn read_stop(path: &Path, runtime_id: &str, expected_owner_uid: u32) Ok(None) => Stop::Absent, Ok(Some(record)) if record.state == StopState::Clear => Stop::Clear(record), Ok(Some(record)) => Stop::Engaged(record), - Err(error) => Stop::Unreadable(format!("{error:#}")), + Err(error) => Stop::Unreadable(format!( + "{error:#}; to replace it run: idunn-provision deployment-brake-clear \ + --store {} --runtime-id {runtime_id} --reason (or deployment-brake-engage)", + path.display() + )), } } /// Write the next record under a compare-exchange of the whole store, so a -/// concurrent verb or a store that moved since it was read is refused, and a -/// foreign or corrupt predecessor is replaced only by an explicit verb. The +/// concurrent verb or a store that moved since it was read is refused. The /// author is the login name of the process uid and the sequence is the -/// predecessor's plus one; neither is an argument. +/// predecessor's plus one (1 when there is none or it cannot be decoded); +/// neither is an argument. +/// +/// This is also the operator's break-glass. A predecessor that is a retired v1 +/// brake, another runtime's record, ambiguous, or zero length is replaced. A +/// file that is not a CultCache store at all (junk bytes) is replaced only +/// when it is a regular file owned by the caller, by removing it first. A +/// symlink or another user's file is refused. The sequence is not validated +/// beyond being positive: nothing reads it for authority. pub(crate) fn replace_stop( path: &Path, runtime_id: &str, @@ -232,14 +269,22 @@ pub(crate) fn replace_stop( now_unix_millis: u64, ) -> Result { let store = SingleFileMessagePackBackingStore::new(path); - let before = store.pull_all_read_only_snapshot()?; + let before = match store.pull_all_read_only_snapshot() { + Ok(entries) => entries, + Err(undecodable) => { + check_file(path, process_uid()).with_context(|| { + format!("deployment stop store is undecodable ({undecodable:#}) and not the caller's to replace") + })?; + std::fs::remove_file(path)?; + Vec::new() + } + }; let sequence = match before.as_slice() { - [] => 1, [entry] => match DeploymentStop::decode(entry) { Ok(previous) => previous.sequence.saturating_add(1), Err(_) => 1, }, - _ => bail!("deployment stop store is ambiguous"), + _ => 1, }; let record = DeploymentStop { schema_version: DEPLOYMENT_STOP_SCHEMA.into(), @@ -296,8 +341,12 @@ mod tests { engage(&path, "held", 100)?; // Another runtime's record. assert!(!read_stop(&path, "other", me).admits()); - // Owned by someone other than the expected uid. - assert!(!read_stop(&path, "runtime", me + 1).admits()); + // Owned by someone other than the expected uid: even a CLEAR record + // from a foreign owner must not admit. + let cleared = dir.path().join("cleared.cc"); + replace_stop(&cleared, "runtime", StopState::Clear, "thaw", 100)?; + assert!(read_stop(&cleared, "runtime", me).admits()); + assert!(matches!(read_stop(&cleared, "runtime", me + 1), Stop::Unreadable(_))); // A symlink to a valid, clear store. let real = dir.path().join("real.cc"); replace_stop(&real, "runtime", StopState::Clear, "clear", 100)?; @@ -386,4 +435,79 @@ mod tests { assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "moved"); Ok(()) } + + #[test] + fn only_a_missing_file_in_an_existing_directory_is_absent() -> Result<()> { + let dir = tempfile::tempdir()?; + let me = process_uid(); + // Parent is a regular file: ENOTDIR, not absence. + let file = dir.path().join("plain"); + std::fs::write(&file, b"x")?; + assert!(matches!(read_stop(&file.join("stop.cc"), "runtime", me), Stop::Unreadable(_))); + // Parent does not exist. + assert!(matches!( + read_stop(&dir.path().join("nope/stop.cc"), "runtime", me), + Stop::Unreadable(_) + )); + // Zero-length file. + let empty = dir.path().join("empty.cc"); + std::fs::write(&empty, b"")?; + assert!(matches!(read_stop(&empty, "runtime", me), Stop::Unreadable(_))); + // An existing directory with no store is absent. + assert_eq!(read_stop(&dir.path().join("stop.cc"), "runtime", me), Stop::Absent); + // A parent that cannot be searched (skipped as root, which can). + #[cfg(unix)] + if me != 0 { + use std::os::unix::fs::PermissionsExt; + let locked = dir.path().join("locked"); + std::fs::create_dir(&locked)?; + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000))?; + let reading = read_stop(&locked.join("stop.cc"), "runtime", me); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700))?; + assert!(matches!(reading, Stop::Unreadable(_)), "{reading:?}"); + } + Ok(()) + } + + #[test] + fn the_author_is_the_uid_login_never_the_environment() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + // SAFETY: no other test reads USER, LOGNAME or LNAME. + unsafe { + std::env::set_var("USER", "spoof"); + std::env::set_var("LOGNAME", "spoof"); + } + let written = engage(&path, "held", 10)?; + assert_eq!(written.by, login_name(process_uid())?); + assert_ne!(written.by, "spoof"); + Ok(()) + } + + #[test] + fn unreadable_names_the_command_and_a_break_glass_clear_replaces_junk_and_v1() -> Result<()> { + let dir = tempfile::tempdir()?; + let me = process_uid(); + let path = dir.path().join("stop.cc"); + std::fs::write(&path, b"junk that is no store")?; + let Stop::Unreadable(why) = read_stop(&path, "runtime", me) else { + bail!("junk admitted") + }; + assert!(why.contains("idunn-provision deployment-brake-clear --store"), "{why}"); + assert!(why.contains("--runtime-id runtime"), "{why}"); + let cleared = replace_stop(&path, "runtime", StopState::Clear, "operator break-glass", 50)?; + assert_eq!(cleared.sequence, 1); + assert!(read_stop(&path, "runtime", me).admits()); + // A symlink is not the caller's to replace. + let real = dir.path().join("real.cc"); + std::fs::write(&real, b"junk")?; + let link = dir.path().join("link.cc"); + #[cfg(unix)] + { + std::os::unix::fs::symlink(&real, &link)?; + assert!(replace_stop(&link, "runtime", StopState::Clear, "x", 60).is_err()); + assert_eq!(std::fs::read(&real)?, b"junk"); + } + Ok(()) + } } diff --git a/src/control_plane.rs b/src/control_plane.rs index dc71d7a..5b06543 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -8028,26 +8028,10 @@ impl Engine { let mut next = current.value.clone(); match current.value.command_kind { CommandKind::Deploy | CommandKind::Rollback => { - match self.deployment_stop(current)? { - Stop::Absent | Stop::Clear(_) => { - next.deployment_admitted_at_unix_millis = Some(now) - } - Stop::Engaged(held) => { - return self.record_gate_wait( - current, - &format!( - "deployment brake engaged by {}: {}", - held.by, held.reason - ), - ); - } - Stop::Unreadable(why) => { - return self.record_gate_wait( - current, - &format!("deployment brake is unreadable and fails closed: {why}"), - ); - } + if let Some(note) = self.stop_note(current)? { + return self.record_gate_wait(current, ¬e); } + next.deployment_admitted_at_unix_millis = Some(now); } CommandKind::Continuity => { if !self.lifecycle_allows(current, now)? { @@ -8176,6 +8160,11 @@ impl Engine { let Some(flip) = ¤t.value.flip else { // Nothing has moved yet, so a refusal here ends the transaction // honestly: the release is still what was sealed, or it is not. + // The flip intent is the last point a stop can halt a static + // release: once it is recorded the flip runs to its end. + if let Some(note) = self.stop_note(current)? { + return self.record_gate_wait(current, ¬e); + } self.static_release.verify_installed(workload, release)?; let previous_release_id = self.static_release.observe_current(workload)?; let release_id = release.sealed_release_id.clone(); @@ -8188,6 +8177,7 @@ impl Engine { flipped_at_unix_millis: None, }); next.updated_at_unix_millis = now; + next.last_error = None; Ok(()) }); }; @@ -8418,6 +8408,14 @@ impl Engine { }); } + // Fencing is the last pre-fencing abort-able point of a deploy: an + // engaged stop halts here, before the incumbent is fenced. A fenced + // deploy runs to its end. Continuity is never gated by this stop. + if current.value.command_kind == CommandKind::Deploy + && let Some(note) = self.stop_note(current)? + { + return self.record_gate_wait(current, ¬e); + } self.transition(current, DeploymentPhase::Fencing) } @@ -9343,6 +9341,22 @@ impl Engine { .context("no private candidate port remains in the operator range") } + /// Why the deployment stop holds this transaction, or None when it admits. + /// Read live at each point that is still reversible: Sealing, the static + /// flip intent and the entry into Fencing. + fn stop_note(&self, current: &Stored) -> Result> { + Ok(match self.deployment_stop(current)? { + Stop::Absent | Stop::Clear(_) => None, + Stop::Engaged(held) => Some(format!( + "deployment brake engaged by {}: {}", + held.by, held.reason + )), + Stop::Unreadable(why) => Some(format!( + "deployment brake is unreadable and fails closed: {why}" + )), + }) + } + /// What the binding's deployment stop says about this transaction's /// runtime. Read live; never cached, never consumed. fn deployment_stop(&self, current: &Stored) -> Result { @@ -14111,6 +14125,10 @@ mod tests { .split_once("[brakes]") .context("binding has no brakes table")? .1 + ) + .replace( + "/var/lib/gamecult/idunn/service-deployment-brake.cc", + &world.root.join("service-deployment-brake.cc").display().to_string(), ), )?; let recipe = RECIPE @@ -19999,6 +20017,66 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(deploy) } + /// An engaged stop halts a deploy before it fences, the last pre-fencing + /// abort-able point; clearing lets the same transaction fence. + #[test] + fn an_engaged_stop_halts_entry_into_fencing_and_a_clear_lets_it_through() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + let plan = deploy.plan.as_ref().context("no plan")?; + set_stop(plan, StopState::Engaged, "freeze before fencing")?; + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + let held = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(held.phase, DeploymentPhase::Warming); + assert!(held.pre_fencing_abort.is_none() && held.completion.is_none()); + assert!( + held.last_error.as_deref().is_some_and(|note| note.contains("deployment brake engaged by") + && note.contains("freeze before fencing")), + "{:?}", + held.last_error + ); + set_stop(plan, StopState::Clear, "thaw")?; + assert!(world.engine.run_scheduler_tick()?); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + Ok(()) + } + + /// An Unreadable stop holds a Sealing deploy with a message that names the + /// remedy; it never admits. + #[test] + fn an_unreadable_stop_keeps_a_sealing_deploy_held_with_the_remedy() -> Result<()> { + let world = EngineFixture::new()?; + let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + make_stop_first(&world, &deploy.transaction_id)?; + let sealing = resident_id(&world, &deploy.transaction_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_admitted_at_unix_millis = None; + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + let store = tail.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; + std::fs::create_dir_all(store.parent().context("no parent")?)?; + for bytes in [&b"junk that is no store"[..], &b""[..]] { + std::fs::write(&store, bytes)?; + let held = resident_id(&world, &deploy.transaction_id)?; + rephase(&world, &deploy.transaction_id, DeploymentPhase::Sealing)?; + let held_now = resident_id(&world, &deploy.transaction_id)?; + world.engine.advance_transaction(&held_now)?; + let after = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(after.phase, DeploymentPhase::Sealing, "{:?}", held.value.last_error); + assert!(!after.deployment_admitted()); + assert!( + after.last_error.as_deref().is_some_and(|note| note.contains("fails closed") + && note.contains("idunn-provision deployment-brake-clear")), + "{:?}", + after.last_error + ); + } + Ok(()) + } + #[test] fn a_continuity_minted_as_a_deployment_takes_the_target_is_resolved_next_pass() -> Result<()> { // (a) The deployment fences first in the tick, so supervision reads its @@ -26239,6 +26317,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide )?; std::fs::create_dir_all(site.join("www"))?; std::fs::create_dir_all(site.join("config"))?; + std::fs::create_dir_all(site.join("authority"))?; std::fs::write(site.join("config/site.json"), SITE_CONFIG_ONE)?; *world.engine.runner_stand_in.lock().unwrap() = Some(Arc::new(SealingRunner { tree: tree.clone() })); @@ -26404,6 +26483,51 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + #[cfg(unix)] + #[test] + fn a_stop_engaged_after_admission_halts_the_static_flip_until_it_clears() -> Result<()> { + let w = static_world()?; + let first = w.deploy("up-service")?; + let incumbent = w.current_release().context("no incumbent")?; + assert!(first.is_terminal()); + enqueue_deploy(&w.world, "up-service-2")?; + let id = w.transaction_of("up-service-2")?; + // Admitted by the absent-or-clear stop, flip intent not yet recorded. + w.tick_until(&id, "admission", |t| t.phase == DeploymentPhase::Starting)?; + assert!(latest_id(&w.world, &id)?.flip.is_none()); + w.hold()?; + for _ in 0..4 { + w.world.engine.resume_backoff.lock().unwrap().clear(); + let _ = w.world.engine.run_scheduler_tick(); + } + let held = latest_id(&w.world, &id)?; + assert_eq!(held.phase, DeploymentPhase::Starting); + assert!(held.flip.is_none(), "a halted flip recorded an intent"); + assert!(held.pre_fencing_abort.is_none() && held.completion.is_none()); + assert!(held.last_error.as_deref().is_some_and(|n| n.contains("deployment brake engaged by")), "{:?}", held.last_error); + assert_eq!(w.current_release().as_deref(), Some(incumbent.as_str()), "the link moved under a stop"); + w.release_brake(&held)?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert_ne!(w.current_release().as_deref(), Some(incumbent.as_str())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_stop_engaged_after_the_flip_intent_does_not_halt_the_flip() -> Result<()> { + let w = static_world()?; + let id = w.transaction_of("up-service")?; + w.tick_until(&id, "the flip intent", |t| t.flip.is_some())?; + w.hold()?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert!(done.flip.as_ref().is_some_and(FlipEvidence::landed)); + Ok(()) + } + #[cfg(unix)] #[test] fn static_release_failed_flip_leaves_current_and_ends_before_fencing() -> Result<()> { -- 2.47.3