diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index c7407ca..1c45594 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -45,6 +45,7 @@ ExecStart=/usr/local/bin/idunn serve \ --idunn-identity-store /var/lib/gamecult/idunn/idunn-service-identity.cc \ --odin-correlation-store /var/lib/gamecult/odin/topology.cc \ --odin-trust-anchor /etc/gamecult/idunn/odin-topology-anchor.cc \ + --deployment-brake-operator-anchor /etc/gamecult/idunn/deployment-brake-operator-anchor.cc \ --host-actuator-bind 10.77.0.1:17890 \ --source-uid 986 \ --source-gid 978 diff --git a/deploy/legacy/idunn-yggdrasil b/deploy/legacy/idunn-yggdrasil index 62a18eb..26970c0 100644 --- a/deploy/legacy/idunn-yggdrasil +++ b/deploy/legacy/idunn-yggdrasil @@ -16,6 +16,7 @@ target="${2:-}" compose=/srv/compose/yggdrasil-apps.yaml brake_store=/var/lib/gamecult/idunn-authority/deployment-brake.cc brake_lock=/var/lib/gamecult/idunn-authority/deployment-brake.cc.lock +brake_anchor=/etc/gamecult/idunn/deployment-brake-operator-anchor.cc generation_lock=/var/lib/gamecult/idunn-authority/idunn-generation.lock idunn_provision=/usr/local/bin/idunn-provision @@ -153,7 +154,7 @@ deployment_target_policy() { esac } -require_deployment_admission() { +require_signed_deployment_grant() { source_commit="$3" deployment_request_id="$4" release_authority_id="$5" @@ -179,17 +180,28 @@ require_deployment_admission() { [ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; } require_root_owned_resolved_executable "$idunn_provision" + require_root_owned_regular_path "$brake_store" + require_root_owned_regular_path "$brake_lock" + require_root_owned_regular_path "$brake_anchor" - # This validation is the deployment-admission point. An absent or clear stop - # admits; an engaged or unreadable one makes the status verb exit nonzero. - # The root actuator remains the sole shared-lock holder through the - # synchronous consequence, so an engage or clear cannot race or rewrite it. - # The lock file is created here when no store has ever been written. - exec 9>>"$brake_lock" + # This validation is the deployment-admission point. The signed expiry is a + # start-before deadline: it denies a later transaction, but does not become a + # timer-shaped veto over one already admitted. The root actuator remains the + # sole shared-lock holder through the synchronous consequence, so an explicit + # engage/release transition cannot race or rewrite that transaction. + exec 9<"$brake_lock" /usr/bin/flock --shared 9 + now_unix_millis="$(/usr/bin/date +%s%3N)" + case "$now_unix_millis" in + ''|*[!0-9]*) echo "root actuator clock did not produce Unix milliseconds" >&2; exit 1 ;; + esac "$idunn_provision" deployment-brake-status \ --store "$brake_store" \ - --runtime-id yggdrasil + --operator-anchor "$brake_anchor" \ + --runtime-id yggdrasil \ + --release-id "$source_commit" \ + --deployment-id "$deployment_request_id" \ + --now-unix-millis "$now_unix_millis" if [ "$minimum_source_revision_required" = true ]; then require_root_owned_resolved_executable /usr/local/bin/idunn @@ -310,7 +322,7 @@ case "$action:$target" in manifest="/srv/odin/deploy-manifests/$target" require_root_owned_regular_path "$manifest" [ -x "$manifest" ] || { echo "root actuator manifest is not executable: $manifest" >&2; exit 1; } - require_deployment_admission "$@" + require_signed_deployment_grant "$@" export IDUNN_ACTUATOR=1 export IDUNN_COMMAND_AUTHORITY=idunn-daemon export IDUNN_SOURCE_COMMIT="$source_commit" diff --git a/docs/authority-map.md b/docs/authority-map.md index 001b644..c8a8546 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -240,7 +240,7 @@ Fencing retries its phase forward on a capped backoff and never stops its live candidate, since that is the only running copy of the admitted body; the deadline opens the same incident. The one exception is yielding: a continuity whose route proof has not succeeded gives way, through its own post-fencing -abort, to a deployment the deployment stop admits and that awaits the +abort, to a deployment the deployment brake has released and that awaits the handover; a continuity that has proven its route never yields. The `ContinuityExhausted` incident (six charged attempts in an hour, `restarts_page`) pages and gates nothing. When continuity next acts is derived @@ -257,14 +257,9 @@ held until a lift succeeds. It is a migration with a deletion line (follow-up still owe dependency failures not growing the streak and the dead-generation route refresh. -The deployment brake is a stop, not a per-deploy gate: it gates changes to -source, release, configuration, schema, unit, or authority binding only while -engaged. Its one reader and writer is `src/brake.rs` (`idunn.deployment_stop.v1`: -engaged or clear, who, why, when, sequence). An absent or clear store admits; -a store that is present but unreadable, foreign, for another runtime, a symlink -or owned by another uid fails closed. Nothing is signed, bound to a release or -consumed, and one clear admits every deploy until it is engaged again. The -lifecycle brake separately gates restart of an already-admitted incarnation. Neither target brake may gate Idunn itself or an +The deployment brake gates changes to source, release, configuration, schema, +unit, or authority binding. The lifecycle brake separately gates restart of an +already-admitted incarnation. Neither target brake may gate Idunn itself or an unrelated service. If Odin is unavailable, Idunn preserves existing routes and continuity for diff --git a/docs/deploy-provenance-target.md b/docs/deploy-provenance-target.md index e1086e8..6b6293d 100644 --- a/docs/deploy-provenance-target.md +++ b/docs/deploy-provenance-target.md @@ -4,15 +4,6 @@ Status: target, Self, 2026-10-19 (campaign `idunn-deploy-provenance` in Eureka's mind). It states the ends. The map will live in `deploy-provenance-map.md`. -Live as of cut `brake-is-a-stop` (2026-11-13): the deployment brake is the -stop of invariant 3 and the signed per-deploy release is gone. `src/brake.rs` -reads and writes it; `idunn-provision deployment-brake-engage|clear|status` -take only `--store`, `--runtime-id` and `--reason`; the writer's login name and -the verb's clock are recorded; an absent or clear store admits. The -transaction stamps `deployment_admitted_at_unix_millis` (record v6). Still -planned: the typed authority provenance on the transaction (invariant 2), and a -steward-scoped store. - ## Why Operator, 2026-10-19: diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index 048e784..1c5a7f4 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -48,8 +48,8 @@ and health. CultMesh carries typed projections between them. scaling, restart, host reboot recovery, and crash recovery use the same source-to-recipe-to-artifact provenance and the same distinct write-lease and route-membership primitives. - Deployment may change that chain only while the deployment stop admits it - (absent or clear; an engaged stop halts). Continuity may restart only the already-admitted chain and + Deployment may change that chain only while the deployment brake admits the + exact transaction. Continuity may restart only the already-admitted chain and is governed by a separate lifecycle brake. - **Deletion line:** hard-coded target copies, raw `deploy_command` and `restart_command` state, `sh -c` actuation, root Git inspection, staged @@ -209,8 +209,8 @@ non-regular carry source fails the deployment before anything moves. What `current_link` names is observed from the symlink and confirmed by the latest Admitted static transaction's `flip.release_id`; it is stored nowhere else. A rollback is a flip back to `flip.previous_release_id`: release -directories stay, and Idunn prunes none. The deployment stop is read exactly as for a -service, by the binding's runtime id. Release ids are per transaction, so +directories stay, and Idunn prunes none. The brake is read exactly as for a +service, naming the binding's runtime id and the sealed release id. Release ids are per transaction, so redeploying an unchanged tarball with a changed site file is a new release that carries the new file, never an in-place re-carry of an existing one: the carry is part of a release. If `current_link` is replaced by a directory or by a link @@ -230,8 +230,9 @@ transaction (`CommandKind::Rollback`) that reuses the previous release exactly: the plan, sealed release and installed release are copied from the transaction that flipped it, so there is no source, no build and no source cleanup, and it creates no admitted generation. It starts at Sealing with that evidence and -reaches the same deployment stop as a deploy, which admits it unless engaged; -then `advance_static` flips, with the same three durable steps, the same +reaches the same deployment brake as a deploy, which the operator releases for +the rollback transaction's id and the previous sealed release id; then +`advance_static` flips, with the same three durable steps, the same nothing-aborts-after-the-rename rule and the same stuck-flip behavior. The previous release is derived, never stored: the newest Admitted static @@ -299,82 +300,6 @@ names both content addresses and the executable artifact digest. Present and Ready remain Odin-owned correlations over independently verified runtime evidence; the deployment-plan module deliberately does not model them. -## The deployment stop - -`src/brake.rs` is the only reader and writer of the stop. A stop halts work -that has not crossed an irreversible step, and it is read again at each such -step: Sealing (`advance_sealing`), the static flip intent (`advance_static`) and -the entry into Fencing for a deploy (`advance_warming`). An engaged stop there -makes the transaction wait with a note naming who engaged it and why; nothing -aborts and nothing moves. After the flip intent is recorded, or once a deploy -has fenced, the transaction runs to its end: a stop halts before the -irreversible step, never mid-fence or mid-flip. Continuity is never gated. -The sequence of a stop record is positive and advances by one per verb; nothing -reads it for authority, so it is not validated further. - -Absent means the store file is missing in an existing directory. Anything else -Idunn cannot judge (a symlink, a zero-length or undecodable file, another -runtime's record, another owner, an unsearchable or non-directory parent) reads -as unreadable and fails closed; the wait note carries the -`idunn-provision deployment-brake-clear` command that replaces it. - -A binding may name a second stop, the target's steward: - -```toml -[brakes.steward] -user = "mind-streampixels" # a login, never root -store = "/var/lib/gamecult/idunn-steward/streampixels/deployment-brake.cc" -``` - -Deploys admit only when every stop admits. The steward's store is read as owned -by that user's uid: the owner uid proves who wrote the file, and the wait note -names the stop by the binding's steward user, never by the record's `by` (which -is the verb's own text). Idunn never writes the steward directory and no key, -group or ACL is involved. The operator's store is a separate stop the steward -cannot clear, and the wait note names each holding stop. - -The steward owns its directory, so it could unlink any file in it. Binding -validation therefore refuses a steward store whose directory is shared with, -inside or around the directory of any other Idunn authority store (deployment -and lifecycle brakes, trust anchor, write-lease record, release authority). -Validation cannot see the file system: the root who provisions the directory -makes it contain the steward's store and nothing else. - -A steward store owned by another uid, or a steward user that does not resolve -to a non-root uid, is unreadable and fails closed. Only the steward (the -owner uid) can repair it: a clear run as root leaves a root-owned file that is -still unreadable. - -### Provisioning a steward (root, once per target) - -1. Create the store directory, owned by the steward and holding nothing else: - `install -d -o mind-streampixels -g mind-streampixels -m 0755 - /var/lib/gamecult/idunn-steward/streampixels`. It must not be, contain or sit - inside any other authority directory, and no workload `ReadWritePaths` may - cover it. Idunn (root) reads it without further grants; no group or ACL. -2. Add the stanza above to the target's operator binding, after `[brakes]`, - with the steward's real login. Deploys planned afterwards see it. -3. The steward engages and clears as itself, never through root: - `idunn-provision deployment-brake-engage --store --runtime-id - --reason ` and `deployment-brake-clear` with the same flags. -4. The operator reads it with `idunn-provision deployment-brake-status --store - --runtime-id --owner-user mind-streampixels`. - -The binding is sealed into each plan. A deploy already planned keeps the -binding it was sealed with: editing the stanza does not apply to it. If the -steward's login is deleted or its directory is lost, that deploy stays -unreadable-held at Sealing (or the static flip intent). `idunn cancel` withdraws -only a deploy held for a readiness disagreement before Fencing, and `idunn -expire` only one in Warming through Committing, so neither frees a deploy held -by the stop. Recovery is to restore the steward's login or recreate the -directory owned by it (and have the steward clear its store). - -Record v6 (`deployment_admitted_at_unix_millis`, key 38) is forward-only: an -Idunn that predates v6 cannot read a migrated `control.cc`, so, as for v5 (see -`route-continuity-cut.md`, "An older Idunn refuses the v5 store"), take a -byte-compared pre-install backup of `control.cc` and `history.cc` with Idunn -stopped, and roll back by restoring the backup, never by swapping the binary. - ## Initial drivers and replaceable boundary The first Yggdrasil implementation uses: @@ -413,7 +338,7 @@ For a singleton incarnation: declaration, binding, launch contract, and input digests. 5. Derive the release-bound Expected incarnation from the validated sealed release and its exact compiled plan. Publish it only once the deployment - stop has admitted this transaction (stamping `deployment_admitted_at_unix_millis`), under the candidate's own + brake has admitted this exact transaction, under the candidate's own incarnation key, beside whatever the target's admitted incarnation projects. 6. Issue one opaque runtime-instance activation bound to the Expected digest, pass the exact Expected and activation documents through the actuator's @@ -436,8 +361,8 @@ For a singleton incarnation: and stop the incumbent. Sealing resolves the source, freezes it, runs every recipe step, and installs -the release before the deployment stop is consulted at all (`advance_sealing`). -The stop gates step 5, the first Verse-visible change: publishing the +the release before the deployment brake is consulted at all (`advance_sealing`). +The brake gates step 5, the first Verse-visible change: publishing the candidate's Expected incarnation on the move into `Starting`. Build and test already run unbraked, inside the sealing phase. @@ -468,7 +393,7 @@ identity, and route membership. Every restart receives a new runtime-instance activation and, for a state writer, a new process-bound lease only after the new warming presence is observed. A lease from the dead process is never reused. A continuity restart does not fetch a branch, rebuild, rerun a migration, alter -the launch contract, or pass the deployment stop. An explicit lifecycle +the launch contract, or consume the deployment brake. An explicit lifecycle brake may suspend this actuation without changing deployment authority. Continuity never stops. Six charged attempts in an hour only open the @@ -501,7 +426,7 @@ the admitted body. Both operator cases open an `operator-required` incident on. The one yield: a continuity whose route proof has not succeeded gives way to a -deployment the deployment stop admits that awaits the handover of the +deployment the deployment brake has released that awaits the handover of the target. It begins its own post-fencing abort and the deployment fences what is admitted. A continuity that has proven its route is serving never yields. diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 7b2573b..a278165 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -390,8 +390,6 @@ read-only command on Yggdrasil on 2026-09-29. Nothing was written to the host. (`:3670`, `:3715`), and topology refresh (`:3780`). After three refused continuity restarts, supervision stops restarting (`:3522-3528`) but keeps supervising the route of a dead incarnation. Read. - Resolved by `cut-liveness-gates-presence`: route challenge and topology - refresh now run only for an incarnation observed running. - F9. The continuity restart limit counts refused restarts per generation (`is_refused_restart`, `:3469-3488`). A restart that succeeds admits a new generation and resets the count. A unit that starts, is admitted, and then @@ -1162,7 +1160,7 @@ target's generation or to its `TargetSupervision` ends that target's iteration: - **Why no existing owner can serve:** - The generation does not exist for a first deploy (the ruling). - The transaction dies with each attempt. - - The deployment brake is a stop that a steward or the operator writes (`src/brake.rs`); Idunn writes no brake. + - Brakes are operator-signed and are not Idunn's to write. - **What it replaces:** two carry compensators, `for_new_incarnation` and the backoff copy in `from_transaction`, plus a history scan. diff --git a/docs/verify-transaction-cut.md b/docs/verify-transaction-cut.md index 5bc21db..87387dd 100644 --- a/docs/verify-transaction-cut.md +++ b/docs/verify-transaction-cut.md @@ -155,7 +155,7 @@ cleaned up. `idunn redeploy --daemon …`. The installed binary has no `redeploy` command (probe of its usage text), so the script is dead and is **not** the current pattern. The current pattern is `idunn up` followed by - `idunn-provision deployment-brake-clear` when a stop is engaged (an absent or clear store admits) + `idunn-provision deployment-brake-release` (`runbooks/idunn-host-raven.md`, "Deploy Muninn to Raven"). - F24. Build placement (`gamecult-ops/inventory.md:559-566`): build on or for the target's host. Yggdrasil has 16 vCPU and 62 GiB and carries live services, so diff --git a/src/brake.rs b/src/brake.rs deleted file mode 100644 index 2dbee13..0000000 --- a/src/brake.rs +++ /dev/null @@ -1,701 +0,0 @@ -//! The deployment stop: the one reader and the one writer of the record that -//! halts deployments of a runtime. Engaged halts; absent or clear admits. Who -//! engaged it and why are on the record, and nothing is signed, issued, bound -//! to a release or consumed: the brake is a stop, not a per-deploy gate (ruling -//! brake-is-a-stop). The lifecycle brake is a different record and a different -//! store; nothing here touches it. -//! -//! The reader fails closed. A store that is present but foreign, ambiguous, -//! noncanonical, for another runtime, a symlink, or owned by anyone but the -//! expected uid reads as `Unreadable`, which does not admit. - -use std::path::Path; - -use crate::deployment::BrakeBinding; - -use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcache_rs::{CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore}; -use serde::{Deserialize, Serialize}; - -pub(crate) const DEPLOYMENT_STOP_SCHEMA: &str = "idunn.deployment_stop.v1"; -pub(crate) const DEPLOYMENT_STOP_KEY: &str = "deployment-stop"; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum StopState { - Engaged, - Clear, -} - -impl StopState { - pub(crate) fn name(self) -> &'static str { - match self { - Self::Engaged => "engaged", - Self::Clear => "clear", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache(type = "idunn.deployment_stop", schema = "idunn.deployment_stop.v1")] -pub(crate) struct DeploymentStop { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) runtime_id: String, - #[cultcache(key = 2)] - pub(crate) state: StopState, - /// The login name of the user who ran the verb, read from the process uid. - #[cultcache(key = 3)] - pub(crate) by: String, - #[cultcache(key = 4)] - pub(crate) reason: String, - /// The verb's clock. - #[cultcache(key = 5)] - pub(crate) at_unix_millis: u64, - /// The previous record's sequence plus one; 1 for the first. - #[cultcache(key = 6)] - pub(crate) sequence: u64, -} - -fn bounded(value: &str, label: &str, max: usize) -> Result<()> { - ensure!( - !value.trim().is_empty() && value.len() <= max && !value.chars().any(char::is_control), - "deployment stop {label} is empty, oversized, or contains control characters" - ); - Ok(()) -} - -impl DeploymentStop { - pub(crate) fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == DEPLOYMENT_STOP_SCHEMA, - "deployment stop schema is unsupported" - ); - bounded(&self.runtime_id, "runtime id", 256)?; - bounded(&self.by, "author", 256)?; - bounded(&self.reason, "reason", 1024)?; - ensure!( - self.at_unix_millis > 0 && self.sequence > 0, - "deployment stop time or sequence is zero" - ); - Ok(()) - } - - fn envelope(&self) -> Result { - self.validate()?; - let stored_at = chrono::DateTime::from_timestamp_millis(i64::try_from(self.at_unix_millis)?) - .ok_or_else(|| anyhow!("deployment stop time is out of range"))? - .to_rfc3339(); - Ok(CultCacheEnvelope { - key: DEPLOYMENT_STOP_KEY.into(), - r#type: Self::TYPE.into(), - payload: rmp_serde::to_vec(self)?, - stored_at, - schema_id: Some(DEPLOYMENT_STOP_SCHEMA.into()), - }) - } - - fn decode(envelope: &CultCacheEnvelope) -> Result { - ensure!( - envelope.r#type == Self::TYPE - && envelope.key == DEPLOYMENT_STOP_KEY - && envelope.schema_id.as_deref() == Some(DEPLOYMENT_STOP_SCHEMA), - "deployment stop store holds a foreign record" - ); - let record: Self = rmp_serde::from_slice(&envelope.payload)?; - ensure!( - rmp_serde::to_vec(&record)? == envelope.payload, - "deployment stop is noncanonical" - ); - record.validate()?; - Ok(record) - } -} - -/// What one store says about a runtime's deployments. -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum Stop { - /// No store, or an empty one: nothing halts. - Absent, - Clear(DeploymentStop), - Engaged(DeploymentStop), - /// Present but not a record this runtime can trust. Does not admit. - Unreadable(String), -} - -impl Stop { - pub(crate) fn admits(&self) -> bool { - matches!(self, Self::Absent | Self::Clear(_)) - } -} - -/// One stop a binding names, with how to name it in a wait note. -pub(crate) struct NamedStop { - pub(crate) label: String, - /// The login the file system proves wrote the store: the steward user, whose - /// uid owns the file. None for the operator's store, whose record carries - /// only the verb's own claim of `by`. - pub(crate) proven_writer: Option, - pub(crate) stop: Stop, -} - -impl NamedStop { - /// The wait-note line for a stop that does not admit; None when it admits. - /// A steward stop is named by the proven owner, never by the record's `by`. - pub(crate) fn note(&self) -> Option { - let label = &self.label; - match &self.stop { - Stop::Absent | Stop::Clear(_) => None, - Stop::Engaged(held) => Some(match &self.proven_writer { - Some(_) => format!("{label} engaged: {}", held.reason), - None => format!("{label} engaged by {}: {}", held.by, held.reason), - }), - Stop::Unreadable(why) => Some(format!("{label} is unreadable and fails closed: {why}")), - } - } -} - -/// Every stop a binding names, each with the words that identify its store in -/// a wait note: the operator's, then the steward's when the binding names one. -/// Deployments admit iff every one admits. The operator's store is read as -/// owned by this process's uid; the steward's as owned by the steward user's -/// uid, so the file system proves who wrote it. A steward that does not -/// resolve to a non-root uid is an unreadable stop, never an absent one. -pub(crate) fn read_stops(brakes: &BrakeBinding, runtime_id: &str) -> Vec { - let mut stops = vec![NamedStop { - label: "deployment brake".to_string(), - proven_writer: None, - stop: read_stop(&brakes.deployment_store, runtime_id, process_uid()), - }]; - if let Some(steward) = &brakes.steward { - let label = format!("steward {} deployment brake", steward.user); - let stop = match crate::steward_orders::user_uid(&steward.user) { - Ok(uid) => read_stop_with_remedy(&steward.store, runtime_id, uid, |path| { - // Only the owner uid can mend it: a clear run as root leaves a - // root-owned file, which this reader refuses again. - format!( - "only {} (the store's owner uid) can repair it, by running as that user: \ - idunn-provision deployment-brake-clear --store {} --runtime-id {runtime_id} \ - --reason ; a clear run as root stays unreadable", - steward.user, - path.display() - ) - }), - Err(why) => Stop::Unreadable(format!( - "{why:#}; the binding's steward user {} must be a non-root login", - steward.user - )), - }; - stops.push(NamedStop { - label, - proven_writer: Some(steward.user.clone()), - stop, - }); - } - stops -} - -#[cfg(unix)] -pub(crate) fn process_uid() -> u32 { - // SAFETY: geteuid has no failure mode and no arguments. - unsafe { libc::geteuid() } -} - -#[cfg(not(unix))] -pub(crate) fn process_uid() -> u32 { - 0 -} - -#[cfg(unix)] -pub(crate) fn login_name(uid: u32) -> Result { - // SAFETY: getpwuid returns null or a pointer into libc's static buffer, - // copied before any other call; the verbs are single threaded. - unsafe { - let entry = libc::getpwuid(uid); - ensure!(!entry.is_null(), "uid {uid} has no login name"); - Ok(std::ffi::CStr::from_ptr((*entry).pw_name) - .to_string_lossy() - .into_owned()) - } -} - -#[cfg(not(unix))] -pub(crate) fn login_name(_: u32) -> Result { - std::env::var("USERNAME").context("USERNAME is unset") -} - -/// Open without following a final symlink and judge the file by fstat before -/// anything is read from it. -#[cfg(unix)] -fn check_file(path: &Path, expected_owner_uid: u32) -> Result<()> { - use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; - let file = std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_NOFOLLOW) - .open(path) - .with_context(|| format!("opening deployment stop store {}", path.display()))?; - let metadata = file.metadata()?; - ensure!( - metadata.is_file(), - "deployment stop store is not a regular file" - ); - ensure!( - metadata.uid() == expected_owner_uid, - "deployment stop store is owned by uid {}, not {expected_owner_uid}", - metadata.uid() - ); - Ok(()) -} - -#[cfg(not(unix))] -fn check_file(path: &Path, _: u32) -> Result<()> { - ensure!( - std::fs::metadata(path)?.is_file(), - "deployment stop store is not a regular file" - ); - Ok(()) -} - -/// Absent means exactly this: the final component does not exist and its -/// parent is an existing directory. Every other failure to look (permission, -/// a parent that is a file, a symlink loop) is a store this runtime cannot -/// judge, and fails closed. -fn is_absent(path: &Path) -> Result { - match std::fs::symlink_metadata(path) { - Ok(_) => Ok(false), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - let parent = path.parent().filter(|p| !p.as_os_str().is_empty()); - let parent = parent.unwrap_or_else(|| Path::new(".")); - ensure!( - std::fs::metadata(parent).is_ok_and(|m| m.is_dir()), - "deployment stop store directory {} does not exist", - parent.display() - ); - Ok(true) - } - Err(error) => Err(error) - .with_context(|| format!("looking for deployment stop store {}", path.display())), - } -} - -fn read_checked(path: &Path, runtime_id: &str, owner: u32) -> Result> { - if is_absent(path)? { - return Ok(None); - } - check_file(path, owner)?; - ensure!( - std::fs::metadata(path)?.len() > 0, - "deployment stop store is zero length" - ); - let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - match entries.as_slice() { - [] => bail!("deployment stop store holds no record"), - [entry] => { - let record = DeploymentStop::decode(entry)?; - ensure!( - record.runtime_id == runtime_id, - "deployment stop belongs to runtime {}", - record.runtime_id - ); - Ok(Some(record)) - } - _ => bail!("deployment stop store is ambiguous"), - } -} - -pub(crate) fn read_stop(path: &Path, runtime_id: &str, expected_owner_uid: u32) -> Stop { - read_stop_with_remedy(path, runtime_id, expected_owner_uid, |path| { - format!( - "to replace it run: idunn-provision deployment-brake-clear \ - --store {} --runtime-id {runtime_id} --reason (or deployment-brake-engage)", - path.display() - ) - }) -} - -fn read_stop_with_remedy( - path: &Path, - runtime_id: &str, - expected_owner_uid: u32, - remedy: impl FnOnce(&Path) -> String, -) -> Stop { - match read_checked(path, runtime_id, expected_owner_uid) { - Ok(None) => Stop::Absent, - Ok(Some(record)) if record.state == StopState::Clear => Stop::Clear(record), - Ok(Some(record)) => Stop::Engaged(record), - Err(error) => Stop::Unreadable(format!("{error:#}; {}", remedy(path))), - } -} - -/// Write the next record under a compare-exchange of the whole store, so a -/// concurrent verb or a store that moved since it was read is refused. The -/// author is the login name of the process uid and the sequence is the -/// predecessor's plus one (1 when there is none or it cannot be decoded); -/// neither is an argument. -/// -/// This is also the operator's break-glass. A predecessor that is a retired v1 -/// brake, another runtime's record, ambiguous, or zero length is replaced. A -/// file that is not a CultCache store at all (junk bytes) is replaced only -/// when it is a regular file owned by the caller, by removing it first. A -/// symlink or another user's file is refused. The sequence is not validated -/// beyond being positive: nothing reads it for authority. -pub(crate) fn replace_stop( - path: &Path, - runtime_id: &str, - state: StopState, - reason: &str, - now_unix_millis: u64, -) -> Result { - let store = SingleFileMessagePackBackingStore::new(path); - let before = match store.pull_all_read_only_snapshot() { - Ok(entries) => entries, - Err(undecodable) => { - check_file(path, process_uid()).with_context(|| { - format!("deployment stop store is undecodable ({undecodable:#}) and not the caller's to replace") - })?; - std::fs::remove_file(path)?; - Vec::new() - } - }; - let sequence = match before.as_slice() { - [entry] => match DeploymentStop::decode(entry) { - Ok(previous) => previous.sequence.saturating_add(1), - Err(_) => 1, - }, - _ => 1, - }; - let record = DeploymentStop { - schema_version: DEPLOYMENT_STOP_SCHEMA.into(), - runtime_id: runtime_id.into(), - state, - by: login_name(process_uid())?, - reason: reason.into(), - at_unix_millis: now_unix_millis, - sequence, - }; - let next = record.envelope()?; - ensure!( - store.compare_exchange_snapshot(&before, &[next])?, - "deployment stop changed during transition" - ); - Ok(record) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn engage(path: &Path, reason: &str, at: u64) -> Result { - replace_stop(path, "runtime", StopState::Engaged, reason, at) - } - - #[test] - fn absent_and_empty_stores_admit_a_clear_one_admits_an_engaged_one_does_not() -> Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("stop.cc"); - let me = process_uid(); - assert_eq!(read_stop(&path, "runtime", me), Stop::Absent); - engage(&path, "freeze for the audit", 100)?; - let Stop::Engaged(held) = read_stop(&path, "runtime", me) else { - bail!("an engaged stop did not read as engaged") - }; - assert_eq!(held.reason, "freeze for the audit"); - assert_eq!(held.by, login_name(me)?); - assert!(!Stop::Engaged(held).admits()); - replace_stop(&path, "runtime", StopState::Clear, "audit done", 200)?; - let cleared = read_stop(&path, "runtime", me); - assert!(matches!(&cleared, Stop::Clear(record) if record.sequence == 2)); - assert!(cleared.admits()); - // Clear is not consumed: the same record admits again. - assert!(read_stop(&path, "runtime", me).admits()); - Ok(()) - } - - #[test] - fn corrupt_foreign_other_runtime_symlinked_or_foreign_owned_stores_stop() -> Result<()> { - let dir = tempfile::tempdir()?; - let me = process_uid(); - let path = dir.path().join("stop.cc"); - engage(&path, "held", 100)?; - // Another runtime's record. - assert!(!read_stop(&path, "other", me).admits()); - // Owned by someone other than the expected uid: even a CLEAR record - // from a foreign owner must not admit. - let cleared = dir.path().join("cleared.cc"); - replace_stop(&cleared, "runtime", StopState::Clear, "thaw", 100)?; - assert!(read_stop(&cleared, "runtime", me).admits()); - assert!(matches!(read_stop(&cleared, "runtime", me + 1), Stop::Unreadable(_))); - // A symlink to a valid, clear store. - let real = dir.path().join("real.cc"); - replace_stop(&real, "runtime", StopState::Clear, "clear", 100)?; - let link = dir.path().join("link.cc"); - #[cfg(unix)] - std::os::unix::fs::symlink(&real, &link)?; - #[cfg(unix)] - assert!(matches!(read_stop(&link, "runtime", me), Stop::Unreadable(_))); - // Bytes that are no store at all. - let junk = dir.path().join("junk.cc"); - std::fs::write(&junk, b"not msgpack at all")?; - assert!(matches!(read_stop(&junk, "runtime", me), Stop::Unreadable(_))); - // A record of another type under another key (the retired v1 brake). - let foreign = dir.path().join("foreign.cc"); - let store = SingleFileMessagePackBackingStore::new(&foreign); - let envelope = CultCacheEnvelope { - key: "deployment".into(), - r#type: "idunn.deployment_brake".into(), - payload: vec![0x90], - stored_at: "1970-01-01T00:00:00+00:00".into(), - schema_id: Some("idunn.deployment_brake.v1".into()), - }; - assert!(store.compare_exchange_snapshot(&[], &[envelope])?); - assert!(matches!(read_stop(&foreign, "runtime", me), Stop::Unreadable(_))); - // A noncanonical payload under the right identity. - let mut stop = DeploymentStop::decode(&store_entry(&path)?)?; - stop.sequence = 9; - let mut loose = stop.envelope()?; - loose.payload.push(0xc0); - let noncanonical = dir.path().join("noncanonical.cc"); - assert!(SingleFileMessagePackBackingStore::new(&noncanonical) - .compare_exchange_snapshot(&[], &[loose])?); - assert!(matches!(read_stop(&noncanonical, "runtime", me), Stop::Unreadable(_))); - Ok(()) - } - - /// A binding whose steward is the user running the test, so the steward's - /// store can really be owned by the steward. None when run as root, which - /// no steward may be. - fn binding_with_steward(dir: &Path, user: &str) -> BrakeBinding { - BrakeBinding { - deployment_store: dir.join("operator.cc"), - lifecycle_store: dir.join("lifecycle.cc"), - steward: Some(crate::deployment::StewardBrakeBinding { - user: user.into(), - store: dir.join("steward.cc"), - }), - } - } - - fn admits_all(stops: &[NamedStop]) -> bool { - stops.iter().all(|named| named.stop.admits()) - } - - #[test] - fn the_steward_store_is_a_second_stop_and_the_operator_store_is_one_it_cannot_clear() -> Result<()> { - let me = process_uid(); - if me == 0 { - eprintln!("SKIPPED: a steward is never root, and this run is root"); - return Ok(()); - } - let dir = tempfile::tempdir()?; - let binding = binding_with_steward(dir.path(), &login_name(me)?); - let (operator, steward) = (&binding.deployment_store, &binding.steward.as_ref().unwrap().store); - // Both absent: admits. - assert!(admits_all(&read_stops(&binding, "runtime"))); - // Steward engaged alone: halts, and the stop names the steward's store. - replace_stop(steward, "runtime", StopState::Engaged, "steward freeze", 10)?; - let stops = read_stops(&binding, "runtime"); - assert!(!admits_all(&stops)); - assert!(matches!(&stops[1].stop, Stop::Engaged(held) if held.reason == "steward freeze")); - // The note names the proven owner, and never the record's claimed `by`. - assert_eq!( - stops[1].note().unwrap(), - format!("steward {} deployment brake engaged: steward freeze", login_name(me)?) - ); - // Both engaged, then the steward clears its own: the operator's holds. - replace_stop(operator, "runtime", StopState::Engaged, "operator hold", 11)?; - replace_stop(steward, "runtime", StopState::Clear, "steward thaw", 12)?; - let stops = read_stops(&binding, "runtime"); - assert!(matches!(&stops[0].stop, Stop::Engaged(_)) && matches!(&stops[1].stop, Stop::Clear(_))); - assert!(!admits_all(&stops)); - // The other order: operator clear, steward engaged still halts. - replace_stop(operator, "runtime", StopState::Clear, "operator thaw", 13)?; - replace_stop(steward, "runtime", StopState::Engaged, "steward again", 14)?; - assert!(!admits_all(&read_stops(&binding, "runtime"))); - // Both clear: admits. - replace_stop(steward, "runtime", StopState::Clear, "steward thaw", 15)?; - assert!(admits_all(&read_stops(&binding, "runtime"))); - Ok(()) - } - - #[test] - fn steward_store_owned_by_another_uid_stops() -> Result<()> { - let me = process_uid(); - let dir = tempfile::tempdir()?; - // Steward is "nobody" (a real non-root user that is not the writer); - // the file is written by this process, so even a Clear record fails. - if me == 65534 || user_missing("nobody") { - eprintln!("SKIPPED: needs a user `nobody` other than the caller"); - return Ok(()); - } - let binding = binding_with_steward(dir.path(), "nobody"); - let steward = &binding.steward.as_ref().unwrap().store; - replace_stop(steward, "runtime", StopState::Clear, "forged clear", 10)?; - let stops = read_stops(&binding, "runtime"); - assert!(matches!(&stops[1].stop, Stop::Unreadable(why) if why.contains("is owned by uid") - && why.contains("only nobody (the store's owner uid) can repair it") - && why.contains("stays unreadable") - && why.contains("idunn-provision deployment-brake-clear"))); - // The operator's stop keeps the operator's remedy. - replace_stop(&binding.deployment_store, "other-runtime", StopState::Clear, "x", 11)?; - assert!(matches!(&read_stops(&binding, "runtime")[0].stop, Stop::Unreadable(why) - if why.contains("to replace it run:") && !why.contains("only "))); - assert!(!admits_all(&stops)); - Ok(()) - } - - fn user_missing(name: &str) -> bool { - crate::steward_orders::user_uid(name).is_err() - } - - #[test] - fn an_unresolvable_or_root_steward_user_is_an_unreadable_stop_not_an_absent_one() -> Result<()> { - let dir = tempfile::tempdir()?; - for user in ["no-such-login-zz", "root"] { - let binding = binding_with_steward(dir.path(), user); - let stops = read_stops(&binding, "runtime"); - assert!(matches!(&stops[1].stop, Stop::Unreadable(_)), "{user}"); - assert!(!admits_all(&stops)); - } - Ok(()) - } - - #[test] - fn a_binding_without_a_steward_reads_only_the_operator_store() -> Result<()> { - let dir = tempfile::tempdir()?; - let mut binding = binding_with_steward(dir.path(), "nobody"); - binding.steward = None; - let stops = read_stops(&binding, "runtime"); - assert_eq!(stops.len(), 1); - assert!(admits_all(&stops)); - Ok(()) - } - - fn store_entry(path: &Path) -> Result { - let mut entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - entries.pop().context("no entry") - } - - #[test] - fn a_verb_replaces_a_retired_brake_record_and_the_sequence_advances_by_one() -> Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("stop.cc"); - let store = SingleFileMessagePackBackingStore::new(&path); - let retired = CultCacheEnvelope { - key: "deployment".into(), - r#type: "idunn.deployment_brake".into(), - payload: vec![0x90], - stored_at: "1970-01-01T00:00:00+00:00".into(), - schema_id: Some("idunn.deployment_brake.v1".into()), - }; - assert!(store.compare_exchange_snapshot(&[], &[retired])?); - assert_eq!(engage(&path, "first", 10)?.sequence, 1); - assert_eq!(store.pull_all_read_only_snapshot()?.len(), 1); - assert_eq!(engage(&path, "second", 11)?.sequence, 2); - assert_eq!( - replace_stop(&path, "runtime", StopState::Clear, "third", 12)?.sequence, - 3 - ); - assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "third"); - Ok(()) - } - - #[test] - fn a_stale_exchange_is_refused_and_a_record_without_a_reason_is_not_written() -> Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("stop.cc"); - engage(&path, "held", 10)?; - let store = SingleFileMessagePackBackingStore::new(&path); - let stale = store.pull_all_read_only_snapshot()?; - engage(&path, "moved", 11)?; - let next = DeploymentStop { - schema_version: DEPLOYMENT_STOP_SCHEMA.into(), - runtime_id: "runtime".into(), - state: StopState::Clear, - by: "someone".into(), - reason: "late".into(), - at_unix_millis: 12, - sequence: 2, - } - .envelope()?; - assert!(!store.compare_exchange_snapshot(&stale, &[next])?); - assert!(replace_stop(&path, "runtime", StopState::Clear, " ", 13).is_err()); - assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "moved"); - Ok(()) - } - - #[test] - fn only_a_missing_file_in_an_existing_directory_is_absent() -> Result<()> { - let dir = tempfile::tempdir()?; - let me = process_uid(); - // Parent is a regular file: ENOTDIR, not absence. - let file = dir.path().join("plain"); - std::fs::write(&file, b"x")?; - assert!(matches!(read_stop(&file.join("stop.cc"), "runtime", me), Stop::Unreadable(_))); - // Parent does not exist. - assert!(matches!( - read_stop(&dir.path().join("nope/stop.cc"), "runtime", me), - Stop::Unreadable(_) - )); - // Zero-length file. - let empty = dir.path().join("empty.cc"); - std::fs::write(&empty, b"")?; - assert!(matches!(read_stop(&empty, "runtime", me), Stop::Unreadable(_))); - // An existing directory with no store is absent. - assert_eq!(read_stop(&dir.path().join("stop.cc"), "runtime", me), Stop::Absent); - // A parent that cannot be searched (skipped as root, which can). - #[cfg(unix)] - if me != 0 { - use std::os::unix::fs::PermissionsExt; - let locked = dir.path().join("locked"); - std::fs::create_dir(&locked)?; - std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000))?; - let reading = read_stop(&locked.join("stop.cc"), "runtime", me); - std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700))?; - assert!(matches!(reading, Stop::Unreadable(_)), "{reading:?}"); - } - Ok(()) - } - - #[test] - fn the_author_is_the_uid_login_never_the_environment() -> Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("stop.cc"); - // SAFETY: no other test reads USER, LOGNAME or LNAME. - unsafe { - std::env::set_var("USER", "spoof"); - std::env::set_var("LOGNAME", "spoof"); - } - let written = engage(&path, "held", 10)?; - assert_eq!(written.by, login_name(process_uid())?); - assert_ne!(written.by, "spoof"); - Ok(()) - } - - #[test] - fn unreadable_names_the_command_and_a_break_glass_clear_replaces_junk_and_v1() -> Result<()> { - let dir = tempfile::tempdir()?; - let me = process_uid(); - let path = dir.path().join("stop.cc"); - std::fs::write(&path, b"junk that is no store")?; - let Stop::Unreadable(why) = read_stop(&path, "runtime", me) else { - bail!("junk admitted") - }; - assert!(why.contains("idunn-provision deployment-brake-clear --store"), "{why}"); - assert!(why.contains("--runtime-id runtime"), "{why}"); - let cleared = replace_stop(&path, "runtime", StopState::Clear, "operator break-glass", 50)?; - assert_eq!(cleared.sequence, 1); - assert!(read_stop(&path, "runtime", me).admits()); - // A symlink is not the caller's to replace. - let real = dir.path().join("real.cc"); - std::fs::write(&real, b"junk")?; - let link = dir.path().join("link.cc"); - #[cfg(unix)] - { - std::os::unix::fs::symlink(&real, &link)?; - assert!(replace_stop(&link, "runtime", StopState::Clear, "x", 60).is_err()); - assert_eq!(std::fs::read(&real)?, b"junk"); - } - Ok(()) - } -} diff --git a/src/control_plane.rs b/src/control_plane.rs index 83a266b..8e96a93 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14,23 +14,22 @@ use cultcache_rs::{ }; use cultnet_rs::{ AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, - IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, - IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, + IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, + IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, correlate_runtime_presence_claim, derive_service_identity_id, - evaluate_idunn_continuity_restart, open_service_identity_at, verify_runtime_authority, + evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, + verify_idunn_deployment_brake_authorization, verify_runtime_authority, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; -use crate::brake::{NamedStop, read_stops}; -#[cfg(test)] -use crate::brake::StopState; use crate::deployment::{ DependencyKind, OperatorBinding, RolloutStrategy, RouteBinding, WorkloadBinding, capability_compatible, @@ -57,8 +56,7 @@ use crate::host_actuator::{ }; const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2"; -const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v6"; -const DEPLOYMENT_TRANSACTION_SCHEMA_V5: &str = "idunn.deployment_transaction.v5"; +const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v5"; const DEPLOYMENT_TRANSACTION_SCHEMA_V4: &str = "idunn.deployment_transaction.v4"; const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3"; const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2"; @@ -245,12 +243,8 @@ enum DeploymentPhase { Complete, } -/// The signed brake release a Deploy once consumed. Never written and never -/// verified now: the deployment brake is a stop (`brake.rs`). It stays so the -/// terminal records that hold one in key 16 still decode. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -#[allow(dead_code)] struct DeploymentAuthorization { authorization_id: String, brake_sha256: String, @@ -258,6 +252,27 @@ struct DeploymentAuthorization { authorized_at_unix_millis: u64, } +impl DeploymentAuthorization { + fn validate_shape(&self) -> Result<()> { + require_id(&self.authorization_id, "deployment authorization id")?; + ensure!( + !self.canonical_brake_bytes.is_empty() + && self.brake_sha256 == sha256_id(&self.canonical_brake_bytes) + && self.authorized_at_unix_millis > 0, + "deployment authorization receipt is incomplete" + ); + let record: IdunnDeploymentBrakeRecord = + rmp_serde::from_slice(&self.canonical_brake_bytes)?; + record.validate()?; + ensure!( + rmp_serde::to_vec(&record)? == self.canonical_brake_bytes + && record.authorization_id.as_deref() == Some(&self.authorization_id), + "deployment authorization receipt is noncanonical or mismatched" + ); + Ok(()) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct TopologyEvidence { @@ -1502,7 +1517,7 @@ mod cleanup_evidence { } } - /// The schema migration's one entry point: read a stored v2, v3, v4 or v5 + /// The schema migration's one entry point: read a stored v2, v3 or v4 /// transaction (lifting older ones) and return its key and current-shape /// envelope. The record may carry the marker the lift just set. pub(super) fn migrate_transaction_record( @@ -1663,7 +1678,7 @@ impl PostCommitCleanup { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v6" + schema = "idunn.deployment_transaction.v5" )] struct DeploymentTransaction { #[cultcache(key = 0)] @@ -1749,155 +1764,6 @@ struct DeploymentTransaction { /// predates it, which decodes as None: no schema bump (precedent key 34). #[cultcache(key = 37, default)] flip: Option, - /// When the deployment stop admitted this Deploy or Rollback at Sealing: - /// the stamp that replaced the signed release in key 16. Absent on every - /// record that predates it. - #[cultcache(key = 38, default)] - deployment_admitted_at_unix_millis: Option, -} - -/// The v5 transaction layout: the one before `deployment_admitted_at_unix_millis` -/// (key 38), which replaced the signed brake release. Records are decoded by it -/// and lifted, because the control store refuses a record whose bytes differ -/// from its re-encoding. Migration to v6 is forward-only: an Idunn that predates -/// v6 cannot read a migrated store. `history.cc` is never migrated, so this -/// struct lives until history is pruned. -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v5" -)] -struct LegacyDeploymentTransactionV5 { - #[cultcache(key = 0)] - schema_version: String, - #[cultcache(key = 1)] - transaction_id: String, - #[cultcache(key = 2)] - command_id: String, - #[cultcache(key = 3)] - command_kind: CommandKind, - #[cultcache(key = 4)] - target: String, - #[cultcache(key = 5)] - ordinal: u32, - #[cultcache(key = 6)] - phase: DeploymentPhase, - #[cultcache(key = 7)] - created_at_unix_millis: u64, - #[cultcache(key = 8)] - updated_at_unix_millis: u64, - #[cultcache(key = 9)] - incumbent_generation_id: Option, - #[cultcache(key = 10)] - plan: Option, - #[cultcache(key = 11)] - frozen_source: Option, - #[cultcache(key = 12)] - sealed_release: Option, - #[cultcache(key = 13)] - installed_release: Option, - #[cultcache(key = 14)] - expected: Option, - #[cultcache(key = 15)] - expected_publication_sha256: Option, - #[cultcache(key = 16)] - deployment_authorization: Option, - #[cultcache(key = 17)] - lifecycle_authorized_at_unix_millis: Option, - #[cultcache(key = 18)] - activation: Option, - #[cultcache(key = 19)] - workload: Option, - #[cultcache(key = 20)] - activation_publication_sha256: Option, - #[cultcache(key = 21)] - latest_odin_observation: Option, - #[cultcache(key = 22)] - warming: Option, - #[cultcache(key = 23)] - route_preflight: Option, - #[cultcache(key = 24)] - isolation: Option, - #[cultcache(key = 25)] - fencing: Option, - #[cultcache(key = 26)] - leasing: Option, - #[cultcache(key = 27)] - ready: Option, - #[cultcache(key = 28)] - routing: Option, - #[cultcache(key = 29)] - odin_publisher_sequence_cursor: u64, - #[cultcache(key = 30)] - last_error: Option, - #[cultcache(key = 31)] - completion: Option, - #[cultcache(key = 32)] - pre_fencing_abort: Option, - #[cultcache(key = 33)] - post_commit_cleanup: Option, - // because every transaction already durable predates this field: - // an absent key 34 decodes as None, which is exactly "this transaction has - // no post-fencing abort". Without it the daemon refuses to read its own - // control store and crashloops. - #[cultcache(key = 34, default)] - post_fencing_abort: Option, - /// Set by `transition` on entry to each post-fencing phase. - #[cultcache(key = 35)] - phase_deadline: Option, - #[cultcache(key = 36)] - lease_adoption: Option, - /// A static release's carry and flip. Absent on every record that - /// predates it, which decodes as None: no schema bump (precedent key 34). - #[cultcache(key = 37, default)] - flip: Option, -} - - -impl LegacyDeploymentTransactionV5 { - fn into_current(self) -> DeploymentTransaction { - DeploymentTransaction { - schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), - transaction_id: self.transaction_id, - command_id: self.command_id, - command_kind: self.command_kind, - target: self.target, - ordinal: self.ordinal, - phase: self.phase, - created_at_unix_millis: self.created_at_unix_millis, - updated_at_unix_millis: self.updated_at_unix_millis, - incumbent_generation_id: self.incumbent_generation_id, - plan: self.plan, - frozen_source: self.frozen_source, - sealed_release: self.sealed_release, - installed_release: self.installed_release, - expected: self.expected, - expected_publication_sha256: self.expected_publication_sha256, - deployment_authorization: self.deployment_authorization, - lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, - activation: self.activation, - workload: self.workload, - activation_publication_sha256: self.activation_publication_sha256, - latest_odin_observation: self.latest_odin_observation, - warming: self.warming, - route_preflight: self.route_preflight, - isolation: self.isolation, - fencing: self.fencing, - leasing: self.leasing, - ready: self.ready, - routing: self.routing, - odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, - last_error: self.last_error, - completion: self.completion, - pre_fencing_abort: self.pre_fencing_abort, - post_commit_cleanup: self.post_commit_cleanup, - post_fencing_abort: self.post_fencing_abort, - phase_deadline: self.phase_deadline, - lease_adoption: self.lease_adoption, - flip: self.flip, - deployment_admitted_at_unix_millis: None, - } - } } /// The v4 transaction layout: the current one before a static release's @@ -2034,7 +1900,6 @@ impl LegacyDeploymentTransactionV4 { phase_deadline: self.phase_deadline, lease_adoption: self.lease_adoption, flip: None, - deployment_admitted_at_unix_millis: None, } } } @@ -2286,9 +2151,9 @@ impl DeploymentTransaction { self.flip.is_some() && error.downcast_ref::().is_none() } - /// The runtime and the sealed release this transaction names, for the - /// status listing. One reading for both kinds of target: the binding's - /// runtime identity and the sealed release. + /// The runtime and the sealed release a deployment brake release must name. + /// One reading for both kinds of target: the binding's runtime identity and + /// the sealed release, which a service's Expected projection copies. fn brake_subject(&self) -> Result<(String, String)> { let plan = required(&self.plan, "transaction plan")?; let release = required(&self.sealed_release, "sealed release")?; @@ -2354,7 +2219,6 @@ impl DeploymentTransaction { phase_deadline: None, lease_adoption: None, flip: None, - deployment_admitted_at_unix_millis: None, }; transaction.validate()?; Ok(transaction) @@ -2536,12 +2400,6 @@ impl DeploymentTransaction { /// What a static release transaction may hold. It never runs a workload, so /// every piece of evidence of one is absent, and it uses three phases: /// Sealing, Starting (the flip, behind the brake) and Complete. - /// Whether the deployment stop admitted this transaction: stamped at - /// Sealing now, or a legacy record that holds a consumed signed release. - fn deployment_admitted(&self) -> bool { - self.deployment_admitted_at_unix_millis.is_some() || self.deployment_authorization.is_some() - } - fn validate_static(&self) -> Result<()> { ensure!( self.command_kind != CommandKind::Continuity, @@ -2585,13 +2443,13 @@ impl DeploymentTransaction { && self.sealed_release.is_some() && self.installed_release.is_some() && (self.command_kind == CommandKind::Rollback || self.frozen_source.is_some()) - && self.deployment_admitted(), - "static release entered Starting without sealed release evidence and the stop's admission" + && self.deployment_authorization.is_some(), + "static release entered Starting without sealed release evidence and brake consumption" ); } else { ensure!( - self.flip.is_none() && !self.deployment_admitted(), - "static release flipped or was admitted by the stop before Starting" + self.flip.is_none() && self.deployment_authorization.is_none(), + "static release flipped or consumed the brake before Starting" ); } if self.phase == DeploymentPhase::Complete { @@ -2803,6 +2661,9 @@ impl DeploymentTransaction { "lease adoption does not name the granted lease" ); } + if let Some(authorization) = &self.deployment_authorization { + authorization.validate_shape()?; + } let is_static = self.is_static(); if let Some(flip) = &self.flip { ensure!(is_static, "a flip belongs to a static release transaction"); @@ -2957,13 +2818,13 @@ impl DeploymentTransaction { ); match self.command_kind { CommandKind::Deploy => ensure!( - self.frozen_source.is_some() && self.deployment_admitted(), - "deployment entered Starting without frozen source and the stop's admission" + self.frozen_source.is_some() && self.deployment_authorization.is_some(), + "deployment entered Starting without frozen source and brake consumption" ), CommandKind::Rollback => bail!("a rollback is a static release transaction"), CommandKind::Continuity => ensure!( self.lifecycle_authorized_at_unix_millis.is_some() - && !self.deployment_admitted(), + && self.deployment_authorization.is_none(), "continuity entered Starting without its lifecycle gate" ), } @@ -3341,6 +3202,7 @@ struct RuntimeOptions { odin_correlation_store: PathBuf, odin_trust_anchor: PathBuf, idunn_identity_store: PathBuf, + deployment_brake_operator_anchor: PathBuf, source_identity: Option, topology_maximum_age_millis: u64, topology_maximum_future_skew_millis: u64, @@ -3370,6 +3232,9 @@ impl Default for RuntimeOptions { idunn_identity_store: PathBuf::from( "/var/lib/gamecult/idunn/idunn-service-identity.cc", ), + deployment_brake_operator_anchor: PathBuf::from( + "/etc/gamecult/idunn/deployment-brake-operator-anchor.cc", + ), source_identity: None, topology_maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, topology_maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, @@ -3540,6 +3405,9 @@ fn parse_serve(mut args: impl Iterator) -> Result { "--idunn-identity-store" => { options.idunn_identity_store = path_value(&mut args, &argument)? } + "--deployment-brake-operator-anchor" => { + options.deployment_brake_operator_anchor = path_value(&mut args, &argument)? + } "--topology-maximum-age-millis" => { options.topology_maximum_age_millis = u64_value(&mut args, &argument)? } @@ -4038,6 +3906,17 @@ impl ControlSnapshot { } } } + let mut authorizations = BTreeSet::new(); + for authorization in self + .transactions + .iter() + .filter_map(|stored| stored.value.deployment_authorization.as_ref()) + { + ensure!( + authorizations.insert(authorization.authorization_id.as_str()), + "deployment authorization was consumed more than once" + ); + } Ok(()) } @@ -4355,7 +4234,6 @@ impl LegacyDeploymentTransaction { phase_deadline: None, lease_adoption: None, flip: None, - deployment_admitted_at_unix_millis: None, } } } @@ -4564,19 +4442,10 @@ impl LegacyAdmittedGenerationV3 { } } -/// Decode a v2, v3, v4 or v5 transaction and lift it to the current shape, without +/// Decode a v2, v3 or v4 transaction and lift it to the current shape, without /// validating it. History reads stop here; the control store validates. fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result { let schema = match envelope.schema_id.as_deref() { - Some(schema @ DEPLOYMENT_TRANSACTION_SCHEMA_V5) => { - let value: LegacyDeploymentTransactionV5 = rmp_serde::from_slice(&envelope.payload) - .with_context(|| format!("decoding a {schema} deployment transaction"))?; - ensure!( - value.schema_version == schema, - "stored transaction schema differs from its envelope" - ); - return Ok(value.into_current()); - } Some(schema @ DEPLOYMENT_TRANSACTION_SCHEMA_V4) => { let value: LegacyDeploymentTransactionV4 = rmp_serde::from_slice(&envelope.payload) .with_context(|| format!("decoding a {schema} deployment transaction"))?; @@ -4604,7 +4473,7 @@ fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result Result { DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3 | DEPLOYMENT_TRANSACTION_SCHEMA_V4 - | DEPLOYMENT_TRANSACTION_SCHEMA_V5 ) )) || (envelope.r#type == AdmittedGeneration::TYPE @@ -5046,7 +4914,6 @@ fn decode_history_transactions( DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3 | DEPLOYMENT_TRANSACTION_SCHEMA_V4 - | DEPLOYMENT_TRANSACTION_SCHEMA_V5 ) => { lift_legacy_transaction(&envelope) } @@ -5525,23 +5392,14 @@ fn render_supervision( } if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { lines.push(format!( - " route {} consecutive-failures {} last-challenge-at {}{} next-challenge-at {}{}", match route.degraded_since_unix_millis { + " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", + match route.degraded_since_unix_millis { Some(since) => format!("degraded-since {since}"), None => "healthy".to_owned(), }, route.consecutive_failures, at(route.last_challenge_at_unix_millis), - route - .last_challenge_at_unix_millis - .map_or(String::new(), |last| format!(" ({} ms ago)", now.saturating_sub(last))), - at(route.next_challenge_at_unix_millis), - // Status sees the stored schedule, not why it is not being kept - // (a refused write, a transaction, a daemon that is down): it - // says only how overdue the challenge is. - route - .next_challenge_at_unix_millis - .filter(|next| *next < now) - .map_or(String::new(), |next| format!(" (past due {} ms)", now - next)) + at(route.next_challenge_at_unix_millis) )); } if generation.is_none_or(|value| value.route_supervision.is_some()) @@ -5693,11 +5551,6 @@ struct Engine { /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. fault_reports: Mutex>, - /// The key of every `report_once` line that reached stderr: what a test - /// counts, since stderr is not observable from inside it, and a key's - /// own entry is dropped when its fault clears. - #[cfg(test)] - printed_reports: Mutex>, /// Retry pacing for a transaction whose step keeps failing, by /// transaction id: consecutive failures and the earliest next attempt. /// Process-local on purpose: a restart retries at once, and no schema @@ -5825,8 +5678,6 @@ impl Engine { runner_stand_in: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), - #[cfg(test)] - printed_reports: Mutex::default(), resume_backoff: Mutex::default(), lift_held: Mutex::default(), lift_unsettled: std::sync::atomic::AtomicBool::new(false), @@ -5945,8 +5796,6 @@ impl Engine { .or_default() .offer(Some(message)); if let Some(message) = offered { - #[cfg(test)] - self.printed_reports.lock().unwrap().push(key.to_string()); eprintln!("{message}"); } } @@ -6181,15 +6030,41 @@ impl Engine { /// and authorize nothing further, so re-proving them only creates ways for /// history to refuse a boot. On yggdrasil that is 259 of 260 records. /// - /// The deployment stop is not durable authority: it is read live at - /// Sealing and gates nothing already admitted, so no brake artifact is - /// read here and none can refuse a boot. + /// The operator anchor is read on demand rather than up front. Reading it + /// unconditionally meant an absent brake artifact refused startup against + /// an *empty* store -- a deployment brake gating Idunn itself, which + /// `F:\Projects\CLAUDE.md` forbids outright: a target brake "may never gate + /// Idunn itself or unrelated service lifecycle". fn validate_durable_authority(&self, snapshot: &ControlSnapshot) -> Result<()> { + let mut operator_anchor = None; for stored in &snapshot.transactions { let transaction = &stored.value; if transaction.is_terminal() { continue; } + if let Some(authorization) = &transaction.deployment_authorization { + authorization.validate_shape()?; + let record: IdunnDeploymentBrakeRecord = + rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + if operator_anchor.is_none() { + operator_anchor = + Some(read_trust_anchor::( + &self.options.deployment_brake_operator_anchor, + )?); + } + verify_idunn_deployment_brake_authorization( + &record, + operator_anchor.as_ref().expect("read directly above"), + )?; + let (runtime_id, sealed_release_id) = transaction.brake_subject()?; + ensure!( + record.authorized_release_id.as_deref() == Some(sealed_release_id.as_str()) + && record.authorized_deployment_id.as_deref() + == Some(transaction.transaction_id.as_str()) + && record.runtime_id == runtime_id, + "durable deployment authorization names another release or transaction" + ); + } let lease = transaction .leasing .as_ref() @@ -6466,11 +6341,16 @@ impl Engine { { return LiftDecision::KeepAborting; } - if self - .deploy_the_continuity_yields_to(snapshot, transaction) - .is_some() - { - return LiftDecision::KeepAborting; + match now_millis() { + Ok(now) => { + if self + .deploy_the_continuity_yields_to(snapshot, transaction, now) + .is_some() + { + return LiftDecision::KeepAborting; + } + } + Err(error) => return LiftDecision::Undecided(error), } let lifted = (|| -> Result { let plan = required(&transaction.plan, "transaction plan")?; @@ -6748,23 +6628,10 @@ impl Engine { } fn clear_fault(&self, transaction_id: &str) { - let mut reports = self.fault_reports.lock().expect("fault report mutex"); - reports.remove(transaction_id); - reports.remove(&route_proof_report_key(transaction_id)); - } - - /// Say why the stable route's proof failed. The Routing phase retries the - /// proof until its deadline, and the deadline's own cause names only the - /// clock, so this line is where an operator learns what the listener - /// actually did. Said once per distinct reason while the fault lasts. - fn report_route_proof_failure(&self, transaction_id: &str, error: &anyhow::Error) { - self.report_once( - &route_proof_report_key(transaction_id), - format!( - "Idunn route proof for {transaction_id} failed: {}", - route_proof_reason(error) - ), - ); + self.fault_reports + .lock() + .expect("fault report mutex") + .remove(transaction_id); } /// Write `detail` as the record's `last_error`, touching nothing else and @@ -7284,18 +7151,37 @@ impl Engine { || snapshot.on_target(¤t.value.target).any(|stored| { stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity }); + match self.supervise_admitted_route(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + } + match self.refresh_admitted_topology(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn preserved admitted {} after rejecting topology observation: {error:#}", + current.value.target + ), + } match self.retire_stale_incarnations(&snapshot, current) { Ok(true) => { progressed = true; continue; } - Ok(false) => self.clear_fault(&format!("stale-incarnation:{}", current.value.target)), - Err(error) => self.report_once( - &format!("stale-incarnation:{}", current.value.target), - format!( - "Idunn preserved a stale {} incarnation projection: {error:#}", - current.value.target - ), + Ok(false) => {} + Err(error) => eprintln!( + "Idunn preserved a stale {} incarnation projection: {error:#}", + current.value.target ), } // Restore the admitted Expected before anything is asked of the @@ -7324,16 +7210,11 @@ impl Engine { } Ok(()) })(); - let repair_key = format!("projection-repair:{}", current.value.target); - match repair { - Ok(()) => self.clear_fault(&repair_key), - Err(error) => self.report_once( - &repair_key, - format!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target - ), - ), + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -7344,7 +7225,6 @@ impl Engine { ) }) { Ok(observation) => { - self.clear_fault(&format!("observe:{}", current.value.target)); let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { let lease_health = (|| -> Result { let lease_path = current @@ -7365,10 +7245,6 @@ impl Engine { } bail!("write-lease store contains unexpected authority") })(); - let lease_key = format!("write-lease:{}", current.value.target); - if lease_health.is_ok() { - self.clear_fault(&lease_key); - } match lease_health { Ok(true) => false, Ok(false) => { @@ -7377,12 +7253,9 @@ impl Engine { true } Err(error) => { - self.report_once( - &lease_key, - format!( - "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", - current.value.target - ), + eprintln!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target ); continue; } @@ -7402,20 +7275,13 @@ impl Engine { // silence, not a death; counting it as one burned every // continuity attempt in the reattach window on // 2026-09-11 and left a healthy Muninn demoted. - // Said once while it lasts, and cleared the moment the host - // answers: a reattach window is not a line per tick. - self.report_once( - &format!("observe:{}", current.value.target), - format!( - "Idunn cannot observe admitted {} this tick: {error:#}", - current.value.target - ), + eprintln!( + "Idunn cannot observe admitted {} this tick: {error:#}", + current.value.target ); continue; } Err(error) => { - // The host answered, with a death: no longer silence. - self.clear_fault(&format!("observe:{}", current.value.target)); operational_error = Some(error); None } @@ -7454,42 +7320,6 @@ impl Engine { ) }), ); - // Route and topology supervision are about a running incarnation; - // workload liveness decides whether there is one (route-continuity-cut F8, - // resolved by this order). Topology cannot bind without an observed - // activation, so a HostUnobservable tick skips both too. - let route_key = format!("route-supervision:{}", current.value.target); - match self.supervise_admitted_route(&snapshot, current) { - Ok(true) => { - self.clear_fault(&route_key); - progressed = true; - continue; - } - Ok(false) => self.clear_fault(&route_key), - Err(error) => self.report_once( - &route_key, - format!( - "Idunn rejected admitted {} route continuity: {error:#}", - current.value.target - ), - ), - } - let topology_key = format!("topology:{}", current.value.target); - match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => { - self.clear_fault(&topology_key); - progressed = true; - continue; - } - Ok(false) => self.clear_fault(&topology_key), - Err(error) => self.report_once( - &topology_key, - format!( - "Idunn preserved admitted {} after rejecting topology observation: {error:#}", - current.value.target - ), - ), - } continue; } let workload_error = operational_error @@ -7627,17 +7457,10 @@ impl Engine { // could not resolve. So a failed demotion mints nothing. The // failure is written to the generation, and the next attempt is // not before the deferral ends. - let demotion_key = format!("demotion:{}", current.value.target); - if demotion.is_ok() { - self.clear_fault(&demotion_key); - } if let Err(error) = demotion { - self.report_once( - &demotion_key, - format!( - "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", - current.value.target - ), + eprintln!( + "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", + current.value.target ); let mut deferred = supervision.clone(); deferred.defer_continuity(now, &bounded_detail(&format!("{error:#}"))); @@ -7764,13 +7587,17 @@ impl Engine { "admitted route receipt names another incarnation" ); let binding = current.value.plan.parsed_inputs()?.1; - let lease_driver = if current.value.leasing.lease().is_some() { + let lease_driver = if let Some(lease) = current.value.leasing.lease() { let lease_path = &binding .process_write_lease .as_ref() .context("stateful admitted generation has no write-lease binding")? .record_path; let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + ensure!( + driver.observe_exact(lease)?, + "admitted process write lease is no longer exact" + ); Some(driver) } else { None @@ -8099,10 +7926,13 @@ impl Engine { let mut next = current.value.clone(); match current.value.command_kind { CommandKind::Deploy | CommandKind::Rollback => { - if let Some(note) = self.stop_note(current)? { - return self.record_gate_wait(current, ¬e); - } - next.deployment_admitted_at_unix_millis = Some(now); + let Some(authorization) = self.deployment_authorization(current, now)? else { + return self.record_gate_wait( + current, + "deployment brake has not released this exact transaction", + ); + }; + next.deployment_authorization = Some(authorization); } CommandKind::Continuity => { if !self.lifecycle_allows(current, now)? { @@ -8231,11 +8061,6 @@ impl Engine { let Some(flip) = ¤t.value.flip else { // Nothing has moved yet, so a refusal here ends the transaction // honestly: the release is still what was sealed, or it is not. - // The flip intent is the last point a stop can halt a static - // release: once it is recorded the flip runs to its end. - if let Some(note) = self.stop_note(current)? { - return self.record_gate_wait(current, ¬e); - } self.static_release.verify_installed(workload, release)?; let previous_release_id = self.static_release.observe_current(workload)?; let release_id = release.sealed_release_id.clone(); @@ -8248,7 +8073,6 @@ impl Engine { flipped_at_unix_millis: None, }); next.updated_at_unix_millis = now; - next.last_error = None; Ok(()) }); }; @@ -8479,14 +8303,6 @@ impl Engine { }); } - // Fencing is the last pre-fencing abort-able point of a deploy: an - // engaged stop halts here, before the incumbent is fenced. A fenced - // deploy runs to its end. Continuity is never gated by this stop. - if current.value.command_kind == CommandKind::Deploy - && let Some(note) = self.stop_note(current)? - { - return self.record_gate_wait(current, ¬e); - } self.transition(current, DeploymentPhase::Fencing) } @@ -8935,21 +8751,18 @@ impl Engine { rollback_allowed, &gate, )?; - let observation = match self.prove_routed_candidate_through_handoff( + let observation = match self.prove_stable_route( &admitted.value, &driver, membership_sha256, - preflight, ) { Ok(observation) => observation, Err(proof_error) if !rollback_allowed => { - self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return Err(proof_error).context( "incumbent authority was fenced; candidate route remains installed for fail-closed retry", ); } Err(proof_error) => { - self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return match driver.rollback( expected, &activation.runtime_instance_id, @@ -9415,43 +9228,66 @@ impl Engine { .context("no private candidate port remains in the operator range") } - /// Why the deployment stop holds this transaction, or None when it admits. - /// Read live at each point that is still reversible: Sealing, the static - /// flip intent and the entry into Fencing. - fn stop_note(&self, current: &Stored) -> Result> { - let notes: Vec = self - .deployment_stops(current)? - .iter() - .filter_map(NamedStop::note) - .collect(); - Ok((!notes.is_empty()).then(|| notes.join("; "))) - } - - /// What the binding's stops say about this transaction's runtime: the - /// operator's store and, when the binding names one, the steward's. Read - /// live; never cached, never consumed. - fn deployment_stops( + fn deployment_authorization( &self, current: &Stored, - ) -> Result> { + now: u64, + ) -> Result> { + let (runtime_id, sealed_release_id) = current.value.brake_subject()?; let plan = required(¤t.value.plan, "transaction plan")?; let binding = plan.parsed_inputs()?.1; - Ok(read_stops( - &binding.brakes, - &binding.runtime_identity.runtime_id, - )) + let Some((record, canonical_bytes)) = + read_deployment_brake(&binding.brakes.deployment_store)? + else { + return Ok(None); + }; + let anchor = read_trust_anchor::( + &self.options.deployment_brake_operator_anchor, + )?; + if evaluate_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&record), + &anchor, + &runtime_id, + &sealed_release_id, + ¤t.value.transaction_id, + now, + ) + .is_err() + { + return Ok(None); + } + let authorization_id = record + .authorization_id + .clone() + .context("released brake has no authorization id")?; + let snapshot = ControlSnapshot::read(&self.options.state_store)?; + ensure!( + !snapshot.transactions.iter().any(|stored| { + stored + .value + .deployment_authorization + .as_ref() + .is_some_and(|used| used.authorization_id == authorization_id) + }), + "deployment authorization was already consumed" + ); + Ok(Some(DeploymentAuthorization { + authorization_id, + brake_sha256: sha256_id(&canonical_bytes), + canonical_brake_bytes: canonical_bytes, + authorized_at_unix_millis: now, + })) } /// Whether a deployment awaiting the handover is free to take it: the - /// deployment stop admits it, so the stop a yield makes is followed by the - /// deployment's own landing. A Deploy at its Sealing tail has not passed - /// the stop yet and is read live; one that is still building, or whose - /// stop is engaged or cannot be read, is held. An operator's + /// deployment brake has released its exact transaction, so the stop a yield + /// makes is followed by the deployment's own landing. A Deploy at its + /// Sealing tail has not passed the brake yet and is read live; one that is + /// still building, or whose brake cannot be read, is held. An operator's /// hold is a reason to leave a serving or retrying body alone. - fn deploy_can_move(&self, deploy: &Stored) -> bool { - deploy.value.deployment_admitted() - || self.deployment_stops(deploy) - .is_ok_and(|stops| stops.iter().all(|named| named.stop.admits())) + fn deploy_can_move(&self, deploy: &Stored, now: u64) -> bool { + deploy.value.deployment_authorization.is_some() + || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) } /// The deployment a continuity gives way to: one on its target that awaits @@ -9463,12 +9299,13 @@ impl Engine { &self, snapshot: &'a ControlSnapshot, continuity: &DeploymentTransaction, + now: u64, ) -> Option<&'a Stored> { if continuity.serving_proven() { return None; } snapshot.on_target(&continuity.target).find(|other| { - other.value.awaits_handover() && self.deploy_can_move(other) + other.value.awaits_handover() && self.deploy_can_move(other, now) }) } @@ -9864,42 +9701,6 @@ impl Engine { ) } - /// Routing's proof, waiting out nginx's reload handoff. `systemctl reload` - /// only signals the master: the old workers keep answering the stable - /// listener until the new configuration is parsed and they are told to - /// quit, so Idunn's first challenge can reach the incumbent. An answer - /// that names the preflight's incumbent is therefore not-yet for - /// `ROUTE_RELOAD_HANDOFF`; it can delay and never admit (the id is read - /// unauthenticated, and the candidate's proof is checked as before). - /// Anything else fails the attempt at once. One challenge per poll and - /// nothing else: no install, rollback, route gate or persistence. - fn prove_routed_candidate_through_handoff( - &self, - transaction: &DeploymentTransaction, - driver: &NginxRouteDriver, - membership_sha256: String, - preflight: &crate::drivers::RoutePreflightReceipt, - ) -> Result { - let candidate = &required(&transaction.activation, "route activation")?.runtime_instance_id; - let until = handoff_now() + ROUTE_RELOAD_HANDOFF; - loop { - match self.prove_stable_route(transaction, driver, membership_sha256.clone()) { - Err(error) - if preflight.changes_route() - && handoff_now() < until - && error.downcast_ref::().is_some_and(|answer| { - preflight.incumbent_runtime_instance_id.as_deref() - == Some(answer.answered_instance.as_str()) - && answer.answered_instance != *candidate - }) => - { - handoff_sleep(ROUTE_RELOAD_HANDOFF_POLL); - } - result => return result, - } - } - } - fn prove_stable_route_against( &self, expected: &IdunnExpectedIncarnationRecord, @@ -9924,15 +9725,7 @@ impl Engine { challenged_at_unix_millis, received_at_unix_millis, &response.canonical_presence, - ) - .map_err(|error| match presence_answer_ids(&response.canonical_presence) { - Some((answered_instance, answered_activation_signer)) => error.context(RouteProofAnswer { - answered_instance: route_proof_label(&answered_instance), - answered_activation_signer: route_proof_label(&answered_activation_signer), - candidate_instance: route_proof_label(&activation.runtime_instance_id), - }), - None => error, - })?; + )?; let observation = RouteObservation { route_id: driver.binding.route_id.clone(), runtime_instance_id: activation.runtime_instance_id.clone(), @@ -10678,15 +10471,8 @@ impl Engine { .iter() .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { - // The step's latest failure is the reason the phase never - // finished, and the abort that ends it must not drop it. - let last_attempt = transaction - .last_error - .as_deref() - .map(|error| format!("; its last attempt failed: {error}")) - .unwrap_or_default(); format!( - "the {:?} phase passed its deadline at unix ms {}{last_attempt}", + "the {:?} phase passed its deadline at unix ms {}", transaction.phase, deadline.deadline_at_unix_millis ) } else if requested { @@ -10717,9 +10503,10 @@ impl Engine { } PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; + let now = now_millis()?; if let Some(deploy) = transaction .yields_to_a_deploy() - .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction)) + .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction, now)) .flatten() { self.begin_post_fencing_abort( @@ -11372,6 +11159,24 @@ fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation && record.disagreements.is_empty() } +fn read_deployment_brake(path: &Path) -> Result)>> { + let Some(envelope) = read_single_envelope(path)? else { + return Ok(None); + }; + ensure!( + envelope.r#type == IdunnDeploymentBrakeRecord::TYPE + && envelope.schema_id.as_deref() == Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA), + "deployment brake store contains a foreign record" + ); + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&envelope.payload)?; + record.validate()?; + ensure!( + rmp_serde::to_vec(&record)? == envelope.payload && envelope.key == record.brake_id, + "deployment brake is noncanonical or keyed by another authority" + ); + Ok(Some((record, envelope.payload))) +} + fn read_lifecycle_brake(path: &Path) -> Result> { let Some(envelope) = read_single_envelope(path)? else { return Ok(None); @@ -11551,109 +11356,6 @@ fn runner_failure_journal_line( } } -fn route_proof_report_key(transaction_id: &str) -> String { - format!("route-proof:{transaction_id}") -} - -/// How long Routing waits for nginx's old workers to let go of the stable -/// listener after a reload, derived from two ceilings: one Forward actuation -/// must suffice per attempt (12 an hour), and the 120 s Routing deadline must -/// keep room for at least four more backoff-spaced attempts if the route -/// genuinely never switches. -const ROUTE_RELOAD_HANDOFF: Duration = - if cfg!(test) { Duration::from_millis(400) } else { Duration::from_secs(10) }; -const ROUTE_RELOAD_HANDOFF_POLL: Duration = - if cfg!(test) { Duration::from_millis(20) } else { Duration::from_millis(250) }; - -/// The handoff loop's clock. Production reads the real one; a test may install -/// a virtual one on its own thread, so the window and the poll cadence are -/// measured exactly, whatever the host load. -#[cfg(not(test))] -fn handoff_now() -> std::time::Instant { - std::time::Instant::now() -} -#[cfg(not(test))] -fn handoff_sleep(pause: Duration) { - thread::sleep(pause); -} -#[cfg(test)] -thread_local! { - /// (virtual offset, sleeps taken, clock reads) while a test holds it. - static HANDOFF_CLOCK: std::cell::RefCell, usize)>> = - const { std::cell::RefCell::new(None) }; -} -#[cfg(test)] -fn handoff_now() -> std::time::Instant { - static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); - let base = *BASE.get_or_init(std::time::Instant::now); - HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { - Some((offset, _, reads)) => { - *reads += 1; - assert!(*reads < 10_000, "the handoff loop spins without sleeping"); - base + *offset - } - None => std::time::Instant::now(), - }) -} -#[cfg(test)] -fn handoff_sleep(pause: Duration) { - HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { - Some((offset, sleeps, _)) => { - *offset += pause; - sleeps.push(pause); - } - None => thread::sleep(pause), - }) -} - -/// Who answered a stable route challenge that failed authentication, as the -/// answer claimed itself: ids only, unauthenticated, never signature bytes. -/// A label on a rejection, and the one thing that may say "challenge again". -#[derive(Debug)] -struct RouteProofAnswer { - answered_instance: String, - answered_activation_signer: String, - candidate_instance: String, -} - -impl std::fmt::Display for RouteProofAnswer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "stable route answered as runtime instance {}, activation signer {} (unauthenticated); the candidate is {}", - self.answered_instance, self.answered_activation_signer, self.candidate_instance - ) - } -} - -/// The ids a presence claims, if it decodes at all. No re-encode check and no -/// validation: this labels a rejection and is never evidence. -fn presence_answer_ids(canonical: &[u8]) -> Option<(String, String)> { - let presence = - rmp_serde::from_slice::(canonical).ok()?; - Some((presence.runtime_instance_id, presence.activation_signer_identity_id)) -} - -/// An id cut to 96 bytes at a char boundary, control characters stripped. -fn route_proof_label(id: &str) -> String { - let id: String = id.chars().filter(|c| !c.is_control()).collect(); - id[..id.floor_char_boundary(96)].to_owned() -} - -/// The stable route proof's failure in words: whether the listener said -/// nothing or answered wrongly, then the specific refusal (HTTP status line, -/// presence not active, challenge or authority disagreement). The texts it -/// carries name states and codes, never keys or signature bytes. A rejected -/// answer names the instance and activation signer it claimed, unauthenticated, -/// ids only. -fn route_proof_reason(error: &anyhow::Error) -> String { - match error.downcast_ref::() { - Some(ChallengeFailure::Silent(_)) => format!("silent, nothing answered: {error:#}"), - Some(ChallengeFailure::Refused(_)) => format!("refused: {error:#}"), - None => format!("{error:#}"), - } -} - fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -11957,6 +11659,7 @@ mod tests { odin_correlation_store: root.join("odin-correlation.cc"), odin_trust_anchor: odin_anchor, idunn_identity_store: idunn, + deployment_brake_operator_anchor: root.join("brake-anchor.cc"), #[cfg(unix)] route_actuators, ..RuntimeOptions::default() @@ -12180,6 +11883,27 @@ mod tests { Ok(()) } + #[test] + fn an_absent_brake_anchor_does_not_gate_an_empty_store() -> Result<()> { + // `F:\Projects\CLAUDE.md`: a target brake "may never gate Idunn itself + // or unrelated service lifecycle". Reading the operator anchor up front + // meant a missing brake artifact refused startup against a store with + // nothing in it -- the deployment authority held hostage by an artifact + // describing one target's consent. + let world = EngineFixture::new()?; + assert!( + !world + .engine + .options + .deployment_brake_operator_anchor + .exists(), + "fixture must not have written a brake anchor" + ); + let snapshot = ControlSnapshot::read(&world.state_store)?; + world.engine.validate_durable_authority(&snapshot)?; + Ok(()) + } + #[test] fn one_unreadable_record_faults_its_tick_without_touching_the_store() -> Result<()> { let world = EngineFixture::new()?; @@ -14255,18 +13979,77 @@ mod tests { if incumbent.is_some() { 14 } else { 4 } } - /// Write the deployment stop of the binding `plan` names, as the verb - /// would: engaged halts that runtime's deployments, clear admits them. - fn set_stop(plan: &CompiledDeploymentPlan, state: StopState, reason: &str) -> Result<()> { - let binding = plan.parsed_inputs()?.1; - crate::brake::replace_stop( - &binding.brakes.deployment_store, - &binding.runtime_identity.runtime_id, - state, - reason, - now_millis()?, - )?; - Ok(()) + /// A released deployment brake receipt, signed by an operator identity + /// enrolled in the world: what a Deploy's `deployment_authorization` holds. + fn deployment_authorization( + world: &EngineFixture, + expected: &IdunnExpectedIncarnationRecord, + transaction_id: &str, + now: u64, + ) -> Result { + released_brake(world, &expected.runtime_id, &expected.sealed_release_id, transaction_id, now) + } + + /// The release receipt for one runtime, release and transaction: all a + /// brake release names, so a target with no Expected can be released too. + fn released_brake( + world: &EngineFixture, + runtime_id: &str, + sealed_release_id: &str, + transaction_id: &str, + now: u64, + ) -> Result { + use cultnet_rs::{ + IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, + IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, + IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, + }; + // One operator per world, however many deployments it releases. + let signer_path = world.root.join("identities/brake-operator.cc"); + let signer = if signer_path.exists() { + open_service_identity_at::(&signer_path)? + } else { + let signer = + enroll_service_identity_at::(&signer_path)?; + export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; + signer + }; + let authorization_id = format!("authorization-{transaction_id}"); + let mut record = IdunnDeploymentBrakeRecord { + schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), + brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), + runtime_id: runtime_id.to_owned(), + status: "released".into(), + scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), + reason: "test rollout".into(), + observed_at_unix_millis: now, + expires_at_unix_millis: Some(now + 600_000), + authorization_id: Some(authorization_id.clone()), + authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), + authorized_release_id: Some(sealed_release_id.to_owned()), + authorized_deployment_id: Some(transaction_id.to_owned()), + authorized_by: Some(signer.trust_anchor()?.identity_id), + authorization_issued_at_unix_millis: Some(now), + authorization_expires_at_unix_millis: Some(now + 600_000), + signature_algorithm: Some("ed25519".into()), + signature: None, + private_state_exposed: false, + updated_by: "operator/test".into(), + }; + record.signature = Some( + signer + .sign::(&rmp_serde::to_vec(&record)?) + .signature, + ); + record.validate()?; + let canonical_brake_bytes = rmp_serde::to_vec(&record)?; + Ok(DeploymentAuthorization { + authorization_id, + brake_sha256: sha256_id(&canonical_brake_bytes), + canonical_brake_bytes, + authorized_at_unix_millis: now, + }) } /// `binding` with its provider trust anchor and signer pointed at this @@ -14353,14 +14136,6 @@ mod tests { .split_once("[brakes]") .context("binding has no brakes table")? .1 - ) - .replace( - "/var/lib/gamecult/idunn/service-deployment-brake.cc", - &world.root.join("service-deployment-brake.cc").display().to_string(), - ) - .replace( - "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", - &world.root.join("service-lifecycle-brake.cc").display().to_string(), ), )?; let recipe = RECIPE @@ -14458,7 +14233,12 @@ mod tests { plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - transaction.deployment_admitted_at_unix_millis = Some(now); + transaction.deployment_authorization = Some(deployment_authorization( + world, + &expected, + &transaction.transaction_id, + now, + )?); } } transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); @@ -15264,132 +15044,6 @@ mod tests { odin_reports_ready(world, transaction, sequence) } - /// An Odin-correlated generation, admitted over `workload`, with the - /// sequence of the reading it holds. - fn odin_admitted_world( - workload: Arc, - ) -> Result<(EngineFixture, DeploymentTransaction, u64)> { - let world = EngineFixture::with_workload(workload)?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - let first = seeded_sequence(None) + 1; - odin_reports_ready(&world, &seeded, first)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - assert!(world.engine.retire_one_terminal_transaction()?); - Ok((world, seeded, first)) - } - - /// Odin's next reading, about an activation that is not the current one. - fn odin_republishes_another_activation( - world: &EngineFixture, - seeded: &DeploymentTransaction, - sequence: u64, - ) -> Result<()> { - let mut other = seeded.clone(); - other.activation.as_mut().context("no activation")?.runtime_instance_id = - sha256_id(b"another-incarnation"); - odin_republishes(world, &other, sequence) - } - - /// Engage the admitted target's lifecycle brake, at the path its seeded - /// binding names: a dead incarnation mints no continuity, so every later - /// tick reaches the same gates. - fn engage_lifecycle_brake_in(world: &EngineFixture) -> Result<()> { - let generation = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no admitted generation")? - .value - .clone(); - crate::provisioning::replace_lifecycle_brake( - &world.root.join("service-lifecycle-brake.cc"), - IdunnLifecycleBrakeRecord { - schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), - authority: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), - runtime_id: generation.expected.runtime_id.clone(), - target: "service".into(), - scope: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), - status: "engaged".into(), - reason: "test".into(), - updated_at_unix_millis: now_millis()?, - released_until_unix_millis: None, - }, - ) - } - - #[test] - fn a_dead_generation_does_not_take_odin_correlations() -> Result<()> { - let switch = SwitchWorkload::new(); - let (world, seeded, first) = odin_admitted_world(switch.clone())?; - switch.kill(); - // No continuity is minted, so no transaction ever owns the target and - // every tick reaches the topology gate. - engage_lifecycle_brake_in(&world)?; - let envelope = incumbent_envelope(&world)?; - - // A newer reading that binds the admitted activation, and one that does - // not: neither is taken, nor said, about an incarnation that is not there. - for reading in 0..2 { - if reading == 0 { - odin_republishes(&world, &seeded, first + 4)?; - } else { - odin_republishes_another_activation(&world, &seeded, first + 5)?; - } - for _ in 0..100 { - world.engine.supervise_one_admitted_generation()?; - } - assert!( - ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), - "a transaction owns the target: later ticks never reach the gate" - ); - assert_eq!(incumbent_envelope(&world)?, envelope); - assert!( - !world.engine.fault_reports.lock().unwrap().contains_key("topology:service"), - "a dead generation reported a topology rejection" - ); - } - Ok(()) - } - - fn printed(world: &EngineFixture, key: &str) -> usize { - world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() - } - - #[test] - fn a_running_generation_with_a_non_binding_correlation_is_reported_once() -> Result<()> { - let (world, seeded, first) = odin_admitted_world(Arc::new(StillWorkload))?; - odin_republishes_another_activation(&world, &seeded, first + 4)?; - let envelope = incumbent_envelope(&world)?; - for _ in 0..50 { - assert!(!world.engine.supervise_one_admitted_generation()?); - } - assert_eq!(incumbent_envelope(&world)?, envelope); - // The engine said it once over fifty ticks. - assert_eq!(printed(&world, "topology:service"), 1); - assert!(world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); - - // A binding reading is adopted and ends the fault, so the key clears. - odin_republishes(&world, &seeded, first + 6)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); - let adopted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no generation")? - .value - .clone(); - assert_eq!(adopted.odin_publisher_sequence_cursor, first + 6); - - // A recurrence is said again, once; a reading withdrawn ends it too. - odin_republishes_another_activation(&world, &seeded, first + 8)?; - for _ in 0..20 { - world.engine.supervise_one_admitted_generation()?; - } - assert_eq!(printed(&world, "topology:service"), 2); - let store = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); - assert!(store.delete_batch_if_unchanged(&store.pull_all_read_only_snapshot()?)?); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); - Ok(()) - } - fn ready_sequence(transaction: &DeploymentTransaction) -> Option { transaction .ready @@ -17312,60 +16966,6 @@ mod tests { Ok(()) } - #[test] - fn a_route_proof_failure_names_what_the_listener_did() { - let reason = |error: anyhow::Error| route_proof_reason(&error); - let silent = anyhow::Error::new(ChallengeFailure::Silent(anyhow!( - "connecting stable CultNet HTTP route 127.0.0.1:9: refused" - ))); - let text = reason(silent); - assert!(text.contains("silent") && text.contains("127.0.0.1:9"), "{text}"); - - let status = anyhow::Error::new(ChallengeFailure::Refused(anyhow!( - "stable CultNet HTTP route rejected the snapshot challenge: HTTP/1.1 502 Bad Gateway" - ))) - .context("candidate did not answer its stable route challenge"); - let text = reason(status); - assert!(text.contains("refused") && text.contains("502 Bad Gateway"), "{text}"); - - let inactive = anyhow!("route proof runtime is warming, not active"); - assert!(reason(inactive).contains("warming, not active")); - - let disagrees = anyhow::Error::new(PresenceDisagrees { - disagreements: vec![OdinTopologyDisagreement { - code: "runtime-instance-mismatch".into(), - expected: Some("a".into()), - observed: Some("b".into()), - }], - }); - let text = reason(disagrees); - assert!(text.contains("runtime-instance-mismatch (expected a, observed b)"), "{text}"); - - let unbound = anyhow!("route proof response is not bound to the exact challenge"); - assert!(reason(unbound).contains("not bound to the exact challenge")); - } - - #[test] - fn a_route_proof_failure_is_reported_once_per_reason_and_cleared_on_recovery() -> Result<()> { - let world = EngineFixture::new()?; - let key = route_proof_report_key("tx-1"); - let said = |world: &EngineFixture| { - world - .engine - .fault_reports - .lock() - .unwrap() - .get(&key) - .and_then(|report| report.last.lock().unwrap().clone()) - }; - world.engine.report_route_proof_failure("tx-1", &anyhow!("route proof runtime is warming, not active")); - let first = said(&world).context("no report")?; - assert!(first.contains("tx-1") && first.contains("warming, not active"), "{first}"); - world.engine.clear_fault("tx-1"); - assert!(said(&world).is_none(), "recovery left the proof report armed"); - Ok(()) - } - #[test] fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { write_fixture_anchor()?; @@ -19160,12 +18760,11 @@ mod tests { make_stop_first(world, deploy_id)?; let sealing = resident_id(world, deploy_id)?; let mut tail = sealing.value.clone(); - tail.deployment_admitted_at_unix_millis = None; + tail.deployment_authorization = None; replace_transaction(&world.state_store, &sealing, &tail)?; - set_stop(tail.plan.as_ref().context("no plan")?, StopState::Engaged, "test hold")?; let tail = resident_id(world, deploy_id)?.value; assert!(tail.awaits_handover()); - assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?)); + assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?, now_millis()?)); Ok(()) } @@ -19428,14 +19027,13 @@ mod tests { /// for it; once the brake releases, the continuity yields. The yield closes /// the continuity's operator-required incident Recovered. #[test] - fn a_continuity_does_not_yield_to_a_stopped_deploy_and_yields_once_the_stop_clears() -> Result<()> { + fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; make_stop_first(&world, &deploy_id).context("A make_stop_first")?; let sealing = resident_id(&world, &deploy_id).context("B sealing")?; let mut tail = sealing.value.clone(); - tail.deployment_admitted_at_unix_millis = None; + tail.deployment_authorization = None; replace_transaction(&world.state_store, &sealing, &tail)?; - set_stop(tail.plan.as_ref().context("no plan")?, StopState::Engaged, "test hold")?; assert!(resident_id(&world, &deploy_id).context("C awaits")?.value.awaits_handover()); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); @@ -19448,9 +19046,30 @@ mod tests { assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); - // The steward clears the stop; no per-deploy release exists. + // The operator releases the brake for this exact Deploy. let deploy = resident_id(&world, &deploy_id).context("E deploy")?.value; - set_stop(deploy.plan.as_ref().context("no plan")?, StopState::Clear, "test thaw")?; + let authorization = deployment_authorization( + &world, + deploy.expected.as_ref().context("no Expected")?, + &deploy_id, + now_millis()?, + )?; + let store = deploy.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + assert!(SingleFileMessagePackBackingStore::new(&store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + key: record.brake_id.clone(), + current: None, + }], + &[CultCacheEnvelope { + key: record.brake_id.clone(), + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + payload: authorization.canonical_brake_bytes.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), + }], + )?); world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; let yielded = resident_id(&world, &owner_id).context("F yielded")?.value; @@ -19841,7 +19460,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - next.deployment_admitted_at_unix_millis = Some(now); + next.deployment_authorization = + Some(deployment_authorization(world, &expected, id, now)?); } next.installed_release = Some(crate::drivers::InstalledReleaseObservation { sealed_release_id: release.sealed_release_id.clone(), @@ -20171,9 +19791,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let sealing = resident_id(&world, deploy_id)?; world.engine.advance_transaction(&sealing)?; let after = resident_id(&world, deploy_id)?.value; - // The target has no stop, so Sealing passes it and is not held. assert!( - after.last_error.as_deref().is_none_or(|note| !note.contains(&continuity.transaction_id)), + after.last_error.as_deref().is_some_and(|note| !note.contains(&continuity.transaction_id)), "Sealing was held by the owner: {:?}", after.last_error ); @@ -20429,66 +20048,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(deploy) } - /// An engaged stop halts a deploy before it fences, the last pre-fencing - /// abort-able point; clearing lets the same transaction fence. - #[test] - fn an_engaged_stop_halts_entry_into_fencing_and_a_clear_lets_it_through() -> Result<()> { - let workload = SwitchWorkload::new(); - let world = EngineFixture::with_workload(workload.clone())?; - let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; - let plan = deploy.plan.as_ref().context("no plan")?; - set_stop(plan, StopState::Engaged, "freeze before fencing")?; - for _ in 0..3 { - world.engine.run_scheduler_tick()?; - } - let held = resident_id(&world, &deploy.transaction_id)?.value; - assert_eq!(held.phase, DeploymentPhase::Warming); - assert!(held.pre_fencing_abort.is_none() && held.completion.is_none()); - assert!( - held.last_error.as_deref().is_some_and(|note| note.contains("deployment brake engaged by") - && note.contains("freeze before fencing")), - "{:?}", - held.last_error - ); - set_stop(plan, StopState::Clear, "thaw")?; - assert!(world.engine.run_scheduler_tick()?); - assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); - Ok(()) - } - - /// An Unreadable stop holds a Sealing deploy with a message that names the - /// remedy; it never admits. - #[test] - fn an_unreadable_stop_keeps_a_sealing_deploy_held_with_the_remedy() -> Result<()> { - let world = EngineFixture::new()?; - let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; - make_stop_first(&world, &deploy.transaction_id)?; - let sealing = resident_id(&world, &deploy.transaction_id)?; - let mut tail = sealing.value.clone(); - tail.deployment_admitted_at_unix_millis = None; - tail.deployment_authorization = None; - replace_transaction(&world.state_store, &sealing, &tail)?; - let store = tail.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; - std::fs::create_dir_all(store.parent().context("no parent")?)?; - for bytes in [&b"junk that is no store"[..], &b""[..]] { - std::fs::write(&store, bytes)?; - let held = resident_id(&world, &deploy.transaction_id)?; - rephase(&world, &deploy.transaction_id, DeploymentPhase::Sealing)?; - let held_now = resident_id(&world, &deploy.transaction_id)?; - world.engine.advance_transaction(&held_now)?; - let after = resident_id(&world, &deploy.transaction_id)?.value; - assert_eq!(after.phase, DeploymentPhase::Sealing, "{:?}", held.value.last_error); - assert!(!after.deployment_admitted()); - assert!( - after.last_error.as_deref().is_some_and(|note| note.contains("fails closed") - && note.contains("idunn-provision deployment-brake-clear")), - "{:?}", - after.last_error - ); - } - Ok(()) - } - #[test] fn a_continuity_minted_as_a_deployment_takes_the_target_is_resolved_next_pass() -> Result<()> { // (a) The deployment fences first in the tick, so supervision reads its @@ -21978,25 +21537,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } - #[test] - fn status_says_how_overdue_a_route_challenge_is_and_claims_no_cause() -> Result<()> { - let now = 10_000_000_u64; - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.route_supervision = Some(RouteSupervisionState { - last_challenge_at_unix_millis: Some(now - 5_000), - next_challenge_at_unix_millis: Some(now - 4_000), - ..RouteSupervisionState::default() - }); - let lines = render_supervision("service", Some(&generation), None, now).join("\n"); - assert!(lines.contains("last-challenge-at 9995000 (5000 ms ago)"), "{lines}"); - assert!(lines.contains("next-challenge-at 9996000 (past due 4000 ms)"), "{lines}"); - assert!(!lines.contains("waits on"), "{lines}"); - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(now); - let lines = render_supervision("service", Some(&generation), None, now).join("\n"); - assert!(!lines.contains("past due"), "{lines}"); - Ok(()) - } - #[test] fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { let now = 10_000_000_u64; @@ -22096,11 +21636,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation: IdunnRuntimeActivationRecord, provider: cultnet_rs::ServiceIdentitySigner, activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, - /// A second key the transaction's activation record never named. - other_signer: cultnet_rs::IdunnRuntimeActivationSigner, - /// The instance the stable listener answered as before the reload. - incumbent_instance: Mutex>, - stable_answers: AtomicUsize, state: Mutex<&'static str>, capacity: AtomicU32, sequence: AtomicU64, @@ -22130,36 +21665,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// A first Odin candidate answering its Warming challenge: the /// detail names the challenge as `idunn-warming:`. FirstOdinWarming, - /// The stable listener's first n answers come from the incumbent - /// (its instance id, a key the record never named), then Honest. - IncumbentFirst(usize), - /// The candidate's own instance id, signed with the wrong key. - ForeignActivation, - /// Neither the candidate nor the incumbent, wrong key. - StrangerInstance, } impl RuntimeStub { fn answer(&self, message_id: &str) -> Result> { - self.answer_on(message_id, false) - } - - fn answer_on(&self, message_id: &str, stable: bool) -> Result> { let expected = &self.expected; - let own = self.activation.runtime_instance_id.clone(); - let (instance, signer) = match *self.reply.lock().unwrap() { - Reply::IncumbentFirst(n) - if stable && self.stable_answers.fetch_add(1, Ordering::SeqCst) < n => - { - let incumbent = self.incumbent_instance.lock().unwrap().clone(); - (incumbent.context("no incumbent instance set")?, &self.other_signer) - } - Reply::ForeignActivation if stable => (own, &self.other_signer), - Reply::StrangerInstance if stable => { - (sha256_id(b"stranger instance"), &self.other_signer) - } - _ => (own, &self.activation_signer), - }; let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), target: expected.target.clone(), @@ -22171,7 +21681,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide state_schema_generation: expected.state_schema_generation.clone(), state_contract_sha256: expected.state_contract_sha256.clone(), runtime_id: expected.runtime_id.clone(), - runtime_instance_id: instance, + runtime_instance_id: self.activation.runtime_instance_id.clone(), bound_endpoint: expected .route .as_ref() @@ -22198,7 +21708,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide observed_at_unix_millis: now_millis()?, signature_algorithm: "ed25519".into(), signature: Vec::new(), - activation_signer_identity_id: signer.identity_id(), + activation_signer_identity_id: self.activation_signer.identity_id(), activation_signature: Vec::new(), }; let proof = presence.canonical_proof_payload()?; @@ -22206,7 +21716,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .provider .sign::(&proof) .signature; - presence.activation_signature = signer.sign_presence_proof(&presence)?; + presence.activation_signature = + self.activation_signer.sign_presence_proof(&presence)?; Ok(rmp_serde::to_vec(&presence)?) } @@ -22214,7 +21725,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide *self.state.lock().unwrap() = state; } - fn serve_one(&self, mut stream: TcpStream, stable: bool) -> Result<()> { + fn serve_one(&self, mut stream: TcpStream) -> Result<()> { stream.set_read_timeout(Some(Duration::from_secs(3)))?; let mut request = Vec::new(); let mut chunk = [0_u8; 4096]; @@ -22261,7 +21772,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide record_key: self.expected.target.clone(), stored_at: rfc3339_millis(now_millis()?)?, payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, - payload: self.answer_on(&message_id, stable)?, + payload: self.answer(&message_id)?, source_runtime_id: None, source_agent_id: None, source_role: None, @@ -22303,7 +21814,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide hook(); } if !stub.hang_up.load(Ordering::SeqCst) { - let _ = stub.serve_one(stream, !candidate); + let _ = stub.serve_one(stream); } } Err(_) => std::thread::sleep(Duration::from_millis(5)), @@ -22415,10 +21926,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .replace( "/etc/nginx/idunn-stream-routes/service.conf", &config.display().to_string().replace('\\', "/"), - ) - .replace( - "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", - &world.root.join("service-lifecycle-brake.cc").display().to_string(), ); let (binding, recipe_source) = if stateful { std::fs::create_dir_all(world.root.join("lease"))?; @@ -22512,7 +22019,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - transaction.deployment_admitted_at_unix_millis = Some(now); + transaction.deployment_authorization = Some(deployment_authorization( + &world, + &expected, + &transaction.transaction_id, + now, + )?); } } let launch = IdunnRuntimeActivationLaunch::issue( @@ -22523,14 +22035,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide )?; let mut seed = Vec::new(); let activation = launch.write_credential(&mut seed)?; - let mut other_seed = Vec::new(); - IdunnRuntimeActivationLaunch::issue( - &expected, - sha256_id(b"unrecorded activation"), - now, - &world.engine.idunn_signer, - )? - .write_credential(&mut other_seed)?; let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; let mut workload = recorded.workload.clone().context("no recorded workload")?; match &mut workload { @@ -22562,11 +22066,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( &seed[..], )?, - other_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( - &other_seed[..], - )?, - incumbent_instance: Mutex::new(None), - stable_answers: AtomicUsize::new(0), state: Mutex::new("warming"), capacity: AtomicU32::new(minimum_capacity), sequence: AtomicU64::new(0), @@ -24546,325 +24045,30 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[cfg(unix)] #[test] - fn a_dead_unit_is_restarted_without_a_route_challenge() -> Result<()> { + fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.admit()?; routed.workload.kill(); routed.stub.hang_up.store(true, Ordering::SeqCst); routed.drift()?; - let before = admitted(&routed)?.route_supervision; - let reloads = routed.reloads(); - let survivals = meters_of(&routed.world, "service")?.route_survivals; - // Tick 1: restart, not route repair, is the response to a dead - // incarnation. The continuity is minted and no route survival is - // charged for a proof nobody could have passed. + // Tick 1: the repair charges the target's meters and the proof + // fails. Both are writes, so they end the pass: nothing else is + // decided from the snapshot they made stale. + assert!(routed.tick()?); + let state = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(state.consecutive_failures, 1); + assert!(state.degraded_since_unix_millis.is_some()); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_survivals.len(), 2); + assert!(meters.route_actuations.is_empty()); + + // Tick 2: the route waits, and the dead unit is restarted from a + // fresh read, counted in the same write that mints it. assert!(routed.tick()?); assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); - assert_eq!(meters_of(&routed.world, "service")?.route_survivals, survivals); - assert_eq!(admitted(&routed)?.route_supervision, before); - assert_eq!(routed.reloads(), reloads, "the dead unit's route was repaired"); - Ok(()) - } - - /// A target with a process write lease, admitted and running. - #[cfg(unix)] - fn admitted_stateful_world() -> Result { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; - let granted_lease = routed - .transaction()? - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("no granted lease")? - .to_owned(); - routed.stub.set_state("active"); - *routed.stub.lease.lock().unwrap() = Some(granted_lease); - routed.step()?; - routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; - routed.promote()?; - routed.drive_until(|transaction| transaction.is_terminal())?; - assert!(routed.world.engine.retire_one_terminal_transaction()?); - Ok(routed) - } - - /// Engage the target's lifecycle brake: the dead path may not mint. - fn engage_lifecycle_brake(routed: &RoutedWorld) -> Result<()> { - engage_lifecycle_brake_in(&routed.world) - } - #[cfg(unix)] - #[test] - fn a_parked_routed_target_is_not_route_supervised() -> Result<()> { - let routed = admitted_stateful_world()?; - let generation = admitted(&routed)?; - let lease = generation.leasing.lease().context("no lease")?.clone(); - CultCacheWriteLeaseDriver::new( - "service", - routed.world.root.join("lease/process-write-lease.cc"), - ) - .revoke_exact(Some(&lease))?; - routed.workload.kill(); - engage_lifecycle_brake(&routed)?; - routed.drift()?; - let envelope = incumbent_envelope(&routed.world)?; - let reloads = routed.reloads(); - let programs = routed.world.route_stubs.count(""); - - for _ in 0..100 { - routed.tick()?; - } - assert_eq!(routed.reloads(), reloads, "a parked target's route was reloaded"); - assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); - assert_eq!(incumbent_envelope(&routed.world)?, envelope); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let reports = routed.world.engine.fault_reports.lock().unwrap(); - assert!( - !reports.keys().any(|key| key.starts_with("route-supervision:") || key.starts_with("route:")), - "{:?}", - reports.keys().collect::>() - ); - Ok(()) - } - - fn fault_keys(routed: &RoutedWorld) -> Vec { - routed.world.engine.fault_reports.lock().unwrap().keys().cloned().collect() - } - - /// Soul's probe: a route record that cannot be written leaves the - /// challenge past due on a running incarnation. Status shows the age - /// and does not name a cause it cannot see. - #[cfg(unix)] - #[test] - fn status_of_a_past_due_route_on_a_running_incarnation_claims_no_wait() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - routed.drift()?; - assert!(routed.tick()?); - assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); - routed.stub.hang_up.store(false, Ordering::SeqCst); - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); - })?; - let fragment = routed.fragment_path(); - std::fs::remove_file(&fragment)?; - std::fs::create_dir(&fragment)?; - for _ in 0..20 { - routed.tick()?; - } - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - assert!(fault_keys(&routed).contains(&"route-supervision:service".to_owned())); - let generation = admitted(&routed)?; - let lines = render_supervision("service", Some(&generation), None, now_millis()?).join("\n"); - assert!(lines.contains("(past due "), "{lines}"); - assert!(!lines.contains("waits on"), "{lines}"); - Ok(()) - } - - /// A target whose lease store is empty while its process runs is a - /// dead-path case: the route is not challenged for it. - #[cfg(unix)] - #[test] - fn a_running_target_with_an_empty_lease_is_not_route_challenged() -> Result<()> { - let routed = admitted_stateful_world()?; - let lease = admitted(&routed)?.leasing.lease().context("no lease")?.clone(); - CultCacheWriteLeaseDriver::new( - "service", - routed.world.root.join("lease/process-write-lease.cc"), - ) - .revoke_exact(Some(&lease))?; - engage_lifecycle_brake(&routed)?; - routed.drift()?; - let envelope = incumbent_envelope(&routed.world)?; - let reloads = routed.reloads(); - let programs = routed.world.route_stubs.count(""); - for _ in 0..50 { - routed.tick()?; - } - assert_eq!(routed.reloads(), reloads, "an empty-lease target's route was reloaded"); - assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); - assert_eq!(incumbent_envelope(&routed.world)?, envelope); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - assert!( - !fault_keys(&routed).iter().any(|key| key.starts_with("route")), - "{:?}", - fault_keys(&routed) - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_host_that_answers_with_a_death_ends_the_silence_fault() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - engage_lifecycle_brake(&routed)?; - routed.workload.silence(true); - for _ in 0..5 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, "observe:service"), 1); - routed.workload.silence(false); - routed.workload.kill(); - routed.tick()?; - assert!(!fault_keys(&routed).contains(&"observe:service".to_owned())); - Ok(()) - } - - /// Replace the topology store with a directory: every read of it fails. - #[cfg(unix)] - fn break_topology(routed: &RoutedWorld) -> Result<()> { - let store = routed.world.root.join("topology.cc"); - if store.exists() { - std::fs::rename(&store, routed.world.root.join("topology.aside"))?; - } - std::fs::create_dir(&store)?; - Ok(()) - } - - #[cfg(unix)] - fn mend_topology(routed: &RoutedWorld) -> Result<()> { - let store = routed.world.root.join("topology.cc"); - std::fs::remove_dir(&store)?; - let aside = routed.world.root.join("topology.aside"); - if aside.exists() { - std::fs::rename(aside, &store)?; - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_stale_incarnation_and_a_projection_repair_that_fail_are_each_said_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let keys = ["stale-incarnation:service", "projection-repair:service"]; - for round in 1..=2 { - break_topology(&routed)?; - for _ in 0..30 { - routed.tick()?; - } - for key in keys { - assert_eq!(printed_in(&routed, key), round, "{key}"); - assert!(fault_keys(&routed).contains(&key.to_owned()), "{key}"); - } - mend_topology(&routed)?; - routed.tick()?; - for key in keys { - assert!(!fault_keys(&routed).contains(&key.to_owned()), "{key}"); - } - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_refused_demotion_is_said_once_and_cleared_when_it_succeeds() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.workload.kill(); - break_topology(&routed)?; - // Each attempt is let through by forgetting the deferral it wrote. - for _ in 0..6 { - set_meters(&routed.world, &TargetSupervision::new("service"))?; - routed.tick()?; - } - assert_eq!(printed_in(&routed, "demotion:service"), 1); - assert!(fault_keys(&routed).contains(&"demotion:service".to_owned())); - mend_topology(&routed)?; - set_meters(&routed.world, &TargetSupervision::new("service"))?; - routed.tick()?; - assert!(!fault_keys(&routed).contains(&"demotion:service".to_owned())); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_unsafe_write_lease_store_is_said_once_and_cleared_when_it_is_exact() -> Result<()> { - let routed = admitted_stateful_world()?; - let store = routed.world.root.join("lease/process-write-lease.cc"); - let exact = std::fs::read(&store)?; - for round in 1..=2 { - std::fs::write(&store, b"not a lease store")?; - for _ in 0..30 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, "write-lease:service"), round); - assert!(fault_keys(&routed).contains(&"write-lease:service".to_owned())); - std::fs::write(&store, &exact)?; - routed.tick()?; - assert!(!fault_keys(&routed).contains(&"write-lease:service".to_owned())); - } - Ok(()) - } - - fn printed_in(routed: &RoutedWorld, key: &str) -> usize { - routed.world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() - } - - #[cfg(unix)] - #[test] - fn a_running_route_whose_fragment_cannot_be_read_is_reported_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let key = "route-supervision:service"; - let fragment = routed.fragment_path(); - let bytes = std::fs::read(&fragment)?; - // The fragment is a directory: reading it fails, and nothing is - // written, so every tick meets the same error. - std::fs::remove_file(&fragment)?; - std::fs::create_dir(&fragment)?; - for _ in 0..30 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, key), 1); - assert!(routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); - - // The fragment is back: the fault ends and the key clears. - std::fs::remove_dir(&fragment)?; - std::fs::write(&fragment, &bytes)?; - routed.tick()?; - assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); - - // A recurrence is said again, once. - std::fs::remove_file(&fragment)?; - std::fs::create_dir(&fragment)?; - for _ in 0..30 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, key), 2); - Ok(()) - } - - /// Skipping route and topology on a tick the host could not be asked is - /// the steward's ruling (no activation can be bound without an - /// observation; no routed target runs on a host actuator today). - #[cfg(unix)] - #[test] - fn a_host_that_cannot_be_asked_skips_route_supervision_and_is_said_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.drift()?; - let envelope = incumbent_envelope(&routed.world)?; - let reloads = routed.reloads(); - routed.workload.silence(true); - for _ in 0..30 { - routed.tick()?; - } - assert_eq!(routed.reloads(), reloads, "route supervision ran without an observation"); - assert_eq!(incumbent_envelope(&routed.world)?, envelope); - assert_eq!(printed_in(&routed, "observe:service"), 1); - - routed.workload.silence(false); - routed.tick()?; - assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key("observe:service")); - routed.workload.silence(true); - for _ in 0..5 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, "observe:service"), 2); Ok(()) } @@ -25029,39 +24233,9 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.run_to_routing()?; let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; routed.stub.hang_up.store(true, Ordering::SeqCst); - // The Routing tick itself reports why the proof failed, once per - // distinct reason, and recovery clears it. - let proof_key = route_proof_report_key(&routed.transaction()?.transaction_id); - let said = |routed: &RoutedWorld| -> Option { - routed - .world - .engine - .fault_reports - .lock() - .unwrap() - .get(&proof_key) - .and_then(|report| report.last.lock().unwrap().clone()) - }; - assert!(said(&routed).is_none(), "a proof report existed before any failure"); - routed.resume()?; - let first = said(&routed).context("the failing Routing tick reported no proof failure")?; - assert!(first.contains("route proof") && first.contains("silent, nothing answered"), "{first}"); - // The hung-up stub fails the same way each attempt, but the OS - // words the tail either as an early EOF or a broken pipe, so a - // retry may honestly report a second distinct reason. What may - // not change is everything up to that tail, and a repeat must not - // mint a new line each attempt. - let head = |text: &str| text.split("answered:").next().unwrap().to_owned(); - let mut seen = vec![first.clone()]; - for _ in 0..2 { + for _ in 0..3 { routed.resume()?; - let again = said(&routed).context("a retry cleared the proof report")?; - assert_eq!(head(&again), head(&first), "a retry reworded the report"); - if !seen.contains(&again) { - seen.push(again); - } } - assert!(seen.len() <= 2, "the same failure was reported afresh on every retry: {seen:#?}"); // Before the deadline each attempt installs and rolls back, and no // resume moves the deadline. assert_eq!(routed.reloads(), 6); @@ -25104,200 +24278,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(records.len(), 1); assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); - assert!(said(&routed).is_none(), "recovery left the proof report armed"); - Ok(()) - } - - // ----------------------------------------------------------------- - // The reload handoff: nginx's old workers may answer the stable - // listener for a moment after `systemctl reload` returns. - // ----------------------------------------------------------------- - - /// A deploy world at Routing whose preflight found an incumbent route - /// (another instance, another membership), so the install changes it. - #[cfg(unix)] - fn handoff_world(reply: Reply) -> Result<(RoutedWorld, String)> { - handoff_world_with(reply, false) - } - - /// `same_membership`: the incumbent's membership equals the - /// candidate's, so the install changes no route. - #[cfg(unix)] - fn handoff_world_with(reply: Reply, same_membership: bool) -> Result<(RoutedWorld, String)> { - let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - let incumbent = sha256_id(b"incumbent instance"); - let current = resident(&routed.world)?; - let mut next = current.value.clone(); - let mut on_disk = b"incumbent route\n".to_vec(); - if same_membership { - on_disk = NginxRouteDriver::new( - next.plan.as_ref().unwrap().parsed_inputs()?.1.route.context("no route binding")?, - ) - .render(next.expected.as_ref().context("no Expected")?)?; - } - let preflight = next.route_preflight.as_mut().context("no preflight")?; - preflight.incumbent_runtime_instance_id = Some(incumbent.clone()); - preflight.incumbent_membership_sha256 = Some(sha256_id(&on_disk)); - preflight.incumbent_configuration = Some(on_disk.clone()); - replace_transaction(&routed.world.state_store, ¤t, &next)?; - std::fs::write(routed.fragment_path(), &on_disk)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - *routed.stub.incumbent_instance.lock().unwrap() = Some(incumbent.clone()); - *routed.stub.reply.lock().unwrap() = reply; - Ok((routed, incumbent)) - } - - fn forwards(routed: &RoutedWorld) -> Result { - Ok(meters_of(&routed.world, "service")?.route_actuations.len()) - } - - fn proof_report(routed: &RoutedWorld) -> Result> { - let key = route_proof_report_key(&routed.transaction()?.transaction_id); - Ok(routed - .world - .engine - .fault_reports - .lock() - .unwrap() - .get(&key) - .and_then(|report| report.last.lock().unwrap().clone())) - } - - /// Holds a virtual handoff clock on this thread: the wait is measured - /// in the loop's own sleeps, so no host load can move it. - struct VirtualHandoffClock; - - impl VirtualHandoffClock { - fn hold() -> Self { - HANDOFF_CLOCK.with(|c| *c.borrow_mut() = Some((Duration::ZERO, Vec::new(), 0))); - Self - } - - /// The pauses the loop took so far. - fn sleeps(&self) -> Vec { - HANDOFF_CLOCK.with(|c| c.borrow().as_ref().unwrap().1.clone()) - } - } - - impl Drop for VirtualHandoffClock { - fn drop(&mut self) { - HANDOFF_CLOCK.with(|c| *c.borrow_mut() = None); - } - } - - #[cfg(unix)] - #[test] - fn routing_waits_out_the_incumbent_answering_after_reload() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, _) = handoff_world(Reply::IncumbentFirst(3))?; - let before = routed.reloads(); - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - routed.promote()?; - assert_eq!(routed.reloads() - before, 1, "an install and no rollback"); - assert_eq!(forwards(&routed)?, 1); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 4); - assert_eq!(clock.sleeps(), vec![ROUTE_RELOAD_HANDOFF_POLL; 3]); - assert!(proof_report(&routed)?.is_none(), "a route proof report remains"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn routing_fails_once_when_the_incumbent_never_lets_go() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, incumbent) = handoff_world(Reply::IncumbentFirst(usize::MAX))?; - let before = routed.reloads(); - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - let error = routed.step().unwrap_err(); - // The window is exactly ROUTE_RELOAD_HANDOFF of polls at exactly - // the poll cadence: one challenge before each pause and one after - // the last. - let sleeps = clock.sleeps(); - let polls = (ROUTE_RELOAD_HANDOFF.as_millis() / ROUTE_RELOAD_HANDOFF_POLL.as_millis()) as usize; - assert_eq!(sleeps, vec![ROUTE_RELOAD_HANDOFF_POLL; polls]); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, polls + 1); - assert_eq!(routed.reloads() - before, 2, "install and one rollback"); - assert_eq!(forwards(&routed)?, 1); - let said = proof_report(&routed)?.context("no route proof report")?; - let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; - let signer = routed.stub.other_signer.identity_id(); - for text in [format!("{error:#}"), said] { - assert!(text.contains(&format!("runtime instance {incumbent}")), "{text}"); - assert!(text.contains(&format!("activation signer {signer}")), "{text}"); - assert!(text.contains(&format!("the candidate is {candidate}")), "{text}"); - } - routed.step().unwrap_err(); - assert_eq!(forwards(&routed)?, 2, "the second attempt charges one more Forward"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn routing_fails_at_once_when_the_candidate_answers_with_a_foreign_activation() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, _) = handoff_world(Reply::ForeignActivation)?; - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - let error = routed.step().unwrap_err(); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); - assert!(clock.sleeps().is_empty()); - let text = format!("{error:#}"); - let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; - assert!(text.contains("runtime activation proof verification failed"), "{text}"); - assert!(text.contains(&format!("runtime instance {candidate}")), "{text}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn routing_fails_at_once_for_a_stranger_instance() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, _) = handoff_world(Reply::StrangerInstance)?; - let before = routed.reloads(); - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - routed.step().unwrap_err(); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); - assert!(clock.sleeps().is_empty()); - assert_eq!(routed.reloads() - before, 2); - Ok(()) - } - - /// An incumbent whose membership equals the candidate's has no route - /// to hand off, so its answer is a plain failure, never waited on. - #[cfg(unix)] - #[test] - fn routing_does_not_wait_when_the_route_is_unchanged() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, _) = handoff_world_with(Reply::IncumbentFirst(1), true)?; - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - routed.step().unwrap_err(); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); - assert!(clock.sleeps().is_empty()); - Ok(()) - } - - #[test] - fn route_proof_answer_names_ids_only() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let mut presence = rmp_serde::from_slice::( - &routed.stub.answer("probe")?, - )?; - presence.runtime_instance_id = format!("a\u{7}{}", "é".repeat(5_000)); - presence.activation_signer_identity_id = "b".repeat(10_240); - presence.signature = vec![0xAB; 64]; - presence.activation_signature = vec![0xCD; 64]; - let (instance, signer) = presence_answer_ids(&rmp_serde::to_vec(&presence)?) - .context("a decodable presence gave no ids")?; - let text = RouteProofAnswer { - answered_instance: route_proof_label(&instance), - answered_activation_signer: route_proof_label(&signer), - candidate_instance: route_proof_label(&"c".repeat(10_240)), - } - .to_string(); - assert!(text.len() <= 400, "{} bytes", text.len()); - assert!(!text.contains('\u{7}')); - assert!(!text.contains("abab") && !text.contains("cdcd")); - assert!(presence_answer_ids(b"not msgpack presence").is_none()); Ok(()) } @@ -25906,16 +24886,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .post_fencing_abort .context("the passed deadline did not end a deployment")?; assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); - assert!( - abort.error.contains("its last attempt failed"), - "the deadline abort dropped the proof's failure: {}", - abort.error - ); - assert!( - abort.error.contains("stable CultNet route stopped before its headers ended"), - "the deadline abort dropped the proof's own failure text: {}", - abort.error - ); assert!( routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), "the boot lift took a deployment's abort" @@ -26865,7 +25835,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); if envelope.r#type == DeploymentTransaction::TYPE { transactions += 1; - assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v6")); + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v5")); } } } @@ -26904,12 +25874,9 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; /// The v4 bytes of the same record, captured at base 804ed75: a legacy - /// fixture. v5 added key 37 (`flip`) and v6 key 38, so v4 is two elements shorter. + /// fixture. v5 adds key 37 (`flip`), so it is one element longer and names v5. const PINNED_TRANSACTION_HEX_V4: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; - const PINNED_TRANSACTION_HEX_V5: &str = "dc0026bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7635a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0c0"; - /// The v6 bytes: v5 with key 38 (`deployment_admitted_at_unix_millis`) appended, so - /// one element longer and naming v6. - const PINNED_TRANSACTION_HEX: &str = "dc0027bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7636a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0c0c0"; + const PINNED_TRANSACTION_HEX: &str = "dc0026bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7635a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0c0"; fn pinned_v4_envelope() -> Result { let payload = (0..PINNED_TRANSACTION_HEX_V4.len()) @@ -26926,15 +25893,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } #[test] - fn a_pinned_v4_transaction_lifts_and_reencodes_as_the_pinned_v6() -> Result<()> { + fn a_pinned_v4_transaction_lifts_and_reencodes_as_the_pinned_v5() -> Result<()> { let lifted = read_transaction_record(&pinned_v4_envelope()?)?; assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); assert_eq!(lifted.flip, None); - assert_eq!(lifted.deployment_admitted_at_unix_millis, None); let bytes = encode_transaction(&lifted, 100)?.payload; let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); assert_eq!(hex, PINNED_TRANSACTION_HEX); - // Read as v6 the v4 bytes are refused, never silently accepted. + // Read as v5 the v4 bytes are refused, never silently accepted. let mut mislabeled = pinned_v4_envelope()?; mislabeled.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()); assert!(read_transaction_record(&mislabeled).is_err()); @@ -26943,7 +25909,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// Always a copy in a temp dir: never a live control.cc. #[test] - fn a_v4_control_store_boots_into_v6_and_a_v6_store_is_left_alone() -> Result<()> { + fn a_v4_control_store_boots_into_v5_and_a_v5_store_is_left_alone() -> Result<()> { let temporary = tempfile::tempdir()?; let path = temporary.path().join("control.cc"); let store = SingleFileMessagePackBackingStore::new(&path); @@ -26963,68 +25929,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(after[0].stored_at, v4.stored_at); let migrated: String = after[0].payload.iter().map(|byte| format!("{byte:02x}")).collect(); assert_eq!(migrated, PINNED_TRANSACTION_HEX); - // Negative: an already-v6 store is untouched, byte for byte. - let file_before = std::fs::read(&path)?; - assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); - assert_eq!(std::fs::read(&path)?, file_before); - read_transaction_record(&after[0])?; - Ok(()) - } - fn pinned_v5_envelope() -> Result { - let payload = (0..PINNED_TRANSACTION_HEX_V5.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&PINNED_TRANSACTION_HEX_V5[i..i + 2], 16)) - .collect::, _>>()?; - Ok(CultCacheEnvelope { - key: "tx-pinned".into(), - r#type: DeploymentTransaction::TYPE.into(), - payload, - stored_at: rfc3339_millis(100)?, - schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA_V5.into()), - }) - } - - #[test] - fn a_pinned_v5_transaction_lifts_and_reencodes_as_the_pinned_v6() -> Result<()> { - let lifted = read_transaction_record(&pinned_v5_envelope()?)?; - assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); - assert_eq!(lifted.deployment_admitted_at_unix_millis, None); - let bytes = encode_transaction(&lifted, 100)?.payload; - let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); - assert_eq!(hex, PINNED_TRANSACTION_HEX); - // Read as v6 the v5 bytes are refused, never silently accepted. - let mut mislabeled = pinned_v5_envelope()?; - mislabeled.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()); - assert!(read_transaction_record(&mislabeled).is_err()); - // History is never migrated: it lifts a v5 record the same way. - let (history, undecodable) = decode_history_transactions(vec![pinned_v5_envelope()?]); - assert!(undecodable.is_empty(), "{undecodable:?}"); - assert_eq!(history, vec![lifted]); - Ok(()) - } - - /// Always a copy in a temp dir: never a live control.cc. - #[test] - fn a_v5_control_store_copy_boots_into_v6_and_a_v6_store_is_left_alone() -> Result<()> { - let temporary = tempfile::tempdir()?; - let path = temporary.path().join("control.cc"); - let store = SingleFileMessagePackBackingStore::new(&path); - let v5 = pinned_v5_envelope()?; - assert!(store.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: v5.r#type.clone(), - key: v5.key.clone(), - current: None, - }], - std::slice::from_ref(&v5), - )?); - assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); - let after = store.pull_all_read_only_snapshot()?; - assert_eq!(after.len(), 1); - assert_eq!(after[0].schema_id.as_deref(), Some(DEPLOYMENT_TRANSACTION_SCHEMA)); - assert_eq!(after[0].stored_at, v5.stored_at); - let migrated: String = after[0].payload.iter().map(|byte| format!("{byte:02x}")).collect(); - assert_eq!(migrated, PINNED_TRANSACTION_HEX); + // Negative: an already-v5 store is untouched, byte for byte. let file_before = std::fs::read(&path)?; assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); assert_eq!(std::fs::read(&path)?, file_before); @@ -27202,23 +26107,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide latest_id(&self.world, &id) } - /// Engage the stop for this world's runtime, as a steward would. - fn hold(&self) -> Result<()> { - crate::brake::replace_stop( - &self.site.join("authority/deployment-brake.cc"), - "site-yggdrasil", - StopState::Engaged, - "test hold", - now_millis()?, - )?; - Ok(()) - } - /// Submit a rollback command and tick until the daemon freezes it. - /// A rollback is born at Sealing with its evidence, so a test that - /// wants it held at the stop engages the stop first. fn submit_rollback(&self, command_id: &str) -> Result { - self.hold()?; commit_deployment_command( &self.world.state_store, CommandKind::Rollback, @@ -27270,32 +26160,51 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(error) } - /// The steward clears the stop. It is not bound to this transaction and - /// not consumed by it. + /// The operator releases the brake for this exact transaction. fn release_brake(&self, transaction: &DeploymentTransaction) -> Result<()> { - set_stop( - transaction.plan.as_ref().context("no plan")?, - StopState::Clear, - "test thaw", - ) + let (runtime_id, release_id) = transaction.brake_subject()?; + let authorization = released_brake( + &self.world, + &runtime_id, + &release_id, + &transaction.transaction_id, + now_millis()?, + )?; + let store = self.site.join("authority/deployment-brake.cc"); + let record: IdunnDeploymentBrakeRecord = + rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; + let backing = SingleFileMessagePackBackingStore::new(&store); + let current = backing + .pull_all_read_only_snapshot() + .ok() + .and_then(|entries| entries.into_iter().next()); + ensure!(backing.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + key: record.brake_id.clone(), + current, + }], + &[CultCacheEnvelope { + key: record.brake_id.clone(), + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + payload: authorization.canonical_brake_bytes.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), + }], + )?); + Ok(()) } - /// Run a Deploy or a Rollback up to the stop and say where it waits. + /// Run a Deploy up to the brake and say where it waits. fn run_to_brake(&self, id: &str) -> Result { - self.hold()?; - self.tick_until(id, "the sealed release", |t| t.installed_release.is_some())?; + self.tick_until(id, "the brake", |t| t.installed_release.is_some())?; for _ in 0..2 { self.world.engine.resume_backoff.lock().unwrap().clear(); let _ = self.world.engine.run_scheduler_tick(); } let waiting = latest_id(&self.world, id)?; assert_eq!(waiting.phase, DeploymentPhase::Sealing, "{waiting:?}"); - assert!(!waiting.deployment_admitted()); - assert!( - waiting.last_error.as_deref().is_some_and(|why| why.contains("deployment brake engaged by")), - "{:?}", - waiting.last_error - ); + assert!(waiting.deployment_authorization.is_none()); Ok(waiting) } } @@ -27304,13 +26213,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// that seals it, and a stand-in site beside it. #[cfg(unix)] fn static_world() -> Result { - static_world_with_steward(None) - } - - /// The static world, its binding naming `steward` (a login) as a second - /// stop whose store is `steward/brake.cc`. - #[cfg(unix)] - fn static_world_with_steward(steward: Option<&str>) -> Result { use crate::deployment::tests::{STATIC_BINDING, STATIC_RECIPE}; let tools = TempDir::new()?; let tree = tools.path().join("tree"); @@ -27324,24 +26226,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide STATIC_RECIPE, |root| { site = root.join("site"); - let text = STATIC_BINDING.replace("@SITE@", &site.display().to_string()); - match steward { - None => text, - Some(user) => text.replace( - "[rollout]", - &format!( - "[brakes.steward]\nuser = \"{user}\"\nstore = \"{}/steward/brake.cc\"\n\n[rollout]", - site.display() - ), - ), - } + STATIC_BINDING.replace("@SITE@", &site.display().to_string()) }, Arc::new(InstallingWorkload), )?; std::fs::create_dir_all(site.join("www"))?; std::fs::create_dir_all(site.join("config"))?; - std::fs::create_dir_all(site.join("authority"))?; - std::fs::create_dir_all(site.join("steward"))?; std::fs::write(site.join("config/site.json"), SITE_CONFIG_ONE)?; *world.engine.runner_stand_in.lock().unwrap() = Some(Arc::new(SealingRunner { tree: tree.clone() })); @@ -27396,7 +26286,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(flip.previous_release_id, None); assert_eq!(flip.carried_sha256.as_deref(), Some(sha256_of(SITE_CONFIG_TWO).as_str())); assert_eq!(flip.carry_source, w.site_config()); - assert!(done.deployment_admitted_at_unix_millis.is_some(), "the stop did not stamp its admission"); + assert!(done.deployment_authorization.is_some(), "the brake was not consumed"); assert_eq!(w.current_release().as_deref(), Some(sealed.sealed_release_id.as_str())); let carried = w.current_link().join("config.json"); assert_eq!(std::fs::read(&carried)?, SITE_CONFIG_TWO); @@ -27461,152 +26351,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } - #[cfg(unix)] - #[test] - fn absent_store_admits_and_stamps_admission() -> Result<()> { - let w = static_world()?; - let id = w.transaction_of("up-service")?; - w.tick_until(&id, "the end", |t| t.is_terminal())?; - assert!(!w.site.join("authority/deployment-brake.cc").exists()); - let done = latest_id(&w.world, &id)?; - assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); - assert!(done.deployment_admitted_at_unix_millis.is_some()); - assert!(done.deployment_authorization.is_none()); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_steward_stop_holds_a_deploy_beside_the_operators_and_names_who() -> Result<()> { - let me = crate::brake::process_uid(); - if me == 0 { - eprintln!("SKIPPED: a steward is never root, and this run is root"); - return Ok(()); - } - let user = crate::brake::login_name(me)?; - let w = static_world_with_steward(Some(&user))?; - let steward_store = w.site.join("steward/brake.cc"); - let op_store = w.site.join("authority/deployment-brake.cc"); - let stop = |store: &Path, state, reason: &str| -> Result<()> { - crate::brake::replace_stop(store, "site-yggdrasil", state, reason, now_millis()?)?; - Ok(()) - }; - stop(&steward_store, StopState::Engaged, "steward hold")?; - let id = w.transaction_of("up-service")?; - w.tick_until(&id, "the sealed release", |t| t.installed_release.is_some())?; - let settle = |w: &StaticWorld| -> Result { - for _ in 0..2 { - w.world.engine.resume_backoff.lock().unwrap().clear(); - let _ = w.world.engine.run_scheduler_tick(); - } - latest_id(&w.world, &id) - }; - let held = settle(&w)?; - assert_eq!(held.phase, DeploymentPhase::Sealing); - assert_eq!( - held.last_error.as_deref(), - Some(format!("steward {user} deployment brake engaged: steward hold").as_str()) - ); - // The operator engages too; the steward clears its own. The operator's holds. - stop(&op_store, StopState::Engaged, "operator hold")?; - stop(&steward_store, StopState::Clear, "steward thaw")?; - let held = settle(&w)?; - assert!(!held.deployment_admitted()); - assert_eq!( - held.last_error.as_deref(), - Some(format!("deployment brake engaged by {user}: operator hold").as_str()) - ); - // Both engaged: both named. - stop(&steward_store, StopState::Engaged, "steward again")?; - let held = settle(&w)?; - assert!(held.last_error.as_deref().is_some_and(|note| note.contains("operator hold") - && note.contains("steward again"))); - // The operator clears, the steward still holds; then both clear. - stop(&op_store, StopState::Clear, "operator thaw")?; - assert!(!settle(&w)?.deployment_admitted()); - stop(&steward_store, StopState::Clear, "steward thaw")?; - w.tick_until(&id, "the end", |t| t.is_terminal())?; - assert!(latest_id(&w.world, &id)?.deployment_admitted()); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn one_clear_admits_many_deploys_and_an_engaged_stop_names_who() -> Result<()> { - let w = static_world()?; - let first_id = w.transaction_of("up-service")?; - let first = w.run_to_brake(&first_id)?; - let held = first.last_error.clone().context("no wait note")?; - let me = crate::brake::process_uid(); - let store = w.site.join("authority/deployment-brake.cc"); - let crate::brake::Stop::Engaged(engaged) = crate::brake::read_stop(&store, "site-yggdrasil", me) else { - bail!("the stop is not engaged") - }; - assert_eq!(held, format!("deployment brake engaged by {}: {}", engaged.by, engaged.reason)); - w.release_brake(&first)?; - w.tick_until(&first_id, "the first end", |t| t.is_terminal())?; - // No engage in between: one Clear admits the second deploy too, with - // no release id or transaction id in sight. - enqueue_deploy(&w.world, "up-service-2")?; - let second_id = w.transaction_of("up-service-2")?; - w.tick_until(&second_id, "the second end", |t| t.is_terminal())?; - let crate::brake::Stop::Clear(record) = crate::brake::read_stop(&store, "site-yggdrasil", me) else { - bail!("the stop did not stay clear") - }; - assert_eq!(record.sequence, 2); - for id in [&first_id, &second_id] { - let done = latest_id(&w.world, id)?; - assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); - assert!(done.deployment_admitted_at_unix_millis.is_some()); - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_stop_engaged_after_admission_halts_the_static_flip_until_it_clears() -> Result<()> { - let w = static_world()?; - let first = w.deploy("up-service")?; - let incumbent = w.current_release().context("no incumbent")?; - assert!(first.is_terminal()); - enqueue_deploy(&w.world, "up-service-2")?; - let id = w.transaction_of("up-service-2")?; - // Admitted by the absent-or-clear stop, flip intent not yet recorded. - w.tick_until(&id, "admission", |t| t.phase == DeploymentPhase::Starting)?; - assert!(latest_id(&w.world, &id)?.flip.is_none()); - w.hold()?; - for _ in 0..4 { - w.world.engine.resume_backoff.lock().unwrap().clear(); - let _ = w.world.engine.run_scheduler_tick(); - } - let held = latest_id(&w.world, &id)?; - assert_eq!(held.phase, DeploymentPhase::Starting); - assert!(held.flip.is_none(), "a halted flip recorded an intent"); - assert!(held.pre_fencing_abort.is_none() && held.completion.is_none()); - assert!(held.last_error.as_deref().is_some_and(|n| n.contains("deployment brake engaged by")), "{:?}", held.last_error); - assert_eq!(w.current_release().as_deref(), Some(incumbent.as_str()), "the link moved under a stop"); - w.release_brake(&held)?; - w.tick_until(&id, "the end", |t| t.is_terminal())?; - let done = latest_id(&w.world, &id)?; - assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); - assert_ne!(w.current_release().as_deref(), Some(incumbent.as_str())); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_stop_engaged_after_the_flip_intent_does_not_halt_the_flip() -> Result<()> { - let w = static_world()?; - let id = w.transaction_of("up-service")?; - w.tick_until(&id, "the flip intent", |t| t.flip.is_some())?; - w.hold()?; - w.tick_until(&id, "the end", |t| t.is_terminal())?; - let done = latest_id(&w.world, &id)?; - assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); - assert!(done.flip.as_ref().is_some_and(FlipEvidence::landed)); - Ok(()) - } - #[cfg(unix)] #[test] fn static_release_failed_flip_leaves_current_and_ends_before_fencing() -> Result<()> { @@ -27756,26 +26500,15 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } - /// A populated record an older Idunn could have written: a static deploy - /// that consumed a signed brake release (key 16, now legacy) and ended - /// before fencing. `drop_keys` is how many trailing nil keys the older - /// layout lacks: v5 lacks key 38, v4 also lacks key 37. #[cfg(unix)] - fn populated_legacy_lifts_to_the_exact_current_field_by_field( - schema: &str, - drop_keys: usize, - ) -> Result<()> { + #[test] + fn a_populated_v4_transaction_lifts_to_the_exact_v5_field_by_field() -> Result<()> { let w = static_world()?; let done = drive_static_to_end(&w)?; + // A populated record a v4 Idunn could have written: a deploy that + // consumed its brake and ended before fencing. let mut expected = done.clone(); expected.flip = None; - expected.deployment_admitted_at_unix_millis = None; - expected.deployment_authorization = Some(DeploymentAuthorization { - authorization_id: "authorization-legacy".into(), - brake_sha256: sha256_id(b"legacy brake bytes"), - canonical_brake_bytes: b"legacy brake bytes".to_vec(), - authorized_at_unix_millis: 99, - }); expected.completion = Some(TransactionCompletion::FailedBeforeFencing { error: "refused".into() }); expected.pre_fencing_abort = Some(PreFencingAbort { error: "refused".into(), @@ -27786,47 +26519,42 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide expected.post_commit_cleanup = None; expected.phase_deadline = None; expected.last_error = Some("refused".into()); - // Starting accepts the legacy receipt as the admission. expected.validate()?; assert_ne!(expected.created_at_unix_millis, expected.updated_at_unix_millis); - // The older bytes: the current encoding with the old schema name and - // `drop_keys` fewer elements, every one a nil. - let mut older = expected.clone(); - older.schema_version = schema.into(); - let mut payload = rmp_serde::to_vec(&older)?; - assert_eq!(&payload[..3], &[0xdc, 0x00, 0x27]); - payload[2] -= u8::try_from(drop_keys)?; - for _ in 0..drop_keys { - assert_eq!(payload.pop(), Some(0xc0)); - } + assert!(expected.completion.is_some() && expected.deployment_authorization.is_some()); + // The v4 bytes of this record: the v5 encoding with the old schema name, + // one element shorter (key 37 is the last, a nil here). + let mut as_v4 = expected.clone(); + as_v4.schema_version = DEPLOYMENT_TRANSACTION_SCHEMA_V4.into(); + let mut payload = rmp_serde::to_vec(&as_v4)?; + assert_eq!(&payload[..3], &[0xdc, 0x00, 0x26]); + payload[2] = 0x25; + assert_eq!(payload.pop(), Some(0xc0)); let envelope = CultCacheEnvelope { key: expected.transaction_id.clone(), r#type: DeploymentTransaction::TYPE.into(), payload, stored_at: rfc3339_millis(100)?, - schema_id: Some(schema.into()), + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA_V4.into()), }; let lifted = read_transaction_record(&envelope)?; assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + assert_eq!(lifted.transaction_id, expected.transaction_id); + assert_eq!(lifted.created_at_unix_millis, expected.created_at_unix_millis); + assert_eq!(lifted.updated_at_unix_millis, expected.updated_at_unix_millis); + assert_eq!(lifted.completion, expected.completion); + assert_eq!(lifted.plan, expected.plan); + assert_eq!(lifted.sealed_release, expected.sealed_release); + assert_eq!(lifted.installed_release, expected.installed_release); + assert_eq!(lifted.frozen_source, expected.frozen_source); assert_eq!(lifted.deployment_authorization, expected.deployment_authorization); - assert_eq!(lifted.deployment_admitted_at_unix_millis, None); + assert_eq!(lifted.post_commit_cleanup, expected.post_commit_cleanup); + assert_eq!(lifted.phase_deadline, expected.phase_deadline); assert_eq!(lifted.flip, None); assert_eq!(lifted, expected, "some field changed in the lift"); Ok(()) } - #[cfg(unix)] - #[test] - fn a_populated_v4_transaction_lifts_to_the_exact_v6_field_by_field() -> Result<()> { - populated_legacy_lifts_to_the_exact_current_field_by_field(DEPLOYMENT_TRANSACTION_SCHEMA_V4, 2) - } - - #[cfg(unix)] - #[test] - fn a_populated_v5_transaction_with_a_consumed_release_lifts_to_the_exact_v6_field_by_field() -> Result<()> { - populated_legacy_lifts_to_the_exact_current_field_by_field(DEPLOYMENT_TRANSACTION_SCHEMA_V5, 1) - } - #[cfg(unix)] #[test] fn static_release_with_a_foreign_current_retries_visibly_and_cannot_be_cancelled() -> Result<()> { @@ -27888,7 +26616,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(flip.carried_sha256.as_deref(), Some(sha256_of(SITE_CONFIG_THREE).as_str())); assert_eq!(w.current_release().as_deref(), Some(first.as_str())); assert_eq!(std::fs::read(w.current_link().join("config.json"))?, SITE_CONFIG_THREE); - assert!(done.deployment_admitted_at_unix_millis.is_some(), "the stop did not stamp its admission"); + assert!(done.deployment_authorization.is_some(), "the brake was not consumed"); // Negatives: no source, no source cleanup, no generation, no pruning. assert!(done.frozen_source.is_none()); assert_eq!( diff --git a/src/deployment.rs b/src/deployment.rs index 265e59f..21cf234 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -753,20 +753,6 @@ impl ProcessWriteLeaseBinding { pub struct BrakeBinding { pub deployment_store: PathBuf, pub lifecycle_store: PathBuf, - /// A second deployment stop, written by the target's steward in a - /// directory only the steward can write. Absent means the operator's store - /// is the only stop. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub steward: Option, -} - -/// The steward's own stop: the file's owner uid proves who wrote it. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StewardBrakeBinding { - /// Login name; must resolve to a non-root uid when the store is read. - pub user: String, - pub store: PathBuf, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1444,39 +1430,6 @@ impl OperatorBinding { self.brakes.deployment_store != self.brakes.lifecycle_store, "deployment and lifecycle brakes are identical" ); - if let Some(steward) = &self.brakes.steward { - require_absolute_path(&steward.store, "steward deployment brake")?; - ensure!( - crate::steward_orders::login_grammar_ok(&steward.user) && steward.user != "root", - "steward brake user is invalid or root" - ); - ensure!( - steward.store != self.brakes.deployment_store - && steward.store != self.brakes.lifecycle_store, - "steward brake store is identical to another brake store" - ); - // The steward owns its store's directory, so it could unlink any - // file there. Every other authority store must therefore live - // outside that directory, and the steward's directory must be - // clear of every directory an operator store lives in. - let steward_dir = steward.store.parent().unwrap_or(&steward.store); - let others = [ - Some(&self.brakes.deployment_store), - Some(&self.brakes.lifecycle_store), - Some(&self.runtime_identity.trust_anchor_store), - self.process_write_lease.as_ref().map(|lease| &lease.record_path), - self.repository.release_authority_store.as_ref(), - ]; - for other in others.into_iter().flatten() { - let other_dir = other.parent().unwrap_or(other); - ensure!( - !paths_overlap(steward_dir, other_dir), - "steward brake directory {} is shared with or nested in the directory of Idunn authority {}: the steward owns its directory and could remove that file", - steward_dir.display(), - other.display() - ); - } - } self.workload.validate_launch_environment()?; let workload = match &self.workload { WorkloadBinding::HostActuator(workload) => { @@ -1509,10 +1462,7 @@ impl OperatorBinding { &self.brakes.deployment_store, &self.brakes.lifecycle_store, &self.runtime_identity.trust_anchor_store, - ] - .into_iter() - .chain(self.brakes.steward.as_ref().map(|steward| &steward.store)) - { + ] { ensure!( !paths_overlap(authority, &workload.release_root) && !paths_overlap(authority, &workload.current_link), @@ -1581,7 +1531,6 @@ impl OperatorBinding { self.brakes.deployment_store.clone(), self.brakes.lifecycle_store.clone(), ]; - authority_paths.extend(self.brakes.steward.as_ref().map(|steward| steward.store.clone())); if let Some(write_lease) = &self.process_write_lease { authority_paths.push(write_lease.record_path.clone()); authority_paths.push(write_lease.lock_path()); @@ -3116,65 +3065,6 @@ nodes = ["yggdrasil"] text } - const STEWARD: &str = "\n[brakes.steward]\nuser = \"mind-streampixels\"\nstore = \"/var/lib/gamecult/idunn-steward/streampixels/deployment-brake.cc\"\n"; - - #[test] - fn binding_without_steward_parses_and_admits_on_operator_store_only() { - for text in [BINDING.to_string(), static_binding(&[])] { - assert_eq!(OperatorBinding::parse(&text).unwrap().brakes.steward, None); - } - let steward = OperatorBinding::parse(&BINDING.replace("\n[rollout]", &format!("{STEWARD}\n[rollout]"))) - .unwrap() - .brakes - .steward - .unwrap(); - assert_eq!(steward.user, "mind-streampixels"); - } - - #[test] - fn a_steward_brake_must_be_a_distinct_absolute_non_root_login_and_clear_of_the_workload() { - let with = |steward: &str| BINDING.replace("\n[rollout]", &format!("{steward}\n[rollout]")); - let store = "/var/lib/gamecult/idunn-steward/x/deployment-brake.cc"; - for bad in [ - format!("\n[brakes.steward]\nuser = \"root\"\nstore = \"{store}\"\n"), - format!("\n[brakes.steward]\nuser = \"Bad User\"\nstore = \"{store}\"\n"), - "\n[brakes.steward]\nuser = \"mind\"\nstore = \"relative.cc\"\n".into(), - "\n[brakes.steward]\nuser = \"mind\"\nstore = \"/var/lib/gamecult/idunn-authority/test-service-deployment-brake.cc\"\n".into(), - "\n[brakes.steward]\nuser = \"mind\"\nstore = \"/var/lib/gamecult/idunn-authority/test-service-lifecycle-brake.cc\"\n".into(), - format!("\n[brakes.steward]\nuser = \"mind\"\nstore = \"{store}\"\nextra = 1\n"), - ] { - assert!(OperatorBinding::parse(&with(&bad)).is_err(), "{bad}"); - } - // A steward directory shared with, or enclosing, an operator store - // lets the steward unlink a root-owned file: refused. - for (deployment, steward_store) in [ - ("/var/lib/gamecult/steward-x/op.cc", "/var/lib/gamecult/steward-x/s.cc"), - ("/var/lib/gamecult/steward-x/sub/op.cc", "/var/lib/gamecult/steward-x/s.cc"), - ("/var/lib/gamecult/op.cc", "/var/lib/gamecult/steward-x/s.cc"), - ] { - let shared = BINDING - .replace("/var/lib/gamecult/idunn-authority/test-service-deployment-brake.cc", deployment) - .replace("\n[rollout]", &format!("\n[brakes.steward]\nuser = \"mind\"\nstore = \"{steward_store}\"\n\n[rollout]")); - assert!(OperatorBinding::parse(&shared).is_err(), "{deployment} {steward_store}"); - } - // Disjoint directories are the one admitted shape. - let disjoint = BINDING.replace( - "\n[rollout]", - "\n[brakes.steward]\nuser = \"mind\"\nstore = \"/var/lib/gamecult/steward-x/s.cc\"\n\n[rollout]", - ); - OperatorBinding::parse(&disjoint).unwrap(); - // steward_store_overlap_with_workload_path_refused - let overlapping = BINDING - .replace("\n[rollout]", &format!("\n[brakes.steward]\nuser = \"mind\"\nstore = \"{store}\"\n\n[rollout]")) - .replace( - "cpu_quota_percent = 200\n\n[workload.argument_bindings]", - "cpu_quota_percent = 200\nread_write_paths = [\"/var/lib/gamecult/idunn-steward/x\"]\n\n[workload.argument_bindings]", - ); - assert!(OperatorBinding::parse(&overlapping).is_err()); - let static_overlap = static_binding(&[("[rollout]", "[brakes.steward]\nuser = \"mind\"\nstore = \"/srv/site/releases/x/brake.cc\"\n\n[rollout]")]); - assert!(OperatorBinding::parse(&static_overlap).is_err()); - } - #[test] fn static_recipe_and_binding_admit() { let recipe = TargetDeclaration::parse(STATIC_RECIPE).unwrap(); diff --git a/src/drivers.rs b/src/drivers.rs index 7dfeaee..46cb916 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -762,7 +762,7 @@ impl StaticReleasePort for FilesystemStaticRelease { binding: &StaticReleaseBinding, release_id: &str, ) -> Result { - use std::os::unix::fs::{OpenOptionsExt, PermissionsExt, symlink}; + use std::os::unix::fs::{OpenOptionsExt, symlink}; // Everything up to and including the rename can refuse, and only that // refuses: the link has not moved when `FlipRefused` is returned. @@ -823,10 +823,6 @@ impl StaticReleasePort for FilesystemStaticRelease { .create_new(true) .mode(0o444) .open(&temporary_carry)?; - // The creation mode is masked by the service's umask - // (UMask=0027 yields 0440), and the web server reads this file - // as "other"; say 0444 on the open handle. - file.set_permissions(fs::Permissions::from_mode(0o444))?; file.write_all(&bytes)?; file.sync_all()?; fs::rename(&temporary_carry, &carried)?; @@ -17049,43 +17045,6 @@ processes 9 Ok(()) } - /// The service runs with UMask=0027, which masks the create mode to 0440 - /// and hides the carry from the web user. The umask is process-global, so - /// the proof runs in a child copy of this test binary and no sibling test - /// ever sees it. - #[cfg(unix)] - #[test] - fn static_release_carry_is_0444_under_the_service_umask() -> Result<()> { - use std::os::unix::fs::PermissionsExt; - const CHILD: &str = "IDUNN_TEST_CARRY_UMASK_CHILD"; - const NAME: &str = "static_release_carry_is_0444_under_the_service_umask"; - if std::env::var_os(CHILD).is_none() { - let path = format!("{}::{NAME}", module_path!().split_once("::").map_or("", |(_, rest)| rest)); - let output = std::process::Command::new(std::env::current_exe()?) - .args(["--exact", path.as_str(), "--nocapture", "--test-threads=1"]) - .env(CHILD, "1") - .output()?; - assert!( - output.status.success(), - "child under umask 0027 failed:\n{}\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - assert!( - String::from_utf8_lossy(&output.stdout).contains("1 passed"), - "the child filter matched no test:\n{}", - String::from_utf8_lossy(&output.stdout) - ); - return Ok(()); - } - unsafe { libc::umask(0o027) }; - let site = stand_in_site()?; - FilesystemStaticRelease.carry_and_flip(&site.binding, RELEASE_A)?; - let carried = site.binding.current_link.join("config.json"); - assert_eq!(fs::metadata(&carried)?.permissions().mode() & 0o777, 0o444); - Ok(()) - } - #[cfg(unix)] #[test] fn static_release_missing_site_config_fails_before_the_link_moves() -> Result<()> { diff --git a/src/lib.rs b/src/lib.rs index 6b034d9..b5499a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,3 @@ -mod brake; pub mod control_plane; pub mod deployment; pub mod deployment_plan; diff --git a/src/provisioning.rs b/src/provisioning.rs index 324785e..1c74a83 100644 --- a/src/provisioning.rs +++ b/src/provisioning.rs @@ -1,5 +1,4 @@ -use crate::brake::{Stop, StopState, process_uid, read_stop, replace_stop}; -use anyhow::{Context, Result, anyhow, bail, ensure}; +use anyhow::{Context, Result, anyhow, bail}; use cultcache_rs::{ CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, }; @@ -7,13 +6,15 @@ use cultnet_rs::{ GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, GameCultServiceTrustAnchorRecord, IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA, IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SIGNING_PURPOSE, - IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA, - IDUNN_LIFECYCLE_BRAKE_AUTHORITY, + IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, + IDUNN_DEPLOYMENT_BRAKE_ID, IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_SCOPE, + IDUNN_DEPLOYMENT_RELEASE_PURPOSE, IDUNN_LIFECYCLE_BRAKE_AUTHORITY, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCOPE, IdunnDaemonHealthTrustBindingRecord, - IdunnLifecycleBrakeObservation, + IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, + IdunnDeploymentBrakeRecord, IdunnDeploymentBrakeReleasePurpose, IdunnLifecycleBrakeObservation, IdunnLifecycleBrakeRecord, IdunnServiceIdentity, OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentityTrustAnchor, derive_service_identity_id, enroll_service_identity_at, - evaluate_idunn_continuity_restart, + evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, export_service_identity_trust_anchor, open_service_identity_at, }; use std::collections::{BTreeMap, BTreeSet}; @@ -79,8 +80,17 @@ pub fn run(args: impl IntoIterator) -> Result<()> { "export-odin-topology-public-anchor" => { export_identity_anchor::(&options, "Odin topology public anchor")? } - "deployment-brake-engage" => deployment_brake_write(&options, StopState::Engaged)?, - "deployment-brake-clear" => deployment_brake_write(&options, StopState::Clear)?, + "enroll-deployment-brake-operator" => { + enroll_identity::(&options)? + } + "export-deployment-brake-operator-anchor" => { + export_identity_anchor::( + &options, + "deployment brake operator anchor", + )? + } + "deployment-brake-engage" => deployment_brake_engage(&options)?, + "deployment-brake-release" => deployment_brake_release(&options)?, "deployment-brake-status" => deployment_brake_status(&options)?, "lifecycle-brake-engage" => lifecycle_brake_engage(&options)?, "lifecycle-brake-release" => lifecycle_brake_release(&options)?, @@ -147,57 +157,150 @@ fn provider_health_public_key(options: &BTreeMap) -> Result, state: StopState) -> Result<()> { - require_only(options, &["store", "runtime-id", "reason"])?; - let now = u64::try_from( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH)? - .as_millis(), +fn deployment_brake_engage(options: &BTreeMap) -> Result<()> { + require_only( + options, + &[ + "store", + "runtime-id", + "owner", + "reason", + "observed-at-unix-millis", + ], )?; - let record = replace_stop( - &path(options, "store")?, - required(options, "runtime-id")?, - state, - required(options, "reason")?, - now, - )?; - println!( - "{} runtime={} by={} sequence={}", - record.state.name(), - record.runtime_id, - record.by, - record.sequence - ); - Ok(()) + let observed = parse_u64(options, "observed-at-unix-millis")?; + let record = IdunnDeploymentBrakeRecord { + schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), + brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), + runtime_id: required(options, "runtime-id")?.into(), + status: "engaged".into(), + scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), + reason: required(options, "reason")?.into(), + observed_at_unix_millis: observed, + expires_at_unix_millis: None, + authorization_id: None, + authorization_purpose: None, + authorized_release_id: None, + authorized_deployment_id: None, + authorized_by: None, + authorization_issued_at_unix_millis: None, + authorization_expires_at_unix_millis: None, + signature_algorithm: None, + signature: None, + private_state_exposed: false, + updated_by: required(options, "owner")?.into(), + }; + record.validate()?; + replace_brake(&path(options, "store")?, record, observed) } -/// Print the stop and exit nonzero unless deployments admit, so a script can -/// gate on it as it could on the retired exact-actuation validation. -fn deployment_brake_status(options: &BTreeMap) -> Result<()> { - allow_only(options, &["store", "runtime-id", "owner-user"])?; - // A steward's store is owned by the steward: the operator names that user - // to read it, and the owner check is the same one Idunn makes. - let owner = match options.get("owner-user") { - Some(user) => crate::steward_orders::user_uid(user)?, - None => process_uid(), +fn deployment_brake_release(options: &BTreeMap) -> Result<()> { + require_only( + options, + &[ + "store", + "private-store", + "runtime-id", + "owner", + "reason", + "authorization-id", + "release-id", + "deployment-id", + "issued-at-unix-millis", + "expires-at-unix-millis", + ], + )?; + let issued = parse_u64(options, "issued-at-unix-millis")?; + let expires = parse_u64(options, "expires-at-unix-millis")?; + let signer = open_service_identity_at::(&path( + options, + "private-store", + )?)?; + let mut record = IdunnDeploymentBrakeRecord { + schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), + brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), + runtime_id: required(options, "runtime-id")?.into(), + status: "released".into(), + scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), + reason: required(options, "reason")?.into(), + observed_at_unix_millis: issued, + expires_at_unix_millis: Some(expires), + authorization_id: Some(required(options, "authorization-id")?.into()), + authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), + authorized_release_id: Some(required(options, "release-id")?.into()), + authorized_deployment_id: Some(required(options, "deployment-id")?.into()), + authorized_by: Some(signer.trust_anchor()?.identity_id), + authorization_issued_at_unix_millis: Some(issued), + authorization_expires_at_unix_millis: Some(expires), + signature_algorithm: Some("ed25519".into()), + signature: None, + private_state_exposed: false, + updated_by: required(options, "owner")?.into(), }; - let stop = read_stop(&path(options, "store")?, required(options, "runtime-id")?, owner); - match &stop { - Stop::Absent => println!("absent: deployments admit"), - Stop::Unreadable(why) => println!("unreadable: deployments wait ({why})"), - Stop::Clear(record) | Stop::Engaged(record) => println!( - "{} runtime={} by={} at={} sequence={} reason={}", - record.state.name(), - record.runtime_id, - record.by, - record.at_unix_millis, - record.sequence, - record.reason - ), + record.signature = Some( + signer + .sign::(&rmp_serde::to_vec(&record)?) + .signature, + ); + record.validate()?; + replace_brake(&path(options, "store")?, record, issued) +} + +fn deployment_brake_status(options: &BTreeMap) -> Result<()> { + allow_only( + options, + &[ + "store", + "operator-anchor", + "runtime-id", + "release-id", + "deployment-id", + "now-unix-millis", + ], + )?; + let exact_actuation_fields = ["release-id", "deployment-id", "now-unix-millis"]; + let exact_actuation_field_count = exact_actuation_fields + .into_iter() + .filter(|field| options.contains_key(*field)) + .count(); + if exact_actuation_field_count != 0 + && exact_actuation_field_count != exact_actuation_fields.len() + { + bail!("exact deployment-brake actuation validation requires release, deployment, and time"); } - ensure!(stop.admits(), "the deployment brake does not admit deployments"); + let record = read_brake(&path(options, "store")?)?; + if record.runtime_id != required(options, "runtime-id")? { + bail!("deployment brake belongs to another runtime") + } + if record.status == "engaged" { + if exact_actuation_field_count != 0 { + bail!("engaged deployment brake denies exact actuation validation"); + } + println!( + "engaged runtime={} scope={} owner={} reason={}", + record.runtime_id, record.scope, record.updated_by, record.reason + ); + return Ok(()); + } + let anchor = read_operator_anchor(&path(options, "operator-anchor")?)?; + evaluate_idunn_deployment_brake( + IdunnDeploymentBrakeObservation::Present(&record), + &anchor, + required(options, "runtime-id")?, + required(options, "release-id")?, + required(options, "deployment-id")?, + parse_u64(options, "now-unix-millis")?, + ) + .map_err(|d| anyhow!("deployment brake denied actuation: {d:?}"))?; + println!( + "released runtime={} release={} deployment={} owner={}", + record.runtime_id, + record.authorized_release_id.unwrap(), + record.authorized_deployment_id.unwrap(), + record.updated_by + ); Ok(()) } @@ -304,7 +407,7 @@ fn lifecycle_brake_status(options: &BTreeMap) -> Result<()> { Ok(()) } -pub(crate) fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { +fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { record.validate()?; let store = SingleFileMessagePackBackingStore::new(path); let entries = store.pull_all_read_only_snapshot()?; @@ -379,6 +482,70 @@ fn lifecycle_identifier<'a>(options: &'a BTreeMap, name: &str) - Ok(value) } +fn replace_brake(path: &Path, record: IdunnDeploymentBrakeRecord, millis: u64) -> Result<()> { + let store = SingleFileMessagePackBackingStore::new(path); + let entries = store.pull_all_read_only_snapshot()?; + let current = match entries.as_slice() { + [] => None, + [entry] + if entry.r#type == IdunnDeploymentBrakeRecord::TYPE + && entry.key == IDUNN_DEPLOYMENT_BRAKE_ID => + { + Some(entry.clone()) + } + _ => bail!("deployment brake store is corrupt or ambiguous"), + }; + let next = typed_envelope( + IDUNN_DEPLOYMENT_BRAKE_ID, + &record, + IDUNN_DEPLOYMENT_BRAKE_SCHEMA, + millis, + )?; + if !store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + key: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + current, + }], + &[next], + )? { + bail!("deployment brake changed during transition"); + } + Ok(()) +} + +fn read_brake(path: &Path) -> Result { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let [entry] = entries.as_slice() else { + bail!("deployment brake is missing or ambiguous") + }; + if entry.r#type != IdunnDeploymentBrakeRecord::TYPE + || entry.key != IDUNN_DEPLOYMENT_BRAKE_ID + || entry.schema_id.as_deref() != Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA) + { + bail!("deployment brake is foreign") + } + let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&entry.payload)?; + if rmp_serde::to_vec(&record)? != entry.payload { + bail!("deployment brake is noncanonical") + } + record.validate()?; + Ok(record) +} + +fn read_operator_anchor(path: &Path) -> Result { + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + let [entry] = entries.as_slice() else { + bail!("operator anchor is missing or ambiguous") + }; + if entry.r#type != IdunnDeploymentBrakeOperatorIdentity::TRUST_ANCHOR_TYPE + || entry.key != IdunnDeploymentBrakeOperatorIdentity::TRUST_ANCHOR_KEY + { + bail!("operator anchor is foreign") + } + Ok(rmp_serde::from_slice(&entry.payload)?) +} + fn validate_health_binding_store(path: &Path) -> Result<()> { let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; validated_health_bindings(&entries)?; @@ -960,7 +1127,7 @@ fn normalized(path: &Path) -> Result { } fn usage() -> &'static str { - "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision export-provider-health-public-anchor --private-store --public-anchor \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-odin-topology-identity --private-store \n idunn-provision export-odin-topology-public-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --reason \n idunn-provision deployment-brake-clear --store --runtime-id --reason \n idunn-provision deployment-brake-status --store --runtime-id [--owner-user ]\n idunn-provision lifecycle-brake-engage --store --runtime-id --target --reason --updated-at-unix-millis \n idunn-provision lifecycle-brake-release --store --runtime-id --target --reason --updated-at-unix-millis [--released-until-unix-millis ]\n idunn-provision lifecycle-brake-status --store --runtime-id --target --now-unix-millis \n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " + "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision export-provider-health-public-anchor --private-store --public-anchor \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-odin-topology-identity --private-store \n idunn-provision export-odin-topology-public-anchor --private-store --public-anchor \n idunn-provision enroll-deployment-brake-operator --private-store \n idunn-provision export-deployment-brake-operator-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --owner --reason --observed-at-unix-millis \n idunn-provision deployment-brake-release --store --private-store --runtime-id --owner --reason --authorization-id --release-id --deployment-id --issued-at-unix-millis --expires-at-unix-millis \n idunn-provision deployment-brake-status --store --operator-anchor --runtime-id [--release-id --deployment-id --now-unix-millis ]\n idunn-provision lifecycle-brake-engage --store --runtime-id --target --reason --updated-at-unix-millis \n idunn-provision lifecycle-brake-release --store --runtime-id --target --reason --updated-at-unix-millis [--released-until-unix-millis ]\n idunn-provision lifecycle-brake-status --store --runtime-id --target --now-unix-millis \n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " } #[cfg(test)] @@ -1990,74 +2157,120 @@ mod tests { } #[test] - fn provision_engage_clear_sequence() -> Result<()> { + fn brake_operator_can_issue_one_exact_bounded_rollout_grant() -> Result<()> { let temp = TempDir::new()?; + let private = temp.path().join("operator-private.cc"); + let anchor = temp.path().join("operator-public.cc"); let store = temp.path().join("brake.cc"); - let me = process_uid(); - let engage = |verb: &str, reason: &str| { - invoke(&[ - verb, - "--store", - store.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--reason", - reason, - ]) - }; - assert_eq!(read_stop(&store, "yggdrasil", me), Stop::Absent); invoke(&[ - "deployment-brake-status", + "enroll-deployment-brake-operator", + "--private-store", + private.to_str().unwrap(), + ])?; + invoke(&[ + "export-deployment-brake-operator-anchor", + "--private-store", + private.to_str().unwrap(), + "--public-anchor", + anchor.to_str().unwrap(), + ])?; + invoke(&[ + "deployment-brake-engage", "--store", store.to_str().unwrap(), "--runtime-id", "yggdrasil", + "--owner", + "operator/metacrat", + "--reason", + "sleep mode", + "--observed-at-unix-millis", + "100", ])?; - engage("deployment-brake-engage", "freeze")?; - let status = || { + assert_eq!(read_brake(&store)?.status, "engaged"); + invoke(&[ + "deployment-brake-status", + "--store", + store.to_str().unwrap(), + "--operator-anchor", + anchor.to_str().unwrap(), + "--runtime-id", + "yggdrasil", + ])?; + assert!( invoke(&[ "deployment-brake-status", "--store", store.to_str().unwrap(), + "--operator-anchor", + anchor.to_str().unwrap(), "--runtime-id", "yggdrasil", + "--release-id", + "commit-4", + "--deployment-id", + "request-4", + "--now-unix-millis", + "150", ]) - }; - assert!(status().is_err(), "an engaged stop must not pass status"); - let Stop::Engaged(first) = read_stop(&store, "yggdrasil", me) else { - bail!("engage did not engage") - }; - assert_eq!((first.sequence, first.reason.as_str()), (1, "freeze")); - assert!(!first.by.is_empty() && first.at_unix_millis > 0); - engage("deployment-brake-clear", "thawed")?; - status()?; - let Stop::Clear(second) = read_stop(&store, "yggdrasil", me) else { - bail!("clear did not clear") - }; - assert_eq!((second.sequence, second.by), (2, first.by)); - assert!(second.at_unix_millis >= first.at_unix_millis); - // The author and the time are not arguments, and the retired - // signing arguments are refused. - for stray in ["--owner", "--observed-at-unix-millis", "--private-store"] { - assert!( - invoke(&[ - "deployment-brake-engage", - "--store", - store.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--reason", - "x", - stray, - "y", - ]) - .is_err(), - "{stray}" - ); - } - for gone in ["deployment-brake-release", "enroll-deployment-brake-operator"] { - assert!(invoke(&[gone]).is_err(), "{gone}"); - } + .is_err() + ); + invoke(&[ + "deployment-brake-release", + "--store", + store.to_str().unwrap(), + "--private-store", + private.to_str().unwrap(), + "--runtime-id", + "yggdrasil", + "--owner", + "operator/metacrat", + "--reason", + "one rollout", + "--authorization-id", + "auth/r4", + "--release-id", + "commit-4", + "--deployment-id", + "request-4", + "--issued-at-unix-millis", + "200", + "--expires-at-unix-millis", + "800", + ])?; + invoke(&[ + "deployment-brake-status", + "--store", + store.to_str().unwrap(), + "--operator-anchor", + anchor.to_str().unwrap(), + "--runtime-id", + "yggdrasil", + "--release-id", + "commit-4", + "--deployment-id", + "request-4", + "--now-unix-millis", + "500", + ])?; + assert!( + invoke(&[ + "deployment-brake-status", + "--store", + store.to_str().unwrap(), + "--operator-anchor", + anchor.to_str().unwrap(), + "--runtime-id", + "yggdrasil", + "--release-id", + "substituted", + "--deployment-id", + "request-4", + "--now-unix-millis", + "500" + ]) + .is_err() + ); Ok(()) } @@ -2096,6 +2309,7 @@ mod tests { assert_eq!(engaged.status, "engaged"); assert_eq!(engaged.runtime_id, "idunn-yggdrasil"); assert_eq!(engaged.target, "ghostlight"); + assert!(read_brake(&store).is_err()); assert!( invoke(&[ "lifecycle-brake-status", @@ -2212,4 +2426,57 @@ mod tests { assert_eq!(std::fs::read(&store)?, released_bytes); Ok(()) } + + #[test] + fn brake_compare_exchange_rejects_stale_transition_snapshot() -> Result<()> { + let temp = TempDir::new()?; + let store_path = temp.path().join("brake.cc"); + let base = IdunnDeploymentBrakeRecord { + schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), + brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), + runtime_id: "yggdrasil".into(), + status: "engaged".into(), + scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), + reason: "first".into(), + observed_at_unix_millis: 100, + expires_at_unix_millis: None, + authorization_id: None, + authorization_purpose: None, + authorized_release_id: None, + authorized_deployment_id: None, + authorized_by: None, + authorization_issued_at_unix_millis: None, + authorization_expires_at_unix_millis: None, + signature_algorithm: None, + signature: None, + private_state_exposed: false, + updated_by: "operator/a".into(), + }; + replace_brake(&store_path, base.clone(), 100)?; + let stale = SingleFileMessagePackBackingStore::new(&store_path) + .pull_all_read_only_snapshot()? + .remove(0); + let mut changed = base.clone(); + changed.reason = "racing writer".into(); + changed.observed_at_unix_millis = 101; + replace_brake(&store_path, changed, 101)?; + let candidate = typed_envelope( + IDUNN_DEPLOYMENT_BRAKE_ID, + &base, + IDUNN_DEPLOYMENT_BRAKE_SCHEMA, + 102, + )?; + assert!( + !SingleFileMessagePackBackingStore::new(&store_path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: IdunnDeploymentBrakeRecord::TYPE.into(), + key: IDUNN_DEPLOYMENT_BRAKE_ID.into(), + current: Some(stale) + }], + &[candidate] + )? + ); + Ok(()) + } } diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 379e09c..dba9a02 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -1528,7 +1528,12 @@ impl StewardBinding { !self.principal.starts_with('-') && !self.principal.contains('/'), "steward principal is invalid" ); - let user_ok = login_grammar_ok(&self.user); + let user_ok = !self.user.is_empty() + && self.user.len() <= 32 + && self.user.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) + && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); ensure!(user_ok && self.user != "root", "steward user is invalid or root"); ensure!( self.mind_control.is_absolute(), @@ -1576,18 +1581,7 @@ fn load_binding(path: &Path) -> Result { StewardBinding::parse(&text) } -/// A login name Idunn will resolve: the one grammar for every binding that names -/// a user. -pub(crate) fn login_grammar_ok(user: &str) -> bool { - !user.is_empty() - && user.len() <= 32 - && user.bytes().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') - }) - && !user.starts_with(|first: char| first.is_ascii_digit() || first == '-') -} - -pub(crate) fn user_uid(name: &str) -> Result { +fn user_uid(name: &str) -> Result { let name = std::ffi::CString::new(name)?; // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in // `buffer`, which outlives every read of them below (none are read).