From e50a721dbca3324db6b9a31b4eac9746385289c9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:01 +0100 Subject: [PATCH 01/23] Steward orders: typed package, unit and deploy orders from one steward, in a sibling root unit idunn orders serve takes orders over a root-only socket from the steward uid, records order/start/outcome/post in its own orders.cc, and posts acceptance and result to #operator as a retried projection. Unit orders may act on any unit except the compiled always-refused set (idunn-*, mind-self@*, mind-persona@*), matched on a unit's Id and every name, and for stop and restart on the units systemd stops with it. A refusal names a field and a code, never the value. A started order is never run twice. Deploy orders commit through the same primitive as idunn up. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 1 + Cargo.toml | 1 + src/control_plane.rs | 112 +- src/lib.rs | 2 + src/runner_evidence.rs | 4 +- src/steward_orders.rs | 2932 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 3036 insertions(+), 16 deletions(-) create mode 100644 src/steward_orders.rs diff --git a/Cargo.lock b/Cargo.lock index dfc3087..44c9457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "libc", "rmp-serde", "serde", + "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index dcaca2e..ee797ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } +serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/src/control_plane.rs b/src/control_plane.rs index e43a7ac..cd49073 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -2934,6 +2934,10 @@ enum Command { recipe: PathBuf, binding: Option, }, + #[cfg(unix)] + OrdersServe(crate::steward_orders::ServeOptions), + #[cfg(unix)] + Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { @@ -2973,6 +2977,10 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + #[cfg(unix)] + Command::OrdersServe(options) => crate::steward_orders::serve(options), + #[cfg(unix)] + Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -3025,6 +3033,10 @@ fn parse(args: impl Iterator) -> Result { "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), + #[cfg(unix)] + "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), + #[cfg(unix)] + "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -4730,20 +4742,21 @@ fn resolve_selector( Ok(targets) } -fn submit( +/// The one commit primitive for a deployment command: `idunn up` and a steward +/// deploy order both write theirs here. Returns the commit time. +pub(crate) fn commit_deployment_command( store_path: &Path, + command_id: &str, selector: &str, requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { +) -> Result { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), + command_id: command_id.into(), kind: CommandKind::Deploy, selector: selector.into(), requested_by: requested_by.into(), @@ -4761,7 +4774,33 @@ fn submit( )?, "deployment command id collided" ); - println!("{}", command.command_id); + Ok(now) +} + +/// A command's present state as `idunn status --command` derives it, for the +/// steward order poster: `queued` and `running` are not terminal. +#[cfg(unix)] +pub(crate) fn command_state( + state_store: &Path, + command_id: &str, +) -> Result<(&'static str, String)> { + let snapshot = ControlSnapshot::read(state_store)?; + let archived = read_history_transactions(state_store); + let mut transactions = snapshot.transaction_for_command(command_id); + transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); + Ok(derived_command_status(&transactions)) +} + +fn submit( + store_path: &Path, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + let command_id = format!("up-{}", Uuid::new_v4()); + let now = commit_deployment_command(store_path, &command_id, selector, requested_by)?; + println!("{command_id}"); if !wait { return Ok(()); } @@ -4772,10 +4811,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command.command_id), + .any(|stored| stored.value.command_id == command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command.command_id); + let transactions = snapshot.transaction_for_command(&command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4800,7 +4839,7 @@ fn submit( } } -fn status( +pub(crate) fn status( out: &mut impl Write, store_path: &Path, incident_store: &Path, @@ -10592,7 +10631,13 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -fn now_millis() -> Result { +/// The control store `idunn up` and `idunn serve` use unless told otherwise. +#[cfg(unix)] +pub(crate) fn default_state_store() -> PathBuf { + RuntimeOptions::default().state_store +} + +pub(crate) fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -10612,12 +10657,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -fn string_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -fn path_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -10633,7 +10678,7 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -fn require_selector(value: &str) -> Result<()> { +pub(crate) fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } @@ -10757,7 +10802,10 @@ fn usage() -> &'static str { idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\ + idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ + idunn order --reason TEXT\n\ + idunn order show ID | idunn order list\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -11969,9 +12017,45 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["order", "run", "sh", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], + vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], + vec!["orders", "serve", "--allow-units", "nginx.service"], + vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } + // The declarative order forms parse, and only as order commands. + #[cfg(unix)] + for arguments in [ + vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r"], + vec!["order", "list"], + vec!["orders", "serve"], + ] { + let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); + assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); + } + } + + #[test] + fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + commit_deployment_command(&store, "up-one", "vili", "steward:eureka order ord-1")?; + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.commands.len(), 1); + let command = &snapshot.commands[0].value; + assert_eq!(command.command_id, "up-one"); + assert_eq!(command.selector, "vili"); + assert_eq!(command.requested_by, "steward:eureka order ord-1"); + assert!(commit_deployment_command(&store, "up-one", "other", "x").is_err()); + assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); + Ok(()) } #[test] diff --git a/src/lib.rs b/src/lib.rs index 4f365a3..b5499a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,3 +9,5 @@ pub mod launcher; mod incident; mod runner_evidence; pub mod provisioning; +#[cfg(unix)] +mod steward_orders; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs index 9e9e5ee..30dd1d2 100644 --- a/src/runner_evidence.rs +++ b/src/runner_evidence.rs @@ -366,7 +366,7 @@ pub(crate) fn show_status( /// printable ASCII and tab pass; every other code point is a visible /// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline /// inside a line is escaped too: this function writes the only ones. -fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { +pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { for line in lines { out.write_all(escape(line).as_bytes())?; out.write_all(b"\n")?; @@ -374,7 +374,7 @@ fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> Ok(()) } -fn escape(bytes: &[u8]) -> String { +pub(crate) fn escape(bytes: &[u8]) -> String { let mut shown = String::with_capacity(bytes.len()); for chunk in bytes.utf8_chunks() { for character in chunk.valid().chars() { diff --git a/src/steward_orders.rs b/src/steward_orders.rs new file mode 100644 index 0000000..2aded30 --- /dev/null +++ b/src/steward_orders.rs @@ -0,0 +1,2932 @@ +//! Typed orders from Idunn's steward. +//! +//! `idunn orders serve` is a second root unit from the same binary as +//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints +//! order ids, decides acceptance and refusal, actuates, and records outcomes. +//! The survival daemon takes no part: `control.cc` never learns an order type. +//! +//! An order is a closed enum, never a command string: install packages, act on +//! a systemd unit, or deploy a target through the same commit primitive +//! `idunn up` uses. Every unit is allowed except the always-refused set, which +//! is compiled in here and read from nowhere: no binding field, record or flag +//! holds it. The `#operator` post is a retried projection of the store. +//! +//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the +//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and +//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE +//! `mind-control check`; START record before actuation, at most once; RUN; +//! RESULT outcome; POST accepted and result. +//! +//! No refusal or error here carries a value the issuer supplied: it names the +//! field and a code. A refused unit that a record names was resolved by +//! systemd and passed the unit grammar. + +use std::collections::{BTreeSet, VecDeque}; +use std::fmt; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::control_plane::{ + command_state, commit_deployment_command, decode_record, default_state_store, now_millis, + path_value, require_id, require_selector, string_value, typed_envelope, +}; +use crate::runner_evidence::{emit, escape}; + +const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; +const START_SCHEMA: &str = "idunn.steward_order_start.v1"; +const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; +const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; + +const REQUEST_FRAME_LIMIT: usize = 16 * 1024; +const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; +const TAIL_BYTES: usize = 8 * 1024; +const REASON_LIMIT: usize = 300; +const PACKAGE_LIMIT: usize = 16; +const UNIT_LIMIT: usize = 200; +const CLOSURE_LIMIT: usize = 512; +const LIST_LIMIT: usize = 50; +const UNIT_SUFFIXES: [&str; 6] = [".service", ".socket", ".timer", ".path", ".mount", ".target"]; + +/// The always-refused set: Idunn's own units and the minds' own. Matched on +/// every name a unit answers to, ASCII case-insensitively (systemd units are +/// case-sensitive, so the fold only widens the refusal). +fn is_refused_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + ["idunn-", "mind-self@", "mind-persona@"] + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +// ---------------------------------------------------------------- grammar -- + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Code { + Empty, + TooLong, + TooMany, + BadCharacter, + BadShape, + Malformed, + Unauthorized, + Unknown, + Unavailable, +} + +/// A refusal names the field and a code. It never carries the offending value. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Refusal { + pub(crate) field: String, + pub(crate) code: Code, +} + +impl Refusal { + fn new(field: &'static str, code: Code) -> Self { + Self { + field: field.to_owned(), + code, + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{} refused: {:?}", self.field, self.code) + } +} + +impl std::error::Error for Refusal {} + +fn validate_package(spec: &str) -> Result<(), Refusal> { + const FIELD: &str = "package"; + if spec.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if spec.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let (name, version) = match spec.split_once('=') { + Some((name, version)) => (name, Some(version)), + None => (spec, None), + }; + let bytes = name.as_bytes(); + if bytes.len() < 2 { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); + let rest = bytes[1..].iter().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') + }); + if !head || !rest { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + // apt reads `name-` as a removal and `name.deb` as a local file. + if name.ends_with('-') || name.ends_with(".deb") { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if let Some(version) = version { + if version.is_empty() { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if !version.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') + }) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + } + Ok(()) +} + +fn validate_unit(unit: &str) -> Result<(), Refusal> { + const FIELD: &str = "unit"; + if unit.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if unit.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let allowed = unit.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') + }); + if !allowed || unit.starts_with('-') { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + match UNIT_SUFFIXES + .iter() + .find_map(|suffix| unit.strip_suffix(suffix)) + { + Some(stem) if !stem.is_empty() => Ok(()), + _ => Err(Refusal::new(FIELD, Code::BadShape)), + } +} + +fn validate_reason(reason: &str) -> Result<(), Refusal> { + const FIELD: &str = "reason"; + if reason.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if reason.len() > REASON_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Verb { + Start, + Stop, + Restart, + Reload, +} + +impl Verb { + fn as_str(self) -> &'static str { + match self { + Verb::Start => "start", + Verb::Stop => "stop", + Verb::Restart => "restart", + Verb::Reload => "reload", + } + } + + fn parse(word: &str) -> Option { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + .into_iter() + .find(|verb| verb.as_str() == word) + } + + /// Verbs systemd propagates down the dependency graph. + fn propagates(self) -> bool { + matches!(self, Verb::Stop | Verb::Restart) + } +} + +/// The closed set of things an order can ask for. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Operation { + PackageInstall { packages: Vec }, + UnitAction { verb: Verb, unit: String }, + Deploy { selector: String }, +} + +/// One fixed argv for `systemd-run`, built from the typed operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Step { + pub(crate) argv: Vec, + pub(crate) environment: Vec<(String, String)>, +} + +impl Operation { + fn validate(&self) -> Result<(), Refusal> { + match self { + Operation::PackageInstall { packages } => { + if packages.is_empty() { + return Err(Refusal::new("package", Code::Empty)); + } + if packages.len() > PACKAGE_LIMIT { + return Err(Refusal::new("package", Code::TooMany)); + } + packages.iter().try_for_each(|package| validate_package(package)) + } + Operation::UnitAction { unit, .. } => validate_unit(unit), + Operation::Deploy { selector } => require_selector(selector) + .map_err(|_| Refusal::new("selector", Code::BadShape)), + } + } + + fn steps(&self) -> Vec { + let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); + let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; + match self { + Operation::PackageInstall { packages } => { + let mut install: Vec = words(&["apt-get", "install", "-y", "-q", "--"]); + install.extend(packages.iter().cloned()); + vec![ + Step { + argv: words(&["apt-get", "update", "-q"]), + environment: apt_environment.clone(), + }, + Step { + argv: install, + environment: apt_environment, + }, + ] + } + Operation::UnitAction { verb, unit } => vec![Step { + argv: vec![ + "systemctl".into(), + verb.as_str().into(), + "--".into(), + unit.clone(), + ], + environment: Vec::new(), + }], + Operation::Deploy { .. } => Vec::new(), + } + } + + fn describe(&self) -> String { + match self { + Operation::PackageInstall { packages } => { + format!("package-install {}", packages.join(" ")) + } + Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), + Operation::Deploy { selector } => format!("deploy {selector}"), + } + } +} + +// ---------------------------------------------------------------- records -- + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] +pub(crate) struct OrderRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) principal: String, + #[cultcache(key = 3)] + pub(crate) issuer_uid: u32, + #[cultcache(key = 4)] + pub(crate) issuer_unit: String, + #[cultcache(key = 5)] + pub(crate) issuer_invocation: String, + #[cultcache(key = 6)] + pub(crate) issued_at_unix_millis: u64, + #[cultcache(key = 7)] + pub(crate) operation: Operation, + #[cultcache(key = 8)] + pub(crate) reason: String, +} + +impl fmt::Debug for OrderRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OrderRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_start", + schema = "idunn.steward_order_start.v1" +)] +pub(crate) struct StartRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) started_at_unix_millis: u64, + /// The transient unit name, or the deployment command id. + #[cultcache(key = 3)] + pub(crate) actuation: String, +} + +impl fmt::Debug for StartRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StartRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Via { + Name, + Alias, + Propagation, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Refused { + AlwaysRefused { unit: String, via: Via }, + Braked, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Outcome { + Ran, + Failed { + exit_code: Option, + signal: Option, + }, + Refused(Refused), + Lost, + Submitted { command_id: String }, +} + +impl Outcome { + fn describe(&self) -> String { + match self { + Outcome::Ran => "ran (exit 0)".into(), + Outcome::Failed { + exit_code: Some(code), + .. + } => format!("failed (exit {code})"), + Outcome::Failed { + signal: Some(signal), + .. + } => format!("failed (signal {signal})"), + Outcome::Failed { .. } => "failed (not run to an exit)".into(), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( + "refused: always-refused unit {} reached by {}", + escape(unit.as_bytes()), + match via { + Via::Name => "name", + Via::Alias => "alias", + Via::Propagation => "propagation", + } + ), + Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Lost => "lost: the orders service restarted mid-run".into(), + Outcome::Submitted { command_id } => { + format!("submitted deployment {}", escape(command_id.as_bytes())) + } + } + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_outcome", + schema = "idunn.steward_order_outcome.v1" +)] +pub(crate) struct OutcomeRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) ended_at_unix_millis: u64, + #[cultcache(key = 3)] + pub(crate) outcome: Outcome, + #[cultcache(key = 4)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 5)] + pub(crate) stderr_tail: Vec, +} + +/// Tails stay out of `Debug`: they are raw program output. +impl fmt::Debug for OutcomeRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutcomeRecord") + .field("order_id", &self.order_id) + .field("outcome", &self.outcome) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum PostKind { + Accepted, + Result, +} + +impl PostKind { + fn as_str(self) -> &'static str { + match self { + PostKind::Accepted => "accepted", + PostKind::Result => "result", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_post", + schema = "idunn.steward_order_post.v1" +)] +pub(crate) struct PostRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `:`. + #[cultcache(key = 1)] + pub(crate) post_key: String, + #[cultcache(key = 2)] + pub(crate) event_id: String, + #[cultcache(key = 3)] + pub(crate) posted_at_unix_millis: u64, +} + +// ------------------------------------------------------------------ store -- + +/// `orders.cc`: root-only, written by `idunn orders serve` alone. +pub(crate) struct OrderStore { + directory: PathBuf, +} + +#[derive(Default)] +struct Ledger { + orders: Vec, + starts: Vec, + outcomes: Vec, + posts: Vec, +} + +impl Ledger { + fn start_of(&self, order_id: &str) -> Option<&StartRecord> { + self.starts.iter().find(|start| start.order_id == order_id) + } + + fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { + self.outcomes + .iter() + .find(|outcome| outcome.order_id == order_id) + } + + fn posted(&self, order_id: &str, kind: PostKind) -> bool { + let key = post_key(order_id, kind); + self.posts.iter().any(|post| post.post_key == key) + } +} + +fn post_key(order_id: &str, kind: PostKind) -> String { + format!("{order_id}:{}", kind.as_str()) +} + +impl OrderStore { + pub(crate) fn new(directory: &Path) -> Self { + Self { + directory: directory.to_owned(), + } + } + + fn path(&self) -> PathBuf { + self.directory.join("orders.cc") + } + + fn prepare(&self) -> Result<()> { + std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; + std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) + .context("making the orders directory private") + } + + /// Insert a record whose (type, key) is absent. `false` means it was there. + fn put(&self, key: &str, schema: &str, value: &T) -> Result { + self.prepare()?; + let path = self.path(); + let now = now_millis()?; + let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: T::TYPE.into(), + key: key.into(), + current: None, + }], + &[typed_envelope(key, T::TYPE, schema, value, now)?], + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .context("making the orders store private")?; + Ok(inserted) + } + + fn ledger(&self) -> Result { + let path = self.path(); + let mut ledger = Ledger::default(); + if !path.exists() { + return Ok(ledger); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .context("reading the orders store")?; + for envelope in &envelopes { + let schema = envelope.schema_id.as_deref(); + let key = envelope.key.as_str(); + match envelope.r#type.as_str() { + OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { + let record: OrderRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an order is stored under another key"); + ledger.orders.push(record); + } + StartRecord::TYPE if schema == Some(START_SCHEMA) => { + let record: StartRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "a start is stored under another key"); + ledger.starts.push(record); + } + OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { + let record: OutcomeRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an outcome is stored under another key"); + ledger.outcomes.push(record); + } + PostRecord::TYPE if schema == Some(POST_SCHEMA) => { + let record: PostRecord = decode_record(envelope)?; + ensure!(record.post_key == key, "a post is stored under another key"); + ledger.posts.push(record); + } + _ => bail!("the orders store holds a document that is not an order record"), + } + } + ledger + .orders + .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); + Ok(ledger) + } +} + +// ------------------------------------------------------------------ ports -- + +/// What `systemctl show` says about one unit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct UnitFacts { + pub(crate) id: String, + pub(crate) names: Vec, + pub(crate) required_by: Vec, + pub(crate) bound_by: Vec, + pub(crate) consists_of: Vec, +} + +impl UnitFacts { + fn identities(&self) -> impl Iterator { + std::iter::once(&self.id).chain(self.names.iter()) + } + + /// The units systemd stops with this one. + fn dependents(&self) -> impl Iterator { + self.required_by + .iter() + .chain(self.bound_by.iter()) + .chain(self.consists_of.iter()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RunReport { + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stderr_tail: Vec, +} + +pub(crate) trait Systemd: Send + Sync { + fn show(&self, unit: &str) -> Result; + fn invocation(&self, unit: &str) -> Option; + /// Run one step as the transient unit `unit`, to its end. + fn run(&self, unit: &str, step: &Step) -> Result; +} + +pub(crate) struct Peer { + pub(crate) uid: u32, + pub(crate) issuer_unit: String, +} + +pub(crate) trait PeerCredentials: Send + Sync { + fn peer(&self, stream: &UnixStream) -> Result; +} + +pub(crate) trait BrakeCheck: Send + Sync { + /// `true` only when the instance's brake check exits 0. + fn clear(&self, principal: &str) -> bool; +} + +pub(crate) trait Homeserver: Send + Sync { + /// Idempotent by `txn_id`; returns the event id. + fn post(&self, txn_id: &str, text: &str) -> Result; +} + +pub(crate) struct Ctx { + pub(crate) store: OrderStore, + pub(crate) state_store: PathBuf, + pub(crate) steward_uid: u32, + pub(crate) principal: String, + pub(crate) systemd: Arc, + pub(crate) peers: Arc, + pub(crate) brake: Arc, + pub(crate) homeserver: Arc, +} + +// ------------------------------------------------------------------ scope -- + +fn refused(unit: &str, via: Via) -> Refused { + Refused::AlwaysRefused { + unit: unit.to_owned(), + via, + } +} + +/// Would `verb` on `unit` reach an always-refused unit: by its name, by an +/// alias it resolves from, or, for the verbs systemd propagates, by the units +/// systemd stops with it. Resolution reads systemd before anything runs. +fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { + if is_refused_name(unit) { + return Ok(Some(refused(unit, Via::Name))); + } + let facts = systemd.show(unit)?; + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Alias))); + } + if !verb.propagates() { + return Ok(None); + } + let mut seen: BTreeSet = facts.identities().cloned().collect(); + seen.insert(unit.to_owned()); + let mut queue: VecDeque = facts.dependents().cloned().collect(); + while let Some(next) = queue.pop_front() { + if is_refused_name(&next) { + return Ok(Some(refused(&next, Via::Propagation))); + } + if !seen.insert(next.clone()) { + continue; + } + ensure!( + seen.len() <= CLOSURE_LIMIT, + "the units a stop would reach are too many to check" + ); + let facts = systemd.show(&next)?; + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Propagation))); + } + seen.extend(facts.identities().cloned()); + queue.extend(facts.dependents().cloned()); + } + Ok(None) +} + +// -------------------------------------------------------------- execution -- + +fn tail(bytes: &[u8]) -> Vec { + bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() +} + +fn conclude( + ctx: &Ctx, + order_id: &str, + outcome: Outcome, + stdout_tail: &[u8], + stderr_tail: &[u8], +) -> Result<()> { + ctx.store.put( + order_id, + OUTCOME_SCHEMA, + &OutcomeRecord { + schema_version: OUTCOME_SCHEMA.into(), + order_id: order_id.into(), + ended_at_unix_millis: now_millis()?, + outcome, + stdout_tail: tail(stdout_tail), + stderr_tail: tail(stderr_tail), + }, + )?; + Ok(()) +} + +fn order_uuid(order_id: &str) -> &str { + order_id.strip_prefix("ord-").unwrap_or(order_id) +} + +pub(crate) fn requester(principal: &str, order_id: &str) -> String { + format!("steward:{principal} order {order_id}") +} + +/// Run one accepted order. A start record is written before anything is +/// actuated and an order that has one is never run again. +pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { + let ledger = ctx.store.ledger()?; + let order = ledger + .orders + .iter() + .find(|order| order.order_id == order_id) + .context("the order is not in the store")?; + if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { + return Ok(()); + } + if order.operation.validate().is_err() { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the stored order no longer satisfies the grammar", + ); + } + if let Operation::UnitAction { verb, unit } = &order.operation { + match refusal_for(ctx.systemd.as_ref(), unit, *verb) { + Ok(None) => {} + Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), + Err(_) => { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the unit could not be resolved, so it was not acted on", + ); + } + } + } + if !ctx.brake.clear(&order.principal) { + return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); + } + let uuid = order_uuid(order_id); + let command_id = format!("up-{uuid}"); + let steps = order.operation.steps(); + let actuation = match &order.operation { + Operation::Deploy { .. } => command_id.clone(), + _ => format!("idunn-order-{uuid}"), + }; + let started = StartRecord { + schema_version: START_SCHEMA.into(), + order_id: order_id.into(), + started_at_unix_millis: now_millis()?, + actuation, + }; + if !ctx.store.put(order_id, START_SCHEMA, &started)? { + return Ok(()); + } + if let Operation::Deploy { selector } = &order.operation { + let outcome = match commit_deployment_command( + &ctx.state_store, + &command_id, + selector, + &requester(&order.principal, order_id), + ) { + Ok(_) => Outcome::Submitted { command_id }, + Err(_) => Outcome::Failed { + exit_code: None, + signal: None, + }, + }; + return conclude(ctx, order_id, outcome, b"", b""); + } + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let mut outcome = Outcome::Ran; + for (index, step) in steps.iter().enumerate() { + let unit = if steps.len() == 1 { + format!("idunn-order-{uuid}") + } else { + format!("idunn-order-{uuid}-{}", index + 1) + }; + match ctx.systemd.run(&unit, step) { + Ok(report) => { + stdout = report.stdout_tail; + stderr = report.stderr_tail; + if report.exit_code != Some(0) { + outcome = Outcome::Failed { + exit_code: report.exit_code, + signal: report.signal, + }; + break; + } + } + Err(_) => { + outcome = Outcome::Failed { + exit_code: None, + signal: None, + }; + stderr = b"the transient unit could not be started".to_vec(); + break; + } + } + } + conclude(ctx, order_id, outcome, &stdout, &stderr) +} + +/// On service start: an order with a start record and no outcome ran, or began +/// to, in a life that ended. It is Lost and never run again. Returns the +/// orders that never started, oldest first. +pub(crate) fn recover(ctx: &Ctx) -> Result> { + let ledger = ctx.store.ledger()?; + for start in &ledger.starts { + if ledger.outcome_of(&start.order_id).is_none() { + conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; + } + } + Ok(ledger + .orders + .iter() + .filter(|order| { + ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() + }) + .map(|order| order.order_id.clone()) + .collect()) +} + +// ---------------------------------------------------------------- posting -- + +struct Due { + order_id: String, + kind: PostKind, + text: String, +} + +fn quoted(reason: &str) -> String { + format!("\"{}\"", escape(reason.as_bytes())) +} + +fn accepted_text(order: &OrderRecord) -> String { + format!( + "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", + order.order_id, + escape(order.operation.describe().as_bytes()), + escape(order.issuer_unit.as_bytes()), + escape(order.issuer_invocation.as_bytes()), + quoted(&order.reason) + ) +} + +/// A deploy's command has finished once its state is neither queued nor running. +fn deploy_finished(state: &str) -> bool { + !matches!(state, "queued" | "running") +} + +fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { + let mut due = Vec::new(); + for order in &ledger.orders { + let id = &order.order_id; + if !ledger.posted(id, PostKind::Accepted) { + due.push(Due { + order_id: id.clone(), + kind: PostKind::Accepted, + text: accepted_text(order), + }); + } + if ledger.posted(id, PostKind::Result) { + continue; + } + let Some(record) = ledger.outcome_of(id) else { + continue; + }; + let mut line = record.outcome.describe(); + if let Outcome::Submitted { command_id } = &record.outcome { + match command_state(&ctx.state_store, command_id) { + Ok((state, detail)) if deploy_finished(state) => { + line = format!("{line}: {state} {}", escape(detail.as_bytes())); + } + _ => continue, + } + } + due.push(Due { + order_id: id.clone(), + kind: PostKind::Result, + text: format!( + "Steward order {id}: {line}\n{}", + escape(order.operation.describe().as_bytes()) + ), + }); + } + due +} + +/// Post what is due, in order, stopping at the first failure. Returns how many +/// went out. +pub(crate) fn post_pass(ctx: &Ctx) -> Result { + let ledger = ctx.store.ledger()?; + let mut sent = 0; + for due in due_posts(ctx, &ledger) { + let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); + let event_id = ctx.homeserver.post(&transaction, &due.text)?; + ctx.store.put( + &post_key(&due.order_id, due.kind), + POST_SCHEMA, + &PostRecord { + schema_version: POST_SCHEMA.into(), + post_key: post_key(&due.order_id, due.kind), + event_id, + posted_at_unix_millis: now_millis()?, + }, + )?; + sent += 1; + } + Ok(sent) +} + +// ----------------------------------------------------------------- frames -- + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct OrderRequest { + pub(crate) operation: Operation, + pub(crate) reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ShowRequest { + pub(crate) id: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Request { + Order(OrderRequest), + Show(ShowRequest), + List, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderView { + pub(crate) order: OrderRecord, + pub(crate) start: Option, + pub(crate) outcome: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderLine { + pub(crate) order_id: String, + pub(crate) status: String, + pub(crate) summary: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Response { + Accepted { order_id: String }, + Refused(Refusal), + Show(Box), + List(Vec), +} + +fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { + let bytes = rmp_serde::to_vec_named(value)?; + stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; + stream.write_all(&bytes)?; + Ok(stream.flush()?) +} + +fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { + let mut length = [0u8; 4]; + stream.read_exact(&mut length)?; + let length = u32::from_be_bytes(length) as usize; + ensure!(length <= limit, "a frame exceeds its limit"); + let mut bytes = vec![0u8; length]; + stream.read_exact(&mut bytes)?; + Ok(bytes) +} + +fn status_word(view: &OrderView) -> &'static str { + match (&view.outcome, &view.start) { + (Some(record), _) => match record.outcome { + Outcome::Ran => "ran", + Outcome::Failed { .. } => "failed", + Outcome::Refused(_) => "refused", + Outcome::Lost => "lost", + Outcome::Submitted { .. } => "submitted", + }, + (None, Some(_)) => "running", + (None, None) => "accepted", + } +} + +fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { + OrderView { + order: order.clone(), + start: ledger.start_of(&order.order_id).cloned(), + outcome: ledger.outcome_of(&order.order_id).cloned(), + } +} + +fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { + write_frame(stream, &response) +} + +fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { + answer(stream, Response::Refused(Refusal::new(field, code))) +} + +/// Serve one connection. The peer is authenticated before its frame is read. +pub(crate) fn handle( + ctx: &Ctx, + stream: &mut UnixStream, + queue: &mpsc::Sender, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + let peer = ctx.peers.peer(stream)?; + if peer.uid != ctx.steward_uid { + eprintln!("steward socket: refused a peer that is not the steward"); + return refuse(stream, "peer", Code::Unauthorized); + } + let request = read_frame(stream, REQUEST_FRAME_LIMIT) + .ok() + .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); + let Some(request) = request else { + eprintln!("steward socket: refused a malformed frame"); + return refuse(stream, "frame", Code::Malformed); + }; + match request { + Request::Order(request) => { + let checked = + validate_reason(&request.reason).and_then(|()| request.operation.validate()); + if let Err(refusal) = checked { + eprintln!("steward socket: refused an order at the grammar"); + return answer(stream, Response::Refused(refusal)); + } + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: format!("ord-{}", Uuid::new_v4()), + principal: ctx.principal.clone(), + issuer_uid: peer.uid, + issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), + issuer_unit: peer.issuer_unit, + issued_at_unix_millis: now_millis()?, + operation: request.operation, + reason: request.reason, + }; + match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { + Ok(true) => {} + _ => return refuse(stream, "store", Code::Unavailable), + } + queue + .send(order.order_id.clone()) + .map_err(|_| anyhow!("the executor has stopped"))?; + answer( + stream, + Response::Accepted { + order_id: order.order_id, + }, + ) + } + Request::Show(request) => { + if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) + { + return refuse(stream, "id", Code::BadShape); + } + let ledger = ctx.store.ledger()?; + match ledger.orders.iter().find(|order| order.order_id == request.id) { + Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), + None => refuse(stream, "id", Code::Unknown), + } + } + Request::List => { + let ledger = ctx.store.ledger()?; + let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); + let lines = ledger + .orders + .iter() + .skip(skip) + .map(|order| { + let view = view_of(&ledger, order); + OrderLine { + order_id: order.order_id.clone(), + status: status_word(&view).into(), + summary: order.operation.describe(), + } + }) + .collect(); + answer(stream, Response::List(lines)) + } + } +} + +// ------------------------------------------------------------- real ports -- + +/// `systemctl` and `systemd-run`, found by name. +pub(crate) struct HostSystemd; + +fn parse_names(value: &str) -> Vec { + value.split_whitespace().map(str::to_owned).collect() +} + +fn parse_show(output: &str) -> Result { + let mut facts = UnitFacts::default(); + for line in output.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + match key { + "Id" => facts.id = value.to_owned(), + "Names" => facts.names = parse_names(value), + "RequiredBy" => facts.required_by = parse_names(value), + "BoundBy" => facts.bound_by = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), + _ => {} + } + } + ensure!(!facts.id.is_empty(), "systemctl named no unit"); + Ok(facts) +} + +impl Systemd for HostSystemd { + fn show(&self, unit: &str) -> Result { + let output = Command::new("systemctl") + .args([ + "show", + "--no-pager", + "--property=Id,Names,RequiredBy,BoundBy,ConsistsOf", + "--", + unit, + ]) + .stdin(Stdio::null()) + .output() + .context("running systemctl show")?; + ensure!(output.status.success(), "systemctl show failed"); + parse_show(&String::from_utf8_lossy(&output.stdout)) + } + + fn invocation(&self, unit: &str) -> Option { + let output = Command::new("systemctl") + .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) + .arg(unit) + .stdin(Stdio::null()) + .output() + .ok()?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + (output.status.success() && !value.is_empty()).then_some(value) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + let mut command = Command::new("systemd-run"); + command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); + command.args(["-p", "RuntimeMaxSec=15min"]); + for (name, value) in &step.environment { + command.arg(format!("--setenv={name}={value}")); + } + let output = command + .arg("--") + .args(&step.argv) + .stdin(Stdio::null()) + .output() + .context("running systemd-run")?; + Ok(RunReport { + exit_code: output.status.code(), + signal: output.status.signal(), + stdout_tail: tail(&output.stdout), + stderr_tail: tail(&output.stderr), + }) + } +} + +/// The last `.service` or `.scope` in a process's cgroup path names its issuer. +fn unit_from_cgroup(text: &str) -> String { + let line = text + .lines() + .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) + .unwrap_or_default(); + let path = line.rsplit_once(':').map_or("", |(_, path)| path); + path.rsplit('/') + .find(|segment| segment.ends_with(".service") || segment.ends_with(".scope")) + .filter(|segment| segment.len() <= UNIT_LIMIT) + .map_or_else(|| "unknown".to_owned(), str::to_owned) +} + +/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. +pub(crate) struct KernelPeers; + +impl PeerCredentials for KernelPeers { + fn peer(&self, stream: &UnixStream) -> Result { + use std::os::fd::AsRawFd; + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = std::mem::size_of::() as libc::socklen_t; + // SAFETY: the descriptor is a live socket and the pointer and length + // describe `credentials`, which outlives the call. + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&mut credentials as *mut libc::ucred).cast(), + &mut length, + ) + }; + ensure!( + status == 0 && length as usize == std::mem::size_of::(), + "peer credentials are unavailable" + ); + let cgroup = + std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); + Ok(Peer { + uid: credentials.uid, + issuer_unit: unit_from_cgroup(&cgroup), + }) + } +} + +/// `mind-control --instance check`; a missing binary is a held brake. +pub(crate) struct MindControl { + pub(crate) program: PathBuf, +} + +impl BrakeCheck for MindControl { + fn clear(&self, principal: &str) -> bool { + Command::new(&self.program) + .args(["--instance", principal, "check"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} + +/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is +/// never chunked. The token is read from its credential file for each post. +pub(crate) struct Synapse { + pub(crate) address: std::net::SocketAddr, + pub(crate) room: String, + pub(crate) token_file: PathBuf, +} + +fn percent_encode(text: &str) -> String { + let mut encoded = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +impl Homeserver for Synapse { + fn post(&self, txn_id: &str, text: &str) -> Result { + let token = std::fs::read_to_string(&self.token_file) + .context("the homeserver token is unreadable")?; + let token = token.trim(); + ensure!( + !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), + "the homeserver token is not a bearer token" + ); + let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; + let head = format!( + "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ + Host: {}\r\nAuthorization: Bearer {token}\r\n\ + Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", + percent_encode(&self.room), + self.address, + body.len() + ); + let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) + .context("the homeserver is unreachable")?; + socket.set_read_timeout(Some(Duration::from_secs(10)))?; + socket.set_write_timeout(Some(Duration::from_secs(10)))?; + socket.write_all(head.as_bytes())?; + socket.write_all(&body)?; + let mut reply = Vec::new(); + socket + .take(RESPONSE_FRAME_LIMIT as u64) + .read_to_end(&mut reply) + .context("the homeserver reply was cut short")?; + let split = reply + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("the homeserver reply has no body")?; + let status = String::from_utf8_lossy(&reply[..split]); + ensure!( + status.split_whitespace().nth(1) == Some("200"), + "the homeserver refused the post" + ); + #[derive(Deserialize)] + struct Sent { + event_id: String, + } + let sent: Sent = serde_json::from_slice(&reply[split + 4..]) + .map_err(|_| anyhow!("the homeserver reply has no event id"))?; + Ok(sent.event_id) + } +} + +// ---------------------------------------------------------------- binding -- + +/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. +/// It lists no unit of any kind, and any field outside this set is refused. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct StewardBinding { + pub(crate) principal: String, + pub(crate) user: String, + pub(crate) mind_control: PathBuf, + pub(crate) room: String, + pub(crate) homeserver: String, + pub(crate) token_credential: String, +} + +impl StewardBinding { + pub(crate) fn parse(text: &str) -> Result { + let binding: Self = toml::from_str(text) + .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; + binding.validate()?; + Ok(binding) + } + + fn validate(&self) -> Result<()> { + require_id(&self.principal, "steward principal")?; + ensure!( + !self.principal.starts_with('-') && !self.principal.contains('/'), + "steward principal is invalid" + ); + let user_ok = !self.user.is_empty() + && self.user.len() <= 32 + && self.user.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) + && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); + ensure!(user_ok && self.user != "root", "steward user is invalid or root"); + ensure!( + self.mind_control.is_absolute(), + "mind_control must be an absolute path" + ); + let room_ok = self.room.starts_with('!') + && self.room.len() <= 255 + && self.room.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') + }); + ensure!(room_ok, "room must be a Matrix room id"); + self.address()?; + let credential_ok = !self.token_credential.is_empty() + && !self.token_credential.starts_with('.') + && self.token_credential.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') + }); + ensure!(credential_ok, "token_credential is not a credential name"); + Ok(()) + } + + fn address(&self) -> Result { + let address: std::net::SocketAddr = self + .homeserver + .strip_prefix("http://") + .and_then(|rest| rest.parse().ok()) + .context("homeserver must be http://:")?; + ensure!(address.ip().is_loopback(), "homeserver must be loopback"); + Ok(address) + } +} + +/// Root-admitted: owned by root and writable by no one else. +fn admitted(owner: u32, mode: u32) -> bool { + owner == 0 && mode & 0o022 == 0 +} + +fn load_binding(path: &Path) -> Result { + let metadata = std::fs::metadata(path).context("reading the steward binding")?; + ensure!( + metadata.is_file() && admitted(metadata.uid(), metadata.mode()), + "the steward binding is not a root-owned file closed to writes" + ); + let text = std::fs::read_to_string(path).context("reading the steward binding")?; + StewardBinding::parse(&text) +} + +fn user_uid(name: &str) -> Result { + let name = std::ffi::CString::new(name)?; + // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in + // `buffer`, which outlives every read of them below (none are read). + let (status, found, uid) = unsafe { + let mut passwd: libc::passwd = std::mem::zeroed(); + let mut buffer = vec![0 as libc::c_char; 4096]; + let mut result: *mut libc::passwd = std::ptr::null_mut(); + let status = libc::getpwnam_r( + name.as_ptr(), + &mut passwd, + buffer.as_mut_ptr(), + buffer.len(), + &mut result, + ); + (status, !result.is_null(), passwd.pw_uid) + }; + ensure!(status == 0 && found, "the steward user is unknown"); + ensure!(uid != 0, "the steward user resolves to root"); + Ok(uid) +} + +// ------------------------------------------------------------------ serve -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ServeOptions { + pub(crate) state_store: PathBuf, + pub(crate) store: PathBuf, + pub(crate) steward_binding: PathBuf, + pub(crate) socket: PathBuf, +} + +const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; +const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; +const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; + +pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { + ensure!( + args.next().as_deref() == Some("serve"), + "idunn orders takes the subcommand serve" + ); + let mut options = ServeOptions { + state_store: default_state_store(), + store: DEFAULT_STORE.into(), + steward_binding: DEFAULT_BINDING.into(), + socket: DEFAULT_SOCKET.into(), + }; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, + "--store" => options.store = path_value(&mut args, "--store")?, + "--steward-binding" => { + options.steward_binding = path_value(&mut args, "--steward-binding")? + } + "--socket" => options.socket = path_value(&mut args, "--socket")?, + _ => bail!("an orders serve option is not part of the grammar"), + } + } + Ok(options) +} + +/// Run a thread whose end ends the service: systemd restarts it, and the +/// start recovery settles what it left. +fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { + struct ExitOnDrop(&'static str); + impl Drop for ExitOnDrop { + fn drop(&mut self) { + eprintln!("steward orders: the {} thread ended", self.0); + std::process::exit(1); + } + } + thread::Builder::new() + .name(name.into()) + .spawn(move || { + let _guard = ExitOnDrop(name); + work(); + }) + .expect("a thread can be spawned"); +} + +fn bind_socket(path: &Path) -> Result { + if let Ok(metadata) = std::fs::symlink_metadata(path) { + use std::os::unix::fs::FileTypeExt; + ensure!( + metadata.file_type().is_socket(), + "the socket path holds something that is not a socket" + ); + std::fs::remove_file(path).context("removing a stale socket")?; + } + let listener = UnixListener::bind(path).context("binding the steward socket")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) + .context("setting the steward socket mode")?; + Ok(listener) +} + +pub(crate) fn serve(options: ServeOptions) -> Result<()> { + let binding = load_binding(&options.steward_binding)?; + let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") + .map(PathBuf::from) + .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; + let ctx = Arc::new(Ctx { + store: OrderStore::new(&options.store), + state_store: options.state_store, + steward_uid: user_uid(&binding.user)?, + principal: binding.principal.clone(), + systemd: Arc::new(HostSystemd), + peers: Arc::new(KernelPeers), + brake: Arc::new(MindControl { + program: binding.mind_control.clone(), + }), + homeserver: Arc::new(Synapse { + address: binding.address()?, + room: binding.room.clone(), + token_file: credentials.join(&binding.token_credential), + }), + }); + ctx.store.prepare()?; + let pending = recover(&ctx)?; + let listener = bind_socket(&options.socket)?; + let (queue, orders) = mpsc::channel::(); + for order_id in pending { + queue.send(order_id)?; + } + let executor = Arc::clone(&ctx); + vital("executor", move || { + for order_id in orders { + if let Err(error) = execute(&executor, &order_id) { + eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); + } + } + }); + let poster = Arc::clone(&ctx); + vital("poster", move || { + let mut failures = 0u32; + loop { + let pause = match post_pass(&poster) { + Ok(_) => { + failures = 0; + 2 + } + Err(error) => { + failures = failures.saturating_add(1); + eprintln!("steward orders: posting is delayed: {error:#}"); + (2u64 << failures.min(5)).min(60) + } + }; + thread::sleep(Duration::from_secs(pause)); + } + }); + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + if let Err(error) = handle(&ctx, &mut stream, &queue) { + eprintln!("steward socket: a connection ended in error: {error:#}"); + } + } + Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), + } + } + bail!("the steward socket stopped accepting") +} + +// ----------------------------------------------------------------- client -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OrderAction { + Issue { operation: Operation, reason: String }, + Show { id: String }, + List, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct OrderCommand { + pub(crate) socket: PathBuf, + pub(crate) action: OrderAction, +} + +pub(crate) fn parse_order(mut args: impl Iterator) -> Result { + let kind = args.next().context("idunn order requires an operation")?; + let mut socket = PathBuf::from(DEFAULT_SOCKET); + let mut reason = None; + let mut positional = Vec::new(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--socket" => socket = path_value(&mut args, "--socket")?, + "--reason" => reason = Some(string_value(&mut args, "--reason")?), + word if word.starts_with('-') => bail!("an order option is not part of the grammar"), + _ => positional.push(argument), + } + } + let action = match kind.as_str() { + "list" => { + ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); + OrderAction::List + } + "show" => { + ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); + OrderAction::Show { + id: positional.remove(0), + } + } + "package-install" | "unit" | "deploy" => { + let operation = match kind.as_str() { + "package-install" => Operation::PackageInstall { + packages: positional, + }, + "unit" => { + ensure!(positional.len() == 2, "unit takes a verb and a unit"); + let verb = Verb::parse(&positional[0]) + .context("the unit verb is not start, stop, restart or reload")?; + Operation::UnitAction { + verb, + unit: positional.remove(1), + } + } + _ => { + ensure!(positional.len() == 1, "deploy takes one selector"); + Operation::Deploy { + selector: positional.remove(0), + } + } + }; + let reason = reason.context("an order requires --reason")?; + validate_reason(&reason)?; + operation.validate()?; + OrderAction::Issue { operation, reason } + } + _ => bail!("the order operation is not part of the grammar"), + }; + Ok(OrderCommand { socket, action }) +} + +fn call(socket: &Path, request: &Request) -> Result { + let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + write_frame(&mut stream, request)?; + let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; + rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) +} + +fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); + let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); +} + +/// Raw lines for the terminal; `emit` is their only way out. +fn render_view(view: &OrderView) -> Vec> { + let order = &view.order; + let mut lines: Vec> = vec![ + format!("order {}", order.order_id).into_bytes(), + format!("status {}", status_word(view)).into_bytes(), + format!("operation {}", order.operation.describe()).into_bytes(), + format!( + "issuer {} (invocation {}, uid {})", + order.issuer_unit, order.issuer_invocation, order.issuer_uid + ) + .into_bytes(), + format!("reason {}", quoted(&order.reason)).into_bytes(), + ]; + if let Some(start) = &view.start { + lines.push(format!("started {}", start.actuation).into_bytes()); + } + if let Some(record) = &view.outcome { + lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); + render_tail(&mut lines, "stdout", &record.stdout_tail); + render_tail(&mut lines, "stderr", &record.stderr_tail); + } + lines +} + +pub(crate) fn run_order(command: OrderCommand) -> Result<()> { + let request = match command.action { + OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), + OrderAction::Show { id } => Request::Show(ShowRequest { id }), + OrderAction::List => Request::List, + }; + let mut out = std::io::stdout().lock(); + match call(&command.socket, &request)? { + Response::Accepted { order_id } => writeln!(out, "{order_id}")?, + Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), + Response::Show(view) => emit(&mut out, &render_view(&view))?, + Response::List(lines) => { + let lines: Vec> = lines + .iter() + .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) + .collect(); + emit(&mut out, &lines)? + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tempfile::TempDir; + + use super::*; + + const STEWARD: u32 = 1000; + const CANARY: &str = "CANARY-7f3a91"; + + struct FakeSystemd { + units: Mutex>, + runs: Mutex>, + report: Mutex, + } + + impl Systemd for FakeSystemd { + fn show(&self, unit: &str) -> Result { + let known = self.units.lock().unwrap().get(unit).cloned(); + Ok(known.unwrap_or_else(|| UnitFacts { + id: unit.into(), + names: vec![unit.into()], + ..UnitFacts::default() + })) + } + + fn invocation(&self, _unit: &str) -> Option { + Some("invocation-1".into()) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + self.runs.lock().unwrap().push((unit.into(), step.clone())); + Ok(self.report.lock().unwrap().clone()) + } + } + + struct FakePeers(u32); + + impl PeerCredentials for FakePeers { + fn peer(&self, _stream: &UnixStream) -> Result { + Ok(Peer { + uid: self.0, + issuer_unit: "mind-eureka-self.service".into(), + }) + } + } + + struct FakeBrake(AtomicBool); + + impl BrakeCheck for FakeBrake { + fn clear(&self, _principal: &str) -> bool { + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct FakeHome { + failing: AtomicBool, + sent: Mutex>, + } + + impl Homeserver for FakeHome { + fn post(&self, txn_id: &str, text: &str) -> Result { + ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); + let mut sent = self.sent.lock().unwrap(); + sent.push((txn_id.into(), text.into())); + Ok(format!("$event{}", sent.len())) + } + } + + struct Harness { + directory: TempDir, + ctx: Ctx, + systemd: Arc, + brake: Arc, + home: Arc, + queue: mpsc::Sender, + _orders: mpsc::Receiver, + } + + fn harness_for(peer_uid: u32) -> Harness { + let directory = TempDir::new().unwrap(); + let systemd = Arc::new(FakeSystemd { + units: Mutex::new(BTreeMap::new()), + runs: Mutex::new(Vec::new()), + report: Mutex::new(RunReport { + exit_code: Some(0), + ..RunReport::default() + }), + }); + let brake = Arc::new(FakeBrake(AtomicBool::new(true))); + let home = Arc::new(FakeHome::default()); + let (queue, orders) = mpsc::channel(); + let ctx = Ctx { + store: OrderStore::new(&directory.path().join("orders")), + state_store: directory.path().join("state").join("control.cc"), + steward_uid: STEWARD, + principal: "eureka".into(), + systemd: systemd.clone(), + peers: Arc::new(FakePeers(peer_uid)), + brake: brake.clone(), + homeserver: home.clone(), + }; + Harness { + directory, + ctx, + systemd, + brake, + home, + queue, + _orders: orders, + } + } + + fn harness() -> Harness { + harness_for(STEWARD) + } + + impl Harness { + fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { + let (mut client, mut server) = UnixStream::pair().unwrap(); + write(&mut client); + handle(&self.ctx, &mut server, &self.queue).unwrap(); + let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); + rmp_serde::from_slice(&bytes).unwrap() + } + + fn ask(&self, request: &Request) -> Response { + self.exchange(|client| write_frame(client, request).unwrap()) + } + + fn issue(&self, operation: Operation) -> Response { + self.ask(&Request::Order(OrderRequest { + operation, + reason: "probe".into(), + })) + } + + fn issue_unit(&self, verb: Verb, unit: &str) -> Response { + self.issue(Operation::UnitAction { + verb, + unit: unit.into(), + }) + } + + fn accept(&self, response: Response) -> String { + match response { + Response::Accepted { order_id } => order_id, + other => panic!("expected an accepted order, got {other:?}"), + } + } + + fn outcome(&self, order_id: &str) -> Outcome { + self.ctx + .store + .ledger() + .unwrap() + .outcome_of(order_id) + .expect("the order has an outcome") + .outcome + .clone() + } + + fn runs(&self) -> Vec<(String, Step)> { + self.systemd.runs.lock().unwrap().clone() + } + + fn describe_unit(&self, name: &str, facts: UnitFacts) { + self.systemd.units.lock().unwrap().insert(name.into(), facts); + } + } + + fn all_verbs() -> [Verb; 4] { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + } + + #[test] + fn foreign_uid_is_refused_at_the_door() { + let h = harness_for(STEWARD + 1); + let response = h.issue_unit(Verb::Restart, "nginx.service"); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + let response = h.ask(&Request::List); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); + assert!(h.runs().is_empty()); + } + + #[derive(Serialize)] + struct ForgedBody { + operation: Operation, + reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + } + + #[derive(Serialize)] + struct ForgedFrame { + order: ForgedBody, + } + + #[test] + fn order_id_and_requester_are_minted_not_declared() { + let h = harness(); + let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { + let frame = ForgedFrame { + order: ForgedBody { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into(), + }, + reason: "probe".into(), + order_id: order_id.map(str::to_owned), + requested_by: requested_by.map(str::to_owned), + command: command.map(str::to_owned), + }, + }; + h.exchange(|client| write_frame(client, &frame).unwrap()) + }; + // The same frame without the extra field is a good order. + let honest = h.accept(forge(None, None, None)); + for forged in [ + forge(Some("ord-declared"), None, None), + forge(None, Some("steward:someone-else order x"), None), + forge(None, None, Some("reboot-now")), + ] { + assert_eq!( + forged, + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + } + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); + let order = &ledger.orders[0]; + assert_eq!(order.order_id, honest); + let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); + assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); + assert_eq!(order.issuer_unit, "mind-eureka-self.service"); + assert_eq!(order.issuer_invocation, "invocation-1"); + assert_eq!(order.principal, "eureka"); + assert_eq!(order.issuer_uid, STEWARD); + assert_eq!( + requester(&order.principal, &order.order_id), + format!("steward:eureka order {}", order.order_id) + ); + } + + #[test] + fn package_names_outside_grammar_are_refused() { + let h = harness(); + let bad = [ + "-y", + "--allow-unauthenticated", + "-", + "a", + "", + "Foo", + "a/b", + "/tmp/x", + "./x", + "x.deb", + "foo.deb", + "foo bar", + "foo;reboot", + "foo$(id)", + "foo`id`", + "foo|bar", + "foo&bar", + "foo\nbar", + "foo*", + "foo?", + "pkg:amd64", + "pkg/stable", + "libc6-", + "foo=", + "foo=1 2", + "foo=1;2", + "foo=1=2", + "fo\u{ff0f}o", + "f\u{43e}o", + ]; + for name in bad { + let response = h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into(), name.into()], + }); + assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); + } + let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); + assert_eq!( + h.issue(Operation::PackageInstall { packages: many }), + Response::Refused(Refusal::new("package", Code::TooMany)) + ); + assert_eq!( + h.issue(Operation::PackageInstall { packages: vec![] }), + Response::Refused(Refusal::new("package", Code::Empty)) + ); + assert!(!h.ctx.store.path().exists(), "no refusal left a record"); + + let good = [ + "git-lfs", + "matrix-synapse-py3", + "libnss-systemd", + "bubblewrap", + "socat", + "g++", + "libstdc++6", + "python3.11", + "nginx=1.24.0-2ubuntu7", + "foo=1:2.3-4+b1~rc1", + ]; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: good.iter().map(|name| (*name).to_owned()).collect(), + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + let runs = h.runs(); + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); + let mut install = vec!["apt-get", "install", "-y", "-q", "--"]; + install.extend(good); + assert_eq!(runs[1].1.argv, install); + for (unit, step) in &runs { + assert!(unit.starts_with("idunn-order-")); + assert_eq!( + step.environment, + [("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + ); + } + assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + } + + #[test] + fn any_unit_outside_the_refused_set_runs() { + let h = harness(); + let units = [ + "wg-quick@wg0.service", + "nginx.service", + "matrix-synapse.service", + "ssh.socket", + "huginn@eureka.service", + "claude-connector.service", + "persona-matrix-mouth.service", + "mind-heartbeat.service", + "apt-daily.timer", + "multi-user.target", + // Near misses: the prefix is the rule, not a substring or a stem. + "midunn-x.service", + "not-idunn-x.service", + "idunn.service", + "mind-self.service", + "mind-selfx@a.service", + "mind-persona.service", + "mind-personal@a.service", + ]; + for verb in all_verbs() { + for unit in units { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); + let (transient, step) = h.runs().pop().unwrap(); + assert!(transient.starts_with("idunn-order-")); + assert_eq!(step.argv, ["systemctl", verb.as_str(), "--", unit]); + } + } + assert_eq!(h.runs().len(), units.len() * 4); + } + + #[test] + fn refused_set_is_refused_by_name_alias_and_propagation() { + let h = harness(); + h.describe_unit( + "backup-name.service", + UnitFacts { + id: "idunn-yggdrasil.service".into(), + names: vec!["idunn-yggdrasil.service".into(), "backup-name.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "quiet-alias.service", + UnitFacts { + id: "quiet-alias.service".into(), + names: vec!["quiet-alias.service".into(), "mind-persona@void.service".into()], + ..UnitFacts::default() + }, + ); + // fuse <- wrapper <- (bound by) mind-self@eureka + h.describe_unit( + "fuse.service", + UnitFacts { + id: "fuse.service".into(), + names: vec!["fuse.service".into()], + required_by: vec!["wrapper.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "wrapper.service", + UnitFacts { + id: "wrapper.service".into(), + names: vec!["wrapper.service".into()], + bound_by: vec!["mind-self@eureka.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "stack.target", + UnitFacts { + id: "stack.target".into(), + names: vec!["stack.target".into()], + consists_of: vec!["mind-persona@void.service".into()], + ..UnitFacts::default() + }, + ); + // A cycle that reaches an alias of a refused unit on its second hop. + h.describe_unit( + "loop-a.service", + UnitFacts { + id: "loop-a.service".into(), + names: vec!["loop-a.service".into()], + required_by: vec!["loop-b.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "loop-b.service", + UnitFacts { + id: "loop-b.service".into(), + names: vec!["loop-b.service".into()], + required_by: vec!["loop-a.service".into(), "hidden.service".into()], + ..UnitFacts::default() + }, + ); + h.describe_unit( + "hidden.service", + UnitFacts { + id: "idunn-orders.service".into(), + names: vec!["idunn-orders.service".into(), "hidden.service".into()], + ..UnitFacts::default() + }, + ); + + let direct = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-x.service", + "idunn-order-3f2c.service", + "idunn-yggdrasil.timer", + "idunn-x.socket", + "idunn-.service", + "idunn-yggdrasil.service.service", + "idunn-x@y.service", + "mind-self@eureka.service", + "mind-persona@void.service", + "mind-self@.service", + "mind-persona@a@b.service", + // Case folds: a lookalike is still refused, never run. + "IDUNN-yggdrasil.service", + "Idunn-Orders.service", + "MIND-SELF@eureka.service", + "Mind-Persona@Void.service", + ]; + for verb in all_verbs() { + for unit in direct { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + match h.outcome(&id) { + Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { + assert_eq!(hit, unit, "{verb:?} {unit}"); + assert_eq!(via, Via::Name); + } + other => panic!("{verb:?} {unit} was {other:?}"), + } + } + } + for verb in all_verbs() { + for (unit, hit) in [ + ("backup-name.service", "idunn-yggdrasil.service"), + ("quiet-alias.service", "mind-persona@void.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Alias + }), + "{verb:?} {unit}" + ); + } + } + for verb in [Verb::Stop, Verb::Restart] { + for (unit, hit) in [ + ("fuse.service", "mind-self@eureka.service"), + ("stack.target", "mind-persona@void.service"), + ("loop-a.service", "idunn-orders.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Propagation + }), + "{verb:?} {unit}" + ); + } + } + assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); + // start and reload propagate nothing, so the same units are not refused. + for verb in [Verb::Start, Verb::Reload] { + let id = h.accept(h.issue_unit(verb, "fuse.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + } + // Every refusal is recorded and has a result post due. + let ledger = h.ctx.store.ledger().unwrap(); + let refusals = ledger + .outcomes + .iter() + .filter(|record| matches!(record.outcome, Outcome::Refused(_))) + .count(); + assert_eq!(refusals, direct.len() * 4 + 8 + 6); + let due = due_posts(&h.ctx, &ledger); + let results = due + .iter() + .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) + .count(); + assert_eq!(results, refusals); + // A refused order never started. + assert!(ledger.starts.len() == 2); + } + + #[test] + fn globs_slices_and_scopes_are_not_units() { + let h = harness(); + let bad = [ + "idunn*", + "idunn-*.service", + "idunn-?ggdrasil.service", + "mind-?elf@eureka.service", + "mind-self@*.service", + "mind-[s]elf@eureka.service", + "system.slice", + "user.slice", + "session-1.scope", + "init.scope", + "idunn-yggdrasil", + "mind-self@eureka", + "nginx", + ".service", + "", + "-idunn-x.service", + "-x.service", + "/etc/systemd/system/idunn-yggdrasil.service", + "../idunn-yggdrasil.service", + "idunn-yggdrasil.service\n", + "idunn-yggdrasil.service ", + " idunn-yggdrasil.service", + "idunn-yggdrasil.service\0", + "idunn-yggdrasil.service;reboot", + "idunn\\x2dyggdrasil.service", + "idunn%2dyggdrasil.service", + // Unicode lookalikes of a refused name. + "\u{131}dunn-yggdrasil.service", + "idunn\u{2010}yggdrasil.service", + "idunn\u{ff0d}yggdrasil.service", + "idunn-yggdrasil.service\u{200b}", + "\u{200b}idunn-yggdrasil.service", + "idunn-yggdrasil\u{2024}service", + "mind-self\u{ff20}eureka.service", + ]; + for unit in bad { + for verb in all_verbs() { + assert!( + matches!(h.issue_unit(verb, unit), Response::Refused(_)), + "{unit:?} was not refused by the grammar" + ); + } + } + let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); + assert_eq!( + h.issue_unit(Verb::Start, &long), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); + } + + #[test] + fn binding_with_a_unit_list_is_refused() { + let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ + mind_control = \"/usr/local/bin/mind-control\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ + token_credential = \"idunn-matrix-token\"\n"; + let binding = StewardBinding::parse(valid).unwrap(); + assert_eq!(binding.principal, "eureka"); + assert_eq!(binding.address().unwrap().port(), 8008); + for extra in [ + "units = [\"nginx.service\"]", + "allow = [\"nginx.service\"]", + "deny = [\"idunn-*\"]", + "denylist = [\"idunn-*\"]", + "refused = [\"idunn-*\"]", + "unit = \"nginx.service\"", + "[units]\nnginx = true", + ] { + assert!( + StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), + "{extra} loaded" + ); + } + let swap = |from: &str, to: &str| valid.replace(from, to); + for broken in [ + swap("user = \"mind-eureka-self\"", "user = \"root\""), + swap("user = \"mind-eureka-self\"", "user = \"\""), + swap("/usr/local/bin/mind-control", "mind-control"), + swap("!abc:example.org", "#alias:example.org"), + swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), + swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), + swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), + swap("idunn-matrix-token", "../token"), + swap("principal = \"eureka\"\n", ""), + ] { + assert!(StewardBinding::parse(&broken).is_err()); + } + assert!(admitted(0, 0o100600)); + assert!(admitted(0, 0o100644)); + assert!(!admitted(0, 0o100664)); + assert!(!admitted(0, 0o100666)); + assert!(!admitted(0, 0o100602)); + assert!(!admitted(1000, 0o100600)); + } + + #[test] + fn held_brake_refuses_at_start() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); + assert!(h.runs().is_empty()); + // The deploy path is gated the same way and writes no command. + let deploy = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &deploy).unwrap(); + assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); + assert!(!h.ctx.state_store.exists()); + // A brake that cannot be asked is a held brake. + let directory = TempDir::new().unwrap(); + assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); + } + + #[test] + fn brake_check_is_the_exit_status_of_mind_control_check() { + let directory = TempDir::new().unwrap(); + let program = |name: &str, body: &str| { + let path = directory.path().join(name); + std::fs::write(&path, body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + MindControl { program: path } + }; + let recorder = directory.path().join("argv"); + let clear = program( + "clear", + &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), + ); + assert!(clear.clear("eureka")); + assert_eq!( + std::fs::read_to_string(&recorder).unwrap().trim(), + "--instance eureka check" + ); + assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); + } + + #[test] + fn started_order_is_never_rerun() { + let h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The first order began and its service died: a start with no outcome. + h.ctx + .store + .put( + &id, + START_SCHEMA, + &StartRecord { + schema_version: START_SCHEMA.into(), + order_id: id.clone(), + started_at_unix_millis: 1, + actuation: format!("idunn-order-{}", order_uuid(&id)), + }, + ) + .unwrap(); + let pending = recover(&h.ctx).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Lost); + assert_eq!(pending, [waiting.clone()], "an order that never started waits"); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); + // Recovery is idempotent, and a finished order is not run again. + assert_eq!(recover(&h.ctx).unwrap(), [waiting.clone()]); + execute(&h.ctx, &waiting).unwrap(); + execute(&h.ctx, &waiting).unwrap(); + assert_eq!(h.runs().len(), 1); + assert!(recover(&h.ctx).unwrap().is_empty()); + } + + #[test] + fn deploy_order_uses_the_up_primitive() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let command_id = format!("up-{}", order_uuid(&id)); + assert_eq!( + h.outcome(&id), + Outcome::Submitted { + command_id: command_id.clone() + } + ); + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); + assert!(h.runs().is_empty(), "a deploy runs no transient unit"); + let mut shown = Vec::new(); + crate::control_plane::status( + &mut shown, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + Some(&command_id), + ) + .unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); + assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); + // Re-executing writes no second command. + execute(&h.ctx, &id).unwrap(); + let mut again = Vec::new(); + crate::control_plane::status( + &mut again, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + None, + ) + .unwrap(); + assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); + // The result post waits for the transactions: queued is not finished. + let due = due_posts(&h.ctx, &ledger); + assert_eq!(due.len(), 1); + assert_eq!(due[0].kind, PostKind::Accepted); + assert!(!deploy_finished("queued") && !deploy_finished("running")); + assert!(deploy_finished("succeeded") && deploy_finished("failed")); + assert!(deploy_finished("complete")); + } + + #[test] + fn chat_outage_delays_posts_not_orders() { + let h = harness(); + h.home.failing.store(true, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + assert_eq!(h.runs().len(), 1); + assert!(post_pass(&h.ctx).is_err()); + assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); + h.home.failing.store(false, Ordering::SeqCst); + assert_eq!(post_pass(&h.ctx).unwrap(), 2); + assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); + let sent = h.home.sent.lock().unwrap().clone(); + assert_eq!(sent.len(), 2); + assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); + assert_eq!(sent[1].0, format!("idunn-{id}-result")); + assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); + assert!(sent[0].1.contains("mind-eureka-self.service")); + assert!(sent[0].1.contains("reason: \"probe\"")); + assert!(sent[1].1.contains("ran (exit 0)")); + } + + #[test] + fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { + let h = harness(); + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(100), + signal: None, + stdout_tail: vec![b'o'; 3 * TAIL_BYTES], + stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), + }; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into()], + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Failed { + exit_code: Some(100), + signal: None + } + ); + assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); + let ledger = h.ctx.store.ledger().unwrap(); + let record = ledger.outcome_of(&id).unwrap(); + assert_eq!(record.stdout_tail.len(), TAIL_BYTES); + assert!(!format!("{record:?}").contains("secret-looking")); + // Neither the post nor the journal text carries a tail. + let results: Vec<_> = due_posts(&h.ctx, &ledger) + .into_iter() + .filter(|due| due.kind == PostKind::Result) + .collect(); + assert_eq!(results.len(), 1); + assert!(results[0].text.contains("failed (exit 100)")); + assert!(!results[0].text.contains("secret-looking")); + // The steward reads them through the escape sink and nothing raw. + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("expected a show") + }; + let mut shown = Vec::new(); + emit(&mut shown, &render_view(&view)).unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert!(shown.contains("secret-looking")); + assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); + assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); + assert!(shown.contains("status failed")); + // A show of an unknown or malformed id names the field only. + assert_eq!( + h.ask(&Request::Show(ShowRequest { + id: format!("ord-{}", Uuid::new_v4()) + })), + Response::Refused(Refusal::new("id", Code::Unknown)) + ); + assert_eq!( + h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), + Response::Refused(Refusal::new("id", Code::BadShape)) + ); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("expected a list") + }; + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].status, "failed"); + } + + #[test] + fn a_refusal_or_error_never_echoes_an_input_value() { + let h = harness(); + let mut seen: Vec> = Vec::new(); + let mut note = |response: Response| { + let bytes = rmp_serde::to_vec_named(&response).unwrap(); + if let Response::Refused(refusal) = &response { + seen.push(refusal.to_string().into_bytes()); + } + seen.push(bytes); + }; + for unit in [ + format!("{CANARY}*.service"), + format!("{CANARY}/../idunn-orders.service"), + format!("idunn-{CANARY};x.service"), + format!("{CANARY}"), + ] { + note(h.issue_unit(Verb::Restart, &unit)); + } + note(h.issue(Operation::PackageInstall { + packages: vec![format!("{CANARY}/x")], + })); + note(h.issue(Operation::PackageInstall { + packages: vec![format!("pkg={CANARY} 1")], + })); + note(h.issue(Operation::Deploy { + selector: format!("{CANARY} bad"), + })); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: format!("{CANARY}\n"), + }))); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: CANARY.repeat(40), + }))); + note(h.ask(&Request::Show(ShowRequest { + id: format!("ord-{CANARY}"), + }))); + // A frame whose own keys and values are the canary, and plain garbage. + let named: std::collections::BTreeMap> = + [( + CANARY.to_owned(), + [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), + )] + .into_iter() + .collect(); + note(h.exchange(|client| write_frame(client, &named).unwrap())); + note(h.exchange(|client| { + let garbage = format!("{CANARY}{CANARY}"); + client + .write_all(&(garbage.len() as u32).to_be_bytes()) + .unwrap(); + client.write_all(garbage.as_bytes()).unwrap(); + })); + note(h.exchange(|client| { + client.write_all(&u32::MAX.to_be_bytes()).unwrap(); + })); + // The client grammar fails the same way. + let mut messages = Vec::new(); + let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); + for line in [ + words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), + words(&["unit", CANARY, "nginx.service", "--reason", "r"]), + words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), + words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), + words(&[CANARY]), + words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), + words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), + words(&["show"]), + ] { + messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); + } + for line in [ + words(&[CANARY]), + words(&["serve", &format!("--{CANARY}")]), + words(&["serve", "--store"]), + ] { + messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); + } + for bytes in &seen { + assert!( + !String::from_utf8_lossy(bytes).contains(CANARY), + "a response carried an input value" + ); + } + for message in &messages { + assert!(!message.contains(CANARY), "an error carried an input value: {message}"); + } + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn cgroup_names_the_issuer_unit() { + assert_eq!( + unit_from_cgroup("0::/system.slice/claude-connector.service\n"), + "claude-connector.service" + ); + assert_eq!( + unit_from_cgroup("0::/system.slice/huginn@eureka.service/sub/leaf\n"), + "huginn@eureka.service" + ); + assert_eq!( + unit_from_cgroup("0::/user.slice/user-1000.slice/session-3.scope\n"), + "session-3.scope" + ); + assert_eq!( + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.service\n"), + "a.service" + ); + assert_eq!(unit_from_cgroup("0::/\n"), "unknown"); + assert_eq!(unit_from_cgroup(""), "unknown"); + } + + #[test] + fn systemctl_show_output_is_read_into_facts() { + let facts = parse_show( + "Id=a.service\nNames=a.service b.service\nRequiredBy=c.service\n\ + BoundBy=\nConsistsOf=d.target e.service\nOther=x\n", + ) + .unwrap(); + assert_eq!(facts.id, "a.service"); + assert_eq!(facts.names, ["a.service", "b.service"]); + assert_eq!(facts.required_by, ["c.service"]); + assert!(facts.bound_by.is_empty()); + assert_eq!(facts.consists_of, ["d.target", "e.service"]); + assert!(parse_show("Names=a.service\n").is_err()); + } + + #[test] + fn the_store_refuses_a_second_record_and_a_foreign_document() { + let h = harness(); + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: "ord-x".into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }; + assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert_eq!( + std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); + SingleFileMessagePackBackingStore::new(&h.ctx.store.path()) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: "idunn.other".into(), + key: "k".into(), + current: None, + }], + &[foreign], + ) + .unwrap(); + assert!(h.ctx.store.ledger().is_err()); + } + + #[test] + fn synapse_post_is_one_idempotent_put_on_loopback() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (sent, requests) = mpsc::channel(); + let server = thread::spawn(move || { + for reply in [ + "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", + "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", + ] { + let (mut socket, _) = listener.accept().unwrap(); + let mut received = Vec::new(); + let mut chunk = [0u8; 1024]; + while !received.windows(4).any(|window| window == b"\r\n\r\n") + || !String::from_utf8_lossy(&received).contains("\"body\"") + { + let count = socket.read(&mut chunk).unwrap(); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); + socket.write_all(reply.as_bytes()).unwrap(); + } + }); + let synapse = Synapse { + address, + room: "!room:example.org".into(), + token_file, + }; + assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); + let request = requests.recv().unwrap(); + assert!(request.starts_with( + "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" + )); + assert!(request.contains("Authorization: Bearer tok-123\r\n")); + assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); + let error = synapse.post("txn-2", "x").unwrap_err(); + assert!(!format!("{error:#}").contains("tok-123")); + server.join().unwrap(); + assert!( + Synapse { + address, + room: "!r:e".into(), + token_file: directory.path().join("absent") + } + .post("t", "x") + .is_err() + ); + } + + #[test] + fn grammar_limits_are_exact_at_their_edges() { + let h = harness(); + let order = |operation: Operation, reason: &str| { + h.ask(&Request::Order(OrderRequest { + operation, + reason: reason.into(), + })) + }; + let packages = |names: Vec| Operation::PackageInstall { packages: names }; + let unit = |name: String| Operation::UnitAction { + verb: Verb::Start, + unit: name, + }; + // Package: two characters is the shortest name, 200 bytes the longest. + assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); + assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); + assert_eq!( + order(packages(vec!["a".repeat(201)]), "r"), + Response::Refused(Refusal::new("package", Code::TooLong)) + ); + assert_eq!( + order(packages(vec!["a".into()]), "r"), + Response::Refused(Refusal::new("package", Code::BadShape)) + ); + let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); + assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); + // Unit: 200 bytes with its suffix is the longest. + let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); + assert_eq!(longest.len(), UNIT_LIMIT); + assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); + let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); + assert_eq!( + order(unit(too_long), "r"), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + // Reason: 300 bytes is the longest, and it must be there. + assert!(matches!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), + Response::Accepted { .. } + )); + assert_eq!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), + Response::Refused(Refusal::new("reason", Code::TooLong)) + ); + assert_eq!( + order(unit("nginx.service".into()), ""), + Response::Refused(Refusal::new("reason", Code::Empty)) + ); + assert_eq!( + order(unit("nginx.service".into()), "caf\u{e9}"), + Response::Refused(Refusal::new("reason", Code::BadCharacter)) + ); + } + + #[test] + fn a_concluded_order_is_never_run_later() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + // The brake is released by someone else; the refused order stays refused. + h.brake.0.store(true, Ordering::SeqCst); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty()); + let ledger = h.ctx.store.ledger().unwrap(); + assert!(ledger.start_of(&id).is_none()); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + } + + #[test] + fn the_orders_cli_has_no_command_string() { + let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); + let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); + assert_eq!( + parsed.action, + OrderAction::Issue { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into() + }, + reason: "why".into() + } + ); + assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); + assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); + assert!(parse(&["show", "ord-x"]).is_ok()); + assert!(parse(&["list"]).is_ok()); + for refused in [ + vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], + vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], + vec!["run", "sh", "--reason", "r"], + vec!["exec", "--reason", "r"], + vec!["unit", "enable", "nginx.service", "--reason", "r"], + vec!["unit", "restart", "--reason", "r"], + vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], + vec!["unit", "restart", "nginx.service"], + vec!["unit", "restart", "nginx*", "--reason", "r"], + vec!["package-install", "--reason", "r"], + vec!["package-install", "-y", "x", "--reason", "r"], + vec!["deploy", "--reason", "r"], + vec!["deploy", "a", "b", "--reason", "r"], + vec!["list", "extra"], + vec!["show"], + vec![], + ] { + assert!(parse(&refused).is_err(), "{refused:?} parsed"); + } + } +} From 1a3634e04696bfaee40b8c4e4c317c2202e6357f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:02 +0100 Subject: [PATCH 02/23] Document idunn order and add the idunn-orders unit Co-Authored-By: Claude Sonnet 5.5 --- README.md | 29 ++++++++++++++++++++++ deploy/idunn-orders.service | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 deploy/idunn-orders.service diff --git a/README.md b/README.md index a5c48bc..6af4527 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,35 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. +### Steward orders + +A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), +takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. +It has its own store, `orders.cc`, and its own binding, `steward.toml`; the +survival daemon takes no order. There is still no command string: + +```bash +idunn order package-install [=version]... --reason TEXT # apt-get update, then install +idunn order unit --reason TEXT +idunn order deploy --reason TEXT # the same commit as `idunn up` +idunn order show # status and output tails +idunn order list +``` + +A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' +own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and +read from no configuration. A name, an alias that resolves to one of them, or a +`stop` or `restart` whose dependents systemd would stop with it is refused, +recorded and posted. Slices, scopes and globs are not units an order can name. + +An order runs once, after the steward's uid is checked and the instance's +brake check (`mind-control --instance check`) exits 0; an order +that had started when the service died is recorded as lost and never run +again. No order releases a brake. Acceptance and result are posted to +`#operator` as a retried projection of `orders.cc`: a chat outage delays the +posts and not the orders. A refusal names the field and a code, never the value +it refused. + ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service new file mode 100644 index 0000000..cd3c46b --- /dev/null +++ b/deploy/idunn-orders.service @@ -0,0 +1,48 @@ +[Unit] +Description=Idunn steward orders: typed package, unit and deploy orders from the steward +# Deliberately no After= or Requires= on any managed target, Synapse or +# idunn-yggdrasil: orders are how an operator repairs those, so none of them +# may be a condition of this unit starting. Posts are a retried projection and +# wait for Synapse on their own. + +[Service] +Type=simple + +# Root, from the same binary as idunn-yggdrasil, with its own store and socket. +# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: +# no order code runs in `idunn serve`. +# +# The unit name matches idunn-*, which is in the always-refused set, so no +# order can stop or restart this service. +User=root +# The socket is created root:idunn-steward 0660 (UMask below). The steward user +# is the only peer the service answers: it checks SO_PEERCRED on every +# connection, so the group is a second door held shut, not the lock. +Group=idunn-steward +ExecStart=/usr/local/bin/idunn orders serve \ + --store /var/lib/gamecult/idunn-orders \ + --state-store /var/lib/gamecult/idunn/control.cc \ + --steward-binding /etc/gamecult/idunn/steward.toml \ + --socket /run/idunn-orders/steward.sock +Restart=always +RestartSec=5 +TimeoutStopSec=20 +UMask=0117 + +RuntimeDirectory=idunn-orders +RuntimeDirectoryMode=0750 + +# The Synapse access token of @idunn, copied from a root-only file, and read +# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. +LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token + +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +# Its own order store, and the control store a deploy order commits a command +# to. Transient units started with systemd-run run in the manager's namespace, +# so packages and units are not limited by this list. +ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn + +[Install] +WantedBy=multi-user.target From 7d5cd85728b0d9e7dad801f48b15b358a0bd6bc3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 02:33:12 +0100 Subject: [PATCH 03/23] Reword a test comment so the no-brake-release tripwire reads clean Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 2aded30..31a2d06 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2880,7 +2880,7 @@ mod tests { let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); execute(&h.ctx, &id).unwrap(); assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - // The brake is released by someone else; the refused order stays refused. + // Someone else clears the brake; the refused order stays refused. h.brake.0.store(true, Ordering::SeqCst); execute(&h.ctx, &id).unwrap(); assert!(h.runs().is_empty()); From ec6222e2a114d3ebf5287bb326d0b5edc5ea907f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:49:26 +0100 Subject: [PATCH 04/23] Steward unit orders: refuse by the systemd transaction, install exactly the named packages An order is refused when the closure systemd would build for it reaches an always-refused unit: a start pulls Requires/Requisite/BindsTo/Wants/Upholds and stops Conflicts/ConflictedBy, a stop takes down RequiredBy/BoundBy/RequisiteOf/ ConsistsOf/PropagatesStopTo/StopPropagatedFrom, a restart does both. A refused unit already running is left alone by a start. Names are unquoted from systemctl show; a unit systemd cannot show fails the order. Package installs run apt-get --no-remove with ?exact-name() per package and a final dpkg -s step that fails unless every named package is installed. Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 676 +++++++++++++++++++++++++++++++++--------- 1 file changed, 544 insertions(+), 132 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 31a2d06..ce95337 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -213,11 +213,6 @@ impl Verb { .into_iter() .find(|verb| verb.as_str() == word) } - - /// Verbs systemd propagates down the dependency graph. - fn propagates(self) -> bool { - matches!(self, Verb::Stop | Verb::Restart) - } } /// The closed set of things an order can ask for. @@ -259,8 +254,21 @@ impl Operation { let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; match self { Operation::PackageInstall { packages } => { - let mut install: Vec = words(&["apt-get", "install", "-y", "-q", "--"]); - install.extend(packages.iter().cloned()); + // Never a removal, never a pattern: each name selects exactly + // itself, and apt aborts rather than displace an installed package. + let mut install: Vec = + words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); + let mut present: Vec = words(&["dpkg", "-s", "--"]); + for package in packages { + let (name, version) = package.split_once('=').unwrap_or((package, "")); + install.push(match version { + "" => format!("?exact-name({name})"), + version => format!("?exact-name({name})={version}"), + }); + present.push(name.to_owned()); + } + // An exact name that matches nothing installs nothing and exits 0, + // so the last step fails unless every named package is installed. vec![ Step { argv: words(&["apt-get", "update", "-q"]), @@ -270,6 +278,10 @@ impl Operation { argv: install, environment: apt_environment, }, + Step { + argv: present, + environment: Vec::new(), + }, ] } Operation::UnitAction { verb, unit } => vec![Step { @@ -595,14 +607,45 @@ impl OrderStore { // ------------------------------------------------------------------ ports -- +/// Which way systemd moves along a dependency edge when an order is given. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Edge { + /// Units a start brings up with this one. + Pulls, + /// Units a start stops: the unit's `Conflicts=` and the units that conflict with it. + Conflicts, + /// Units a stop takes down with this one. + Stops, +} + +/// The `systemctl show` properties the closure reads, and the way each one +/// moves. This is the only list: the request and the parse both read it. +const EDGES: [(&str, Edge); 13] = [ + ("Requires", Edge::Pulls), + ("Requisite", Edge::Pulls), + ("BindsTo", Edge::Pulls), + ("Wants", Edge::Pulls), + ("Upholds", Edge::Pulls), + ("Conflicts", Edge::Conflicts), + ("ConflictedBy", Edge::Conflicts), + ("RequiredBy", Edge::Stops), + ("BoundBy", Edge::Stops), + ("RequisiteOf", Edge::Stops), + ("ConsistsOf", Edge::Stops), + ("PropagatesStopTo", Edge::Stops), + ("StopPropagatedFrom", Edge::Stops), +]; + /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { pub(crate) id: String, pub(crate) names: Vec, - pub(crate) required_by: Vec, - pub(crate) bound_by: Vec, - pub(crate) consists_of: Vec, + pub(crate) not_found: bool, + pub(crate) running: bool, + pub(crate) pulls: Vec, + pub(crate) conflicts: Vec, + pub(crate) stops: Vec, } impl UnitFacts { @@ -610,12 +653,20 @@ impl UnitFacts { std::iter::once(&self.id).chain(self.names.iter()) } - /// The units systemd stops with this one. - fn dependents(&self) -> impl Iterator { - self.required_by - .iter() - .chain(self.bound_by.iter()) - .chain(self.consists_of.iter()) + fn along(&self, edge: Edge) -> &[String] { + match edge { + Edge::Pulls => &self.pulls, + Edge::Conflicts => &self.conflicts, + Edge::Stops => &self.stops, + } + } + + fn along_mut(&mut self, edge: Edge) -> &mut Vec { + match edge { + Edge::Pulls => &mut self.pulls, + Edge::Conflicts => &mut self.conflicts, + Edge::Stops => &mut self.stops, + } } } @@ -673,40 +724,95 @@ fn refused(unit: &str, via: Via) -> Refused { } } +/// The units systemd would touch for one order, read from `systemctl show`. +struct Transaction<'a> { + systemd: &'a dyn Systemd, + facts: BTreeMap, +} + +impl Transaction<'_> { + fn facts(&mut self, unit: &str) -> Result<&UnitFacts> { + if !self.facts.contains_key(unit) { + ensure!( + self.facts.len() < CLOSURE_LIMIT, + "the units an order would reach are too many to check" + ); + let facts = self.systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + self.facts.insert(unit.to_owned(), facts); + } + Ok(&self.facts[unit]) + } + + /// Every unit reached from `seeds` along `edge`, seeds included. + fn reach(&mut self, seeds: &BTreeSet, edge: Edge) -> Result> { + let mut reached = BTreeSet::new(); + let mut queue: VecDeque = seeds.iter().cloned().collect(); + while let Some(next) = queue.pop_front() { + if !reached.insert(next.clone()) { + continue; + } + queue.extend(self.facts(&next)?.along(edge).iter().cloned()); + } + Ok(reached) + } + + /// Every unit a start of `started` stops through `Conflicts=`, and what + /// that stop takes down with it. + fn conflicted(&mut self, started: &BTreeSet) -> Result> { + let mut seeds = BTreeSet::new(); + for unit in started { + seeds.extend(self.facts(unit)?.along(Edge::Conflicts).iter().cloned()); + } + self.reach(&seeds, Edge::Stops) + } +} + /// Would `verb` on `unit` reach an always-refused unit: by its name, by an -/// alias it resolves from, or, for the verbs systemd propagates, by the units -/// systemd stops with it. Resolution reads systemd before anything runs. +/// alias it resolves from, or through the transaction systemd builds for it. +/// A start pulls in what the unit requires or wants and stops what conflicts; +/// a stop takes down what depends on the unit; a restart does both. A unit +/// the transaction would stop is refused when it is refused; a unit it would +/// start is refused when it is refused and not running. Every unit in the +/// closure must be one systemd can show. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); } - let facts = systemd.show(unit)?; - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + let mut transaction = Transaction { + systemd, + facts: BTreeMap::new(), + }; + if let Some(name) = transaction + .facts(unit)? + .identities() + .find(|name| is_refused_name(name)) + { return Ok(Some(refused(name, Via::Alias))); } - if !verb.propagates() { - return Ok(None); + let root = BTreeSet::from([unit.to_owned()]); + let mut stopped = BTreeSet::new(); + let mut started = BTreeSet::new(); + match verb { + Verb::Reload => {} + Verb::Start => started = transaction.reach(&root, Edge::Pulls)?, + Verb::Stop => stopped = transaction.reach(&root, Edge::Stops)?, + Verb::Restart => { + stopped = transaction.reach(&root, Edge::Stops)?; + started = transaction.reach(&stopped, Edge::Pulls)?; + } } - let mut seen: BTreeSet = facts.identities().cloned().collect(); - seen.insert(unit.to_owned()); - let mut queue: VecDeque = facts.dependents().cloned().collect(); - while let Some(next) = queue.pop_front() { - if is_refused_name(&next) { - return Ok(Some(refused(&next, Via::Propagation))); + stopped.extend(transaction.conflicted(&started)?); + for (units, only_if_stopped) in [(&stopped, false), (&started, true)] { + for member in units { + let facts = transaction.facts(member)?; + if only_if_stopped && facts.running { + continue; + } + if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { + return Ok(Some(refused(name, Via::Propagation))); + } } - if !seen.insert(next.clone()) { - continue; - } - ensure!( - seen.len() <= CLOSURE_LIMIT, - "the units a stop would reach are too many to check" - ); - let facts = systemd.show(&next)?; - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { - return Ok(Some(refused(name, Via::Propagation))); - } - seen.extend(facts.identities().cloned()); - queue.extend(facts.dependents().cloned()); } Ok(None) } @@ -1148,8 +1254,39 @@ pub(crate) fn handle( /// `systemctl` and `systemd-run`, found by name. pub(crate) struct HostSystemd; +/// The names in one `systemctl show` list value. systemd quotes a name that +/// holds a backslash and doubles the backslash inside the quotes. fn parse_names(value: &str) -> Vec { - value.split_whitespace().map(str::to_owned).collect() + let mut names = Vec::new(); + let mut current = String::new(); + let (mut quoted, mut open, mut escaped) = (false, false, false); + for character in value.chars() { + match character { + _ if escaped => { + current.push(character); + escaped = false; + } + '\\' if quoted => escaped = true, + '"' => { + quoted = !quoted; + open = true; + } + _ if character.is_whitespace() && !quoted => { + if open { + names.push(std::mem::take(&mut current)); + open = false; + } + } + _ => { + current.push(character); + open = true; + } + } + } + if open { + names.push(current); + } + names } fn parse_show(output: &str) -> Result { @@ -1161,10 +1298,15 @@ fn parse_show(output: &str) -> Result { match key { "Id" => facts.id = value.to_owned(), "Names" => facts.names = parse_names(value), - "RequiredBy" => facts.required_by = parse_names(value), - "BoundBy" => facts.bound_by = parse_names(value), - "ConsistsOf" => facts.consists_of = parse_names(value), - _ => {} + "LoadState" => facts.not_found = value == "not-found", + "ActiveState" => { + facts.running = matches!(value, "active" | "activating" | "reloading"); + } + _ => { + if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { + facts.along_mut(*edge).extend(parse_names(value)); + } + } } } ensure!(!facts.id.is_empty(), "systemctl named no unit"); @@ -1173,14 +1315,15 @@ fn parse_show(output: &str) -> Result { impl Systemd for HostSystemd { fn show(&self, unit: &str) -> Result { + let properties = ["Id", "Names", "LoadState", "ActiveState"] + .into_iter() + .chain(EDGES.iter().map(|(property, _)| *property)) + .collect::>() + .join(","); let output = Command::new("systemctl") - .args([ - "show", - "--no-pager", - "--property=Id,Names,RequiredBy,BoundBy,ConsistsOf", - "--", - unit, - ]) + .args(["show", "--no-pager"]) + .arg(format!("--property={properties}")) + .args(["--", unit]) .stdin(Stdio::null()) .output() .context("running systemctl show")?; @@ -1750,8 +1893,23 @@ mod tests { const STEWARD: u32 = 1000; const CANARY: &str = "CANARY-7f3a91"; + /// What `systemctl show` prints for a loaded, active unit with no edges, + /// plus the given properties (a given key replaces its default). + fn show_text(id: &str, properties: &[(&str, &str)]) -> String { + let mut text = format!("Id={id}\n"); + for (key, default) in [("Names", id), ("LoadState", "loaded"), ("ActiveState", "active")] { + if !properties.iter().any(|(given, _)| *given == key) { + text += &format!("{key}={default}\n"); + } + } + for (key, value) in properties { + text += &format!("{key}={value}\n"); + } + text + } + struct FakeSystemd { - units: Mutex>, + units: Mutex>, runs: Mutex>, report: Mutex, } @@ -1759,11 +1917,7 @@ mod tests { impl Systemd for FakeSystemd { fn show(&self, unit: &str) -> Result { let known = self.units.lock().unwrap().get(unit).cloned(); - Ok(known.unwrap_or_else(|| UnitFacts { - id: unit.into(), - names: vec![unit.into()], - ..UnitFacts::default() - })) + parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) } fn invocation(&self, _unit: &str) -> Option { @@ -1907,8 +2061,13 @@ mod tests { self.systemd.runs.lock().unwrap().clone() } - fn describe_unit(&self, name: &str, facts: UnitFacts) { - self.systemd.units.lock().unwrap().insert(name.into(), facts); + /// Tell the fake `systemctl show` what it prints for `name`. + fn describe_unit(&self, name: &str, text: String) { + self.systemd.units.lock().unwrap().insert(name.into(), text); + } + + fn describe(&self, name: &str, properties: &[(&str, &str)]) { + self.describe_unit(name, show_text(name, properties)); } } @@ -2065,19 +2224,52 @@ mod tests { execute(&h.ctx, &id).unwrap(); assert_eq!(h.outcome(&id), Outcome::Ran); let runs = h.runs(); - assert_eq!(runs.len(), 2); + assert_eq!(runs.len(), 3); assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); - let mut install = vec!["apt-get", "install", "-y", "-q", "--"]; - install.extend(good); + // Each name selects exactly itself and apt may remove nothing. + let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; + install.extend(["--no-remove".to_owned(), "--".into()]); + install.extend(good.iter().map(|spec| match spec.split_once('=') { + Some((name, version)) => format!("?exact-name({name})={version}"), + None => format!("?exact-name({spec})"), + })); assert_eq!(runs[1].1.argv, install); + assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); + assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); + // The last step fails unless every named package is installed. + let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; + present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); + assert_eq!(runs[2].1.argv, present); for (unit, step) in &runs { assert!(unit.starts_with("idunn-order-")); + let apt = step.argv[0] == "apt-get"; assert_eq!( step.environment, - [("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + if apt { + vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + } else { + Vec::new() + } ); } assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); + + // A package whose name apt would read as a regex or a conflict is + // exactly itself and never a removal. + for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { + let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); + assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); + } + // A step that failed ends the order: the absent package is a failure. + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(1), + ..RunReport::default() + }; + let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); + execute(&h.ctx, &absent).unwrap(); + assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); + assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); } #[test] @@ -2121,74 +2313,28 @@ mod tests { let h = harness(); h.describe_unit( "backup-name.service", - UnitFacts { - id: "idunn-yggdrasil.service".into(), - names: vec!["idunn-yggdrasil.service".into(), "backup-name.service".into()], - ..UnitFacts::default() - }, + show_text( + "idunn-yggdrasil.service", + &[("Names", "idunn-yggdrasil.service backup-name.service")], + ), ); - h.describe_unit( + h.describe( "quiet-alias.service", - UnitFacts { - id: "quiet-alias.service".into(), - names: vec!["quiet-alias.service".into(), "mind-persona@void.service".into()], - ..UnitFacts::default() - }, + &[("Names", "quiet-alias.service mind-persona@void.service")], ); // fuse <- wrapper <- (bound by) mind-self@eureka - h.describe_unit( - "fuse.service", - UnitFacts { - id: "fuse.service".into(), - names: vec!["fuse.service".into()], - required_by: vec!["wrapper.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "wrapper.service", - UnitFacts { - id: "wrapper.service".into(), - names: vec!["wrapper.service".into()], - bound_by: vec!["mind-self@eureka.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "stack.target", - UnitFacts { - id: "stack.target".into(), - names: vec!["stack.target".into()], - consists_of: vec!["mind-persona@void.service".into()], - ..UnitFacts::default() - }, - ); + h.describe("fuse.service", &[("RequiredBy", "wrapper.service")]); + h.describe("wrapper.service", &[("BoundBy", "mind-self@eureka.service")]); + h.describe("stack.target", &[("ConsistsOf", "mind-persona@void.service")]); // A cycle that reaches an alias of a refused unit on its second hop. - h.describe_unit( - "loop-a.service", - UnitFacts { - id: "loop-a.service".into(), - names: vec!["loop-a.service".into()], - required_by: vec!["loop-b.service".into()], - ..UnitFacts::default() - }, - ); - h.describe_unit( - "loop-b.service", - UnitFacts { - id: "loop-b.service".into(), - names: vec!["loop-b.service".into()], - required_by: vec!["loop-a.service".into(), "hidden.service".into()], - ..UnitFacts::default() - }, - ); + h.describe("loop-a.service", &[("RequiredBy", "loop-b.service")]); + h.describe("loop-b.service", &[("RequiredBy", "loop-a.service hidden.service")]); h.describe_unit( "hidden.service", - UnitFacts { - id: "idunn-orders.service".into(), - names: vec!["idunn-orders.service".into(), "hidden.service".into()], - ..UnitFacts::default() - }, + show_text( + "idunn-orders.service", + &[("Names", "idunn-orders.service hidden.service")], + ), ); let direct = [ @@ -2260,7 +2406,7 @@ mod tests { } } assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); - // start and reload propagate nothing, so the same units are not refused. + // start and reload take nothing down, so the same units are not refused. for verb in [Verb::Start, Verb::Reload] { let id = h.accept(h.issue_unit(verb, "fuse.service")); execute(&h.ctx, &id).unwrap(); @@ -2284,6 +2430,248 @@ mod tests { assert!(ledger.starts.len() == 2); } + /// Yggdrasil's `systemctl show` shapes (systemd 257, 2026-11-01), cut down + /// to the units an order can reach: seven refused units all conflict with + /// shutdown.target, and the shutdown services require it. + fn yggdrasil(h: &Harness, idunn_running: bool) { + let state = if idunn_running { "active" } else { "inactive" }; + let refused = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-e0ada640.service", + "mind-self@eureka.service", + "mind-persona@void.service", + ]; + h.describe( + "shutdown.target", + &[ + ("ActiveState", "inactive"), + ( + "RequiredBy", + "systemd-soft-reboot.service systemd-reboot.service systemd-poweroff.service", + ), + ( + "ConflictedBy", + "nginx.service idunn-yggdrasil.service idunn-orders.service \ + idunn-vili-e0ada640.service mind-self@eureka.service mind-persona@void.service", + ), + ], + ); + for refused in refused { + h.describe( + refused, + &[ + ("Requires", "sysinit.target system.slice -.mount"), + ("Wants", "tmp.mount network-online.target"), + ("Conflicts", "shutdown.target"), + ("ActiveState", if refused == "idunn-yggdrasil.service" { state } else { "inactive" }), + ], + ); + } + h.describe("nginx.service", &[("Wants", "network-online.target"), ("Conflicts", "shutdown.target")]); + for (service, target) in [ + ("systemd-poweroff.service", "poweroff.target"), + ("systemd-reboot.service", "reboot.target"), + ("systemd-soft-reboot.service", "soft-reboot.target"), + ] { + h.describe( + service, + &[ + ("Requires", "shutdown.target final.target system.slice umount.target"), + ("RequiredBy", target), + ("ActiveState", "inactive"), + ], + ); + } + for (target, service) in [ + ("poweroff.target", "systemd-poweroff.service"), + ("reboot.target", "systemd-reboot.service"), + ("soft-reboot.target", "systemd-soft-reboot.service"), + ] { + h.describe(target, &[("Requires", service), ("ActiveState", "inactive")]); + } + h.describe( + "umount.target", + &[ + ("ActiveState", "inactive"), + ("RequiredBy", "systemd-poweroff.service systemd-reboot.service"), + ("ConflictedBy", "boot-efi.mount boot.mount \"blockdev@dev-vda2.target\""), + ], + ); + h.describe( + "multi-user.target", + &[ + ("Requires", "basic.target"), + ("Wants", "cron.service ssh.service idunn-yggdrasil.service"), + ("Conflicts", "shutdown.target rescue.target rescue.service"), + ], + ); + h.describe( + "voidbot.service", + &[ + ("Requires", "docker.service -.mount sysinit.target system.slice"), + ("Wants", "idunn-yggdrasil.service network-online.target"), + ("Conflicts", "shutdown.target"), + ], + ); + } + + fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + h.outcome(&id) + } + + fn hit(outcome: &Outcome) -> Option<&str> { + match outcome { + Outcome::Refused(Refused::AlwaysRefused { unit, via: Via::Propagation }) => Some(unit.as_str()), + _ => None, + } + } + + #[test] + fn a_transaction_that_reaches_a_refused_unit_is_refused() { + let h = harness(); + yggdrasil(&h, true); + // The targets and services that power the host down reach every refused + // unit through Conflicts=, however many requirements away. + for (verb, unit) in [ + (Verb::Start, "shutdown.target"), + (Verb::Start, "poweroff.target"), + (Verb::Restart, "poweroff.target"), + (Verb::Start, "reboot.target"), + (Verb::Restart, "reboot.target"), + (Verb::Start, "soft-reboot.target"), + (Verb::Start, "systemd-soft-reboot.service"), + (Verb::Restart, "systemd-poweroff.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert!(hit(&outcome).is_some(), "{verb:?} {unit} was {outcome:?}"); + } + assert!(h.runs().is_empty(), "no refused order ran anything"); + // A stop of shutdown.target takes down what requires it and runs no start. + let outcome = outcome_of_unit(&h, Verb::Stop, "shutdown.target"); + assert_eq!(outcome, Outcome::Ran); + + // A start that wants a refused unit that is stopped would start it. + h.describe("idunn-yggdrasil.service", &[("ActiveState", "inactive"), ("Conflicts", "shutdown.target")]); + for (verb, unit) in [ + (Verb::Start, "multi-user.target"), + (Verb::Restart, "multi-user.target"), + (Verb::Start, "voidbot.service"), + (Verb::Restart, "voidbot.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert_eq!(hit(&outcome), Some("idunn-yggdrasil.service"), "{verb:?} {unit}"); + } + // The same orders start nothing while the refused unit runs. + h.describe("idunn-yggdrasil.service", &[("ActiveState", "active"), ("Conflicts", "shutdown.target")]); + for (verb, unit) in [ + (Verb::Start, "multi-user.target"), + (Verb::Restart, "voidbot.service"), + (Verb::Start, "voidbot.service"), + ] { + assert_eq!(outcome_of_unit(&h, verb, unit), Outcome::Ran, "{verb:?} {unit}"); + } + // A unit that wants only what is not refused runs, and so does a target + // that unmounts nothing a refused unit requires. + for verb in [Verb::Start, Verb::Restart, Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "nginx.service"), Outcome::Ran); + } + assert_eq!(outcome_of_unit(&h, Verb::Start, "umount.target"), Outcome::Ran); + } + + #[test] + fn the_closure_follows_every_edge_and_every_hop() { + let h = harness(); + // Start: three hops of Wants/Requires/Requisite/BindsTo/Upholds to a + // stopped refused unit. + h.describe("hop-1.service", &[("Wants", "hop-2.service")]); + h.describe("hop-2.service", &[("Requires", "hop-3.service")]); + h.describe("hop-3.service", &[("Requisite", "hop-4.service")]); + h.describe("hop-4.service", &[("BindsTo", "hop-5.service")]); + h.describe("hop-5.service", &[("Upholds", "idunn-orders.service")]); + h.describe("idunn-orders.service", &[("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "hop-1.service"); + assert_eq!(hit(&outcome), Some("idunn-orders.service")); + // Conflicts of a pulled-in unit stop it and what requires it: the + // conflict, then two hops of reverse requirement, to a refused unit. + h.describe("rival.service", &[("Conflicts", "victim.service")]); + h.describe("victim.service", &[("RequiredBy", "mid.service")]); + h.describe("mid.service", &[("BoundBy", "mind-self@eureka.service")]); + h.describe("pulls-rival.service", &[("Requires", "rival.service")]); + for unit in ["rival.service", "pulls-rival.service"] { + let outcome = outcome_of_unit(&h, Verb::Start, unit); + assert_eq!(hit(&outcome), Some("mind-self@eureka.service"), "{unit}"); + } + // The reverse Conflicts edge: a unit conflicted by a refused one. + h.describe("incumbent.service", &[("ConflictedBy", "mind-persona@void.service")]); + let outcome = outcome_of_unit(&h, Verb::Start, "incumbent.service"); + assert_eq!(hit(&outcome), Some("mind-persona@void.service")); + // Stop and restart: every stop edge, one at a time. + for (property, reached) in [ + ("RequiredBy", "idunn-vili-x.service"), + ("BoundBy", "idunn-vili-x.service"), + ("RequisiteOf", "idunn-vili-x.service"), + ("ConsistsOf", "idunn-vili-x.service"), + ("PropagatesStopTo", "mind-persona@void.service"), + ("StopPropagatedFrom", "mind-persona@void.service"), + ] { + let name = format!("edge-{property}.service"); + h.describe(&name, &[(property, reached)]); + for verb in [Verb::Stop, Verb::Restart] { + let outcome = outcome_of_unit(&h, verb, &name); + assert_eq!(hit(&outcome), Some(reached), "{verb:?} {property}"); + } + } + // A restart starts what its stopped dependents pull in. + h.describe("base.service", &[("RequiredBy", "dependent.service")]); + h.describe("dependent.service", &[("Wants", "idunn-orders.service")]); + let outcome = outcome_of_unit(&h, Verb::Restart, "base.service"); + assert_eq!(hit(&outcome), Some("idunn-orders.service")); + assert_eq!(outcome_of_unit(&h, Verb::Stop, "base.service"), Outcome::Ran); + assert_eq!(outcome_of_unit(&h, Verb::Reload, "base.service"), Outcome::Ran); + } + + #[test] + fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { + let h = harness(); + // systemctl show quotes a name with a backslash and doubles the backslash. + h.describe( + "huginn.service", + &[("RequiredBy", "\"mind-self@foo\\\\x2dbar.service\"")], + ); + let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); + assert_eq!(hit(&outcome), Some("mind-self@foo\\x2dbar.service")); + h.describe("quiet.service", &[("Wants", "\"idunn-a\\\\x2db.service\" other.service")]); + h.describe("idunn-a\\x2db.service", &[("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); + assert_eq!(hit(&outcome), Some("idunn-a\\x2db.service")); + assert!(h.runs().is_empty()); + } + + #[test] + fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { + let h = harness(); + h.describe("ghost.service", &[("LoadState", "not-found")]); + h.describe("pulls-ghost.service", &[("Wants", "ghost.service")]); + h.describe("needed-by-ghost.service", &[("RequiredBy", "ghost.service")]); + for (verb, unit) in [ + (Verb::Start, "ghost.service"), + (Verb::Start, "pulls-ghost.service"), + (Verb::Stop, "needed-by-ghost.service"), + (Verb::Restart, "needed-by-ghost.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert!( + matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), + "{verb:?} {unit}" + ); + } + assert!(h.runs().is_empty()); + } + #[test] fn globs_slices_and_scopes_are_not_units() { let h = harness(); @@ -2715,16 +3103,40 @@ mod tests { #[test] fn systemctl_show_output_is_read_into_facts() { let facts = parse_show( - "Id=a.service\nNames=a.service b.service\nRequiredBy=c.service\n\ - BoundBy=\nConsistsOf=d.target e.service\nOther=x\n", + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=inactive\n\ + Requires=r.service \"q\\\\x2dr.slice\" -.mount\nRequisite=t.service\nWants=w.service\n\ + BindsTo=b.service\nUpholds=u.service\nConflicts=x.target\nConflictedBy=y.service\n\ + RequiredBy=c.service\nBoundBy=\nRequisiteOf=ro.service\nConsistsOf=d.target e.service\n\ + PropagatesStopTo=p.service\nStopPropagatedFrom=f.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); assert_eq!(facts.names, ["a.service", "b.service"]); - assert_eq!(facts.required_by, ["c.service"]); - assert!(facts.bound_by.is_empty()); - assert_eq!(facts.consists_of, ["d.target", "e.service"]); + assert!(!facts.not_found && !facts.running); + // A quoted name holds one backslash, not two and not a quote mark. + assert_eq!( + facts.pulls, + ["r.service", "q\\x2dr.slice", "-.mount", "t.service", "w.service", "b.service", "u.service"] + ); + assert_eq!(facts.conflicts, ["x.target", "y.service"]); + assert_eq!( + facts.stops, + ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] + ); assert!(parse_show("Names=a.service\n").is_err()); + let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); + assert!(gone.not_found); + for (state, running) in [ + ("active", true), + ("activating", true), + ("reloading", true), + ("inactive", false), + ("deactivating", false), + ("failed", false), + ] { + let facts = parse_show(&format!("Id=a.service\nActiveState={state}\n")).unwrap(); + assert_eq!(facts.running, running, "{state}"); + } } #[test] From cea17c06a79f0e02dc88990805d1b4be10c31798 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:49:36 +0100 Subject: [PATCH 05/23] README: a unit order is refused by the systemd transaction; a package order installs only what it names Co-Authored-By: Claude Sonnet 5.5 --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6af4527..2bf6675 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ It has its own store, `orders.cc`, and its own binding, `steward.toml`; the survival daemon takes no order. There is still no command string: ```bash -idunn order package-install [=version]... --reason TEXT # apt-get update, then install +idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them idunn order unit --reason TEXT idunn order deploy --reason TEXT # the same commit as `idunn up` idunn order show # status and output tails @@ -75,9 +75,20 @@ idunn order list A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and -read from no configuration. A name, an alias that resolves to one of them, or a -`stop` or `restart` whose dependents systemd would stop with it is refused, -recorded and posted. Slices, scopes and globs are not units an order can name. +read from no configuration. An order is refused, recorded and posted when its +name, an alias it resolves from, or the transaction systemd would build for it +reaches one of them: a start pulls in what the unit requires or wants (a +refused unit that is already running is left alone) and stops what conflicts +with it, a stop takes down what depends on it, and a restart does both. The +transaction is read from `systemctl show`, so `start poweroff.target` is refused +because every refused unit conflicts with `shutdown.target`, not because +targets are banned. A unit `systemctl show` cannot resolve fails the order. +Slices, scopes and globs are not units an order can name. + +A package order installs the packages it names and nothing else: apt runs with +`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` +match themselves, and a last step fails the order unless every named package +is installed. An order runs once, after the steward's uid is checked and the instance's brake check (`mind-control --instance check`) exits 0; an order From be3b95bc2161239bea16d414d4b9511120ce6d9f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:50:20 +0100 Subject: [PATCH 06/23] Import BTreeMap for the transaction facts cache Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index ce95337..127e0e1 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,7 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::{BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; From 27ca875194c0126a09fc0e12d611e36a102989d4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 03:53:41 +0100 Subject: [PATCH 07/23] Steward orders tests: clear four clippy hits in the module Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 127e0e1..fdcaafa 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2836,11 +2836,11 @@ mod tests { .unwrap(); let pending = recover(&h.ctx).unwrap(); assert_eq!(h.outcome(&id), Outcome::Lost); - assert_eq!(pending, [waiting.clone()], "an order that never started waits"); + assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); execute(&h.ctx, &id).unwrap(); assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); // Recovery is idempotent, and a finished order is not run again. - assert_eq!(recover(&h.ctx).unwrap(), [waiting.clone()]); + assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); execute(&h.ctx, &waiting).unwrap(); execute(&h.ctx, &waiting).unwrap(); assert_eq!(h.runs().len(), 1); @@ -2995,7 +2995,7 @@ mod tests { format!("{CANARY}*.service"), format!("{CANARY}/../idunn-orders.service"), format!("idunn-{CANARY};x.service"), - format!("{CANARY}"), + CANARY.to_string(), ] { note(h.issue_unit(Verb::Restart, &unit)); } @@ -3162,7 +3162,7 @@ mod tests { 0o600 ); let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); - SingleFileMessagePackBackingStore::new(&h.ctx.store.path()) + SingleFileMessagePackBackingStore::new(h.ctx.store.path()) .compare_exchange( &[CultCacheExpectedEnvelope { r#type: "idunn.other".into(), From ca439a29ff394f51511eef095a1da234d06a6b62 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 04:17:12 +0100 Subject: [PATCH 08/23] Pin the systemctl show property list and unquoted backslashes Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index fdcaafa..44f28db 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -1289,6 +1289,15 @@ fn parse_names(value: &str) -> Vec { names } +/// The properties one `systemctl show` is asked for: identity, state, edges. +fn show_properties() -> String { + ["Id", "Names", "LoadState", "ActiveState"] + .into_iter() + .chain(EDGES.iter().map(|(property, _)| *property)) + .collect::>() + .join(",") +} + fn parse_show(output: &str) -> Result { let mut facts = UnitFacts::default(); for line in output.lines() { @@ -1315,14 +1324,9 @@ fn parse_show(output: &str) -> Result { impl Systemd for HostSystemd { fn show(&self, unit: &str) -> Result { - let properties = ["Id", "Names", "LoadState", "ActiveState"] - .into_iter() - .chain(EDGES.iter().map(|(property, _)| *property)) - .collect::>() - .join(","); let output = Command::new("systemctl") .args(["show", "--no-pager"]) - .arg(format!("--property={properties}")) + .arg(format!("--property={}", show_properties())) .args(["--", unit]) .stdin(Stdio::null()) .output() @@ -3124,6 +3128,17 @@ mod tests { ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] ); assert!(parse_show("Names=a.service\n").is_err()); + // The request asks for exactly the properties the parse reads. + assert_eq!( + show_properties(), + "Id,Names,LoadState,ActiveState,Requires,Requisite,BindsTo,Wants,Upholds,\ + Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ + PropagatesStopTo,StopPropagatedFrom" + ); + // A backslash outside quotes is a backslash; inside, it escapes one character. + assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); + assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); + assert!(parse_names(" ").is_empty()); let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); for (state, running) in [ From 56edad95c1b211e982b1032a09fb29b74fa28931 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:03:17 +0100 Subject: [PATCH 09/23] Steward orders: one steward gate for fresh and recovered orders; the orders unit leaks no group or umask; root reads; kernel-attested issuer; end-to-end serve test execute refuses, with a record and a post, an order whose principal or issuing uid is not the steward bound now, before grammar, scope and brake; the brake is asked for the bound principal only. idunn-orders.service drops Group= and UMask=: bind_socket gives the socket to the steward uid at 0600 and serve sets Idunn's own umask, so a deploy order writes control.cc 0640. Root may show and list, never order. The issuer unit is the first unit under system.slice, never a name the peer chose below it. Synapse waits for a reply within one deadline. due_posts takes the deploy-state reader so a finished deploy's post is tested. Tests: the gate both ways, root, per-order posts, deploy result, intake and Synapse timeouts, parse_serve, user_uid, socket and store modes, and the whole service as root with a fake systemctl, systemd-run and homeserver. Co-Authored-By: Claude Sonnet 5.5 --- deploy/idunn-orders.service | 11 +- src/steward_orders.rs | 657 ++++++++++++++++++++++++++++++++++-- 2 files changed, 629 insertions(+), 39 deletions(-) diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service index cd3c46b..f2681c2 100644 --- a/deploy/idunn-orders.service +++ b/deploy/idunn-orders.service @@ -15,10 +15,10 @@ Type=simple # The unit name matches idunn-*, which is in the always-refused set, so no # order can stop or restart this service. User=root -# The socket is created root:idunn-steward 0660 (UMask below). The steward user -# is the only peer the service answers: it checks SO_PEERCRED on every -# connection, so the group is a second door held shut, not the lock. -Group=idunn-steward +# No Group= and no UMask=: the service gives its socket to the steward user +# (mode 0600) itself and sets Idunn's own umask for the files it writes, so +# nothing here reaches control.cc. The steward user is the only peer the +# service answers: it checks SO_PEERCRED on every connection. ExecStart=/usr/local/bin/idunn orders serve \ --store /var/lib/gamecult/idunn-orders \ --state-store /var/lib/gamecult/idunn/control.cc \ @@ -27,10 +27,9 @@ ExecStart=/usr/local/bin/idunn orders serve \ Restart=always RestartSec=5 TimeoutStopSec=20 -UMask=0117 RuntimeDirectory=idunn-orders -RuntimeDirectoryMode=0750 +RuntimeDirectoryMode=0755 # The Synapse access token of @idunn, copied from a root-only file, and read # through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 44f28db..e707307 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -381,6 +381,8 @@ pub(crate) enum Via { pub(crate) enum Refused { AlwaysRefused { unit: String, via: Via }, Braked, + /// The order's principal or issuing uid is not the steward bound now. + NotSteward, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -419,6 +421,9 @@ impl Outcome { } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Refused(Refused::NotSteward) => { + "refused: the issuer is not the current steward".into() + } Outcome::Lost => "lost: the orders service restarted mid-run".into(), Outcome::Submitted { command_id } => { format!("submitted deployment {}", escape(command_id.as_bytes())) @@ -865,6 +870,11 @@ pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { return Ok(()); } + // One gate for a fresh order and a recovered one: who may order is what the + // binding says now, not what it said when the order was accepted. + if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { + return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); + } if order.operation.validate().is_err() { return conclude( ctx, @@ -895,7 +905,7 @@ pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { } } } - if !ctx.brake.clear(&order.principal) { + if !ctx.brake.clear(&ctx.principal) { return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); } let uuid = order_uuid(order_id); @@ -1010,7 +1020,12 @@ fn deploy_finished(state: &str) -> bool { !matches!(state, "queued" | "running") } -fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { +/// What is owed to the room. `deploy_state` is a deployment command's derived +/// state and detail, read from the control store by the caller. +fn due_posts( + ledger: &Ledger, + deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, +) -> Vec { let mut due = Vec::new(); for order in &ledger.orders { let id = &order.order_id; @@ -1029,7 +1044,7 @@ fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { }; let mut line = record.outcome.describe(); if let Outcome::Submitted { command_id } = &record.outcome { - match command_state(&ctx.state_store, command_id) { + match deploy_state(command_id) { Ok((state, detail)) if deploy_finished(state) => { line = format!("{line}: {state} {}", escape(detail.as_bytes())); } @@ -1053,7 +1068,8 @@ fn due_posts(ctx: &Ctx, ledger: &Ledger) -> Vec { pub(crate) fn post_pass(ctx: &Ctx) -> Result { let ledger = ctx.store.ledger()?; let mut sent = 0; - for due in due_posts(ctx, &ledger) { + let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); + for due in due_posts(&ledger, &deploy_state) { let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); let event_id = ctx.homeserver.post(&transaction, &due.text)?; ctx.store.put( @@ -1173,7 +1189,9 @@ pub(crate) fn handle( stream.set_read_timeout(Some(Duration::from_secs(5)))?; stream.set_write_timeout(Some(Duration::from_secs(5)))?; let peer = ctx.peers.peer(stream)?; - if peer.uid != ctx.steward_uid { + // The steward may order and read; root, the operator, may only read. + let steward = peer.uid == ctx.steward_uid; + if !steward && peer.uid != 0 { eprintln!("steward socket: refused a peer that is not the steward"); return refuse(stream, "peer", Code::Unauthorized); } @@ -1185,6 +1203,10 @@ pub(crate) fn handle( return refuse(stream, "frame", Code::Malformed); }; match request { + Request::Order(_) if !steward => { + eprintln!("steward socket: refused an order from a peer that is not the steward"); + refuse(stream, "peer", Code::Unauthorized) + } Request::Order(request) => { let checked = validate_reason(&request.reason).and_then(|()| request.operation.validate()); @@ -1368,17 +1390,27 @@ impl Systemd for HostSystemd { } } -/// The last `.service` or `.scope` in a process's cgroup path names its issuer. +/// A system service or scope names the issuer: the first unit under +/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices +/// above it are the manager's, and what a delegated subtree adds below it is +/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. fn unit_from_cgroup(text: &str) -> String { let line = text .lines() .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) .unwrap_or_default(); let path = line.rsplit_once(':').map_or("", |(_, path)| path); - path.rsplit('/') - .find(|segment| segment.ends_with(".service") || segment.ends_with(".scope")) - .filter(|segment| segment.len() <= UNIT_LIMIT) - .map_or_else(|| "unknown".to_owned(), str::to_owned) + let unit = path.strip_prefix("/system.slice/").and_then(|under| { + for segment in under.split('/') { + if segment.ends_with(".slice") { + continue; + } + let unit = segment.ends_with(".service") || segment.ends_with(".scope"); + return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); + } + None + }); + unit.map_or_else(|| "unknown".to_owned(), str::to_owned) } /// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. @@ -1440,6 +1472,8 @@ pub(crate) struct Synapse { pub(crate) address: std::net::SocketAddr, pub(crate) room: String, pub(crate) token_file: PathBuf, + /// The whole wait for a reply, however slowly it drips. + pub(crate) reply_within: Duration, } fn percent_encode(text: &str) -> String { @@ -1474,15 +1508,26 @@ impl Homeserver for Synapse { ); let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) .context("the homeserver is unreachable")?; - socket.set_read_timeout(Some(Duration::from_secs(10)))?; - socket.set_write_timeout(Some(Duration::from_secs(10)))?; + socket.set_write_timeout(Some(self.reply_within))?; socket.write_all(head.as_bytes())?; socket.write_all(&body)?; + let deadline = std::time::Instant::now() + self.reply_within; let mut reply = Vec::new(); - socket - .take(RESPONSE_FRAME_LIMIT as u64) - .read_to_end(&mut reply) - .context("the homeserver reply was cut short")?; + let mut chunk = [0u8; 4096]; + while reply.len() < RESPONSE_FRAME_LIMIT { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|left| !left.is_zero()) + .context("the homeserver reply was too slow")?; + socket.set_read_timeout(Some(remaining))?; + let count = socket + .read(&mut chunk) + .context("the homeserver reply was cut short")?; + if count == 0 { + break; + } + reply.extend_from_slice(&chunk[..count]); + } let split = reply .windows(4) .position(|window| window == b"\r\n\r\n") @@ -1664,7 +1709,9 @@ fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { .expect("a thread can be spawned"); } -fn bind_socket(path: &Path) -> Result { +/// The socket belongs to the steward and no one else may open it; the peer +/// credential check on every connection stays the lock. +fn bind_socket(path: &Path, steward_uid: u32) -> Result { if let Ok(metadata) = std::fs::symlink_metadata(path) { use std::os::unix::fs::FileTypeExt; ensure!( @@ -1674,12 +1721,20 @@ fn bind_socket(path: &Path) -> Result { std::fs::remove_file(path).context("removing a stale socket")?; } let listener = UnixListener::bind(path).context("binding the steward socket")?; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) + std::os::unix::fs::chown(path, Some(steward_uid), None) + .context("giving the steward socket to the steward")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) .context("setting the steward socket mode")?; Ok(listener) } +/// Files this service creates in Idunn's own stores (a deploy order writes +/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. +const IDUNN_UMASK: libc::mode_t = 0o027; + pub(crate) fn serve(options: ServeOptions) -> Result<()> { + // SAFETY: umask only sets the process file-mode mask; it cannot fail. + unsafe { libc::umask(IDUNN_UMASK) }; let binding = load_binding(&options.steward_binding)?; let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") .map(PathBuf::from) @@ -1698,11 +1753,12 @@ pub(crate) fn serve(options: ServeOptions) -> Result<()> { address: binding.address()?, room: binding.room.clone(), token_file: credentials.join(&binding.token_credential), + reply_within: Duration::from_secs(10), }), }); ctx.store.prepare()?; let pending = recover(&ctx)?; - let listener = bind_socket(&options.socket)?; + let listener = bind_socket(&options.socket, ctx.steward_uid)?; let (queue, orders) = mpsc::channel::(); for order_id in pending { queue.send(order_id)?; @@ -1945,10 +2001,12 @@ mod tests { } } - struct FakeBrake(AtomicBool); + /// Whether the brake is clear, and the principals it was asked about. + struct FakeBrake(AtomicBool, Mutex>); impl BrakeCheck for FakeBrake { - fn clear(&self, _principal: &str) -> bool { + fn clear(&self, principal: &str) -> bool { + self.1.lock().unwrap().push(principal.into()); self.0.load(Ordering::SeqCst) } } @@ -1988,7 +2046,7 @@ mod tests { ..RunReport::default() }), }); - let brake = Arc::new(FakeBrake(AtomicBool::new(true))); + let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); let home = Arc::new(FakeHome::default()); let (queue, orders) = mpsc::channel(); let ctx = Ctx { @@ -2424,7 +2482,7 @@ mod tests { .filter(|record| matches!(record.outcome, Outcome::Refused(_))) .count(); assert_eq!(refusals, direct.len() * 4 + 8 + 6); - let due = due_posts(&h.ctx, &ledger); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); let results = due .iter() .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) @@ -2891,7 +2949,7 @@ mod tests { .unwrap(); assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); // The result post waits for the transactions: queued is not finished. - let due = due_posts(&h.ctx, &ledger); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); assert_eq!(due.len(), 1); assert_eq!(due[0].kind, PostKind::Accepted); assert!(!deploy_finished("queued") && !deploy_finished("running")); @@ -2948,7 +3006,7 @@ mod tests { assert_eq!(record.stdout_tail.len(), TAIL_BYTES); assert!(!format!("{record:?}").contains("secret-looking")); // Neither the post nor the journal text carries a tail. - let results: Vec<_> = due_posts(&h.ctx, &ledger) + let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) .into_iter() .filter(|due| due.kind == PostKind::Result) .collect(); @@ -3088,20 +3146,30 @@ mod tests { unit_from_cgroup("0::/system.slice/claude-connector.service\n"), "claude-connector.service" ); + // What a delegated subtree adds below the unit is the peer's to name. assert_eq!( - unit_from_cgroup("0::/system.slice/huginn@eureka.service/sub/leaf\n"), + unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), "huginn@eureka.service" ); assert_eq!( - unit_from_cgroup("0::/user.slice/user-1000.slice/session-3.scope\n"), - "session-3.scope" + unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), + "getty@tty1.service" ); assert_eq!( - unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.service\n"), - "a.service" + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), + "a.scope" ); - assert_eq!(unit_from_cgroup("0::/\n"), "unknown"); - assert_eq!(unit_from_cgroup(""), "unknown"); + // A user manager's units, and paths no manager made, name no one. + for path in [ + "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", + "0::/user.slice/user-1000.slice/session-3.scope\n", + "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", + "0::/system.slice/\n", + "0::/\n", + "", + ] { + assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); + } } #[test] @@ -3223,6 +3291,7 @@ mod tests { address, room: "!room:example.org".into(), token_file, + reply_within: Duration::from_secs(10), }; assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); let request = requests.recv().unwrap(); @@ -3238,7 +3307,8 @@ mod tests { Synapse { address, room: "!r:e".into(), - token_file: directory.path().join("absent") + token_file: directory.path().join("absent"), + reply_within: Duration::from_secs(10), } .post("t", "x") .is_err() @@ -3356,4 +3426,525 @@ mod tests { assert!(parse(&refused).is_err(), "{refused:?} parsed"); } } + + #[test] + fn an_order_runs_only_under_the_steward_bound_now() { + let mut h = harness(); + let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The binding changes while both wait: another uid, then another + // principal with the old uid. Each is refused with a record, before + // the brake is asked and before anything runs. + h.ctx.steward_uid = STEWARD + 7; + execute(&h.ctx, &first).unwrap(); + h.ctx.steward_uid = STEWARD; + h.ctx.principal = "void".into(); + execute(&h.ctx, &second).unwrap(); + for id in [&first, &second] { + assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); + assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); + } + assert!(h.runs().is_empty()); + assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); + // The refusal is posted like any other. + let ledger = h.ctx.store.ledger().unwrap(); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert!(due.iter().any(|due| due.kind == PostKind::Result + && due.text.contains("refused: the issuer is not the current steward"))); + // The other direction: an order the bound steward issued after the + // rebinding, and a recovered one under an unchanged binding, run and + // are gated by the bound principal's brake. + let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); + let resumed = harness(); + let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); + assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&resumed.ctx, &waiting).unwrap(); + execute(&h.ctx, &third).unwrap(); + assert_eq!(resumed.outcome(&waiting), Outcome::Ran); + assert_eq!(h.outcome(&third), Outcome::Ran); + assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); + assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); + } + + #[test] + fn root_reads_orders_and_never_issues_one() { + let mut h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + h.ctx.peers = Arc::new(FakePeers(0)); + let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); + assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("root could not read an order"); + }; + assert_eq!(view.order.order_id, id); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("root could not list orders"); + }; + assert_eq!(lines.len(), 1, "root's refused order left no record"); + h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); + assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); + } + + #[test] + fn every_order_posts_once_in_its_turn() { + let h = harness(); + let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] + .into_iter() + .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) + .collect(); + for id in &ids { + execute(&h.ctx, id).unwrap(); + } + assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); + assert_eq!(post_pass(&h.ctx).unwrap(), 0); + let sent = h.home.sent.lock().unwrap().clone(); + let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); + keys.sort(); + let mut expected: Vec = ids + .iter() + .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) + .collect(); + expected.sort(); + assert_eq!(keys, expected); + // An order accepted later is posted by the next pass alone. + let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + execute(&h.ctx, &late).unwrap(); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + } + + #[test] + fn a_deploys_result_is_posted_once_its_command_has_finished() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let ledger = h.ctx.store.ledger().unwrap(); + let results = |state: Option<(&'static str, &str)>| { + let owed = due_posts(&ledger, &|_| match state { + Some((state, detail)) => Ok((state, detail.to_owned())), + None => Err(anyhow!("the command cannot be read")), + }); + owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() + }; + assert!(results(Some(("queued", ""))).is_empty()); + assert!(results(Some(("running", "vili:Sealing"))).is_empty()); + assert!(results(None).is_empty()); + for state in ["succeeded", "failed"] { + let owed = results(Some((state, "vili"))); + assert_eq!(owed.len(), 1, "{state}"); + assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); + assert!(owed[0].text.contains("submitted deployment up-")); + } + } + + #[test] + fn intake_gives_up_on_a_peer_that_sends_no_frame() { + let h = harness(); + let (mut client, mut server) = UnixStream::pair().unwrap(); + // Half a length prefix, then nothing: the connection must end by itself. + client.write_all(&[0, 0]).unwrap(); + let (done, ended) = mpsc::channel(); + let worker = thread::spawn(move || { + let result = handle(&h.ctx, &mut server, &h.queue); + let _ = done.send(result.is_ok()); + h + }); + let answered = ended + .recv_timeout(Duration::from_secs(20)) + .expect("intake never gave up on a silent peer"); + assert!(answered, "a silent peer is refused as malformed, not left as an error"); + let h = worker.join().unwrap(); + assert_eq!( + rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn serve_options_are_read_from_every_flag() { + let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); + let defaults = parse(&["serve"]).unwrap(); + assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); + assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); + assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); + let given = parse(&[ + "serve", + "--state-store", + "/a/control.cc", + "--store", + "/b", + "--steward-binding", + "/c.toml", + "--socket", + "/d.sock", + ]) + .unwrap(); + assert_eq!( + given, + ServeOptions { + state_store: "/a/control.cc".into(), + store: "/b".into(), + steward_binding: "/c.toml".into(), + socket: "/d.sock".into(), + } + ); + assert!(parse(&["serve", "--store"]).is_err()); + assert!(parse(&["serve", "--other", "/x"]).is_err()); + assert!(parse(&["start"]).is_err()); + } + + #[test] + fn the_steward_user_is_a_real_user_and_never_root() { + assert!(user_uid("root").is_err()); + assert!(user_uid("no-such-user-here").is_err()); + assert!(user_uid("bad\0name").is_err()); + assert_eq!(user_uid("nobody").unwrap(), 65534); + } + + #[test] + fn the_orders_directory_is_made_private() { + let directory = TempDir::new().unwrap(); + let store = OrderStore::new(&directory.path().join("orders")); + store.prepare().unwrap(); + let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + #[test] + fn the_socket_belongs_to_the_steward_alone() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("steward.sock"); + let me = std::fs::metadata(directory.path()).unwrap().uid(); + let _listener = bind_socket(&path, me).unwrap(); + let metadata = std::fs::metadata(&path).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); + // A stale socket is replaced; anything else in the way is not. + let _again = bind_socket(&path, me).unwrap(); + let other = directory.path().join("other"); + std::fs::write(&other, "x").unwrap(); + assert!(bind_socket(&other, me).is_err()); + } + + #[test] + fn synapse_gives_up_on_a_reply_that_never_ends() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + thread::spawn(move || { + // The first connection stays silent; the second drips a byte at a time. + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + thread::spawn(move || { + let mut chunk = [0u8; 4096]; + let _ = socket.read(&mut chunk); + for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { + if index == 0 || socket.write_all(&[*byte]).is_err() { + break; + } + thread::sleep(Duration::from_millis(40)); + } + thread::sleep(Duration::from_secs(5)); + }); + } + }); + let synapse = Synapse { + address, + room: "!r:e".into(), + token_file, + reply_within: Duration::from_millis(600), + }; + for _ in 0..2 { + let started = std::time::Instant::now(); + let error = synapse.post("t", "x").unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error:#}"); + assert!(!format!("{error:#}").contains("tok-123")); + } + } + + /// The whole service, as root, against fake `systemctl`, `systemd-run` + /// and a loopback homeserver, with the steward connecting from another + /// uid. It runs only where the test process is root (the verify container). + #[test] + fn serve_end_to_end_as_root() { + // SAFETY: geteuid reads the process credentials. + if unsafe { libc::geteuid() } != 0 { + eprintln!("serve_end_to_end_as_root: skipped, not root"); + return; + } + let directory = TempDir::new().unwrap(); + let root = directory.path().to_owned(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let bin = root.join("bin"); + std::fs::create_dir(&bin).unwrap(); + let systemctl = format!( + "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ + case \"$*\" in *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + root.display() + ); + let run = format!( + "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", + root.display() + ); + for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { + let path = bin.join(name); + std::fs::write(&path, text).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let credentials = root.join("cred"); + std::fs::create_dir(&credentials).unwrap(); + std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (posted, posts) = mpsc::channel::(); + thread::spawn(move || { + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + let mut received = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + let text = String::from_utf8_lossy(&received).into_owned(); + if let Some(split) = text.find("\r\n\r\n") { + let length = text + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if received.len() >= split + 4 + length { + break; + } + } + let count = socket.read(&mut chunk).unwrap_or(0); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); + let _ = socket.write_all( + format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), + ); + } + }); + let binding = root.join("steward.toml"); + std::fs::write( + &binding, + format!( + "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ + token_credential = \"idunn-matrix-token\"\n" + ), + ) + .unwrap(); + std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); + let run_directory = root.join("run"); + std::fs::create_dir(&run_directory).unwrap(); + std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let socket = run_directory.join("steward.sock"); + let store = root.join("orders"); + let control = root.join("state").join("control.cc"); + // SAFETY: the only test that reads these variables is this one; PATH + // keeps its old entries. + unsafe { + std::env::set_var( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), + ); + std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); + } + let options = ServeOptions { + state_store: control.clone(), + store: store.clone(), + steward_binding: binding, + socket: socket.clone(), + }; + // The unit's umask must not decide the modes of what the service writes. + // SAFETY: umask only sets the process file-mode mask. + let before = unsafe { libc::umask(0o117) }; + let (ended, serve_end) = mpsc::channel::(); + thread::spawn(move || { + let result = serve(options); + let _ = ended.send(format!("{result:?}")); + }); + for _ in 0..100 { + if socket.exists() || serve_end.try_recv().is_ok() { + break; + } + thread::sleep(Duration::from_millis(100)); + } + // SAFETY: as above. + unsafe { libc::umask(before) }; + assert!(socket.exists(), "serve never bound its socket"); + let metadata = std::fs::metadata(&socket).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); + // Root is not the steward: it may read, it may not order. + let unit = |name: &str| { + Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: name.into(), + }, + reason: "end to end".into(), + }) + }; + let as_root = |request: &Request| { + let mut stream = UnixStream::connect(&socket).unwrap(); + write_frame(&mut stream, request).unwrap(); + rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + }; + assert_eq!( + as_root(&unit("nginx.service")), + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert_eq!(as_root(&Request::List), Response::List(Vec::new())); + let orders = OrderStore::new(&store); + let finished = |order_id: &str| { + for _ in 0..200 { + if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { + return record.clone(); + } + thread::sleep(Duration::from_millis(100)); + } + panic!("{order_id} never concluded"); + }; + // The steward is uid 65534 with gid 0: a service that read the gid for + // the uid would take it for root. + let first = steward_asks(&socket, &unit("nginx.service")); + let Response::Accepted { order_id: first } = first else { + panic!("the steward was not accepted: {first:?}"); + }; + assert_eq!(finished(&first).outcome, Outcome::Ran); + std::fs::write(root.join("fail"), "").unwrap(); + let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { + panic!("the second order was not accepted"); + }; + let failed = finished(&second); + assert_eq!( + failed.outcome, + Outcome::Failed { + exit_code: Some(3), + signal: None + } + ); + assert_eq!(failed.stderr_tail, b"boom\n"); + let deploy = Request::Order(OrderRequest { + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "end to end".into(), + }); + let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { + panic!("the deploy was not accepted"); + }; + assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); + let ledger = orders.ledger().unwrap(); + let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); + assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // What the service called, exactly. + let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); + assert!( + run_log.contains(&format!( + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart -- nginx.service", + order_uuid(&first) + )), + "{run_log}" + ); + let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); + assert!( + systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), + "{systemctl_log}" + ); + // Every record is posted; the deploy's result waits for its command. + let mut seen = Vec::new(); + while seen.len() < 5 { + seen.push(posts.recv_timeout(Duration::from_secs(30)).expect("five posts within 30 s")); + } + for (id, kinds) in [ + (&first, ["accepted", "result"].as_slice()), + (&second, &["accepted", "result"]), + (&third, &["accepted"]), + ] { + for kind in kinds { + let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); + assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); + } + } + // Modes: the store is private, and the deploy wrote control.cc under + // Idunn's umask, not the 0117 the process was started with. + let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; + assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); + assert_eq!(mode(&control), 0o640); + } + + /// One request as the steward: uid 65534 and gid 0, from a forked child + /// that only makes system calls. + fn steward_asks(socket: &Path, request: &Request) -> Response { + let body = rmp_serde::to_vec_named(request).unwrap(); + let mut frame = (body.len() as u32).to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + // SAFETY: plain data. + let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { + address.sun_path[index] = *byte as libc::c_char; + } + let mut fds = [0i32; 2]; + // SAFETY: `fds` holds two descriptors. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + // SAFETY: the child allocates nothing and ends with _exit. + let pid = unsafe { libc::fork() }; + if pid == 0 { + // SAFETY: only system calls on buffers made before the fork. + unsafe { + libc::close(fds[0]); + if libc::setgroups(0, std::ptr::null()) != 0 + || libc::setgid(0) != 0 + || libc::setuid(65534) != 0 + { + libc::_exit(2); + } + let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if libc::connect( + fd, + (&address as *const libc::sockaddr_un).cast(), + std::mem::size_of::() as libc::socklen_t, + ) != 0 + { + libc::_exit(3); + } + let mut offset = 0usize; + while offset < frame.len() { + let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); + if count <= 0 { + libc::_exit(4); + } + offset += count as usize; + } + let mut buffer = [0u8; 4096]; + loop { + let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); + if count <= 0 { + break; + } + libc::write(fds[1], buffer.as_ptr().cast(), count as usize); + } + libc::_exit(0); + } + } + // SAFETY: the parent's copy of the write end. + unsafe { libc::close(fds[1]) }; + let mut reply = Vec::new(); + // SAFETY: `fds[0]` is an open descriptor this function now owns. + let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; + pipe.read_to_end(&mut reply).unwrap(); + let mut status = 0; + // SAFETY: `pid` is our child. + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); + rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + } } From 572f9e6aa424661c36abf7e876a467a08d033eb8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:31:34 +0100 Subject: [PATCH 10/23] Steward orders: the end-to-end test keeps serve's umask until it has read control.cc Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index e707307..3f0d229 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3776,8 +3776,6 @@ mod tests { } thread::sleep(Duration::from_millis(100)); } - // SAFETY: as above. - unsafe { libc::umask(before) }; assert!(socket.exists(), "serve never bound its socket"); let metadata = std::fs::metadata(&socket).unwrap(); assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); @@ -3877,7 +3875,10 @@ mod tests { // Idunn's umask, not the 0117 the process was started with. let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); - assert_eq!(mode(&control), 0o640); + let control_mode = mode(&control); + // SAFETY: umask only sets the process file-mode mask. + unsafe { libc::umask(before) }; + assert_eq!(control_mode, 0o640); } /// One request as the steward: uid 65534 and gid 0, from a forked child From 7e15bf1a7b5e9003fd44025dd9e5fae05f18b2b3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:33:23 +0100 Subject: [PATCH 11/23] Steward orders: the Synapse deadline test fails instead of hanging when the deadline is gone Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 3f0d229..261cb24 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3658,11 +3658,20 @@ mod tests { token_file, reply_within: Duration::from_millis(600), }; + let synapse = Arc::new(synapse); for _ in 0..2 { let started = std::time::Instant::now(); - let error = synapse.post("t", "x").unwrap_err(); - assert!(started.elapsed() < Duration::from_secs(3), "{error:#}"); - assert!(!format!("{error:#}").contains("tok-123")); + let (done, ended) = mpsc::channel(); + let poster = Arc::clone(&synapse); + thread::spawn(move || { + let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); + }); + let error = ended + .recv_timeout(Duration::from_secs(4)) + .expect("the post never gave up") + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error}"); + assert!(!error.contains("tok-123")); } } From 8a1beb5c873c96b2741ffb64e7cfb958adbdcd4f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 05:35:30 +0100 Subject: [PATCH 12/23] README: an order is gated by the steward bound at execution; the socket is the steward's; root reads Co-Authored-By: Claude Sonnet 5.5 --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2bf6675..5240106 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,12 @@ match themselves, and a last step fails the order unless every named package is installed. An order runs once, after the steward's uid is checked and the instance's -brake check (`mind-control --instance check`) exits 0; an order -that had started when the service died is recorded as lost and never run -again. No order releases a brake. Acceptance and result are posted to +brake check (`mind-control --instance check`) exits 0. The check +that the order's principal and uid are the bound steward's runs again at every +execution, so an order accepted before `steward.toml` changed is refused and +recorded, not run. The socket belongs to the steward user alone (mode 0600); +root may `show` and `list` orders but never issue one. An order that had +started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to `#operator` as a retried projection of `orders.cc`: a chat outage delays the posts and not the orders. A refusal names the field and a code, never the value it refused. From 9653d270e5d47b6bc7b17a6680666e5529b26e07 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:16:04 +0100 Subject: [PATCH 13/23] Steward orders: pin the ledger schema guards, invocation ids and post backoff; tests fail fast instead of waiting Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 91 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 261cb24..1394891 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -1690,6 +1690,16 @@ pub(crate) fn parse_serve(mut args: impl Iterator) -> Result u64 { + if failures == 0 { + 2 + } else { + (2u64 << failures.min(5)).min(60) + } +} + /// Run a thread whose end ends the service: systemd restarts it, and the /// start recovery settles what it left. fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { @@ -1775,18 +1785,14 @@ pub(crate) fn serve(options: ServeOptions) -> Result<()> { vital("poster", move || { let mut failures = 0u32; loop { - let pause = match post_pass(&poster) { - Ok(_) => { - failures = 0; - 2 - } + match post_pass(&poster) { + Ok(_) => failures = 0, Err(error) => { failures = failures.saturating_add(1); eprintln!("steward orders: posting is delayed: {error:#}"); - (2u64 << failures.min(5)).min(60) } - }; - thread::sleep(Duration::from_secs(pause)); + } + thread::sleep(Duration::from_secs(post_pause(failures))); } }); for stream in listener.incoming() { @@ -2077,6 +2083,7 @@ mod tests { impl Harness { fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); write(&mut client); handle(&self.ctx, &mut server, &self.queue).unwrap(); let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); @@ -3543,6 +3550,7 @@ mod tests { fn intake_gives_up_on_a_peer_that_sends_no_frame() { let h = harness(); let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); // Half a length prefix, then nothing: the connection must end by itself. client.write_all(&[0, 0]).unwrap(); let (done, ended) = mpsc::channel(); @@ -3563,6 +3571,63 @@ mod tests { assert!(!h.ctx.store.path().exists()); } + #[test] + fn a_record_under_another_schema_is_not_read() { + let put = |h: &Harness, schema: &str, key: &str| match schema { + _ if schema.contains("start") => h.ctx.store.put( + key, + schema, + &StartRecord { + schema_version: schema.into(), + order_id: key.into(), + started_at_unix_millis: 1, + actuation: "idunn-order-x".into(), + }, + ), + _ if schema.contains("outcome") => h.ctx.store.put( + key, + schema, + &OutcomeRecord { + schema_version: schema.into(), + order_id: key.into(), + ended_at_unix_millis: 1, + outcome: Outcome::Lost, + stdout_tail: Vec::new(), + stderr_tail: Vec::new(), + }, + ), + _ => h.ctx.store.put( + key, + schema, + &PostRecord { + schema_version: schema.into(), + post_key: key.into(), + event_id: "$e".into(), + posted_at_unix_millis: 1, + }, + ), + }; + for (current, key) in [ + (START_SCHEMA, "ord-a"), + (OUTCOME_SCHEMA, "ord-a"), + (POST_SCHEMA, "ord-a:accepted"), + ] { + let h = harness(); + assert!(put(&h, current, key).unwrap()); + assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); + let h = harness(); + let other = current.replace(".v1", ".v0"); + assert!(put(&h, &other, key).unwrap()); + assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); + } + } + + #[test] + fn posts_are_retried_after_a_widening_pause() { + let pauses: Vec = (0..9).map(post_pause).collect(); + assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); + } + #[test] fn serve_options_are_read_from_every_flag() { let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); @@ -3692,7 +3757,7 @@ mod tests { std::fs::create_dir(&bin).unwrap(); let systemctl = format!( "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ - case \"$*\" in *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", root.display() ); let run = format!( @@ -3800,6 +3865,7 @@ mod tests { }; let as_root = |request: &Request| { let mut stream = UnixStream::connect(&socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); write_frame(&mut stream, request).unwrap(); rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() }; @@ -3851,6 +3917,10 @@ mod tests { let ledger = orders.ledger().unwrap(); let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // An invocation id is what a succeeding systemctl prints, if anything. + assert_eq!(HostSystemd.invocation("quiet.service"), None); + assert_eq!(HostSystemd.invocation("broken.service"), None); + assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); // What the service called, exactly. let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); assert!( @@ -3868,7 +3938,7 @@ mod tests { // Every record is posted; the deploy's result waits for its command. let mut seen = Vec::new(); while seen.len() < 5 { - seen.push(posts.recv_timeout(Duration::from_secs(30)).expect("five posts within 30 s")); + seen.push(posts.recv_timeout(Duration::from_secs(15)).expect("five posts within 15 s")); } for (id, kinds) in [ (&first, ["accepted", "result"].as_slice()), @@ -3911,6 +3981,7 @@ mod tests { // SAFETY: only system calls on buffers made before the fork. unsafe { libc::close(fds[0]); + libc::alarm(10); if libc::setgroups(0, std::ptr::null()) != 0 || libc::setgid(0) != 0 || libc::setuid(65534) != 0 From caacbb4bbaa4de31ffa63f76c1d4eec53a5080ce Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:35:31 +0100 Subject: [PATCH 14/23] Steward orders: the end-to-end test waits 10 s for its posts Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 1394891..688edc8 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3938,7 +3938,7 @@ mod tests { // Every record is posted; the deploy's result waits for its command. let mut seen = Vec::new(); while seen.len() < 5 { - seen.push(posts.recv_timeout(Duration::from_secs(15)).expect("five posts within 15 s")); + seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); } for (id, kinds) in [ (&first, ["accepted", "result"].as_slice()), From d9710c988a0a09adb74bb7f3ae2580ed881c9ef0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 1 Nov 2026 06:35:43 +0100 Subject: [PATCH 15/23] Steward orders: the ledger reads an order only under its own schema too Co-Authored-By: Claude Sonnet 5.5 --- src/steward_orders.rs | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 688edc8..1d72e0e 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3573,8 +3573,25 @@ mod tests { #[test] fn a_record_under_another_schema_is_not_read() { - let put = |h: &Harness, schema: &str, key: &str| match schema { - _ if schema.contains("start") => h.ctx.store.put( + let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { + "order" => h.ctx.store.put( + key, + schema, + &OrderRecord { + schema_version: schema.into(), + order_id: key.into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }, + ), + "start" => h.ctx.store.put( key, schema, &StartRecord { @@ -3584,7 +3601,7 @@ mod tests { actuation: "idunn-order-x".into(), }, ), - _ if schema.contains("outcome") => h.ctx.store.put( + "outcome" => h.ctx.store.put( key, schema, &OutcomeRecord { @@ -3607,17 +3624,18 @@ mod tests { }, ), }; - for (current, key) in [ - (START_SCHEMA, "ord-a"), - (OUTCOME_SCHEMA, "ord-a"), - (POST_SCHEMA, "ord-a:accepted"), + for (kind, current, key) in [ + ("order", ORDER_SCHEMA, "ord-a"), + ("start", START_SCHEMA, "ord-a"), + ("outcome", OUTCOME_SCHEMA, "ord-a"), + ("post", POST_SCHEMA, "ord-a:accepted"), ] { let h = harness(); - assert!(put(&h, current, key).unwrap()); + assert!(put(&h, kind, current, key).unwrap()); assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); let h = harness(); let other = current.replace(".v1", ".v0"); - assert!(put(&h, &other, key).unwrap()); + assert!(put(&h, kind, &other, key).unwrap()); assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); } } From d29bf93be60fdf08412d43299f28e0fbe40016b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:26:29 +0100 Subject: [PATCH 16/23] Steward orders: refuse a start or stop whose unit ends by emergency action or isolates; boot and recovery units are never started from stopped refusal_for now reads SuccessAction, FailureAction, StartLimitAction and JobTimeoutAction (each mapped to the target it runs, so reboot, poweroff, halt, kexec, soft-reboot and exit, with -force and -immediate) and OnFailure and OnSuccess as units a touched unit fires, followed through what they pull in and what those fire. An action it does not know fails the order closed. A stopped DefaultDependencies=no unit in the transaction is refused (Via:: EarlyBoot): rescue.service ends by isolating default.target, which no edge describes. Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 175 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 1d72e0e..b9d64b7 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -374,6 +374,10 @@ pub(crate) enum Via { Name, Alias, Propagation, + /// A unit the order would start from a stopped state that is boot or + /// recovery machinery (`DefaultDependencies=no`): such a unit can end by + /// isolating a target, which no dependency edge describes. + EarlyBoot, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -418,6 +422,7 @@ impl Outcome { Via::Name => "name", Via::Alias => "alias", Via::Propagation => "propagation", + Via::EarlyBoot => "early-boot", } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), @@ -641,6 +646,32 @@ const EDGES: [(&str, Edge); 13] = [ ("StopPropagatedFrom", Edge::Stops), ]; +/// The properties naming what systemd does when a unit ends or its job times +/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). +const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; + +/// The properties naming units systemd starts when a unit ends. +const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; + +/// The target an emergency action runs, `None` for `none`. An action this +/// check does not know is an error: the unit is not checked, so not run. +fn action_target(action: &str) -> Result> { + let base = ["-force", "-immediate"] + .iter() + .find_map(|suffix| action.strip_suffix(suffix)) + .unwrap_or(action); + Ok(match base { + "" | "none" => None, + "reboot" => Some("reboot.target"), + "poweroff" => Some("poweroff.target"), + "halt" => Some("halt.target"), + "kexec" => Some("kexec.target"), + "soft-reboot" => Some("soft-reboot.target"), + "exit" => Some("exit.target"), + _ => bail!("a unit has an emergency action this check does not know"), + }) +} + /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { @@ -648,6 +679,11 @@ pub(crate) struct UnitFacts { pub(crate) names: Vec, pub(crate) not_found: bool, pub(crate) running: bool, + /// `DefaultDependencies=no`: boot, recovery and shutdown machinery. + pub(crate) early_boot: bool, + /// What systemd starts when this unit ends or times out: the targets of + /// its emergency actions and its `OnFailure=` and `OnSuccess=` units. + pub(crate) fires: Vec, pub(crate) pulls: Vec, pub(crate) conflicts: Vec, pub(crate) stops: Vec, @@ -762,6 +798,27 @@ impl Transaction<'_> { Ok(reached) } + /// Every unit the emergency actions and `OnFailure=`/`OnSuccess=` of + /// `touched` would start, with what each pulls in, and in turn what those + /// that are not running would set off. A touched unit ends, so it fires. + fn fired(&mut self, touched: &BTreeSet) -> Result> { + let mut fired = BTreeSet::new(); + let mut seen = BTreeSet::new(); + let mut queue: VecDeque = touched.iter().cloned().collect(); + while let Some(unit) = queue.pop_front() { + if !seen.insert(unit.clone()) { + continue; + } + let seeds: BTreeSet = self.facts(&unit)?.fires.iter().cloned().collect(); + for reached in self.reach(&seeds, Edge::Pulls)? { + if fired.insert(reached.clone()) && !self.facts(&reached)?.running { + queue.push_back(reached); + } + } + } + Ok(fired) + } + /// Every unit a start of `started` stops through `Conflicts=`, and what /// that stop takes down with it. fn conflicted(&mut self, started: &BTreeSet) -> Result> { @@ -807,6 +864,13 @@ fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result Result Vec { /// The properties one `systemctl show` is asked for: identity, state, edges. fn show_properties() -> String { - ["Id", "Names", "LoadState", "ActiveState"] + ["Id", "Names", "LoadState", "ActiveState", "DefaultDependencies"] .into_iter() + .chain(ACTIONS) + .chain(ON_END) .chain(EDGES.iter().map(|(property, _)| *property)) .collect::>() .join(",") @@ -1333,6 +1402,9 @@ fn parse_show(output: &str) -> Result { "ActiveState" => { facts.running = matches!(value, "active" | "activating" | "reloading"); } + "DefaultDependencies" => facts.early_boot = value == "no", + _ if ACTIONS.contains(&key) => facts.fires.extend(action_target(value)?.map(str::to_owned)), + _ if ON_END.contains(&key) => facts.fires.extend(parse_names(value)), _ => { if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { facts.along_mut(*edge).extend(parse_names(value)); @@ -2719,6 +2791,107 @@ mod tests { assert!(h.runs().is_empty()); } + #[test] + fn a_unit_whose_end_reboots_the_host_is_refused_by_its_properties() { + let h = harness(); + yggdrasil(&h, true); + // Each emergency action property alone, in each of systemd's spellings, + // on a unit no name list knows: every verb that starts or stops it. + for (property, action) in [ + ("SuccessAction", "reboot"), + ("FailureAction", "reboot"), + ("StartLimitAction", "poweroff-force"), + ("JobTimeoutAction", "reboot-immediate"), + ("SuccessAction", "poweroff"), + ] { + let name = format!("storagetm-{property}.service"); + h.describe(&name, &[(property, action), ("ActiveState", "inactive")]); + for verb in [Verb::Start, Verb::Restart, Verb::Stop] { + let outcome = outcome_of_unit(&h, verb, &name); + assert!(hit(&outcome).is_some(), "{verb:?} {property}={action} {outcome:?}"); + } + } + // OnFailure= and OnSuccess= start the units they name, and what those + // set off in turn. + for property in ["OnFailure", "OnSuccess"] { + let name = format!("handled-{property}.service"); + h.describe(&name, &[(property, "reboot.target"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, &name); + assert!(hit(&outcome).is_some(), "{property} {outcome:?}"); + } + h.describe("on-failure-chain.service", &[("OnFailure", "handler.service"), ("ActiveState", "inactive")]); + h.describe("handler.service", &[("FailureAction", "reboot"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "on-failure-chain.service"); + assert!(hit(&outcome).is_some(), "{outcome:?}"); + // A unit a start pulls in fires its action too. + h.describe("pulls-storagetm.service", &[("Wants", "storagetm.service")]); + h.describe("storagetm.service", &[("SuccessAction", "reboot"), ("ActiveState", "inactive")]); + let outcome = outcome_of_unit(&h, Verb::Start, "pulls-storagetm.service"); + assert!(hit(&outcome).is_some(), "{outcome:?}"); + // A running unit the order does not stop is not ended by it, and an + // action of none ends nothing. + h.describe("pulls-running.service", &[("Wants", "running-action.service")]); + h.describe("running-action.service", &[("SuccessAction", "reboot")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "pulls-running.service"), Outcome::Ran); + h.describe("quiet-actions.service", &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", "")]); + for verb in [Verb::Start, Verb::Restart, Verb::Stop] { + assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); + } + // An action this check does not know is not checked, so it does not run. + h.describe("odd-action.service", &[("SuccessAction", "implode")]); + let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); + execute(&h.ctx, &id).unwrap(); + assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); + assert!(h.runs().is_empty()); + } + + #[test] + fn a_stopped_boot_or_recovery_unit_is_never_started() { + let h = harness(); + yggdrasil(&h, true); + // rescue.service ends by isolating default.target, which stops every + // unit outside it; no edge says so, its DefaultDependencies=no does. + h.describe( + "rescue.service", + &[("DefaultDependencies", "no"), ("ActiveState", "inactive")], + ); + h.describe("rescue.target", &[("Requires", "rescue.service"), ("ActiveState", "inactive")]); + h.describe("pulls-rescue.service", &[("Wants", "rescue.target")]); + for (verb, unit) in [ + (Verb::Start, "rescue.service"), + (Verb::Restart, "rescue.service"), + (Verb::Start, "rescue.target"), + (Verb::Restart, "rescue.target"), + (Verb::Start, "pulls-rescue.service"), + ] { + let outcome = outcome_of_unit(&h, verb, unit); + assert!( + matches!( + outcome, + Outcome::Refused(Refused::AlwaysRefused { ref unit, via: Via::EarlyBoot }) + if unit == "rescue.service" + ), + "{verb:?} {unit} was {outcome:?}" + ); + } + assert!(h.runs().is_empty()); + // Stopping it, or starting one whose defaults are on, is an ordinary order. + assert_eq!(outcome_of_unit(&h, Verb::Stop, "rescue.service"), Outcome::Ran); + h.describe("ordinary.service", &[("DefaultDependencies", "yes"), ("ActiveState", "inactive")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "ordinary.service"), Outcome::Ran); + // A boot unit that is already running is left alone by a start. + h.describe("journald.service", &[("DefaultDependencies", "no")]); + assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); + } + + #[test] + fn show_asks_for_every_property_the_refusal_reads() { + let asked = show_properties(); + for property in ["DefaultDependencies", "SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction", "OnFailure", "OnSuccess"] { + assert!(asked.split(',').any(|asked| asked == property), "{property}"); + } + } + #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); From 631fb7836b6e875fcbfa64fcc01b0ea78a4a08fd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:29:13 +0100 Subject: [PATCH 17/23] Steward orders: pin the new show properties; the unknown-action probe counts runs instead of expecting none Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index b9d64b7..884afda 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2838,11 +2838,12 @@ mod tests { assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); } // An action this check does not know is not checked, so it does not run. + let ran = h.runs().len(); h.describe("odd-action.service", &[("SuccessAction", "implode")]); let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); execute(&h.ctx, &id).unwrap(); assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); - assert!(h.runs().is_empty()); + assert_eq!(h.runs().len(), ran); } #[test] @@ -2884,14 +2885,6 @@ mod tests { assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); } - #[test] - fn show_asks_for_every_property_the_refusal_reads() { - let asked = show_properties(); - for property in ["DefaultDependencies", "SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction", "OnFailure", "OnSuccess"] { - assert!(asked.split(',').any(|asked| asked == property), "{property}"); - } - } - #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); @@ -3379,8 +3372,9 @@ mod tests { // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), - "Id,Names,LoadState,ActiveState,Requires,Requisite,BindsTo,Wants,Upholds,\ - Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ + "Id,Names,LoadState,ActiveState,DefaultDependencies,SuccessAction,FailureAction,\ + StartLimitAction,JobTimeoutAction,OnFailure,OnSuccess,Requires,Requisite,BindsTo,\ + Wants,Upholds,Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ PropagatesStopTo,StopPropagatedFrom" ); // A backslash outside quotes is a backslash; inside, it escapes one character. From 879b76a8425df76ba18f17a78fff488e87e1bc0a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:50:23 +0100 Subject: [PATCH 18/23] Steward orders: pin every emergency action spelling to its target Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 884afda..69eea41 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -2846,6 +2846,31 @@ mod tests { assert_eq!(h.runs().len(), ran); } + #[test] + fn every_emergency_action_runs_the_target_it_names() { + for (action, target) in [ + ("none", None), + ("", None), + ("reboot", Some("reboot.target")), + ("reboot-force", Some("reboot.target")), + ("reboot-immediate", Some("reboot.target")), + ("poweroff", Some("poweroff.target")), + ("poweroff-force", Some("poweroff.target")), + ("poweroff-immediate", Some("poweroff.target")), + ("halt", Some("halt.target")), + ("halt-force", Some("halt.target")), + ("kexec", Some("kexec.target")), + ("kexec-immediate", Some("kexec.target")), + ("soft-reboot", Some("soft-reboot.target")), + ("soft-reboot-force", Some("soft-reboot.target")), + ("exit", Some("exit.target")), + ("exit-force", Some("exit.target")), + ] { + assert_eq!(action_target(action).unwrap(), target, "{action}"); + } + assert!(action_target("implode").is_err()); + } + #[test] fn a_stopped_boot_or_recovery_unit_is_never_started() { let h = harness(); From 4894ad20df0f98fa308044d82e1f9fa21468ddf7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 21:52:41 +0100 Subject: [PATCH 19/23] README: an order's transaction follows emergency actions and OnFailure/OnSuccess; stopped boot units are refused Co-Authored-By: Claude Opus 5.5 --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5240106..28c1e94 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,14 @@ refused unit that is already running is left alone) and stops what conflicts with it, a stop takes down what depends on it, and a restart does both. The transaction is read from `systemctl show`, so `start poweroff.target` is refused because every refused unit conflicts with `shutdown.target`, not because -targets are banned. A unit `systemctl show` cannot resolve fails the order. +targets are banned. The units a start or stop ends or times out are followed too: +the target each of `SuccessAction=`, `FailureAction=`, `StartLimitAction=` and +`JobTimeoutAction=` runs (`reboot`, `poweroff`, `halt`, `kexec`, `soft-reboot`, +`exit`, with `-force` and `-immediate`), and the units `OnFailure=` and +`OnSuccess=` name, with what each pulls in; an action it does not know fails the +order. A start refuses a stopped `DefaultDependencies=no` unit (boot and recovery +machinery, such as `rescue.service`, which ends by isolating the default +target), whatever pulls it in. A unit `systemctl show` cannot resolve fails the order. Slices, scopes and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with From a859914b0bb481ee9abeb999d73753f9313c28a0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:40:58 +0100 Subject: [PATCH 20/23] Steward orders: run every unit verb as one job and check only the named unit Delete the transaction model (Edge, EDGES, action_target, Transaction, the early-boot rule). Every unit verb runs systemctl --job-mode=ignore-requirements -- ; refusal_for reads one systemctl show of the ordered unit: its names, the host-ending names, what fires when it ends, and who is bound to it. Targets are no longer orderable. Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 806 +++++++++++------------------------------- 1 file changed, 207 insertions(+), 599 deletions(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 69eea41..d5fdcfe 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,7 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::BTreeMap; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -58,9 +58,8 @@ const TAIL_BYTES: usize = 8 * 1024; const REASON_LIMIT: usize = 300; const PACKAGE_LIMIT: usize = 16; const UNIT_LIMIT: usize = 200; -const CLOSURE_LIMIT: usize = 512; const LIST_LIMIT: usize = 50; -const UNIT_SUFFIXES: [&str; 6] = [".service", ".socket", ".timer", ".path", ".mount", ".target"]; +const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; /// The always-refused set: Idunn's own units and the minds' own. Matched on /// every name a unit answers to, ASCII case-insensitively (systemd units are @@ -72,6 +71,21 @@ fn is_refused_name(name: &str) -> bool { .any(|prefix| name.starts_with(prefix)) } +/// Units whose binaries suspend, hibernate or open recovery: no property +/// reveals it, so they are refused by name, on every name they answer to. +const HOST_NAMES: [&str; 6] = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", +]; + +fn is_host_name(name: &str) -> bool { + HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) +} + // ---------------------------------------------------------------- grammar -- #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -288,6 +302,7 @@ impl Operation { argv: vec![ "systemctl".into(), verb.as_str().into(), + "--job-mode=ignore-requirements".into(), "--".into(), unit.clone(), ], @@ -373,11 +388,9 @@ impl fmt::Debug for StartRecord { pub(crate) enum Via { Name, Alias, - Propagation, - /// A unit the order would start from a stopped state that is boot or - /// recovery machinery (`DefaultDependencies=no`): such a unit can end by - /// isolating a target, which no dependency edge describes. - EarlyBoot, + Host, + OnEnd, + Bound, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -421,8 +434,9 @@ impl Outcome { match via { Via::Name => "name", Via::Alias => "alias", - Via::Propagation => "propagation", - Via::EarlyBoot => "early-boot", + Via::Host => "host", + Via::OnEnd => "on-end", + Via::Bound => "bound", } ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), @@ -617,35 +631,6 @@ impl OrderStore { // ------------------------------------------------------------------ ports -- -/// Which way systemd moves along a dependency edge when an order is given. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Edge { - /// Units a start brings up with this one. - Pulls, - /// Units a start stops: the unit's `Conflicts=` and the units that conflict with it. - Conflicts, - /// Units a stop takes down with this one. - Stops, -} - -/// The `systemctl show` properties the closure reads, and the way each one -/// moves. This is the only list: the request and the parse both read it. -const EDGES: [(&str, Edge); 13] = [ - ("Requires", Edge::Pulls), - ("Requisite", Edge::Pulls), - ("BindsTo", Edge::Pulls), - ("Wants", Edge::Pulls), - ("Upholds", Edge::Pulls), - ("Conflicts", Edge::Conflicts), - ("ConflictedBy", Edge::Conflicts), - ("RequiredBy", Edge::Stops), - ("BoundBy", Edge::Stops), - ("RequisiteOf", Edge::Stops), - ("ConsistsOf", Edge::Stops), - ("PropagatesStopTo", Edge::Stops), - ("StopPropagatedFrom", Edge::Stops), -]; - /// The properties naming what systemd does when a unit ends or its job times /// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; @@ -653,62 +638,24 @@ const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction" /// The properties naming units systemd starts when a unit ends. const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; -/// The target an emergency action runs, `None` for `none`. An action this -/// check does not know is an error: the unit is not checked, so not run. -fn action_target(action: &str) -> Result> { - let base = ["-force", "-immediate"] - .iter() - .find_map(|suffix| action.strip_suffix(suffix)) - .unwrap_or(action); - Ok(match base { - "" | "none" => None, - "reboot" => Some("reboot.target"), - "poweroff" => Some("poweroff.target"), - "halt" => Some("halt.target"), - "kexec" => Some("kexec.target"), - "soft-reboot" => Some("soft-reboot.target"), - "exit" => Some("exit.target"), - _ => bail!("a unit has an emergency action this check does not know"), - }) -} - /// What `systemctl show` says about one unit. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct UnitFacts { pub(crate) id: String, pub(crate) names: Vec, pub(crate) not_found: bool, - pub(crate) running: bool, - /// `DefaultDependencies=no`: boot, recovery and shutdown machinery. - pub(crate) early_boot: bool, - /// What systemd starts when this unit ends or times out: the targets of - /// its emergency actions and its `OnFailure=` and `OnSuccess=` units. - pub(crate) fires: Vec, - pub(crate) pulls: Vec, - pub(crate) conflicts: Vec, - pub(crate) stops: Vec, + /// The properties that make the unit act when it ends: each emergency + /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. + pub(crate) on_end: Vec, + /// The units bound to this one at runtime (`BoundBy=`): systemd stops + /// them when this unit stops, whatever the job mode. + pub(crate) bound_by: Vec, } impl UnitFacts { fn identities(&self) -> impl Iterator { std::iter::once(&self.id).chain(self.names.iter()) } - - fn along(&self, edge: Edge) -> &[String] { - match edge { - Edge::Pulls => &self.pulls, - Edge::Conflicts => &self.conflicts, - Edge::Stops => &self.stops, - } - } - - fn along_mut(&mut self, edge: Edge) -> &mut Vec { - match edge { - Edge::Pulls => &mut self.pulls, - Edge::Conflicts => &mut self.conflicts, - Edge::Stops => &mut self.stops, - } - } } #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -765,126 +712,31 @@ fn refused(unit: &str, via: Via) -> Refused { } } -/// The units systemd would touch for one order, read from `systemctl show`. -struct Transaction<'a> { - systemd: &'a dyn Systemd, - facts: BTreeMap, -} - -impl Transaction<'_> { - fn facts(&mut self, unit: &str) -> Result<&UnitFacts> { - if !self.facts.contains_key(unit) { - ensure!( - self.facts.len() < CLOSURE_LIMIT, - "the units an order would reach are too many to check" - ); - let facts = self.systemd.show(unit)?; - ensure!(!facts.not_found, "systemd has no such unit to check"); - self.facts.insert(unit.to_owned(), facts); - } - Ok(&self.facts[unit]) - } - - /// Every unit reached from `seeds` along `edge`, seeds included. - fn reach(&mut self, seeds: &BTreeSet, edge: Edge) -> Result> { - let mut reached = BTreeSet::new(); - let mut queue: VecDeque = seeds.iter().cloned().collect(); - while let Some(next) = queue.pop_front() { - if !reached.insert(next.clone()) { - continue; - } - queue.extend(self.facts(&next)?.along(edge).iter().cloned()); - } - Ok(reached) - } - - /// Every unit the emergency actions and `OnFailure=`/`OnSuccess=` of - /// `touched` would start, with what each pulls in, and in turn what those - /// that are not running would set off. A touched unit ends, so it fires. - fn fired(&mut self, touched: &BTreeSet) -> Result> { - let mut fired = BTreeSet::new(); - let mut seen = BTreeSet::new(); - let mut queue: VecDeque = touched.iter().cloned().collect(); - while let Some(unit) = queue.pop_front() { - if !seen.insert(unit.clone()) { - continue; - } - let seeds: BTreeSet = self.facts(&unit)?.fires.iter().cloned().collect(); - for reached in self.reach(&seeds, Edge::Pulls)? { - if fired.insert(reached.clone()) && !self.facts(&reached)?.running { - queue.push_back(reached); - } - } - } - Ok(fired) - } - - /// Every unit a start of `started` stops through `Conflicts=`, and what - /// that stop takes down with it. - fn conflicted(&mut self, started: &BTreeSet) -> Result> { - let mut seeds = BTreeSet::new(); - for unit in started { - seeds.extend(self.facts(unit)?.along(Edge::Conflicts).iter().cloned()); - } - self.reach(&seeds, Edge::Stops) - } -} - -/// Would `verb` on `unit` reach an always-refused unit: by its name, by an -/// alias it resolves from, or through the transaction systemd builds for it. -/// A start pulls in what the unit requires or wants and stops what conflicts; -/// a stop takes down what depends on the unit; a restart does both. A unit -/// the transaction would stop is refused when it is refused; a unit it would -/// start is refused when it is refused and not running. Every unit in the -/// closure must be one systemd can show. +/// Would `verb` on `unit` be refused. systemd builds the transaction and the +/// order tells it to build one job (`--job-mode=ignore-requirements`), so this +/// reads only what that job mode cannot remove from the one named unit: the +/// name it answers to, what fires when it ends, and who is bound to it. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); } - let mut transaction = Transaction { - systemd, - facts: BTreeMap::new(), - }; - if let Some(name) = transaction - .facts(unit)? - .identities() - .find(|name| is_refused_name(name)) + let facts = systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + for name in facts.identities() { + if is_refused_name(name) { + return Ok(Some(refused(name, Via::Alias))); + } + if is_host_name(name) { + return Ok(Some(refused(name, Via::Host))); + } + } + if verb != Verb::Reload && !facts.on_end.is_empty() { + return Ok(Some(refused(unit, Via::OnEnd))); + } + if matches!(verb, Verb::Stop | Verb::Restart) + && let Some(bound) = facts.bound_by.first() { - return Ok(Some(refused(name, Via::Alias))); - } - let root = BTreeSet::from([unit.to_owned()]); - let mut stopped = BTreeSet::new(); - let mut started = BTreeSet::new(); - match verb { - Verb::Reload => {} - Verb::Start => started = transaction.reach(&root, Edge::Pulls)?, - Verb::Stop => stopped = transaction.reach(&root, Edge::Stops)?, - Verb::Restart => { - stopped = transaction.reach(&root, Edge::Stops)?; - started = transaction.reach(&stopped, Edge::Pulls)?; - } - } - let mut touched = stopped.clone(); - for member in &started { - if !transaction.facts(member)?.running { - touched.insert(member.clone()); - } - } - started.extend(transaction.fired(&touched)?); - stopped.extend(transaction.conflicted(&started)?); - for (units, only_if_stopped) in [(&stopped, false), (&started, true)] { - for member in units { - let facts = transaction.facts(member)?; - if only_if_stopped && facts.running { - continue; - } - if let Some(name) = facts.identities().find(|name| is_refused_name(name)) { - return Ok(Some(refused(name, Via::Propagation))); - } - if only_if_stopped && facts.early_boot { - return Ok(Some(refused(&facts.id, Via::EarlyBoot))); - } - } + return Ok(Some(refused(bound, Via::Bound))); } Ok(None) } @@ -1378,13 +1230,14 @@ fn parse_names(value: &str) -> Vec { names } -/// The properties one `systemctl show` is asked for: identity, state, edges. +/// The properties one `systemctl show` is asked for: identity, what fires at +/// the unit's end, and who is bound to it. fn show_properties() -> String { - ["Id", "Names", "LoadState", "ActiveState", "DefaultDependencies"] + ["Id", "Names", "LoadState"] .into_iter() .chain(ACTIONS) .chain(ON_END) - .chain(EDGES.iter().map(|(property, _)| *property)) + .chain(["BoundBy"]) .collect::>() .join(",") } @@ -1399,17 +1252,16 @@ fn parse_show(output: &str) -> Result { "Id" => facts.id = value.to_owned(), "Names" => facts.names = parse_names(value), "LoadState" => facts.not_found = value == "not-found", - "ActiveState" => { - facts.running = matches!(value, "active" | "activating" | "reloading"); - } - "DefaultDependencies" => facts.early_boot = value == "no", - _ if ACTIONS.contains(&key) => facts.fires.extend(action_target(value)?.map(str::to_owned)), - _ if ON_END.contains(&key) => facts.fires.extend(parse_names(value)), - _ => { - if let Some((_, edge)) = EDGES.iter().find(|(property, _)| *property == key) { - facts.along_mut(*edge).extend(parse_names(value)); + "BoundBy" => facts.bound_by = parse_names(value), + _ if ACTIONS.contains(&key) => { + if !matches!(value, "" | "none") { + facts.on_end.push(key.to_owned()); } } + _ if ON_END.contains(&key) => { + facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); + } + _ => {} } } ensure!(!facts.id.is_empty(), "systemctl named no unit"); @@ -2031,11 +1883,11 @@ mod tests { const STEWARD: u32 = 1000; const CANARY: &str = "CANARY-7f3a91"; - /// What `systemctl show` prints for a loaded, active unit with no edges, + /// What `systemctl show` prints for a loaded unit that acts on nothing, /// plus the given properties (a given key replaces its default). fn show_text(id: &str, properties: &[(&str, &str)]) -> String { let mut text = format!("Id={id}\n"); - for (key, default) in [("Names", id), ("LoadState", "loaded"), ("ActiveState", "active")] { + for (key, default) in [("Names", id), ("LoadState", "loaded")] { if !properties.iter().any(|(given, _)| *given == key) { text += &format!("{key}={default}\n"); } @@ -2049,11 +1901,13 @@ mod tests { struct FakeSystemd { units: Mutex>, runs: Mutex>, + shows: Mutex>, report: Mutex, } impl Systemd for FakeSystemd { fn show(&self, unit: &str) -> Result { + self.shows.lock().unwrap().push(unit.into()); let known = self.units.lock().unwrap().get(unit).cloned(); parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) } @@ -2119,6 +1973,7 @@ mod tests { let systemd = Arc::new(FakeSystemd { units: Mutex::new(BTreeMap::new()), runs: Mutex::new(Vec::new()), + shows: Mutex::new(Vec::new()), report: Mutex::new(RunReport { exit_code: Some(0), ..RunReport::default() @@ -2426,7 +2281,6 @@ mod tests { "persona-matrix-mouth.service", "mind-heartbeat.service", "apt-daily.timer", - "multi-user.target", // Near misses: the prefix is the rule, not a substring or a stem. "midunn-x.service", "not-idunn-x.service", @@ -2443,14 +2297,17 @@ mod tests { assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); let (transient, step) = h.runs().pop().unwrap(); assert!(transient.starts_with("idunn-order-")); - assert_eq!(step.argv, ["systemctl", verb.as_str(), "--", unit]); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] + ); } } assert_eq!(h.runs().len(), units.len() * 4); } #[test] - fn refused_set_is_refused_by_name_alias_and_propagation() { + fn refused_set_is_refused_by_name_and_alias() { let h = harness(); h.describe_unit( "backup-name.service", @@ -2463,21 +2320,6 @@ mod tests { "quiet-alias.service", &[("Names", "quiet-alias.service mind-persona@void.service")], ); - // fuse <- wrapper <- (bound by) mind-self@eureka - h.describe("fuse.service", &[("RequiredBy", "wrapper.service")]); - h.describe("wrapper.service", &[("BoundBy", "mind-self@eureka.service")]); - h.describe("stack.target", &[("ConsistsOf", "mind-persona@void.service")]); - // A cycle that reaches an alias of a refused unit on its second hop. - h.describe("loop-a.service", &[("RequiredBy", "loop-b.service")]); - h.describe("loop-b.service", &[("RequiredBy", "loop-a.service hidden.service")]); - h.describe_unit( - "hidden.service", - show_text( - "idunn-orders.service", - &[("Names", "idunn-orders.service hidden.service")], - ), - ); - let direct = [ "idunn-yggdrasil.service", "idunn-orders.service", @@ -2528,31 +2370,7 @@ mod tests { ); } } - for verb in [Verb::Stop, Verb::Restart] { - for (unit, hit) in [ - ("fuse.service", "mind-self@eureka.service"), - ("stack.target", "mind-persona@void.service"), - ("loop-a.service", "idunn-orders.service"), - ] { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - assert_eq!( - h.outcome(&id), - Outcome::Refused(Refused::AlwaysRefused { - unit: hit.into(), - via: Via::Propagation - }), - "{verb:?} {unit}" - ); - } - } assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); - // start and reload take nothing down, so the same units are not refused. - for verb in [Verb::Start, Verb::Reload] { - let id = h.accept(h.issue_unit(verb, "fuse.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran); - } // Every refusal is recorded and has a result post due. let ledger = h.ctx.store.ledger().unwrap(); let refusals = ledger @@ -2560,7 +2378,7 @@ mod tests { .iter() .filter(|record| matches!(record.outcome, Outcome::Refused(_))) .count(); - assert_eq!(refusals, direct.len() * 4 + 8 + 6); + assert_eq!(refusals, direct.len() * 4 + 8); let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); let results = due .iter() @@ -2568,93 +2386,7 @@ mod tests { .count(); assert_eq!(results, refusals); // A refused order never started. - assert!(ledger.starts.len() == 2); - } - - /// Yggdrasil's `systemctl show` shapes (systemd 257, 2026-11-01), cut down - /// to the units an order can reach: seven refused units all conflict with - /// shutdown.target, and the shutdown services require it. - fn yggdrasil(h: &Harness, idunn_running: bool) { - let state = if idunn_running { "active" } else { "inactive" }; - let refused = [ - "idunn-yggdrasil.service", - "idunn-orders.service", - "idunn-vili-e0ada640.service", - "mind-self@eureka.service", - "mind-persona@void.service", - ]; - h.describe( - "shutdown.target", - &[ - ("ActiveState", "inactive"), - ( - "RequiredBy", - "systemd-soft-reboot.service systemd-reboot.service systemd-poweroff.service", - ), - ( - "ConflictedBy", - "nginx.service idunn-yggdrasil.service idunn-orders.service \ - idunn-vili-e0ada640.service mind-self@eureka.service mind-persona@void.service", - ), - ], - ); - for refused in refused { - h.describe( - refused, - &[ - ("Requires", "sysinit.target system.slice -.mount"), - ("Wants", "tmp.mount network-online.target"), - ("Conflicts", "shutdown.target"), - ("ActiveState", if refused == "idunn-yggdrasil.service" { state } else { "inactive" }), - ], - ); - } - h.describe("nginx.service", &[("Wants", "network-online.target"), ("Conflicts", "shutdown.target")]); - for (service, target) in [ - ("systemd-poweroff.service", "poweroff.target"), - ("systemd-reboot.service", "reboot.target"), - ("systemd-soft-reboot.service", "soft-reboot.target"), - ] { - h.describe( - service, - &[ - ("Requires", "shutdown.target final.target system.slice umount.target"), - ("RequiredBy", target), - ("ActiveState", "inactive"), - ], - ); - } - for (target, service) in [ - ("poweroff.target", "systemd-poweroff.service"), - ("reboot.target", "systemd-reboot.service"), - ("soft-reboot.target", "systemd-soft-reboot.service"), - ] { - h.describe(target, &[("Requires", service), ("ActiveState", "inactive")]); - } - h.describe( - "umount.target", - &[ - ("ActiveState", "inactive"), - ("RequiredBy", "systemd-poweroff.service systemd-reboot.service"), - ("ConflictedBy", "boot-efi.mount boot.mount \"blockdev@dev-vda2.target\""), - ], - ); - h.describe( - "multi-user.target", - &[ - ("Requires", "basic.target"), - ("Wants", "cron.service ssh.service idunn-yggdrasil.service"), - ("Conflicts", "shutdown.target rescue.target rescue.service"), - ], - ); - h.describe( - "voidbot.service", - &[ - ("Requires", "docker.service -.mount sysinit.target system.slice"), - ("Wants", "idunn-yggdrasil.service network-online.target"), - ("Conflicts", "shutdown.target"), - ], - ); + assert!(ledger.starts.is_empty()); } fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { @@ -2663,277 +2395,171 @@ mod tests { h.outcome(&id) } - fn hit(outcome: &Outcome) -> Option<&str> { + fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { match outcome { - Outcome::Refused(Refused::AlwaysRefused { unit, via: Via::Propagation }) => Some(unit.as_str()), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { + Some(unit.as_str()) + } _ => None, } } - #[test] - fn a_transaction_that_reaches_a_refused_unit_is_refused() { - let h = harness(); - yggdrasil(&h, true); - // The targets and services that power the host down reach every refused - // unit through Conflicts=, however many requirements away. - for (verb, unit) in [ - (Verb::Start, "shutdown.target"), - (Verb::Start, "poweroff.target"), - (Verb::Restart, "poweroff.target"), - (Verb::Start, "reboot.target"), - (Verb::Restart, "reboot.target"), - (Verb::Start, "soft-reboot.target"), - (Verb::Start, "systemd-soft-reboot.service"), - (Verb::Restart, "systemd-poweroff.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert!(hit(&outcome).is_some(), "{verb:?} {unit} was {outcome:?}"); - } - assert!(h.runs().is_empty(), "no refused order ran anything"); - // A stop of shutdown.target takes down what requires it and runs no start. - let outcome = outcome_of_unit(&h, Verb::Stop, "shutdown.target"); - assert_eq!(outcome, Outcome::Ran); - - // A start that wants a refused unit that is stopped would start it. - h.describe("idunn-yggdrasil.service", &[("ActiveState", "inactive"), ("Conflicts", "shutdown.target")]); - for (verb, unit) in [ - (Verb::Start, "multi-user.target"), - (Verb::Restart, "multi-user.target"), - (Verb::Start, "voidbot.service"), - (Verb::Restart, "voidbot.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert_eq!(hit(&outcome), Some("idunn-yggdrasil.service"), "{verb:?} {unit}"); - } - // The same orders start nothing while the refused unit runs. - h.describe("idunn-yggdrasil.service", &[("ActiveState", "active"), ("Conflicts", "shutdown.target")]); - for (verb, unit) in [ - (Verb::Start, "multi-user.target"), - (Verb::Restart, "voidbot.service"), - (Verb::Start, "voidbot.service"), - ] { - assert_eq!(outcome_of_unit(&h, verb, unit), Outcome::Ran, "{verb:?} {unit}"); - } - // A unit that wants only what is not refused runs, and so does a target - // that unmounts nothing a refused unit requires. - for verb in [Verb::Start, Verb::Restart, Verb::Stop, Verb::Reload] { - assert_eq!(outcome_of_unit(&h, verb, "nginx.service"), Outcome::Ran); - } - assert_eq!(outcome_of_unit(&h, Verb::Start, "umount.target"), Outcome::Ran); - } - - #[test] - fn the_closure_follows_every_edge_and_every_hop() { - let h = harness(); - // Start: three hops of Wants/Requires/Requisite/BindsTo/Upholds to a - // stopped refused unit. - h.describe("hop-1.service", &[("Wants", "hop-2.service")]); - h.describe("hop-2.service", &[("Requires", "hop-3.service")]); - h.describe("hop-3.service", &[("Requisite", "hop-4.service")]); - h.describe("hop-4.service", &[("BindsTo", "hop-5.service")]); - h.describe("hop-5.service", &[("Upholds", "idunn-orders.service")]); - h.describe("idunn-orders.service", &[("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "hop-1.service"); - assert_eq!(hit(&outcome), Some("idunn-orders.service")); - // Conflicts of a pulled-in unit stop it and what requires it: the - // conflict, then two hops of reverse requirement, to a refused unit. - h.describe("rival.service", &[("Conflicts", "victim.service")]); - h.describe("victim.service", &[("RequiredBy", "mid.service")]); - h.describe("mid.service", &[("BoundBy", "mind-self@eureka.service")]); - h.describe("pulls-rival.service", &[("Requires", "rival.service")]); - for unit in ["rival.service", "pulls-rival.service"] { - let outcome = outcome_of_unit(&h, Verb::Start, unit); - assert_eq!(hit(&outcome), Some("mind-self@eureka.service"), "{unit}"); - } - // The reverse Conflicts edge: a unit conflicted by a refused one. - h.describe("incumbent.service", &[("ConflictedBy", "mind-persona@void.service")]); - let outcome = outcome_of_unit(&h, Verb::Start, "incumbent.service"); - assert_eq!(hit(&outcome), Some("mind-persona@void.service")); - // Stop and restart: every stop edge, one at a time. - for (property, reached) in [ - ("RequiredBy", "idunn-vili-x.service"), - ("BoundBy", "idunn-vili-x.service"), - ("RequisiteOf", "idunn-vili-x.service"), - ("ConsistsOf", "idunn-vili-x.service"), - ("PropagatesStopTo", "mind-persona@void.service"), - ("StopPropagatedFrom", "mind-persona@void.service"), - ] { - let name = format!("edge-{property}.service"); - h.describe(&name, &[(property, reached)]); - for verb in [Verb::Stop, Verb::Restart] { - let outcome = outcome_of_unit(&h, verb, &name); - assert_eq!(hit(&outcome), Some(reached), "{verb:?} {property}"); - } - } - // A restart starts what its stopped dependents pull in. - h.describe("base.service", &[("RequiredBy", "dependent.service")]); - h.describe("dependent.service", &[("Wants", "idunn-orders.service")]); - let outcome = outcome_of_unit(&h, Verb::Restart, "base.service"); - assert_eq!(hit(&outcome), Some("idunn-orders.service")); - assert_eq!(outcome_of_unit(&h, Verb::Stop, "base.service"), Outcome::Ran); - assert_eq!(outcome_of_unit(&h, Verb::Reload, "base.service"), Outcome::Ran); - } - #[test] fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { let h = harness(); // systemctl show quotes a name with a backslash and doubles the backslash. h.describe( "huginn.service", - &[("RequiredBy", "\"mind-self@foo\\\\x2dbar.service\"")], + &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], ); let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); - assert_eq!(hit(&outcome), Some("mind-self@foo\\x2dbar.service")); - h.describe("quiet.service", &[("Wants", "\"idunn-a\\\\x2db.service\" other.service")]); - h.describe("idunn-a\\x2db.service", &[("ActiveState", "inactive")]); + assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); + h.describe( + "quiet.service", + &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], + ); let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); - assert_eq!(hit(&outcome), Some("idunn-a\\x2db.service")); + assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); assert!(h.runs().is_empty()); } #[test] - fn a_unit_whose_end_reboots_the_host_is_refused_by_its_properties() { + fn unit_orders_run_as_one_job() { let h = harness(); - yggdrasil(&h, true); - // Each emergency action property alone, in each of systemd's spellings, - // on a unit no name list knows: every verb that starts or stops it. - for (property, action) in [ - ("SuccessAction", "reboot"), - ("FailureAction", "reboot"), - ("StartLimitAction", "poweroff-force"), - ("JobTimeoutAction", "reboot-immediate"), - ("SuccessAction", "poweroff"), - ] { - let name = format!("storagetm-{property}.service"); - h.describe(&name, &[(property, action), ("ActiveState", "inactive")]); - for verb in [Verb::Start, Verb::Restart, Verb::Stop] { - let outcome = outcome_of_unit(&h, verb, &name); - assert!(hit(&outcome).is_some(), "{verb:?} {property}={action} {outcome:?}"); - } + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); + let (_, step) = h.runs().pop().unwrap(); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] + ); } - // OnFailure= and OnSuccess= start the units they name, and what those - // set off in turn. - for property in ["OnFailure", "OnSuccess"] { - let name = format!("handled-{property}.service"); - h.describe(&name, &[(property, "reboot.target"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, &name); - assert!(hit(&outcome).is_some(), "{property} {outcome:?}"); - } - h.describe("on-failure-chain.service", &[("OnFailure", "handler.service"), ("ActiveState", "inactive")]); - h.describe("handler.service", &[("FailureAction", "reboot"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "on-failure-chain.service"); - assert!(hit(&outcome).is_some(), "{outcome:?}"); - // A unit a start pulls in fires its action too. - h.describe("pulls-storagetm.service", &[("Wants", "storagetm.service")]); - h.describe("storagetm.service", &[("SuccessAction", "reboot"), ("ActiveState", "inactive")]); - let outcome = outcome_of_unit(&h, Verb::Start, "pulls-storagetm.service"); - assert!(hit(&outcome).is_some(), "{outcome:?}"); - // A running unit the order does not stop is not ended by it, and an - // action of none ends nothing. - h.describe("pulls-running.service", &[("Wants", "running-action.service")]); - h.describe("running-action.service", &[("SuccessAction", "reboot")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "pulls-running.service"), Outcome::Ran); - h.describe("quiet-actions.service", &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", "")]); - for verb in [Verb::Start, Verb::Restart, Verb::Stop] { - assert_eq!(outcome_of_unit(&h, verb, "quiet-actions.service"), Outcome::Ran, "{verb:?}"); - } - // An action this check does not know is not checked, so it does not run. - let ran = h.runs().len(); - h.describe("odd-action.service", &[("SuccessAction", "implode")]); - let id = h.accept(h.issue_unit(Verb::Start, "odd-action.service")); - execute(&h.ctx, &id).unwrap(); - assert!(matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None })); - assert_eq!(h.runs().len(), ran); } #[test] - fn every_emergency_action_runs_the_target_it_names() { - for (action, target) in [ - ("none", None), - ("", None), - ("reboot", Some("reboot.target")), - ("reboot-force", Some("reboot.target")), - ("reboot-immediate", Some("reboot.target")), - ("poweroff", Some("poweroff.target")), - ("poweroff-force", Some("poweroff.target")), - ("poweroff-immediate", Some("poweroff.target")), - ("halt", Some("halt.target")), - ("halt-force", Some("halt.target")), - ("kexec", Some("kexec.target")), - ("kexec-immediate", Some("kexec.target")), - ("soft-reboot", Some("soft-reboot.target")), - ("soft-reboot-force", Some("soft-reboot.target")), - ("exit", Some("exit.target")), - ("exit-force", Some("exit.target")), - ] { - assert_eq!(action_target(action).unwrap(), target, "{action}"); - } - assert!(action_target("implode").is_err()); - } - - #[test] - fn a_stopped_boot_or_recovery_unit_is_never_started() { + fn the_rule_reads_only_the_named_unit() { let h = harness(); - yggdrasil(&h, true); - // rescue.service ends by isolating default.target, which stops every - // unit outside it; no edge says so, its DefaultDependencies=no does. + // Whatever the unit lists, no other unit is looked at. h.describe( - "rescue.service", - &[("DefaultDependencies", "no"), ("ActiveState", "inactive")], + "plain.service", + &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], ); - h.describe("rescue.target", &[("Requires", "rescue.service"), ("ActiveState", "inactive")]); - h.describe("pulls-rescue.service", &[("Wants", "rescue.target")]); - for (verb, unit) in [ - (Verb::Start, "rescue.service"), - (Verb::Restart, "rescue.service"), - (Verb::Start, "rescue.target"), - (Verb::Restart, "rescue.target"), - (Verb::Start, "pulls-rescue.service"), - ] { - let outcome = outcome_of_unit(&h, verb, unit); - assert!( - matches!( - outcome, - Outcome::Refused(Refused::AlwaysRefused { ref unit, via: Via::EarlyBoot }) - if unit == "rescue.service" - ), - "{verb:?} {unit} was {outcome:?}" + h.describe("other.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); + } + + #[test] + fn a_unit_that_acts_when_it_ends_is_refused() { + let h = harness(); + let properties = [ + ("SuccessAction", "reboot"), + ("FailureAction", "poweroff-force"), + ("StartLimitAction", "soft-reboot-immediate"), + ("JobTimeoutAction", "exit"), + ("FailureAction", "a-spelling-nobody-knows"), + ("OnFailure", "rescue.service"), + ("OnSuccess", "cleanup.service"), + ]; + for (property, value) in properties { + h.describe("ender.service", &[(property, value)]); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "ender.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "ender.service".into(), + via: Via::OnEnd + }), + "{verb:?} {property}={value}" + ); + } + assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); + } + // `none` and an empty list say the unit does nothing at its end. + h.describe( + "calm.service", + &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.runs().len(), 11); + } + + #[test] + fn host_names_are_refused() { + let h = harness(); + let names = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", + ]; + h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); + for verb in all_verbs() { + for name in names { + assert_eq!( + outcome_of_unit(&h, verb, name), + Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), + "{verb:?} {name}" + ); + } + assert_eq!( + outcome_of_unit(&h, verb, "sleepy.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "systemd-hibernate.service".into(), + via: Via::Host + }), + "{verb:?}" ); } assert!(h.runs().is_empty()); - // Stopping it, or starting one whose defaults are on, is an ordinary order. - assert_eq!(outcome_of_unit(&h, Verb::Stop, "rescue.service"), Outcome::Ran); - h.describe("ordinary.service", &[("DefaultDependencies", "yes"), ("ActiveState", "inactive")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "ordinary.service"), Outcome::Ran); - // A boot unit that is already running is left alone by a start. - h.describe("journald.service", &[("DefaultDependencies", "no")]); - assert_eq!(outcome_of_unit(&h, Verb::Start, "journald.service"), Outcome::Ran); + assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); + } + + #[test] + fn a_unit_others_are_bound_to_only_starts() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service second.service")]); + for verb in [Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "base.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "first.service".into(), + via: Via::Bound + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Start, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); + } } #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); h.describe("ghost.service", &[("LoadState", "not-found")]); - h.describe("pulls-ghost.service", &[("Wants", "ghost.service")]); - h.describe("needed-by-ghost.service", &[("RequiredBy", "ghost.service")]); - for (verb, unit) in [ - (Verb::Start, "ghost.service"), - (Verb::Start, "pulls-ghost.service"), - (Verb::Stop, "needed-by-ghost.service"), - (Verb::Restart, "needed-by-ghost.service"), - ] { - let id = h.accept(h.issue_unit(verb, unit)); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "ghost.service")); execute(&h.ctx, &id).unwrap(); assert!( matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), - "{verb:?} {unit}" + "{verb:?}" ); } assert!(h.runs().is_empty()); } #[test] - fn globs_slices_and_scopes_are_not_units() { + fn globs_slices_scopes_and_targets_are_not_units() { let h = harness(); let bad = [ "idunn*", @@ -2943,6 +2569,8 @@ mod tests { "mind-self@*.service", "mind-[s]elf@eureka.service", "system.slice", + "poweroff.target", + "multi-user.target", "user.slice", "session-1.scope", "init.scope", @@ -3373,34 +3001,24 @@ mod tests { #[test] fn systemctl_show_output_is_read_into_facts() { let facts = parse_show( - "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=inactive\n\ - Requires=r.service \"q\\\\x2dr.slice\" -.mount\nRequisite=t.service\nWants=w.service\n\ - BindsTo=b.service\nUpholds=u.service\nConflicts=x.target\nConflictedBy=y.service\n\ - RequiredBy=c.service\nBoundBy=\nRequisiteOf=ro.service\nConsistsOf=d.target e.service\n\ - PropagatesStopTo=p.service\nStopPropagatedFrom=f.service\nOther=x\n", + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ + SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ + OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ + Requires=r.service\nConflicts=x.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); assert_eq!(facts.names, ["a.service", "b.service"]); - assert!(!facts.not_found && !facts.running); + assert!(!facts.not_found); + assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); // A quoted name holds one backslash, not two and not a quote mark. - assert_eq!( - facts.pulls, - ["r.service", "q\\x2dr.slice", "-.mount", "t.service", "w.service", "b.service", "u.service"] - ); - assert_eq!(facts.conflicts, ["x.target", "y.service"]); - assert_eq!( - facts.stops, - ["c.service", "ro.service", "d.target", "e.service", "p.service", "f.service"] - ); + assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); assert!(parse_show("Names=a.service\n").is_err()); // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), - "Id,Names,LoadState,ActiveState,DefaultDependencies,SuccessAction,FailureAction,\ - StartLimitAction,JobTimeoutAction,OnFailure,OnSuccess,Requires,Requisite,BindsTo,\ - Wants,Upholds,Conflicts,ConflictedBy,RequiredBy,BoundBy,RequisiteOf,ConsistsOf,\ - PropagatesStopTo,StopPropagatedFrom" + "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ + OnFailure,OnSuccess,BoundBy" ); // A backslash outside quotes is a backslash; inside, it escapes one character. assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); @@ -3408,17 +3026,7 @@ mod tests { assert!(parse_names(" ").is_empty()); let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); - for (state, running) in [ - ("active", true), - ("activating", true), - ("reloading", true), - ("inactive", false), - ("deactivating", false), - ("failed", false), - ] { - let facts = parse_show(&format!("Id=a.service\nActiveState={state}\n")).unwrap(); - assert_eq!(facts.running, running, "{state}"); - } + } } #[test] From 8ebe0208a65bb2191acfeec7d22ffe337795a385 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:41:56 +0100 Subject: [PATCH 21/23] Steward orders: pin the one-job rule on the real unit set and a real user manager Co-Authored-By: Claude Opus 5.5 --- src/steward_orders.rs | 118 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/src/steward_orders.rs b/src/steward_orders.rs index d5fdcfe..7f8ced7 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3027,7 +3027,6 @@ mod tests { let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); assert!(gone.not_found); } - } #[test] fn the_store_refuses_a_second_record_and_a_foreign_document() { @@ -3846,4 +3845,121 @@ mod tests { assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() } + + fn host_refusal(verb: Verb, unit: &str) -> Option { + refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) + } + + #[test] + #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] + fn real_host_unit_set() { + for name in [ + "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", + "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", + "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", + "wg-quick@wg-gamecult", "cron", "apt-daily", + ] { + let unit = format!("{name}.service"); + for verb in all_verbs() { + assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); + } + } + for name in [ + "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", + "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", + "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", + "systemd-storagetm", + ] { + let unit = format!("{name}.service"); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + host_refusal(verb, &unit), + Some(refused(&unit, Via::OnEnd)), + "{verb:?} {unit}" + ); + } + } + for unit in HOST_NAMES { + for verb in all_verbs() { + assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); + } + } + for unit in [ + "systemd-networkd.service", + "user-runtime-dir@1000.service", + "user-runtime-dir@105.service", + "user-runtime-dir@978.service", + ] { + assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); + assert!( + matches!( + host_refusal(Verb::Stop, unit), + Some(Refused::AlwaysRefused { via: Via::Bound, .. }) + ), + "{unit}" + ); + } + let listing = Command::new("systemctl") + .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) + .output() + .unwrap(); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let units: Vec<&str> = listing + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|unit| validate_unit(unit).is_ok()) + .collect(); + for verb in all_verbs() { + let mut counts = BTreeMap::new(); + for unit in &units { + let key = match refusal_for(&HostSystemd, unit, verb) { + Ok(None) => "ok".to_owned(), + Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), + Ok(Some(other)) => format!("{other:?}"), + Err(_) => "unresolved".to_owned(), + }; + *counts.entry(key).or_insert(0usize) += 1; + } + println!("{verb:?} over {} units: {counts:?}", units.len()); + } + } + + #[test] + #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] + fn one_job_argv_against_a_real_manager() { + let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); + std::fs::create_dir_all(&directory).unwrap(); + let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); + let body = "[Service]\nExecStart=/bin/sleep 600\n"; + std::fs::write(directory.join(a), body).unwrap(); + std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); + let user = |args: &[&str]| { + let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); + (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) + }; + let outcome = std::panic::catch_unwind(|| { + let one_job = |verb: Verb, unit: &str| { + let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); + let mut argv = step.argv.clone(); + argv.insert(1, "--user".into()); + let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); + assert!(status.success(), "{verb:?} {unit}"); + }; + assert!(user(&["daemon-reload"]).0); + // A start of b leaves a running, though b conflicts with it. + assert!(user(&["start", a]).0); + one_job(Verb::Start, b); + assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); + // A restart of b, with both running, leaves a running. + one_job(Verb::Restart, b); + assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); + }); + user(&["stop", a, b]); + std::fs::remove_file(directory.join(a)).unwrap(); + std::fs::remove_file(directory.join(b)).unwrap(); + user(&["daemon-reload"]); + outcome.unwrap(); + } } From fb1e7b5dd9a0e9f0a77febd66cb43d5ce2f995f4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 22:42:52 +0100 Subject: [PATCH 22/23] Steward orders: README states the one-job rule; pin the end-to-end argv Co-Authored-By: Claude Opus 5.5 --- README.md | 35 +++++++++++++++++------------------ src/steward_orders.rs | 2 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 28c1e94..3a695a4 100644 --- a/README.md +++ b/README.md @@ -73,24 +73,23 @@ idunn order show # status and outp idunn order list ``` -A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' -own (`mind-self@*`, `mind-persona@*`). That set is compiled into the binary and -read from no configuration. An order is refused, recorded and posted when its -name, an alias it resolves from, or the transaction systemd would build for it -reaches one of them: a start pulls in what the unit requires or wants (a -refused unit that is already running is left alone) and stops what conflicts -with it, a stop takes down what depends on it, and a restart does both. The -transaction is read from `systemctl show`, so `start poweroff.target` is refused -because every refused unit conflicts with `shutdown.target`, not because -targets are banned. The units a start or stop ends or times out are followed too: -the target each of `SuccessAction=`, `FailureAction=`, `StartLimitAction=` and -`JobTimeoutAction=` runs (`reboot`, `poweroff`, `halt`, `kexec`, `soft-reboot`, -`exit`, with `-force` and `-immediate`), and the units `OnFailure=` and -`OnSuccess=` name, with what each pulls in; an action it does not know fails the -order. A start refuses a stopped `DefaultDependencies=no` unit (boot and recovery -machinery, such as `rescue.service`, which ends by isolating the default -target), whatever pulls it in. A unit `systemctl show` cannot resolve fails the order. -Slices, scopes and globs are not units an order can name. +A unit order acts on exactly the unit it names: every verb runs +`systemctl --job-mode=ignore-requirements -- `, so systemd builds a +transaction of one job. It does not start what the unit requires or restart what +is part of it: order the dependency first, and order `postgresql@16-main.service` +rather than the `postgresql.service` wrapper. Targets are not orderable. A unit +order may act on any unit except Idunn's own (`idunn-*`) and the minds' own +(`mind-self@*`, `mind-persona@*`), and the units that end or take the host +(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, +`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled +into the binary and read from no configuration. An order is refused, recorded +and posted when the ordered unit, or an alias it resolves from, is one of them; +when it has an emergency action (`SuccessAction=`, `FailureAction=`, +`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` +or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another +unit is `BoundBy=` it, because systemd stops that unit with it whatever the job +mode. A unit `systemctl show` cannot resolve fails the order. Slices, scopes, +targets and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with `--no-remove`, each name is passed as an exact-name pattern so `.` and `+` diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 7f8ced7..3b40c88 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -3742,7 +3742,7 @@ mod tests { let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); assert!( run_log.contains(&format!( - "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart -- nginx.service", + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", order_uuid(&first) )), "{run_log}" From 77e2295e9705c2a9eebb46f27ac40f34b7b49c2a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 2 Nov 2026 23:32:30 +0100 Subject: [PATCH 23/23] Steward orders: refuse Upholds and ConsistsOf units, pin the rule's gaps, README names the residuals A unit with Upholds= is refused for start and restart (systemd would start the upheld unit in a full transaction). A wrapper with ConsistsOf= is refused for stop, restart and reload with a typed Refused::Wrapper naming the first member, so it cannot report ran while touching nothing. Tests pin empty BoundBy, a single show per refusal, no show for a refused name and each posted reason. README: the one-job rule's residual full transactions. Removes the unused BTreeMap import and the never_loop in the host actuator test. Co-Authored-By: Claude Opus 5.5 --- README.md | 15 ++++- src/host_actuator.rs | 2 +- src/steward_orders.rs | 126 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3a695a4..8c6935c 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,16 @@ idunn order show # status and outp idunn order list ``` -A unit order acts on exactly the unit it names: every verb runs +A unit order submits exactly one job, for the unit it names: every verb runs `systemctl --job-mode=ignore-requirements -- `, so systemd builds a transaction of one job. It does not start what the unit requires or restart what is part of it: order the dependency first, and order `postgresql@16-main.service` -rather than the `postgresql.service` wrapper. Targets are not orderable. A unit +rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is +not: systemd still runs full transactions of its own for `Restart=` after a +failure, for a `StopWhenUnneeded=` dependency left without a user, and for a +`Conflicts=` partner of a unit that fails, so an ordered stop of a unit +that something restarts does not last, and a start of a `Restart=always` +unit can report failure and be active seconds later. A unit order may act on any unit except Idunn's own (`idunn-*`) and the minds' own (`mind-self@*`, `mind-persona@*`), and the units that end or take the host (`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, @@ -88,7 +93,11 @@ when it has an emergency action (`SuccessAction=`, `FailureAction=`, `StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another unit is `BoundBy=` it, because systemd stops that unit with it whatever the job -mode. A unit `systemctl show` cannot resolve fails the order. Slices, scopes, +mode; for a start or restart, when it has an `Upholds=` unit, which systemd +starts in a full transaction whatever the job mode; or, for a stop, restart or +reload, when it has a `ConsistsOf=` member, a wrapper such as +`postgresql.service` whose verbs would exit 0 and touch nothing. A unit +`systemctl show` cannot resolve fails the order. Slices, scopes, targets and globs are not units an order can name. A package order installs the packages it names and nothing else: apt runs with diff --git a/src/host_actuator.rs b/src/host_actuator.rs index f7bf4c7..8fd776c 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1151,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - while let Some(frame) = client.receive_once()? { + if let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/steward_orders.rs b/src/steward_orders.rs index 3b40c88..d4be1ee 100644 --- a/src/steward_orders.rs +++ b/src/steward_orders.rs @@ -21,7 +21,6 @@ //! field and a code. A refused unit that a record names was resolved by //! systemd and passed the unit grammar. -use std::collections::BTreeMap; use std::fmt; use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -391,12 +390,16 @@ pub(crate) enum Via { Host, OnEnd, Bound, + Upholds, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub(crate) enum Refused { AlwaysRefused { unit: String, via: Via }, + /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload + /// of the wrapper exits 0 and leaves the member alone. + Wrapper { member: String }, Braked, /// The order's principal or issuing uid is not the steward bound now. NotSteward, @@ -437,8 +440,13 @@ impl Outcome { Via::Host => "host", Via::OnEnd => "on-end", Via::Bound => "bound", + Via::Upholds => "upholds", } ), + Outcome::Refused(Refused::Wrapper { member }) => format!( + "refused: wrapper unit, it only wraps {}", + escape(member.as_bytes()) + ), Outcome::Refused(Refused::Braked) => "refused: braked".into(), Outcome::Refused(Refused::NotSteward) => { "refused: the issuer is not the current steward".into() @@ -650,6 +658,12 @@ pub(crate) struct UnitFacts { /// The units bound to this one at runtime (`BoundBy=`): systemd stops /// them when this unit stops, whatever the job mode. pub(crate) bound_by: Vec, + /// The units this one upholds (`Upholds=`): systemd starts them in a full + /// transaction whenever this unit is active, whatever the job mode. + pub(crate) upholds: Vec, + /// The units this one groups (`ConsistsOf=`): the verbs reach them only + /// through propagation, which the job mode removes. + pub(crate) consists_of: Vec, } impl UnitFacts { @@ -715,7 +729,8 @@ fn refused(unit: &str, via: Via) -> Refused { /// Would `verb` on `unit` be refused. systemd builds the transaction and the /// order tells it to build one job (`--job-mode=ignore-requirements`), so this /// reads only what that job mode cannot remove from the one named unit: the -/// name it answers to, what fires when it ends, and who is bound to it. +/// name it answers to, what fires when it ends, who is bound to it, what it upholds and +/// what it only wraps. fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { if is_refused_name(unit) { return Ok(Some(refused(unit, Via::Name))); @@ -738,6 +753,16 @@ fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result Vec { } /// The properties one `systemctl show` is asked for: identity, what fires at -/// the unit's end, and who is bound to it. +/// the unit's end, who is bound to it, and what it upholds or groups. fn show_properties() -> String { ["Id", "Names", "LoadState"] .into_iter() .chain(ACTIONS) .chain(ON_END) - .chain(["BoundBy"]) + .chain(["BoundBy", "Upholds", "ConsistsOf"]) .collect::>() .join(",") } @@ -1253,6 +1278,8 @@ fn parse_show(output: &str) -> Result { "Names" => facts.names = parse_names(value), "LoadState" => facts.not_found = value == "not-found", "BoundBy" => facts.bound_by = parse_names(value), + "Upholds" => facts.upholds = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), _ if ACTIONS.contains(&key) => { if !matches!(value, "" | "none") { facts.on_end.push(key.to_owned()); @@ -2543,6 +2570,91 @@ mod tests { } } + #[test] + fn a_unit_that_upholds_others_is_refused_where_it_would_start() { + let h = harness(); + h.describe("holder.service", &[("Upholds", "held.service")]); + for verb in [Verb::Start, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "holder.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "holder.service".into(), + via: Via::Upholds + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { + let h = harness(); + h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); + for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { + assert_eq!( + outcome_of_unit(&h, verb, "wrapper.service"), + Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); + } + + #[test] + fn empty_dependency_lists_name_no_one() { + let h = harness(); + // systemctl prints every property it is asked for, empty or not. + h.describe( + "alone.service", + &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service")]); + h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); + assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); + h.systemd.shows.lock().unwrap().clear(); + assert_eq!( + hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), + Some("idunn-ghost.service") + ); + assert!(h.systemd.shows.lock().unwrap().is_empty()); + } + + #[test] + fn each_refusal_is_posted_under_its_own_reason() { + for (via, word) in [ + (Via::Name, "name"), + (Via::Alias, "alias"), + (Via::Host, "host"), + (Via::OnEnd, "on-end"), + (Via::Bound, "bound"), + (Via::Upholds, "upholds"), + ] { + assert_eq!( + Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) + .describe(), + format!("refused: always-refused unit u.service reached by {word}") + ); + } + assert_eq!( + Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), + "refused: wrapper unit, it only wraps m.service" + ); + } + #[test] fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { let h = harness(); @@ -3004,7 +3116,7 @@ mod tests { "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ - Requires=r.service\nConflicts=x.service\nOther=x\n", + Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", ) .unwrap(); assert_eq!(facts.id, "a.service"); @@ -3013,12 +3125,14 @@ mod tests { assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); // A quoted name holds one backslash, not two and not a quote mark. assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); + assert_eq!(facts.upholds, ["u.service"]); + assert_eq!(facts.consists_of, ["m.service", "n.service"]); assert!(parse_show("Names=a.service\n").is_err()); // The request asks for exactly the properties the parse reads. assert_eq!( show_properties(), "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ - OnFailure,OnSuccess,BoundBy" + OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" ); // A backslash outside quotes is a backslash; inside, it escapes one character. assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]);