diff --git a/Cargo.lock b/Cargo.lock index 44c9457..dfc3087 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,7 +482,6 @@ dependencies = [ "libc", "rmp-serde", "serde", - "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index ee797ee..dcaca2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,6 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } -serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/README.md b/README.md index 8c6935c..a5c48bc 100644 --- a/README.md +++ b/README.md @@ -58,64 +58,6 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. -### Steward orders - -A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), -takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. -It has its own store, `orders.cc`, and its own binding, `steward.toml`; the -survival daemon takes no order. There is still no command string: - -```bash -idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them -idunn order unit --reason TEXT -idunn order deploy --reason TEXT # the same commit as `idunn up` -idunn order show # status and output tails -idunn order list -``` - -A unit order submits exactly one job, for the unit it names: every verb runs -`systemctl --job-mode=ignore-requirements -- `, so systemd builds a -transaction of one job. It does not start what the unit requires or restart what -is part of it: order the dependency first, and order `postgresql@16-main.service` -rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is -not: systemd still runs full transactions of its own for `Restart=` after a -failure, for a `StopWhenUnneeded=` dependency left without a user, and for a -`Conflicts=` partner of a unit that fails, so an ordered stop of a unit -that something restarts does not last, and a start of a `Restart=always` -unit can report failure and be active seconds later. A unit -order may act on any unit except Idunn's own (`idunn-*`) and the minds' own -(`mind-self@*`, `mind-persona@*`), and the units that end or take the host -(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, -`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled -into the binary and read from no configuration. An order is refused, recorded -and posted when the ordered unit, or an alias it resolves from, is one of them; -when it has an emergency action (`SuccessAction=`, `FailureAction=`, -`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` -or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another -unit is `BoundBy=` it, because systemd stops that unit with it whatever the job -mode; for a start or restart, when it has an `Upholds=` unit, which systemd -starts in a full transaction whatever the job mode; or, for a stop, restart or -reload, when it has a `ConsistsOf=` member, a wrapper such as -`postgresql.service` whose verbs would exit 0 and touch nothing. A unit -`systemctl show` cannot resolve fails the order. Slices, scopes, -targets and globs are not units an order can name. - -A package order installs the packages it names and nothing else: apt runs with -`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` -match themselves, and a last step fails the order unless every named package -is installed. - -An order runs once, after the steward's uid is checked and the instance's -brake check (`mind-control --instance check`) exits 0. The check -that the order's principal and uid are the bound steward's runs again at every -execution, so an order accepted before `steward.toml` changed is refused and -recorded, not run. The socket belongs to the steward user alone (mode 0600); -root may `show` and `list` orders but never issue one. An order that had -started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to -`#operator` as a retried projection of `orders.cc`: a chat outage delays the -posts and not the orders. A refusal names the field and a code, never the value -it refused. - ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service deleted file mode 100644 index f2681c2..0000000 --- a/deploy/idunn-orders.service +++ /dev/null @@ -1,47 +0,0 @@ -[Unit] -Description=Idunn steward orders: typed package, unit and deploy orders from the steward -# Deliberately no After= or Requires= on any managed target, Synapse or -# idunn-yggdrasil: orders are how an operator repairs those, so none of them -# may be a condition of this unit starting. Posts are a retried projection and -# wait for Synapse on their own. - -[Service] -Type=simple - -# Root, from the same binary as idunn-yggdrasil, with its own store and socket. -# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: -# no order code runs in `idunn serve`. -# -# The unit name matches idunn-*, which is in the always-refused set, so no -# order can stop or restart this service. -User=root -# No Group= and no UMask=: the service gives its socket to the steward user -# (mode 0600) itself and sets Idunn's own umask for the files it writes, so -# nothing here reaches control.cc. The steward user is the only peer the -# service answers: it checks SO_PEERCRED on every connection. -ExecStart=/usr/local/bin/idunn orders serve \ - --store /var/lib/gamecult/idunn-orders \ - --state-store /var/lib/gamecult/idunn/control.cc \ - --steward-binding /etc/gamecult/idunn/steward.toml \ - --socket /run/idunn-orders/steward.sock -Restart=always -RestartSec=5 -TimeoutStopSec=20 - -RuntimeDirectory=idunn-orders -RuntimeDirectoryMode=0755 - -# The Synapse access token of @idunn, copied from a root-only file, and read -# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. -LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token - -PrivateTmp=yes -ProtectHome=yes -ProtectSystem=strict -# Its own order store, and the control store a deploy order commits a command -# to. Transient units started with systemd-run run in the manager's namespace, -# so packages and units are not limited by this list. -ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn - -[Install] -WantedBy=multi-user.target diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md index 95c0eee..195867d 100644 --- a/docs/deploy-provenance-map.md +++ b/docs/deploy-provenance-map.md @@ -199,196 +199,3 @@ on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 terminfo links (T13). If a tree holds a link the T14 rule refuses, the job prints it and stops, and the link is reported to Self, not deleted. - -## Cut `isolation-credential-evidence` - -Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, -at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It -serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, -first deviation): every `idunn up vili` is refused before fencing. - -### Body facts (probed read-only on Yggdrasil, 2026-10-24) - -- I1. `idunn up vili` runs 6 and 7 failed with - `systemd workload isolation properties differ from the Idunn contract`, the - message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, - only one differs on the failed Vili unit: `LoadCredential`. Idunn expects - `""` because Vili has no service credential besides the presence identity - (`service_credential_names` empty); systemd shows `[unprintable]`. Every - other conjunct matches, InaccessiblePaths included (one value across all - five idunn-vili/odin units). -- I2. systemd 257.13 `systemctl show -p LoadCredential` prints - `LoadCredential=[unprintable]` for every unit, whatever the credentials: - `ssh.service` and `systemd-journald.service` (not DynamicUser, zero - LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin - unit (one). The display form carries no information, so the conjunct was - vacuous whenever credentials existed and always false when none did. Odin - and StreamPixels passed only because each carries a regular credential. -- I3. The exact value is readable: D-Bus - `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) - answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 - "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. -- I4. What the workload can actually read is the delivered directory. On the - host, `/run/credentials/` holds a directory only for units with credentials - (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID - root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` - (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` - descriptor, never a credential. -- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn - deployment transaction authority"), with its twin clause in - `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit - touched it. No ruling or finding in any campaign concerns workload isolation - verification; gamecult-chat's foreign-workloads cut touched - `CREDENTIALS_DIRECTORY` reservation, not this check. -- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both - MessagePack encodings, compact (positional) included - (`persisted_systemd_observations_decode_as_the_systemd_variant`, - `:9915-9930`), so deleting the `load_credential` field shifts every later - slot of a live incumbent's record. The field is also in the drift list - (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed - workload: changing what the field holds without leaving that list would make - the next Idunn refuse to stop the live Odin and StreamPixels incumbents. -- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 - properties for a Vili unit (presence identity only) and the Odin unit - (presence identity plus one service credential), captured 2026-10-24 into - this Imagination's scratchpad `fixtures/`. -- I8. The new rule holds for every live systemd incumbent: inside each - MainPID root the delivered entries equal its binding's `secret_files` keys - minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, - StreamPixels web 1; key names compared, no values read). An Idunn restart onto - this cut re-observes them (`:3955-3970`) and passes. - -### Decision - -A property systemd cannot display does not belong in the comparison. The -invariant it reached for, that the workload holds exactly the credentials Idunn -delivered, is observed where it is decided: the entries of -`/run/credentials/` in the MainPID's root equal Idunn's service -credential names, absent directory equal to none. That covers -`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and -needs no new program. The D-Bus read (I3) is exact but sees only one of those -properties and adds a second systemd client and parser; rejected. - -### Authority - -Owner: `observe_unit` decides the credential rule, from the delivered -directory. `LoadCredential`'s display form is no longer read or compared; the -`load_credential` field is a retired wire slot, written empty, out of the drift -list, kept only so persisted records decode. Forbidden writers: the -`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping -the slot waits on the next schema revision of the observation. - -### Deploy - -Idunn's body changes only through gamecult-ops -`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and -bindings unchanged). After it, Vili's Hands reruns from step 4 of -`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. - -## Cut `runner-failure-evidence` - -Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at -`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area -Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, -transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 -UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and -nobody could tell which recipe step failed. - -### Body facts (source read at `804ed75`; host read-only, 2026-10-25) - -- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by - `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and - `Command::output()`: stdout and stderr are each buffered whole in the - daemon's memory, unbounded. On a non-zero exit the error is - `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The - container is gone the moment it exits, so `docker ps -a` and `docker logs` - hold nothing. -- R2. The empty message: the failing process wrote nothing to stderr, and - stdout, the only stream that carried its report, is discarded at - `:2182-2193`. The same interpolation echoes whatever a step does write to - stderr into `last_error`, `pre_fencing_abort.error` and - `completion.error` (`bounded_detail`, 2048 bytes), and from there into - `idunn up`'s `deployment failed:` line and `idunn status`. -- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops - over `declaration.steps` and calls `run_in_workspace` with argv only; - `step.id` is used only in the environment precondition. External inputs - (`curl` in the runner, `:2247-2258`) fail through the same helper with the - same anonymous message. -- R4. Which step failed on 2026-10-25, inferred from the failed transaction's - staging root, which survives (926 MiB, - `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, - `node_modules/.package-lock.json` was written at 00:51:24 (so - `install-vili-dependencies`, `npm ci`, finished), nothing outside - `node_modules` is newer, and `release/vili` (the next step's output) is - absent; the python workspace holds only fetched inputs. The failing step was - `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test - reporter writes to stdout: exactly the stream R1 discards. Inference from - residue, not from output. -- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) - sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which - persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal - on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only - provider-exclusion lines. -- R6. Stores. `control.cc` (root 0640) holds commands, transactions - (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and - expiry requests; `ControlSnapshot::read` refuses any other type - (`:3420`). Records must re-encode byte-exactly (`decode_record`, - `:3717-3727`), so a new transaction field is a schema bump and a v4 lift - (`:4100-4110`). Terminal transactions move verbatim to `history.cc` - (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. - `incidents.cc` lives in the published projection directory and by its own - doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, - because a type added there stops continuity for every target on a rollback. -- R7. The daemon runs as root (`User=root`). Runner secrets reach the - container as read-only file mounts named by env (`:2041-2047`), so any text - a step derives from them can land in its output. -- R8. Staging residue: 475 transaction roots, 55 GiB under - `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces - because `materialize` returns before `remove_tree_inside`. - -### Decision - -The failure becomes a typed value at the runner, `RunnerStepFailure`: the -subject (recipe step or external input, by id), the runner id, the exit code or -signal, and the last 8 KiB of stdout and of stderr with each stream's total -byte count. Its `Display` carries the subject and status, never output. -Capture streams both pipes through bounded tails, so a verbose build no longer -buffers whole in the daemon. - -The tail lives in a new private store, `runner-evidence.cc`, beside -`control.cc` (mode 0600), one record per transaction -(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the -transaction record: that is a v5 bump whose rollback makes every older Idunn -refuse its control store (R6), the scar `incident.rs` already names. Not in -`incidents.cc`: published, and free text is barred there. Not in the journal: -journal readers are a wider set than root, so the journal line names the -transaction, the subject, the status and the store, and no output. - -No redaction. Matching secret bytes would make the daemon read every secret, -and still misses any encoding, prefix or transform of one; a claimed redaction -that leaks is worse than a stated rule. The rule instead is the reader set: the -store's only readers (root) can already read every secret the runner was -given. - -Keeping the failed container (dropping `--rm`) is worse: its output then lives -in Docker's json-file log, unbounded, untyped, outside Idunn's state and -readable by the docker group; containers accumulate with no owner to reap them; -`docker system prune` erases the evidence; and nothing ties a container to the -transaction in `idunn status`. - -### Authority - -Owner: the runner decides what failed and keeps the bounded tail; the engine's -pre-fence failure path is the one writer of `runner-evidence.cc`. -`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the -failure's `Display`. Forbidden writers: the `stderr` interpolation in -`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the -store needs no external owner; staging roots (R8) stay with -`ghostlight-verse:follow_up:idunn-generation-retention`. - -### Deploy - -Idunn's body changes only through gamecult-ops -`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and -bindings unchanged). The next failed Sealing then shows its step and tail on -`sudo idunn status --command `. diff --git a/src/control_plane.rs b/src/control_plane.rs index cd49073..9597d43 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -42,14 +42,11 @@ use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, RunnerStepFailure, SourcePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; -use crate::runner_evidence::{ - self, Kept, RunnerFailureRecord, runner_evidence_store_path, unreadable_store_path, -}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -2934,10 +2931,6 @@ enum Command { recipe: PathBuf, binding: Option, }, - #[cfg(unix)] - OrdersServe(crate::steward_orders::ServeOptions), - #[cfg(unix)] - Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { @@ -2977,10 +2970,6 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), - #[cfg(unix)] - Command::OrdersServe(options) => crate::steward_orders::serve(options), - #[cfg(unix)] - Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -3033,10 +3022,6 @@ fn parse(args: impl Iterator) -> Result { "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), - #[cfg(unix)] - "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), - #[cfg(unix)] - "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -4742,21 +4727,20 @@ fn resolve_selector( Ok(targets) } -/// The one commit primitive for a deployment command: `idunn up` and a steward -/// deploy order both write theirs here. Returns the commit time. -pub(crate) fn commit_deployment_command( +fn submit( store_path: &Path, - command_id: &str, selector: &str, requested_by: &str, -) -> Result { + wait: bool, + timeout_seconds: u64, +) -> Result<()> { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: command_id.into(), + command_id: format!("up-{}", Uuid::new_v4()), kind: CommandKind::Deploy, selector: selector.into(), requested_by: requested_by.into(), @@ -4774,33 +4758,7 @@ pub(crate) fn commit_deployment_command( )?, "deployment command id collided" ); - Ok(now) -} - -/// A command's present state as `idunn status --command` derives it, for the -/// steward order poster: `queued` and `running` are not terminal. -#[cfg(unix)] -pub(crate) fn command_state( - state_store: &Path, - command_id: &str, -) -> Result<(&'static str, String)> { - let snapshot = ControlSnapshot::read(state_store)?; - let archived = read_history_transactions(state_store); - let mut transactions = snapshot.transaction_for_command(command_id); - transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); - Ok(derived_command_status(&transactions)) -} - -fn submit( - store_path: &Path, - selector: &str, - requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { - let command_id = format!("up-{}", Uuid::new_v4()); - let now = commit_deployment_command(store_path, &command_id, selector, requested_by)?; - println!("{command_id}"); + println!("{}", command.command_id); if !wait { return Ok(()); } @@ -4811,10 +4769,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command_id), + .any(|stored| stored.value.command_id == command.command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command_id); + let transactions = snapshot.transaction_for_command(&command.command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4839,7 +4797,7 @@ fn submit( } } -pub(crate) fn status( +fn status( out: &mut impl Write, store_path: &Path, incident_store: &Path, @@ -4919,13 +4877,6 @@ pub(crate) fn status( } if let Some(abort) = &transaction.pre_fencing_abort { writeln!(out, " pre-fencing abort {abort:?}")?; - // The step that failed and its output tail live in their - // own private store. An unreadable store costs its lines. - runner_evidence::show_status( - out, - &runner_evidence_store_path(store_path), - &transaction.transaction_id, - )?; } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -5423,32 +5374,6 @@ impl Engine { IncidentStore::new(&self.options.incident_store) } - /// A failed runner step keeps its output tail in the runner evidence - /// store and says so on the journal, naming no output. The store write is - /// best effort: its failure is named in the line and never stops the - /// abort that follows. - fn keep_runner_failure( - &self, - transaction: &DeploymentTransaction, - error: &anyhow::Error, - ) -> Option { - let failure = error.downcast_ref::()?; - let path = runner_evidence_store_path(&self.options.state_store); - let recorded = now_millis().and_then(|now| { - runner_evidence::record( - &path, - &RunnerFailureRecord::from_failure( - &transaction.transaction_id, - &transaction.command_id, - &transaction.target, - failure, - now, - ), - ) - }); - Some(runner_failure_journal_line(transaction, failure, &path, &recorded)) - } - /// Settle the outcome of one incident operation. A failure never leaves /// this function: it is traced once per episode, under the operation and /// the subject it was decided for (`*` for a pass over the whole store), @@ -6390,9 +6315,6 @@ impl Engine { if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() { - if let Some(line) = self.keep_runner_failure(&latest.value, &error) { - eprintln!("{line}"); - } self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -10631,13 +10553,7 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -/// The control store `idunn up` and `idunn serve` use unless told otherwise. -#[cfg(unix)] -pub(crate) fn default_state_store() -> PathBuf { - RuntimeOptions::default().state_store -} - -pub(crate) fn now_millis() -> Result { +fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -10657,12 +10573,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { +fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { +fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -10678,7 +10594,7 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -pub(crate) fn require_selector(value: &str) -> Result<()> { +fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } @@ -10725,43 +10641,6 @@ const DETAIL_EMPTY: &str = "(no detail)"; /// only judge, and this output always satisfies it. NUL is dropped, empty /// input gets a fixed placeholder, and an over-long value keeps its head and /// its tail (where a build log's error is) cut on char boundaries. -/// The journal line for a failed runner step: the transaction, the subject and -/// status (the failure's `Display`), and where the evidence is. Never output. -fn runner_failure_journal_line( - transaction: &DeploymentTransaction, - failure: &RunnerStepFailure, - store_path: &Path, - recorded: &Result, -) -> String { - let head = format!( - "Idunn transaction {} (command {}, target {}) {failure}", - transaction.transaction_id, transaction.command_id, transaction.target - ); - match recorded { - Ok(kept) => { - let aside = match kept { - Kept::Stored => String::new(), - Kept::SetAside { replaced } => format!( - "; an undecodable evidence store was set aside as {}{}", - unreadable_store_path(store_path).display(), - match replaced { - Some(last_written) => format!( - ", replacing an earlier set-aside store last written at unix millis {last_written}" - ), - None => String::new(), - } - ), - }; - format!( - "{head}; evidence in {}{aside}; read it with idunn status --command {}", - store_path.display(), - transaction.command_id - ) - } - Err(error) => format!("{head}; evidence was not kept: {error:#}"), - } -} - fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -10802,10 +10681,7 @@ fn usage() -> &'static str { idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\ - idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ - idunn order --reason TEXT\n\ - idunn order show ID | idunn order list\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -12017,45 +11893,9 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], - vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], - vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], - vec!["order", "run", "sh", "--reason", "r"], - vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], - vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], - vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], - vec!["orders", "serve", "--allow-units", "nginx.service"], - vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } - // The declarative order forms parse, and only as order commands. - #[cfg(unix)] - for arguments in [ - vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], - vec!["order", "package-install", "git-lfs", "--reason", "r"], - vec!["order", "deploy", "vili", "--reason", "r"], - vec!["order", "list"], - vec!["orders", "serve"], - ] { - let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); - assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); - } - } - - #[test] - fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { - let temp = TempDir::new()?; - let store = temp.path().join("control.cc"); - commit_deployment_command(&store, "up-one", "vili", "steward:eureka order ord-1")?; - let snapshot = ControlSnapshot::read(&store)?; - assert_eq!(snapshot.commands.len(), 1); - let command = &snapshot.commands[0].value; - assert_eq!(command.command_id, "up-one"); - assert_eq!(command.selector, "vili"); - assert_eq!(command.requested_by, "steward:eureka order ord-1"); - assert!(commit_deployment_command(&store, "up-one", "other", "x").is_err()); - assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); - Ok(()) } #[test] @@ -24817,435 +24657,4 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } } - - // BEGIN runner-failure-evidence tests - // A failed runner step is named, its output tail is kept in - // runner-evidence.cc, and no output reaches an error, control.cc or the - // journal line. The runner is the real DockerRunnerDriver driving a fake - // docker program, through the engine's Sealing path. - - const RUNNER_CANARY: &str = "CANARY-7c41e0d9"; - - /// A source that freezes a fixed tree, so a deployment reaches the runner. - struct TreeSource { - recipe: String, - root: PathBuf, - } - - impl SourcePort for TreeSource { - fn resolve( - &self, - _: &OperatorBinding, - _: &str, - _: u64, - ) -> Result { - Ok(crate::drivers::ResolvedSource { - facts: crate::deployment_plan::tests::source(&self.recipe), - recipe_bytes: self.recipe.clone().into_bytes(), - }) - } - - fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result { - Ok(FrozenSourceReceipt { - transaction_id: transaction_id.to_owned(), - plan_id: plan.plan_id.clone(), - snapshot_sha256: format!("sha256-{}", "a".repeat(64)), - }) - } - - fn observe_frozen( - &self, - plan: &CompiledDeploymentPlan, - receipt: &FrozenSourceReceipt, - ) -> Result { - Ok(crate::drivers::FrozenSource::fixture( - receipt.clone(), - plan.source.clone(), - plan.recipe_blob.clone(), - self.root.clone(), - )) - } - - fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { - Ok(()) - } - } - - /// The unrouted recipe with two steps ahead of its build, and no external - /// input: the second step, `test-daemon`, is the one the fake docker fails. - fn two_step_recipe() -> String { - let recipe = unrouted_recipe().replace( - "capability = \"service.runtime\"", - &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), - ); - let (head, rest) = recipe - .split_once("[[external_inputs]]") - .expect("the recipe declares an external input"); - let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts"); - let head = head.replacen( - "[[steps]]\nid = \"build\"", - "[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\ - [[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\ - [[steps]]\nid = \"build\"", - 1, - ); - format!("{head}[[artifacts]]{tail}") - } - - /// A Sealing world whose runner is the real driver over a fake docker: the - /// first call succeeds, the second prints the canary and an escape - /// sequence on both streams and exits 3. - #[cfg(unix)] - fn failing_runner_world() -> Result<(EngineFixture, TempDir)> { - use std::os::unix::fs::PermissionsExt; - let tools = TempDir::new()?; - let docker = tools.path().join("docker"); - std::fs::write( - &docker, - format!( - "#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\ - [ \"$n\" -lt 2 ] && exit 0\n\ - printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n" - ), - )?; - std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?; - let recipe = two_step_recipe(); - let mut world = sealing_world(&recipe)?; - let tree = tools.path().join("tree"); - std::fs::create_dir_all(&tree)?; - std::fs::write(tree.join("README"), b"source")?; - world.engine.source = Arc::new(TreeSource { recipe, root: tree }); - world.engine.docker_runner = DockerRunnerDriver { docker_program: docker }; - Ok((world, tools)) - } - - /// Scheduler ticks until the deployment ends (or the budget is spent). - fn tick_until_ended(world: &EngineFixture) -> Result { - for _ in 0..24 { - world.engine.resume_backoff.lock().unwrap().clear(); - let _ = world.engine.run_scheduler_tick(); - if latest(world)?.completion.is_some() { - break; - } - } - latest(world) - } - - fn printed_status(world: &EngineFixture) -> Result { - let mut printed = Vec::new(); - status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?; - Ok(String::from_utf8(printed)?) - } - - fn canary_failure(stdout: &[u8]) -> RunnerStepFailure { - RunnerStepFailure { - subject: crate::drivers::RunnerSubject::Step("test-daemon".into()), - runner: "rust".into(), - exit_code: Some(3), - signal: None, - stdout_tail: stdout.to_vec(), - stdout_total_bytes: stdout.len() as u64, - stderr_tail: stdout.to_vec(), - stderr_total_bytes: stdout.len() as u64, - } - } - - #[cfg(unix)] - #[test] - fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> { - use std::os::unix::fs::PermissionsExt; - let (world, _tools) = failing_runner_world()?; - let ended = tick_until_ended(&world)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error); - }; - // Named: the second step, on its runner, with the exit status. - assert!( - error.contains("runner step test-daemon on runner rust exited with exit status 3"), - "{error}" - ); - // Never echoed: no error text, no control.cc byte, carries the output. - let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; - for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { - assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); - } - // The terminal transaction has moved to history.cc: both stores. - for store in [world.state_store.clone(), history_store_path(&world.state_store)] { - let bytes = std::fs::read(&store)?; - assert!(!bytes.is_empty(), "{} is empty", store.display()); - assert!( - !bytes.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), - "runner output reached {}", - store.display() - ); - } - // Kept: in the private store, tails of both streams, and on status. - let evidence = runner_evidence_store_path(&world.state_store); - assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); - let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? - .context("no evidence for the failed transaction")?; - assert_eq!(record.subject_id, "test-daemon"); - assert_eq!(record.exit_code, Some(3)); - assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY)); - assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY)); - let printed = printed_status(&world)?; - assert!(printed.contains("failed step test-daemon"), "{printed}"); - assert!(printed.contains(RUNNER_CANARY), "{printed}"); - assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte"); - Ok(()) - } - - #[test] - fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> { - let world = EngineFixture::new()?; - let (transaction, _) = terminal_transaction_with_command("ghostlight")?; - let failure = canary_failure(RUNNER_CANARY.as_bytes()); - let line = world - .engine - .keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing")) - .context("a runner failure produced no journal line")?; - assert!(!line.contains(RUNNER_CANARY), "{line}"); - for needle in [ - transaction.transaction_id.as_str(), - "up-ghostlight", - "ghostlight", - "runner step test-daemon on runner rust exited with exit status 3", - "runner-evidence.cc", - "idunn status --command up-ghostlight", - ] { - assert!(line.contains(needle), "{needle} missing from {line}"); - } - // Not a runner failure: nothing is kept and nothing is said. - assert!( - world - .engine - .keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure")) - .is_none() - ); - // The line a failed write produces says so, with no output. - let unwritable = runner_failure_journal_line( - &transaction, - &failure, - Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"), - &Err(anyhow::anyhow!("disk full")), - ); - assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}"); - assert!(!unwritable.contains(RUNNER_CANARY)); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - // A directory where the store should be: every write to it fails. - std::fs::create_dir(runner_evidence_store_path(&world.state_store))?; - let ended = tick_until_ended(&world)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); - let line = world - .engine - .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b""))) - .context("no journal line")?; - assert!(line.contains("evidence was not kept"), "{line}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_undecodable_evidence_store_does_not_stop_the_next_failure_keeping_its_evidence() -> Result<()> { - let damage = format!("\u{c1} not a store {RUNNER_CANARY}-in-the-damage"); - let (world, _tools) = failing_runner_world()?; - let evidence = runner_evidence_store_path(&world.state_store); - std::fs::write(&evidence, damage.as_bytes())?; - let ended = tick_until_ended(&world)?; - // The transaction's own outcome is the runner failure, not the store. - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the damaged store changed the outcome: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); - assert!(!error.contains("damage"), "{error}"); - // The damage is kept aside; the failure's evidence is in a fresh store. - assert_eq!(std::fs::read(unreadable_store_path(&evidence))?, damage.as_bytes()); - let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? - .context("the failure kept no evidence beside a damaged store")?; - assert_eq!(record.subject_id, "test-daemon"); - assert!(printed_status(&world)?.contains("failed step test-daemon")); - // The journal says it, naming the file and none of its content. - std::fs::write(&evidence, damage.as_bytes())?; - let line = world - .engine - .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"x"))) - .context("no journal line")?; - assert!(line.contains("an undecodable evidence store was set aside as"), "{line}"); - assert!(line.contains("runner-evidence.unreadable.cc"), "{line}"); - assert!(!line.contains("damage") && !line.contains("not a store"), "{line}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - let ended = tick_until_ended(&world)?; - std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?; - let printed = printed_status(&world)?; - assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}"); - assert!(printed.contains("target service phase"), "{printed}"); - assert!(printed.contains("pre-fencing abort"), "{printed}"); - assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); - Ok(()) - } - - /// Everything status prints from the evidence store goes through one - /// allowlist, so a hostile build log reaches the terminal as visible - /// escapes whatever its class. - #[cfg(unix)] - #[test] - fn status_prints_hostile_runner_output_as_visible_escapes_only() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - let ended = tick_until_ended(&world)?; - let hostile: Vec<(&str, &str)> = vec![ - ("\u{1b}[2J", "\\u{1b}[2J"), - ("\u{7}", "\\u{7}"), - ("\r", "\\u{d}"), - ("\u{202e}", "\\u{202e}"), - ("\u{2066}", "\\u{2066}"), - ("\u{85}", "\\u{85}"), - ("\u{e0041}", "\\u{e0041}"), - ("\u{200b}", "\\u{200b}"), - ("\u{feff}", "\\u{feff}"), - ]; - let mut tail: Vec = Vec::new(); - for &(raw, _) in &hostile { - tail.extend_from_slice(b"a"); - tail.extend_from_slice(raw.as_bytes()); - tail.extend_from_slice(b"b\n"); - } - tail.extend_from_slice(b"bad \xff\xc1 bytes\n"); - runner_evidence::record( - &runner_evidence_store_path(&world.state_store), - &RunnerFailureRecord::from_failure( - &ended.transaction_id, - &ended.command_id, - &ended.target, - &canary_failure(&tail), - 9, - ), - )?; - let printed = printed_status(&world)?; - for &(raw, escaped) in &hostile { - assert!(!printed.contains(raw), "{escaped} reached the terminal raw"); - assert!(printed.contains(&format!("a{escaped}b")), "{escaped} missing: {printed}"); - } - assert!(printed.contains("bad \\xff\\xc1 bytes"), "{printed}"); - let section = printed.find(" failed step").context("no evidence section")?; - assert!(printed[section..].is_ascii(), "non-ASCII reached the terminal: {printed}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn status_prints_the_unreadable_store_line_escaped() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - tick_until_ended(&world)?; - let evidence = runner_evidence_store_path(&world.state_store); - // The decode error of this store quotes the stored subject kind. - let mut bytes = std::fs::read(&evidence)?; - let at = bytes - .windows(4) - .position(|window| window == b"step") - .context("no step marker in the evidence store")?; - bytes[at..at + 4].copy_from_slice(b"\x1b[2J"); - std::fs::write(&evidence, bytes)?; - let printed = printed_status(&world)?; - assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); - assert!(!printed.contains('\u{1b}'), "an escape byte reached the terminal"); - assert!(printed.contains("\\u{1b}[2J"), "{printed}"); - Ok(()) - } - - #[test] - fn the_journal_line_names_a_displaced_set_aside_store_by_time_only() -> Result<()> { - let (transaction, _) = terminal_transaction_with_command("ghostlight")?; - let failure = canary_failure(RUNNER_CANARY.as_bytes()); - let path = Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"); - let first = runner_failure_journal_line( - &transaction, - &failure, - path, - &Ok(Kept::SetAside { replaced: None }), - ); - assert!(first.contains("was set aside as"), "{first}"); - assert!(!first.contains("replacing"), "{first}"); - let second = runner_failure_journal_line( - &transaction, - &failure, - path, - &Ok(Kept::SetAside { replaced: Some(1234) }), - ); - assert!( - second.contains("replacing an earlier set-aside store last written at unix millis 1234"), - "{second}" - ); - assert!(!second.contains(RUNNER_CANARY)); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - tick_until_ended(&world)?; - // The store a rolled-back Idunn reads: it refuses any foreign type. - ControlSnapshot::read(&world.state_store)?; - let mut transactions = 0; - for store in [world.state_store.clone(), history_store_path(&world.state_store)] { - for envelope in SingleFileMessagePackBackingStore::new(&store).pull_all_read_only_snapshot()? { - assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); - if envelope.r#type == DeploymentTransaction::TYPE { - transactions += 1; - assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); - } - } - } - assert_eq!(transactions, 1, "the failed transaction is in neither store"); - Ok(()) - } - - /// The records control.cc holds re-encode byte for byte. The hex was - /// captured at base 804ed75, before this cut touched anything: a runner - /// failure adds no field, no type and no schema to the control store. - #[test] - fn control_records_encode_exactly_as_they_did_at_base() -> Result<()> { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-service".into(), - kind: CommandKind::Deploy, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?; - transaction.transaction_id = "tx-pinned".into(); - transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into()); - transaction.pre_fencing_abort = Some(PreFencingAbort { - error: "runner step test-daemon on runner rust exited with exit status 3".into(), - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Complete, - }); - let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); - let command_hex = hex(&command_envelope(&command, 100)?.payload); - let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload); - assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed"); - assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed"); - Ok(()) - } - - const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; - const PINNED_TRANSACTION_HEX: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; - // END runner-failure-evidence tests } diff --git a/src/drivers.rs b/src/drivers.rs index 3ea1145..7f87896 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -157,24 +157,6 @@ pub struct FrozenSourceReceipt { pub snapshot_sha256: String, } -#[cfg(test)] -impl FrozenSource { - /// A frozen source over an existing tree, for tests that drive a runner. - pub(crate) fn fixture( - receipt: FrozenSourceReceipt, - facts: SourceSelectionFacts, - recipe_bytes: Vec, - root: PathBuf, - ) -> Self { - Self { - receipt, - facts, - recipe_bytes, - root, - } - } -} - impl FrozenSourceReceipt { pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> { plan.validate()?; @@ -605,9 +587,7 @@ pub trait WorkloadPort { /// /// Idunn launches candidates as transient units with `Restart=no`, so a /// unit that has failed, or that systemd no longer knows, will not come - /// back on its own. Those units are collected once they fail - /// (`INCARNATION_UNIT_LIFECYCLE`), so a failed candidate usually reads as - /// forgotten rather than failed. Past the fence every error is otherwise treated as + /// back on its own. Past the fence every error is otherwise treated as /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; @@ -2172,115 +2152,6 @@ pub(crate) fn docker_run_args( Ok(args) } -/// Bytes of each output stream that a failed runner step keeps. The daemon -/// never holds more than this per stream while the step runs. -pub(crate) const RUNNER_OUTPUT_TAIL_BYTES: usize = 8192; - -/// What a runner container was running: a recipe step or an external input, -/// by the id the recipe gave it. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum RunnerSubject { - Step(String), - ExternalInput(String), -} - -impl RunnerSubject { - pub(crate) fn id(&self) -> &str { - match self { - Self::Step(id) | Self::ExternalInput(id) => id, - } - } - - fn noun(&self) -> &'static str { - match self { - Self::Step(_) => "step", - Self::ExternalInput(_) => "external input", - } - } -} - -/// A runner container that did not exit successfully: what failed, how, and -/// the last `RUNNER_OUTPUT_TAIL_BYTES` of each stream with its total size. -/// The tails are evidence for `runner-evidence.cc`. `Display` and `Debug` -/// carry none of them, so no error chain, record or log line renders output. -#[derive(Clone, PartialEq, Eq)] -pub(crate) struct RunnerStepFailure { - pub(crate) subject: RunnerSubject, - pub(crate) runner: String, - pub(crate) exit_code: Option, - pub(crate) signal: Option, - pub(crate) stdout_tail: Vec, - pub(crate) stdout_total_bytes: u64, - pub(crate) stderr_tail: Vec, - pub(crate) stderr_total_bytes: u64, -} - -impl RunnerStepFailure { - /// The status as an operator reads it. - pub(crate) fn status(&self) -> String { - runner_status(self.exit_code, self.signal) - } -} - -/// How a container ended, as an operator reads it. -pub(crate) fn runner_status(exit_code: Option, signal: Option) -> String { - match (exit_code, signal) { - (Some(code), _) => format!("exit status {code}"), - (None, Some(signal)) => format!("signal {signal}"), - (None, None) => "an unknown status".to_owned(), - } -} - -impl std::fmt::Display for RunnerStepFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - formatter, - "runner {} {} on runner {} exited with {}; its output tail is kept in runner-evidence.cc", - self.subject.noun(), - self.subject.id(), - self.runner, - self.status() - ) - } -} - -impl std::fmt::Debug for RunnerStepFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RunnerStepFailure") - .field("subject", &self.subject) - .field("runner", &self.runner) - .field("exit_code", &self.exit_code) - .field("signal", &self.signal) - .field("stdout_total_bytes", &self.stdout_total_bytes) - .field("stderr_total_bytes", &self.stderr_total_bytes) - .finish_non_exhaustive() - } -} - -impl std::error::Error for RunnerStepFailure {} - -/// Read `reader` to its end keeping only the last `cap` bytes and counting -/// every byte, so a verbose step never buffers whole in the daemon. -fn drain_tail(mut reader: impl Read, cap: usize) -> std::io::Result<(Vec, u64)> { - let mut tail = Vec::new(); - let mut total = 0u64; - let mut chunk = [0u8; 8192]; - loop { - let read = match reader.read(&mut chunk) { - Ok(0) => return Ok((tail, total)), - Ok(read) => read, - Err(error) if error.kind() == ErrorKind::Interrupted => continue, - Err(error) => return Err(error), - }; - total += read as u64; - tail.extend_from_slice(&chunk[..read]); - if tail.len() > cap { - tail.drain(..tail.len() - cap); - } - } -} - /// Docker is only a runner substrate. Recipe argv stays an argv vector, the /// operator binding selects the exact image/network/mount affordances, and the /// driver returns complete materialization receipts rather than build truth by @@ -2299,7 +2170,7 @@ impl Default for DockerRunnerDriver { } impl DockerRunnerDriver { - fn docker(&self, subject: RunnerSubject, runner_id: &str, args: I) -> Result<()> + fn docker(&self, args: I) -> Result where I: IntoIterator, S: AsRef, @@ -2308,50 +2179,23 @@ impl DockerRunnerDriver { self.docker_program.is_absolute(), "Docker runner program is not absolute" ); - let mut child = Command::new(&self.docker_program) + let output = Command::new(&self.docker_program) .args(args) .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() + .output() .with_context(|| format!("starting Docker runner {}", self.docker_program.display()))?; - let stdout = child.stdout.take().context("Docker runner has no stdout pipe")?; - let stderr = child.stderr.take().context("Docker runner has no stderr pipe")?; - let stdout = thread::spawn(move || drain_tail(stdout, RUNNER_OUTPUT_TAIL_BYTES)); - let stderr = thread::spawn(move || drain_tail(stderr, RUNNER_OUTPUT_TAIL_BYTES)); - let status = child.wait().context("waiting for the Docker runner")?; - let (stdout_tail, stdout_total_bytes) = stdout - .join() - .map_err(|_| anyhow!("the runner stdout reader panicked"))? - .context("reading the runner stdout")?; - let (stderr_tail, stderr_total_bytes) = stderr - .join() - .map_err(|_| anyhow!("the runner stderr reader panicked"))? - .context("reading the runner stderr")?; - if status.success() { - return Ok(()); + if !output.status.success() { + bail!( + "Docker runner exited with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); } - #[cfg(unix)] - let signal = std::os::unix::process::ExitStatusExt::signal(&status); - #[cfg(not(unix))] - let signal = None; - Err(RunnerStepFailure { - subject, - runner: runner_id.to_owned(), - exit_code: status.code(), - signal, - stdout_tail, - stdout_total_bytes, - stderr_tail, - stderr_total_bytes, - } - .into()) + Ok(output) } fn run_in_workspace( &self, - step_id: &str, - runner_id: &str, runner: &DockerRunnerBinding, workspace: &Path, working_directory: &Path, @@ -2368,7 +2212,8 @@ impl DockerRunnerDriver { build_machine_id_file(workspace)?; let spec = ContainerSpec::for_step(runner, required_environment, source_stamp)?; let args = docker_run_args(&spec, workspace, working_directory, argv)?; - self.docker(RunnerSubject::Step(step_id.to_owned()), runner_id, args) + self.docker(args)?; + Ok(()) } fn materialize_external_input( @@ -2410,11 +2255,7 @@ impl DockerRunnerDriver { input.url.clone(), ]; let args = docker_run_args(&spec, workspace, Path::new("."), &curl_argv)?; - self.docker( - RunnerSubject::ExternalInput(input.id.clone()), - &input.runner, - args, - )?; + self.docker(args)?; let bytes = fs::read(&destination).with_context(|| { format!( "reading materialized external input {}", @@ -2536,8 +2377,6 @@ impl RunnerPort for DockerRunnerDriver { ); } self.run_in_workspace( - &step.id, - &step.runner, runner, workspace, &step.working_directory, @@ -2576,13 +2415,6 @@ impl RunnerPort for DockerRunnerDriver { } } -/// How every incarnation unit lives and dies. It runs once (`Restart=no`) -/// and systemd forgets it once it is inactive or failed (`--collect`): the -/// outcome is Idunn's to record in its own store, and a failed transient unit -/// left loaded is only clutter that marks the host degraded. -const INCARNATION_UNIT_LIFECYCLE: [&str; 3] = - ["--collect", "--property=Type=exec", "--property=Restart=no"]; - /// systemd owns process execution. This driver lowers one validated launch /// contract into a transient unit and then proves the native process and /// executable that systemd actually started. It does not decide admission, @@ -3773,9 +3605,8 @@ impl SystemdTransientWorkloadDriver { OsString::from("--no-ask-password"), OsString::from("--expand-environment=no"), OsString::from(format!("--unit={unit}")), - ]; - args.extend(INCARNATION_UNIT_LIFECYCLE.map(OsString::from)); - args.extend([ + OsString::from("--property=Type=exec"), + OsString::from("--property=Restart=no"), OsString::from("--property=KillMode=mixed"), OsString::from("--property=DynamicUser=yes"), OsString::from("--property=SupplementaryGroups="), @@ -3811,7 +3642,7 @@ impl SystemdTransientWorkloadDriver { OsString::from(format!("--working-directory={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", bundle.display())), - ]); + ]; for open_file in parent_only_open_files { args.push(OsString::from(format!("--property=OpenFile={open_file}"))); } @@ -9952,16 +9783,6 @@ Content-Le".to_vec()), |_| {}), Ok(()) } - /// An incarnation unit runs once and is collected when it ends, so a - /// failed candidate does not stay loaded and mark the host degraded. - /// Collection is safe because a forgotten unit already reads as - /// permanently stopped (the test below). - #[test] - fn incarnation_units_run_once_and_are_collected() { - assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--collect")); - assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--property=Restart=no")); - } - /// Admission audit, claim 3: `is_permanently_stopped` now lets the /// Isolation gate skip `prove_isolation` against the incumbent. Enumerate /// what it answers `true` for, using a fake `systemctl show` that replays @@ -16410,292 +16231,4 @@ processes 9 ); Ok(()) } - - // BEGIN runner-failure-evidence tests - // The runner decides what failed and what it kept: these drive the real - // DockerRunnerDriver over a fake docker program, never the capture alone. - - const RUNNER_CANARY: &str = "CANARY-5d2e91b7"; - - #[cfg(unix)] - fn fake_docker(dir: &Path, body: &str) -> Result { - use std::os::unix::fs::PermissionsExt; - let program = dir.join("docker"); - fs::write(&program, format!("#!/bin/sh\n{body}\n"))?; - fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?; - Ok(DockerRunnerDriver { - docker_program: program, - }) - } - - /// One recipe step through the real `run_in_workspace` path. - #[cfg(unix)] - fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> { - let workspace = dir.join("workspace"); - fs::create_dir_all(&workspace)?; - driver.run_in_workspace( - step_id, - "rust", - &fixture_docker_runner_binding(), - &workspace, - Path::new("."), - &["cargo".to_owned(), "test".to_owned()], - &std::collections::BTreeSet::new(), - ("IDUNN_STAMP", "rev"), - ) - } - - #[cfg(unix)] - fn failure_of(result: Result<()>) -> Result { - let error = result.expect_err("the step did not fail"); - error - .downcast_ref::() - .cloned() - .with_context(|| format!("the failure is not typed: {error:#}")) - } - - #[cfg(unix)] - #[test] - fn a_failed_step_is_named_with_both_tails() -> Result<()> { - let temp = tempfile::tempdir()?; - // The first step passes, the second fails: the count is the fake's. - let driver = fake_docker( - temp.path(), - &format!( - "n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\ - [ \"$n\" -lt 2 ] && exit 0\n\ - echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\ - echo 'stderr report' >&2\nexit 3" - ), - )?; - run_step(&driver, temp.path(), "first")?; - let failure = failure_of(run_step(&driver, temp.path(), "second"))?; - assert_eq!(failure.subject, RunnerSubject::Step("second".into())); - assert_eq!(failure.runner, "rust"); - assert_eq!(failure.exit_code, Some(3)); - assert_eq!(failure.signal, None); - assert!( - failure - .stdout_tail - .ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()), - "stdout was dropped: {:?}", - String::from_utf8_lossy(&failure.stdout_tail) - ); - assert_eq!(failure.stderr_tail, b"stderr report\n"); - assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> { - let temp = tempfile::tempdir()?; - let driver = fake_docker(temp.path(), "kill -9 $$")?; - let failure = failure_of(run_step(&driver, temp.path(), "build"))?; - assert_eq!((failure.exit_code, failure.signal), (None, Some(9))); - assert!(failure.to_string().contains("exited with signal 9"), "{failure}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_failed_external_input_is_named_as_one() -> Result<()> { - let temp = tempfile::tempdir()?; - let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?; - let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?; - let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?; - let input_id = declaration.external_inputs[0].id.clone(); - let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]); - fs::create_dir_all(&workspaces["rust"])?; - let error = driver - .materialize_external_input(&declaration, &binding, &workspaces, &input_id) - .expect_err("the download did not fail"); - let failure = error - .downcast_ref::() - .with_context(|| format!("the failure is not typed: {error:#}"))?; - assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone())); - assert_eq!(failure.exit_code, Some(22)); - assert!( - failure.to_string().contains(&format!("runner external input {input_id} on runner rust")), - "{failure}" - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn runner_output_never_reaches_the_error_text() -> Result<()> { - let temp = tempfile::tempdir()?; - let driver = fake_docker( - temp.path(), - &format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"), - )?; - let failure = failure_of(run_step(&driver, temp.path(), "build"))?; - assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY)); - assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY)); - let error = anyhow::Error::new(failure.clone()).context("sealing"); - for text in [ - format!("{error:#}"), - format!("{error:?}"), - format!("{failure}"), - format!("{failure:?}"), - ] { - assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}"); - } - Ok(()) - } - - /// A reader that answers from a script, so a read error the OS rarely - /// produces (an interrupted call) is produced on demand. - struct ScriptedReader(Vec>>); - - impl Read for ScriptedReader { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - if self.0.is_empty() { - return Ok(0); - } - let bytes = self.0.remove(0)?; - buffer[..bytes.len()].copy_from_slice(&bytes); - Ok(bytes.len()) - } - } - - struct BrokenReader; - - impl Read for BrokenReader { - fn read(&mut self, _: &mut [u8]) -> std::io::Result { - Err(std::io::Error::other("persistent read failure")) - } - } - - #[test] - fn an_interrupted_read_is_retried_and_loses_no_byte() { - let reader = ScriptedReader(vec![ - Ok(b"ab".to_vec()), - Err(std::io::Error::from(ErrorKind::Interrupted)), - Ok(b"cd".to_vec()), - Err(std::io::Error::from(ErrorKind::Interrupted)), - ]); - let drained = drain_tail(reader, 3).expect("an interrupted read ended the drain"); - assert_eq!(drained, (b"bcd".to_vec(), 4)); - } - - #[test] - fn a_persistent_read_error_ends_the_drain_instead_of_spinning() { - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let _ = sender.send(drain_tail(BrokenReader, 8).is_err()); - }); - assert_eq!( - receiver.recv_timeout(Duration::from_secs(5)), - Ok(true), - "a persistent read error spun or was swallowed" - ); - } - - #[test] - fn a_non_interrupted_read_error_after_data_is_returned_not_retried() { - let reader = ScriptedReader(vec![ - Ok(b"ab".to_vec()), - Err(std::io::Error::from(ErrorKind::BrokenPipe)), - Ok(b"cd".to_vec()), - ]); - let error = drain_tail(reader, 8).expect_err("a broken pipe was retried past"); - assert_eq!(error.kind(), ErrorKind::BrokenPipe); - } - - #[cfg(unix)] - #[test] - fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { - let temp = tempfile::tempdir()?; - let data = temp.path().join("data"); - for size in [ - 0, - 1, - RUNNER_OUTPUT_TAIL_BYTES - 1, - RUNNER_OUTPUT_TAIL_BYTES, - RUNNER_OUTPUT_TAIL_BYTES + 1, - 1024 * 1024, - ] { - // Distinct neighbours, so a shifted or head-keeping tail differs. - let bytes: Vec = (0..size).map(|index| (index % 251) as u8).collect(); - fs::write(&data, &bytes)?; - let driver = fake_docker( - temp.path(), - &format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()), - )?; - let failure = failure_of(run_step(&driver, temp.path(), "build"))?; - let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..]; - assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes"); - assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes"); - assert_eq!(failure.stdout_total_bytes, size as u64); - assert_eq!(failure.stderr_total_bytes, size as u64); - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> { - let temp = tempfile::tempdir()?; - run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?; - let missing = DockerRunnerDriver { - docker_program: temp.path().join("no-such-docker"), - }; - let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran"); - assert!(error.downcast_ref::().is_none()); - assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); - Ok(()) - } - - /// The 2026-10-25 vili failure: `node --test` reported on stdout alone, - /// stderr stayed empty (the runner's /tmp is a noexec tmpfs) and the step - /// exited 1. The old capture dropped stdout and printed an empty message. - #[cfg(unix)] - #[test] - fn a_step_that_reports_only_on_stdout_keeps_that_report_and_its_name() -> Result<()> { - let temp = tempfile::tempdir()?; - let driver = fake_docker( - temp.path(), - &format!("echo 'not ok 1 - daemon {RUNNER_CANARY}'\necho '# fail 1'\nexit 1"), - )?; - let error = run_step(&driver, temp.path(), "test-daemon").expect_err("the step passed"); - // The error names the step and carries no output. - let text = format!("{error:#}"); - assert!(text.contains("runner step test-daemon on runner rust exited with exit status 1"), "{text}"); - assert!(!text.contains(RUNNER_CANARY) && !text.contains("not ok"), "{text}"); - // The evidence keeps the stdout tail; the empty stderr is a recorded 0. - let failure = error - .downcast_ref::() - .context("the failure is not typed")?; - let store = temp.path().join("runner-evidence.cc"); - let record = crate::runner_evidence::RunnerFailureRecord::from_failure( - "tx-vili", - "up-vili", - "vili", - failure, - 1, - ); - crate::runner_evidence::record(&store, &record)?; - let kept = crate::runner_evidence::for_transaction(&store, "tx-vili")?.context("no evidence kept")?; - assert_eq!(kept.subject_id, "test-daemon"); - assert_eq!(kept.exit_code, Some(1)); - assert_eq!( - String::from_utf8_lossy(&kept.stdout_tail), - format!("not ok 1 - daemon {RUNNER_CANARY}\n# fail 1\n") - ); - assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); - assert!(kept.stderr_tail.is_empty()); - assert_eq!(kept.stderr_total_bytes, 0); - let mut shown = Vec::new(); - crate::runner_evidence::show_status(&mut shown, &store, "tx-vili")?; - let rendered = String::from_utf8(shown)?; - assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); - assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); - // Debug names the failure and prints no tail. - let debug = format!("{failure:?}"); - assert!(debug.starts_with("RunnerStepFailure") && !debug.contains("not ok"), "{debug}"); - Ok(()) - } - // END runner-failure-evidence tests } diff --git a/src/host_actuator.rs b/src/host_actuator.rs index 8fd776c..f7bf4c7 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1151,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - if let Some(frame) = client.receive_once()? { + while let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/lib.rs b/src/lib.rs index b5499a6..e71a17f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,4 @@ pub mod host_actuator; #[cfg(unix)] pub mod launcher; mod incident; -mod runner_evidence; pub mod provisioning; -#[cfg(unix)] -mod steward_orders; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs deleted file mode 100644 index 30dd1d2..0000000 --- a/src/runner_evidence.rs +++ /dev/null @@ -1,805 +0,0 @@ -//! The output tail of a failed runner step. -//! -//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the -//! last bytes of a failed runner container's stdout and stderr, one record per -//! transaction. The engine's pre-fence failure path is its only writer, and -//! `idunn status --command` its only reader. It is never `control.cc`, which -//! refuses any type it does not know, and never `incidents.cc`, which is -//! published and carries no free text. -//! -//! The tail is raw runner output. It is kept unredacted and readable by root -//! alone (mode 0600): root can already read every secret the runner was given. -//! It never enters an error, a transaction record or the journal. The status -//! reader is a terminal, so everything it prints from this store goes through -//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped. - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result, bail, ensure}; -use cultcache_rs::{ - CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, - TryCompareExchangeSnapshotOutcome, -}; -use serde::{Deserialize, Serialize}; - -use crate::control_plane::{decode_record, require_id, typed_envelope}; -use crate::drivers::{ - LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject, - exchange_behind_private_lock, runner_status, -}; - -const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1"; - -/// Records kept; the writer drops the oldest past this, so the store needs no -/// other owner. -pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32; - -/// `runner-evidence.cc`, beside the control store. -pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf { - state_store.with_file_name("runner-evidence.cc") -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum SubjectKind { - Step, - ExternalInput, -} - -#[derive(Clone, PartialEq, Eq, DatabaseEntry)] -#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")] -pub(crate) struct RunnerFailureRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) transaction_id: String, - #[cultcache(key = 2)] - pub(crate) command_id: String, - #[cultcache(key = 3)] - pub(crate) target: String, - #[cultcache(key = 4)] - pub(crate) subject_kind: SubjectKind, - #[cultcache(key = 5)] - pub(crate) subject_id: String, - #[cultcache(key = 6)] - pub(crate) runner: String, - #[cultcache(key = 7)] - pub(crate) exit_code: Option, - #[cultcache(key = 8)] - pub(crate) signal: Option, - #[cultcache(key = 9)] - pub(crate) stdout_tail: Vec, - #[cultcache(key = 10)] - pub(crate) stdout_total_bytes: u64, - #[cultcache(key = 11)] - pub(crate) stderr_tail: Vec, - #[cultcache(key = 12)] - pub(crate) stderr_total_bytes: u64, - #[cultcache(key = 13)] - pub(crate) recorded_at_unix_millis: u64, -} - -/// Tails stay out of `Debug`, so a failed assertion or a logged record never -/// prints runner output. -impl std::fmt::Debug for RunnerFailureRecord { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RunnerFailureRecord") - .field("transaction_id", &self.transaction_id) - .field("subject_id", &self.subject_id) - .finish_non_exhaustive() - } -} - -impl RunnerFailureRecord { - pub(crate) fn from_failure( - transaction_id: &str, - command_id: &str, - target: &str, - failure: &RunnerStepFailure, - now: u64, - ) -> Self { - let (subject_kind, subject_id) = match &failure.subject { - RunnerSubject::Step(id) => (SubjectKind::Step, id), - RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id), - }; - Self { - schema_version: RUNNER_FAILURE_SCHEMA.to_owned(), - transaction_id: transaction_id.to_owned(), - command_id: command_id.to_owned(), - target: target.to_owned(), - subject_kind, - subject_id: subject_id.clone(), - runner: failure.runner.clone(), - exit_code: failure.exit_code, - signal: failure.signal, - stdout_tail: failure.stdout_tail.clone(), - stdout_total_bytes: failure.stdout_total_bytes, - stderr_tail: failure.stderr_tail.clone(), - stderr_total_bytes: failure.stderr_total_bytes, - recorded_at_unix_millis: now, - } - } - - pub(crate) fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == RUNNER_FAILURE_SCHEMA, - "runner failure schema is unsupported" - ); - require_id(&self.transaction_id, "runner failure transaction")?; - require_id(&self.command_id, "runner failure command")?; - require_id(&self.target, "runner failure target")?; - require_id(&self.subject_id, "runner failure subject")?; - require_id(&self.runner, "runner failure runner")?; - for (tail, total, stream) in [ - (&self.stdout_tail, self.stdout_total_bytes, "stdout"), - (&self.stderr_tail, self.stderr_total_bytes, "stderr"), - ] { - ensure!( - tail.len() <= RUNNER_OUTPUT_TAIL_BYTES, - "runner failure {stream} tail exceeds its bound" - ); - ensure!( - tail.len() as u64 <= total, - "runner failure {stream} tail exceeds its stream" - ); - } - Ok(()) - } - - fn envelope(&self) -> Result { - self.validate()?; - typed_envelope( - &self.transaction_id, - Self::TYPE, - RUNNER_FAILURE_SCHEMA, - self, - self.recorded_at_unix_millis, - ) - } -} - -/// One read of the store: the envelopes exactly as stored, and the records -/// decoded from them in the same order. -fn snapshot(path: &Path) -> Result<(Vec, Vec)> { - let envelopes = SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot() - .context("reading the runner evidence store")?; - let mut records = Vec::new(); - for envelope in &envelopes { - ensure!( - envelope.r#type == RunnerFailureRecord::TYPE - && envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA), - "the runner evidence store holds a document that is not a runner failure" - ); - let record: RunnerFailureRecord = decode_record(envelope)?; - record.validate()?; - ensure!( - envelope.key == record.transaction_id, - "runner evidence store key differs from the record's transaction" - ); - records.push(record); - } - Ok((envelopes, records)) -} - -/// How a write left the store. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Kept { - Stored, - /// The store held bytes that do not decode; it was moved to - /// `unreadable_store_path` and the record opened a fresh one. `replaced` - /// is the last-written time (unix milliseconds) of an earlier set-aside - /// store this one displaced, which is gone: one slot, and it says so. - SetAside { replaced: Option }, -} - -/// Where an undecodable store is kept. One slot: the next one replaces it, so -/// the damage a bad store can cost is one extra file. -pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf { - path.with_file_name("runner-evidence.unreadable.cc") -} - -/// Moves the store over the previous set-aside one, root-only, and reports -/// what it displaced. Nothing of the store's content is named: a decode error -/// can quote stored values, and none is carried here. -fn set_aside_unreadable(path: &Path) -> Result> { - let aside = unreadable_store_path(path); - let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| { - earlier - .modified() - .ok() - .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) - .map_or(0, |since| since.as_millis() as u64) - }); - std::fs::rename(path, &aside) - .context("the runner evidence store is undecodable and could not be set aside")?; - keep_private(&aside)?; - Ok(replaced) -} - -/// How the writer reads the store's bytes; tests inject an I/O error here. -type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result>; - -fn unreadable_fault(error: &std::io::Error) -> anyhow::Error { - anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind()) -} - -/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the -/// oldest records go in the same exchange. The file ends at mode 0600. -/// -/// A store is set aside only when bytes that were read do not decode. An I/O -/// error leaves the store in place, keeps no evidence for that failure, and -/// the error names its kind for the journal. -pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { - record_reading(path, record, &|path| std::fs::read(path)) -} - -fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result { - let first = match read(path) { - Ok(bytes) => Some(bytes), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, - Err(error) => return Err(unreadable_fault(&error)), - }; - let (envelopes, records, how) = match snapshot(path) { - Ok((envelopes, records)) => (envelopes, records, Kept::Stored), - Err(_) => { - let Some(first) = first else { - bail!("the runner evidence store could not be decoded"); - }; - // Undecodable only if the same bytes are still there to read: a - // second read that fails, or finds other bytes, is not a verdict. - match read(path) { - Ok(second) if second == first => {} - Ok(_) => bail!("the runner evidence store changed while it was being read"), - Err(error) => return Err(unreadable_fault(&error)), - } - let replaced = set_aside_unreadable(path)?; - (Vec::new(), Vec::new(), Kept::SetAside { replaced }) - } - }; - let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes - .iter() - .zip(&records) - .filter(|(_, stored)| stored.transaction_id != record.transaction_id) - .map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone())) - .collect(); - kept.push((record.recorded_at_unix_millis, record.envelope()?)); - // Stable: records of one instant keep their stored order, so the record - // being written survives a tie. - kept.sort_by_key(|(recorded_at, _)| *recorded_at); - let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS); - let replacements: Vec<_> = kept - .into_iter() - .skip(excess) - .map(|(_, envelope)| envelope) - .collect(); - match exchange_behind_private_lock(path, &envelopes, &replacements)? { - TryCompareExchangeSnapshotOutcome::Exchanged => {} - TryCompareExchangeSnapshotOutcome::Mismatch => { - bail!("the runner evidence store changed while it was being written") - } - TryCompareExchangeSnapshotOutcome::LockContended => { - return Err(LockContended { - store: "runner evidence store", - } - .into()); - } - } - keep_private(path)?; - Ok(how) -} - -#[cfg(unix)] -fn keep_private(path: &Path) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) - .context("making the runner evidence store private") -} - -#[cfg(not(unix))] -fn keep_private(_path: &Path) -> Result<()> { - Ok(()) -} - -/// The one reader. An absent store holds nothing. -pub(crate) fn for_transaction( - path: &Path, - transaction_id: &str, -) -> Result> { - Ok(snapshot(path)? - .1 - .into_iter() - .find(|stored| stored.transaction_id == transaction_id)) -} - -/// The record as `idunn status --command` shows it, as raw lines. They are -/// not text yet: `show_status` is the only caller, and `emit` the only -/// writer. -fn render(record: &RunnerFailureRecord) -> Vec> { - let noun = match record.subject_kind { - SubjectKind::Step => "step", - SubjectKind::ExternalInput => "external input", - }; - let mut lines: Vec> = vec![ - format!(" failed {noun} {}", record.subject_id).into_bytes(), - format!(" runner {}", record.runner).into_bytes(), - format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(), - ]; - for (stream, tail, total) in [ - ("stdout", &record.stdout_tail, record.stdout_total_bytes), - ("stderr", &record.stderr_tail, record.stderr_total_bytes), - ] { - lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes()); - let text = tail.strip_suffix(b"\n").unwrap_or(tail); - if !text.is_empty() { - lines.extend(text.split(|byte| *byte == b'\n').map(|line| { - let mut shown = b" | ".to_vec(); - shown.extend_from_slice(line); - shown - })); - } - } - lines -} - -/// What `idunn status --command` prints for a transaction's runner evidence: -/// its record, or one line naming why the store is unreadable. Nothing is -/// written when the transaction has no record. -pub(crate) fn show_status( - out: &mut impl std::io::Write, - store: &Path, - transaction_id: &str, -) -> std::io::Result<()> { - let lines = match for_transaction(store, transaction_id) { - Ok(Some(record)) => render(&record), - Ok(None) => return Ok(()), - Err(error) => { - vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()] - } - }; - emit(out, &lines) -} - -/// The one sink: every byte the status read prints from the evidence store, -/// or from an error about it, is written here and nowhere else. An allowlist: -/// printable ASCII and tab pass; every other code point is a visible -/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline -/// inside a line is escaped too: this function writes the only ones. -pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { - for line in lines { - out.write_all(escape(line).as_bytes())?; - out.write_all(b"\n")?; - } - Ok(()) -} - -pub(crate) fn escape(bytes: &[u8]) -> String { - let mut shown = String::with_capacity(bytes.len()); - for chunk in bytes.utf8_chunks() { - for character in chunk.valid().chars() { - match character { - ' '..='~' | '\t' => shown.push(character), - _ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))), - } - } - for byte in chunk.invalid() { - shown.push_str(&format!("\\x{byte:02x}")); - } - } - shown -} - -#[cfg(all(test, unix))] -mod tests { - use super::*; - use std::os::unix::fs::PermissionsExt; - - fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure { - RunnerStepFailure { - subject, - runner: "rust".into(), - exit_code: Some(3), - signal: None, - stdout_tail: stdout.to_vec(), - stdout_total_bytes: stdout.len() as u64, - stderr_tail: stderr.to_vec(), - stderr_total_bytes: stderr.len() as u64, - } - } - - /// What status writes for a record, through the one sink. - fn shown(record: &RunnerFailureRecord) -> String { - let mut written = Vec::new(); - emit(&mut written, &render(record)).expect("a Vec takes any write"); - String::from_utf8(written).expect("the sink writes ASCII only") - } - - fn shown_lines(record: &RunnerFailureRecord) -> Vec { - shown(record).lines().map(str::to_owned).collect() - } - - fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { - RunnerFailureRecord::from_failure( - transaction, - "up-service", - "service", - &failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"), - now, - ) - } - - #[test] - fn the_evidence_store_keeps_the_newest_32() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - // The literal, not the constant: the spec's bound is 32. - for index in 0..=32 { - record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; - } - let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); - assert_eq!(kept.len(), 32); - assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); - assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); - assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); - // Rewriting a resident transaction replaces it and drops nothing. - let mut again = record_at("tx-05", 5000); - again.stdout_tail = b"rewritten\n".to_vec(); - again.stdout_total_bytes = 10; - record(&path, &again)?; - let kept = snapshot(&path)?.1; - assert_eq!(kept.len(), 32); - assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); - let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; - assert_eq!(stored.stdout_tail, b"rewritten\n"); - assert_eq!(for_transaction(&path, "tx-00")?, None); - Ok(()) - } - - #[test] - fn a_record_round_trips_byte_for_byte() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - let written = RunnerFailureRecord::from_failure( - "tx-a", - "up-service", - "service", - &failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""), - 77, - ); - record(&path, &written)?; - assert_eq!(for_transaction(&path, "tx-a")?, Some(written)); - assert_eq!(for_transaction(&path, "tx-b")?, None); - Ok(()) - } - - #[test] - fn the_evidence_store_is_private() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - // Whatever the umask, the mode is the store's own rule, on every write. - for index in 0..2 { - record(&path, &record_at(&format!("tx-{index}"), 10 + index))?; - assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); - } - Ok(()) - } - - #[test] - fn a_tail_over_its_bound_or_its_stream_is_refused() { - let mut oversize = record_at("tx-a", 1); - oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1]; - oversize.stdout_total_bytes = u64::MAX; - assert!(oversize.validate().is_err()); - let mut impossible = record_at("tx-a", 1); - impossible.stderr_total_bytes = 1; - assert!(impossible.validate().is_err()); - let mut exact = record_at("tx-a", 1); - exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES]; - exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64; - assert!(exact.validate().is_ok()); - } - - #[test] - fn status_renders_the_tail_with_control_bytes_escaped() { - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec(); - stored.stdout_total_bytes = 5000; - stored.stderr_tail = b"bad\r\x07".to_vec(); - stored.stderr_total_bytes = 6; - let lines = shown_lines(&stored); - let text = lines.join("\n"); - assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); - assert!(!text.contains('\u{7}')); - assert_eq!( - lines, - vec![ - " failed step build", - " runner rust", - " status exit status 3", - " stdout (last 30 of 5000 bytes)", - " | first", - " | \\u{1b}[2Jsecond\\u{0}end\tTab", - " | last", - " stderr (last 5 of 6 bytes)", - " | bad\\u{d}\\u{7}", - ] - ); - } - - #[test] - fn debug_names_the_record_and_prints_no_tail() { - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = b"SECRET-OUTPUT".to_vec(); - stored.stdout_total_bytes = 13; - let debug = format!("{stored:?}"); - assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}"); - } - - #[test] - fn an_empty_tail_renders_only_its_header() { - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = Vec::new(); - stored.stdout_total_bytes = 0; - let lines = shown_lines(&stored); - assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); - assert!(!lines.iter().any(|line| line == " | ")); - } - - #[test] - fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - std::fs::write(&path, b"\xc1 not a store")?; - assert!(for_transaction(&path, "tx-a").is_err()); - Ok(()) - } - - fn files_beside(path: &Path) -> Result> { - let mut names: Vec = std::fs::read_dir(path.parent().context("no parent")?)? - .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned())) - .collect::>()?; - names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock")); - names.sort(); - Ok(names) - } - - /// A store a newer Idunn wrote: well-formed, but a schema this one does not know. - fn write_future_store(path: &Path) -> Result<()> { - let mut future = record_at("tx-future", 1); - future.schema_version = "idunn.runner_failure.v2".into(); - let envelope = typed_envelope( - &future.transaction_id, - RunnerFailureRecord::TYPE, - "idunn.runner_failure.v2", - &future, - 1, - )?; - let outcome = exchange_behind_private_lock(path, &[], &[envelope])?; - ensure!( - matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged), - "the future store was not written" - ); - Ok(()) - } - - #[test] - fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - let damage = b"\xc1 not a store \x00 damage"; - std::fs::write(&path, damage)?; - let kept = record(&path, &record_at("tx-a", 5))?; - assert_eq!(kept, Kept::SetAside { replaced: None }); - // Kept, not deleted: the damage is where the journal says, root-only. - let aside = unreadable_store_path(&path); - assert_eq!(std::fs::read(&aside)?, damage); - assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600); - // The failure's evidence is in a fresh, private store. - assert_eq!( - for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id), - Some("tx-a".to_owned()) - ); - assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); - // A healthy store is not set aside again. - assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored); - assert_eq!(std::fs::read(&aside)?, damage); - Ok(()) - } - - #[test] - fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - write_future_store(&path)?; - let before = std::fs::read(&path)?; - assert!(for_transaction(&path, "tx-future").is_err()); - assert_eq!( - record(&path, &record_at("tx-a", 5))?, - Kept::SetAside { replaced: None } - ); - assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); - assert!(for_transaction(&path, "tx-a")?.is_some()); - assert_eq!(for_transaction(&path, "tx-future")?, None); - Ok(()) - } - - #[test] - fn at_most_one_unreadable_store_is_kept() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"]; - for (index, damage) in damages.into_iter().enumerate() { - std::fs::write(&path, damage)?; - record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?; - assert_eq!( - std::fs::read(unreadable_store_path(&path))?, - damage, - "the newest damage is the one kept" - ); - } - assert_eq!( - files_beside(&path)?, - vec![ - "runner-evidence.cc".to_owned(), - "runner-evidence.unreadable.cc".to_owned() - ] - ); - Ok(()) - } - - #[test] - fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - // A directory where the store should be: unreadable, not undecodable. - std::fs::create_dir(&path)?; - assert!(record(&path, &record_at("tx-a", 1)).is_err()); - assert!(path.is_dir()); - assert!(!unreadable_store_path(&path).exists()); - Ok(()) - } - - #[test] - fn every_code_point_but_printable_ascii_is_a_visible_escape() { - // The allowlist, exhaustively: no class list decides anything. - for code in 0..=0x10ffffu32 { - let Some(character) = char::from_u32(code) else { - continue; - }; - let text = character.to_string(); - let shown = escape(text.as_bytes()); - if matches!(character, ' '..='~' | '\t') { - assert_eq!(shown, text, "U+{code:04X} should pass"); - } else { - assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}"); - } - } - // Invalid UTF-8 is escaped byte by byte; valid text round it passes. - assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82"); - assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}"); - } - - #[test] - fn a_newline_inside_a_line_cannot_forge_a_line() { - let mut stored = record_at("tx-a", 1); - stored.subject_id = "build\n status exit status 0".into(); - let lines = shown_lines(&stored); - assert_eq!(lines[0], " failed step build\\u{a} status exit status 0"); - assert_eq!( - lines.iter().filter(|line| line.starts_with(" status ")).count(), - 1 - ); - } - - /// A store whose first record is well-formed except that its subject kind - /// holds `marker`: the decode error quotes it. - fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> { - record(path, &record_at("tx-a", 1))?; - let mut bytes = std::fs::read(path)?; - let at = bytes - .windows(4) - .position(|window| window == b"step") - .context("no step marker")?; - bytes[at..at + 4].copy_from_slice(marker); - std::fs::write(path, bytes)?; - Ok(()) - } - - #[test] - fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - store_with_hostile_subject_kind(&path, b"\x1b[2J")?; - let mut written = Vec::new(); - show_status(&mut written, &path, "tx-a")?; - assert!(!written.contains(&0x1b), "an escape byte reached the terminal"); - assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n'))); - let text = String::from_utf8(written)?; - assert!(text.starts_with(" runner evidence unreadable: "), "{text}"); - assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}"); - Ok(()) - } - - #[test] - fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - store_with_hostile_subject_kind(&path, b"CNRY")?; - let damage = std::fs::read(&path)?; - // Sanity: this damage decodes to an error that quotes the canary. - assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY")); - // A non-empty directory where the aside goes: rename fails even as root. - let aside = unreadable_store_path(&path); - std::fs::create_dir(&aside)?; - std::fs::write(aside.join("occupied"), b"x")?; - let error = record(&path, &record_at("tx-b", 2)).unwrap_err(); - for text in [format!("{error:#}"), format!("{error:?}")] { - assert!(text.contains("could not be set aside"), "{text}"); - assert!(!text.contains("CNRY"), "stored content reached the error: {text}"); - } - // The failure cost nothing: the store is as it was, with no evidence - // written over it. - assert_eq!(std::fs::read(&path)?, damage); - assert!(aside.join("occupied").exists()); - Ok(()) - } - - #[test] - fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - record(&path, &record_at("tx-a", 1))?; - let before = std::fs::read(&path)?; - let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied); - // The first read fails on a healthy store. - let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err(); - assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); - assert_eq!(std::fs::read(&path)?, before, "the store changed"); - assert!(!unreadable_store_path(&path).exists()); - assert!(for_transaction(&path, "tx-b")?.is_none()); - // The second read, the one that confirms the bytes do not decode, fails. - std::fs::write(&path, b"\xc1 not a store")?; - let calls = std::cell::Cell::new(0usize); - let read = |path: &Path| { - calls.set(calls.get() + 1); - if calls.get() == 2 { - return Err(denied()); - } - std::fs::read(path) - }; - let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err(); - assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); - assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); - assert!(!unreadable_store_path(&path).exists()); - // Bytes that changed between the reads are no verdict either. - let calls = std::cell::Cell::new(0usize); - let read = |path: &Path| -> std::io::Result> { - calls.set(calls.get() + 1); - let mut bytes = std::fs::read(path)?; - bytes.push(calls.get() as u8); - Ok(bytes) - }; - assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err()); - assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); - assert!(!unreadable_store_path(&path).exists()); - Ok(()) - } - - #[test] - fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - std::fs::write(&path, b"\xc1 first")?; - assert_eq!( - record(&path, &record_at("tx-a", 1))?, - Kept::SetAside { replaced: None } - ); - std::fs::write(&path, b"\xc1 second")?; - let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else { - bail!("a displaced set-aside store went unreported"); - }; - assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second"); - Ok(()) - } -} diff --git a/src/steward_orders.rs b/src/steward_orders.rs deleted file mode 100644 index d4be1ee..0000000 --- a/src/steward_orders.rs +++ /dev/null @@ -1,4079 +0,0 @@ -//! Typed orders from Idunn's steward. -//! -//! `idunn orders serve` is a second root unit from the same binary as -//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints -//! order ids, decides acceptance and refusal, actuates, and records outcomes. -//! The survival daemon takes no part: `control.cc` never learns an order type. -//! -//! An order is a closed enum, never a command string: install packages, act on -//! a systemd unit, or deploy a target through the same commit primitive -//! `idunn up` uses. Every unit is allowed except the always-refused set, which -//! is compiled in here and read from nowhere: no binding field, record or flag -//! holds it. The `#operator` post is a retried projection of the store. -//! -//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the -//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and -//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE -//! `mind-control check`; START record before actuation, at most once; RUN; -//! RESULT outcome; POST accepted and result. -//! -//! No refusal or error here carries a value the issuer supplied: it names the -//! field and a code. A refused unit that a record names was resolved by -//! systemd and passed the unit grammar. - -use std::fmt; -use std::io::{Read, Write}; -use std::os::unix::fs::{MetadataExt, PermissionsExt}; -use std::os::unix::net::{UnixListener, UnixStream}; -use std::os::unix::process::ExitStatusExt; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::sync::Arc; -use std::sync::mpsc; -use std::thread; -use std::time::Duration; - -use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcache_rs::{ - CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, -}; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use crate::control_plane::{ - command_state, commit_deployment_command, decode_record, default_state_store, now_millis, - path_value, require_id, require_selector, string_value, typed_envelope, -}; -use crate::runner_evidence::{emit, escape}; - -const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; -const START_SCHEMA: &str = "idunn.steward_order_start.v1"; -const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; -const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; - -const REQUEST_FRAME_LIMIT: usize = 16 * 1024; -const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; -const TAIL_BYTES: usize = 8 * 1024; -const REASON_LIMIT: usize = 300; -const PACKAGE_LIMIT: usize = 16; -const UNIT_LIMIT: usize = 200; -const LIST_LIMIT: usize = 50; -const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; - -/// The always-refused set: Idunn's own units and the minds' own. Matched on -/// every name a unit answers to, ASCII case-insensitively (systemd units are -/// case-sensitive, so the fold only widens the refusal). -fn is_refused_name(name: &str) -> bool { - let name = name.to_ascii_lowercase(); - ["idunn-", "mind-self@", "mind-persona@"] - .iter() - .any(|prefix| name.starts_with(prefix)) -} - -/// Units whose binaries suspend, hibernate or open recovery: no property -/// reveals it, so they are refused by name, on every name they answer to. -const HOST_NAMES: [&str; 6] = [ - "systemd-suspend.service", - "systemd-hibernate.service", - "systemd-hybrid-sleep.service", - "systemd-suspend-then-hibernate.service", - "rescue.service", - "emergency.service", -]; - -fn is_host_name(name: &str) -> bool { - HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) -} - -// ---------------------------------------------------------------- grammar -- - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Code { - Empty, - TooLong, - TooMany, - BadCharacter, - BadShape, - Malformed, - Unauthorized, - Unknown, - Unavailable, -} - -/// A refusal names the field and a code. It never carries the offending value. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct Refusal { - pub(crate) field: String, - pub(crate) code: Code, -} - -impl Refusal { - fn new(field: &'static str, code: Code) -> Self { - Self { - field: field.to_owned(), - code, - } - } -} - -impl fmt::Display for Refusal { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "{} refused: {:?}", self.field, self.code) - } -} - -impl std::error::Error for Refusal {} - -fn validate_package(spec: &str) -> Result<(), Refusal> { - const FIELD: &str = "package"; - if spec.is_empty() { - return Err(Refusal::new(FIELD, Code::Empty)); - } - if spec.len() > UNIT_LIMIT { - return Err(Refusal::new(FIELD, Code::TooLong)); - } - let (name, version) = match spec.split_once('=') { - Some((name, version)) => (name, Some(version)), - None => (spec, None), - }; - let bytes = name.as_bytes(); - if bytes.len() < 2 { - return Err(Refusal::new(FIELD, Code::BadShape)); - } - let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); - let rest = bytes[1..].iter().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') - }); - if !head || !rest { - return Err(Refusal::new(FIELD, Code::BadCharacter)); - } - // apt reads `name-` as a removal and `name.deb` as a local file. - if name.ends_with('-') || name.ends_with(".deb") { - return Err(Refusal::new(FIELD, Code::BadShape)); - } - if let Some(version) = version { - if version.is_empty() { - return Err(Refusal::new(FIELD, Code::BadShape)); - } - if !version.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') - }) { - return Err(Refusal::new(FIELD, Code::BadCharacter)); - } - } - Ok(()) -} - -fn validate_unit(unit: &str) -> Result<(), Refusal> { - const FIELD: &str = "unit"; - if unit.is_empty() { - return Err(Refusal::new(FIELD, Code::Empty)); - } - if unit.len() > UNIT_LIMIT { - return Err(Refusal::new(FIELD, Code::TooLong)); - } - let allowed = unit.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') - }); - if !allowed || unit.starts_with('-') { - return Err(Refusal::new(FIELD, Code::BadCharacter)); - } - match UNIT_SUFFIXES - .iter() - .find_map(|suffix| unit.strip_suffix(suffix)) - { - Some(stem) if !stem.is_empty() => Ok(()), - _ => Err(Refusal::new(FIELD, Code::BadShape)), - } -} - -fn validate_reason(reason: &str) -> Result<(), Refusal> { - const FIELD: &str = "reason"; - if reason.is_empty() { - return Err(Refusal::new(FIELD, Code::Empty)); - } - if reason.len() > REASON_LIMIT { - return Err(Refusal::new(FIELD, Code::TooLong)); - } - if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { - return Err(Refusal::new(FIELD, Code::BadCharacter)); - } - Ok(()) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Verb { - Start, - Stop, - Restart, - Reload, -} - -impl Verb { - fn as_str(self) -> &'static str { - match self { - Verb::Start => "start", - Verb::Stop => "stop", - Verb::Restart => "restart", - Verb::Reload => "reload", - } - } - - fn parse(word: &str) -> Option { - [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] - .into_iter() - .find(|verb| verb.as_str() == word) - } -} - -/// The closed set of things an order can ask for. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Operation { - PackageInstall { packages: Vec }, - UnitAction { verb: Verb, unit: String }, - Deploy { selector: String }, -} - -/// One fixed argv for `systemd-run`, built from the typed operation. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Step { - pub(crate) argv: Vec, - pub(crate) environment: Vec<(String, String)>, -} - -impl Operation { - fn validate(&self) -> Result<(), Refusal> { - match self { - Operation::PackageInstall { packages } => { - if packages.is_empty() { - return Err(Refusal::new("package", Code::Empty)); - } - if packages.len() > PACKAGE_LIMIT { - return Err(Refusal::new("package", Code::TooMany)); - } - packages.iter().try_for_each(|package| validate_package(package)) - } - Operation::UnitAction { unit, .. } => validate_unit(unit), - Operation::Deploy { selector } => require_selector(selector) - .map_err(|_| Refusal::new("selector", Code::BadShape)), - } - } - - fn steps(&self) -> Vec { - let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); - let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; - match self { - Operation::PackageInstall { packages } => { - // Never a removal, never a pattern: each name selects exactly - // itself, and apt aborts rather than displace an installed package. - let mut install: Vec = - words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); - let mut present: Vec = words(&["dpkg", "-s", "--"]); - for package in packages { - let (name, version) = package.split_once('=').unwrap_or((package, "")); - install.push(match version { - "" => format!("?exact-name({name})"), - version => format!("?exact-name({name})={version}"), - }); - present.push(name.to_owned()); - } - // An exact name that matches nothing installs nothing and exits 0, - // so the last step fails unless every named package is installed. - vec![ - Step { - argv: words(&["apt-get", "update", "-q"]), - environment: apt_environment.clone(), - }, - Step { - argv: install, - environment: apt_environment, - }, - Step { - argv: present, - environment: Vec::new(), - }, - ] - } - Operation::UnitAction { verb, unit } => vec![Step { - argv: vec![ - "systemctl".into(), - verb.as_str().into(), - "--job-mode=ignore-requirements".into(), - "--".into(), - unit.clone(), - ], - environment: Vec::new(), - }], - Operation::Deploy { .. } => Vec::new(), - } - } - - fn describe(&self) -> String { - match self { - Operation::PackageInstall { packages } => { - format!("package-install {}", packages.join(" ")) - } - Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), - Operation::Deploy { selector } => format!("deploy {selector}"), - } - } -} - -// ---------------------------------------------------------------- records -- - -#[derive(Clone, PartialEq, Eq, DatabaseEntry)] -#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] -pub(crate) struct OrderRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) order_id: String, - #[cultcache(key = 2)] - pub(crate) principal: String, - #[cultcache(key = 3)] - pub(crate) issuer_uid: u32, - #[cultcache(key = 4)] - pub(crate) issuer_unit: String, - #[cultcache(key = 5)] - pub(crate) issuer_invocation: String, - #[cultcache(key = 6)] - pub(crate) issued_at_unix_millis: u64, - #[cultcache(key = 7)] - pub(crate) operation: Operation, - #[cultcache(key = 8)] - pub(crate) reason: String, -} - -impl fmt::Debug for OrderRecord { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("OrderRecord") - .field("order_id", &self.order_id) - .finish_non_exhaustive() - } -} - -#[derive(Clone, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.steward_order_start", - schema = "idunn.steward_order_start.v1" -)] -pub(crate) struct StartRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) order_id: String, - #[cultcache(key = 2)] - pub(crate) started_at_unix_millis: u64, - /// The transient unit name, or the deployment command id. - #[cultcache(key = 3)] - pub(crate) actuation: String, -} - -impl fmt::Debug for StartRecord { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("StartRecord") - .field("order_id", &self.order_id) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Via { - Name, - Alias, - Host, - OnEnd, - Bound, - Upholds, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Refused { - AlwaysRefused { unit: String, via: Via }, - /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload - /// of the wrapper exits 0 and leaves the member alone. - Wrapper { member: String }, - Braked, - /// The order's principal or issuing uid is not the steward bound now. - NotSteward, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Outcome { - Ran, - Failed { - exit_code: Option, - signal: Option, - }, - Refused(Refused), - Lost, - Submitted { command_id: String }, -} - -impl Outcome { - fn describe(&self) -> String { - match self { - Outcome::Ran => "ran (exit 0)".into(), - Outcome::Failed { - exit_code: Some(code), - .. - } => format!("failed (exit {code})"), - Outcome::Failed { - signal: Some(signal), - .. - } => format!("failed (signal {signal})"), - Outcome::Failed { .. } => "failed (not run to an exit)".into(), - Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( - "refused: always-refused unit {} reached by {}", - escape(unit.as_bytes()), - match via { - Via::Name => "name", - Via::Alias => "alias", - Via::Host => "host", - Via::OnEnd => "on-end", - Via::Bound => "bound", - Via::Upholds => "upholds", - } - ), - Outcome::Refused(Refused::Wrapper { member }) => format!( - "refused: wrapper unit, it only wraps {}", - escape(member.as_bytes()) - ), - Outcome::Refused(Refused::Braked) => "refused: braked".into(), - Outcome::Refused(Refused::NotSteward) => { - "refused: the issuer is not the current steward".into() - } - Outcome::Lost => "lost: the orders service restarted mid-run".into(), - Outcome::Submitted { command_id } => { - format!("submitted deployment {}", escape(command_id.as_bytes())) - } - } - } -} - -#[derive(Clone, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.steward_order_outcome", - schema = "idunn.steward_order_outcome.v1" -)] -pub(crate) struct OutcomeRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) order_id: String, - #[cultcache(key = 2)] - pub(crate) ended_at_unix_millis: u64, - #[cultcache(key = 3)] - pub(crate) outcome: Outcome, - #[cultcache(key = 4)] - pub(crate) stdout_tail: Vec, - #[cultcache(key = 5)] - pub(crate) stderr_tail: Vec, -} - -/// Tails stay out of `Debug`: they are raw program output. -impl fmt::Debug for OutcomeRecord { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("OutcomeRecord") - .field("order_id", &self.order_id) - .field("outcome", &self.outcome) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum PostKind { - Accepted, - Result, -} - -impl PostKind { - fn as_str(self) -> &'static str { - match self { - PostKind::Accepted => "accepted", - PostKind::Result => "result", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.steward_order_post", - schema = "idunn.steward_order_post.v1" -)] -pub(crate) struct PostRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - /// `:`. - #[cultcache(key = 1)] - pub(crate) post_key: String, - #[cultcache(key = 2)] - pub(crate) event_id: String, - #[cultcache(key = 3)] - pub(crate) posted_at_unix_millis: u64, -} - -// ------------------------------------------------------------------ store -- - -/// `orders.cc`: root-only, written by `idunn orders serve` alone. -pub(crate) struct OrderStore { - directory: PathBuf, -} - -#[derive(Default)] -struct Ledger { - orders: Vec, - starts: Vec, - outcomes: Vec, - posts: Vec, -} - -impl Ledger { - fn start_of(&self, order_id: &str) -> Option<&StartRecord> { - self.starts.iter().find(|start| start.order_id == order_id) - } - - fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { - self.outcomes - .iter() - .find(|outcome| outcome.order_id == order_id) - } - - fn posted(&self, order_id: &str, kind: PostKind) -> bool { - let key = post_key(order_id, kind); - self.posts.iter().any(|post| post.post_key == key) - } -} - -fn post_key(order_id: &str, kind: PostKind) -> String { - format!("{order_id}:{}", kind.as_str()) -} - -impl OrderStore { - pub(crate) fn new(directory: &Path) -> Self { - Self { - directory: directory.to_owned(), - } - } - - fn path(&self) -> PathBuf { - self.directory.join("orders.cc") - } - - fn prepare(&self) -> Result<()> { - std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; - std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) - .context("making the orders directory private") - } - - /// Insert a record whose (type, key) is absent. `false` means it was there. - fn put(&self, key: &str, schema: &str, value: &T) -> Result { - self.prepare()?; - let path = self.path(); - let now = now_millis()?; - let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: T::TYPE.into(), - key: key.into(), - current: None, - }], - &[typed_envelope(key, T::TYPE, schema, value, now)?], - )?; - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) - .context("making the orders store private")?; - Ok(inserted) - } - - fn ledger(&self) -> Result { - let path = self.path(); - let mut ledger = Ledger::default(); - if !path.exists() { - return Ok(ledger); - } - let envelopes = SingleFileMessagePackBackingStore::new(&path) - .pull_all_read_only_snapshot() - .context("reading the orders store")?; - for envelope in &envelopes { - let schema = envelope.schema_id.as_deref(); - let key = envelope.key.as_str(); - match envelope.r#type.as_str() { - OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { - let record: OrderRecord = decode_record(envelope)?; - ensure!(record.order_id == key, "an order is stored under another key"); - ledger.orders.push(record); - } - StartRecord::TYPE if schema == Some(START_SCHEMA) => { - let record: StartRecord = decode_record(envelope)?; - ensure!(record.order_id == key, "a start is stored under another key"); - ledger.starts.push(record); - } - OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { - let record: OutcomeRecord = decode_record(envelope)?; - ensure!(record.order_id == key, "an outcome is stored under another key"); - ledger.outcomes.push(record); - } - PostRecord::TYPE if schema == Some(POST_SCHEMA) => { - let record: PostRecord = decode_record(envelope)?; - ensure!(record.post_key == key, "a post is stored under another key"); - ledger.posts.push(record); - } - _ => bail!("the orders store holds a document that is not an order record"), - } - } - ledger - .orders - .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); - Ok(ledger) - } -} - -// ------------------------------------------------------------------ ports -- - -/// The properties naming what systemd does when a unit ends or its job times -/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). -const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; - -/// The properties naming units systemd starts when a unit ends. -const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; - -/// What `systemctl show` says about one unit. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct UnitFacts { - pub(crate) id: String, - pub(crate) names: Vec, - pub(crate) not_found: bool, - /// The properties that make the unit act when it ends: each emergency - /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. - pub(crate) on_end: Vec, - /// The units bound to this one at runtime (`BoundBy=`): systemd stops - /// them when this unit stops, whatever the job mode. - pub(crate) bound_by: Vec, - /// The units this one upholds (`Upholds=`): systemd starts them in a full - /// transaction whenever this unit is active, whatever the job mode. - pub(crate) upholds: Vec, - /// The units this one groups (`ConsistsOf=`): the verbs reach them only - /// through propagation, which the job mode removes. - pub(crate) consists_of: Vec, -} - -impl UnitFacts { - fn identities(&self) -> impl Iterator { - std::iter::once(&self.id).chain(self.names.iter()) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct RunReport { - pub(crate) exit_code: Option, - pub(crate) signal: Option, - pub(crate) stdout_tail: Vec, - pub(crate) stderr_tail: Vec, -} - -pub(crate) trait Systemd: Send + Sync { - fn show(&self, unit: &str) -> Result; - fn invocation(&self, unit: &str) -> Option; - /// Run one step as the transient unit `unit`, to its end. - fn run(&self, unit: &str, step: &Step) -> Result; -} - -pub(crate) struct Peer { - pub(crate) uid: u32, - pub(crate) issuer_unit: String, -} - -pub(crate) trait PeerCredentials: Send + Sync { - fn peer(&self, stream: &UnixStream) -> Result; -} - -pub(crate) trait BrakeCheck: Send + Sync { - /// `true` only when the instance's brake check exits 0. - fn clear(&self, principal: &str) -> bool; -} - -pub(crate) trait Homeserver: Send + Sync { - /// Idempotent by `txn_id`; returns the event id. - fn post(&self, txn_id: &str, text: &str) -> Result; -} - -pub(crate) struct Ctx { - pub(crate) store: OrderStore, - pub(crate) state_store: PathBuf, - pub(crate) steward_uid: u32, - pub(crate) principal: String, - pub(crate) systemd: Arc, - pub(crate) peers: Arc, - pub(crate) brake: Arc, - pub(crate) homeserver: Arc, -} - -// ------------------------------------------------------------------ scope -- - -fn refused(unit: &str, via: Via) -> Refused { - Refused::AlwaysRefused { - unit: unit.to_owned(), - via, - } -} - -/// Would `verb` on `unit` be refused. systemd builds the transaction and the -/// order tells it to build one job (`--job-mode=ignore-requirements`), so this -/// reads only what that job mode cannot remove from the one named unit: the -/// name it answers to, what fires when it ends, who is bound to it, what it upholds and -/// what it only wraps. -fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { - if is_refused_name(unit) { - return Ok(Some(refused(unit, Via::Name))); - } - let facts = systemd.show(unit)?; - ensure!(!facts.not_found, "systemd has no such unit to check"); - for name in facts.identities() { - if is_refused_name(name) { - return Ok(Some(refused(name, Via::Alias))); - } - if is_host_name(name) { - return Ok(Some(refused(name, Via::Host))); - } - } - if verb != Verb::Reload && !facts.on_end.is_empty() { - return Ok(Some(refused(unit, Via::OnEnd))); - } - if matches!(verb, Verb::Stop | Verb::Restart) - && let Some(bound) = facts.bound_by.first() - { - return Ok(Some(refused(bound, Via::Bound))); - } - if matches!(verb, Verb::Start | Verb::Restart) && !facts.upholds.is_empty() { - return Ok(Some(refused(unit, Via::Upholds))); - } - if verb != Verb::Start - && let Some(member) = facts.consists_of.first() - { - return Ok(Some(Refused::Wrapper { - member: member.clone(), - })); - } - Ok(None) -} - -// -------------------------------------------------------------- execution -- - -fn tail(bytes: &[u8]) -> Vec { - bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() -} - -fn conclude( - ctx: &Ctx, - order_id: &str, - outcome: Outcome, - stdout_tail: &[u8], - stderr_tail: &[u8], -) -> Result<()> { - ctx.store.put( - order_id, - OUTCOME_SCHEMA, - &OutcomeRecord { - schema_version: OUTCOME_SCHEMA.into(), - order_id: order_id.into(), - ended_at_unix_millis: now_millis()?, - outcome, - stdout_tail: tail(stdout_tail), - stderr_tail: tail(stderr_tail), - }, - )?; - Ok(()) -} - -fn order_uuid(order_id: &str) -> &str { - order_id.strip_prefix("ord-").unwrap_or(order_id) -} - -pub(crate) fn requester(principal: &str, order_id: &str) -> String { - format!("steward:{principal} order {order_id}") -} - -/// Run one accepted order. A start record is written before anything is -/// actuated and an order that has one is never run again. -pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { - let ledger = ctx.store.ledger()?; - let order = ledger - .orders - .iter() - .find(|order| order.order_id == order_id) - .context("the order is not in the store")?; - if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { - return Ok(()); - } - // One gate for a fresh order and a recovered one: who may order is what the - // binding says now, not what it said when the order was accepted. - if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { - return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); - } - if order.operation.validate().is_err() { - return conclude( - ctx, - order_id, - Outcome::Failed { - exit_code: None, - signal: None, - }, - b"", - b"the stored order no longer satisfies the grammar", - ); - } - if let Operation::UnitAction { verb, unit } = &order.operation { - match refusal_for(ctx.systemd.as_ref(), unit, *verb) { - Ok(None) => {} - Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), - Err(_) => { - return conclude( - ctx, - order_id, - Outcome::Failed { - exit_code: None, - signal: None, - }, - b"", - b"the unit could not be resolved, so it was not acted on", - ); - } - } - } - if !ctx.brake.clear(&ctx.principal) { - return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); - } - let uuid = order_uuid(order_id); - let command_id = format!("up-{uuid}"); - let steps = order.operation.steps(); - let actuation = match &order.operation { - Operation::Deploy { .. } => command_id.clone(), - _ => format!("idunn-order-{uuid}"), - }; - let started = StartRecord { - schema_version: START_SCHEMA.into(), - order_id: order_id.into(), - started_at_unix_millis: now_millis()?, - actuation, - }; - if !ctx.store.put(order_id, START_SCHEMA, &started)? { - return Ok(()); - } - if let Operation::Deploy { selector } = &order.operation { - let outcome = match commit_deployment_command( - &ctx.state_store, - &command_id, - selector, - &requester(&order.principal, order_id), - ) { - Ok(_) => Outcome::Submitted { command_id }, - Err(_) => Outcome::Failed { - exit_code: None, - signal: None, - }, - }; - return conclude(ctx, order_id, outcome, b"", b""); - } - let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); - let mut outcome = Outcome::Ran; - for (index, step) in steps.iter().enumerate() { - let unit = if steps.len() == 1 { - format!("idunn-order-{uuid}") - } else { - format!("idunn-order-{uuid}-{}", index + 1) - }; - match ctx.systemd.run(&unit, step) { - Ok(report) => { - stdout = report.stdout_tail; - stderr = report.stderr_tail; - if report.exit_code != Some(0) { - outcome = Outcome::Failed { - exit_code: report.exit_code, - signal: report.signal, - }; - break; - } - } - Err(_) => { - outcome = Outcome::Failed { - exit_code: None, - signal: None, - }; - stderr = b"the transient unit could not be started".to_vec(); - break; - } - } - } - conclude(ctx, order_id, outcome, &stdout, &stderr) -} - -/// On service start: an order with a start record and no outcome ran, or began -/// to, in a life that ended. It is Lost and never run again. Returns the -/// orders that never started, oldest first. -pub(crate) fn recover(ctx: &Ctx) -> Result> { - let ledger = ctx.store.ledger()?; - for start in &ledger.starts { - if ledger.outcome_of(&start.order_id).is_none() { - conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; - } - } - Ok(ledger - .orders - .iter() - .filter(|order| { - ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() - }) - .map(|order| order.order_id.clone()) - .collect()) -} - -// ---------------------------------------------------------------- posting -- - -struct Due { - order_id: String, - kind: PostKind, - text: String, -} - -fn quoted(reason: &str) -> String { - format!("\"{}\"", escape(reason.as_bytes())) -} - -fn accepted_text(order: &OrderRecord) -> String { - format!( - "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", - order.order_id, - escape(order.operation.describe().as_bytes()), - escape(order.issuer_unit.as_bytes()), - escape(order.issuer_invocation.as_bytes()), - quoted(&order.reason) - ) -} - -/// A deploy's command has finished once its state is neither queued nor running. -fn deploy_finished(state: &str) -> bool { - !matches!(state, "queued" | "running") -} - -/// What is owed to the room. `deploy_state` is a deployment command's derived -/// state and detail, read from the control store by the caller. -fn due_posts( - ledger: &Ledger, - deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, -) -> Vec { - let mut due = Vec::new(); - for order in &ledger.orders { - let id = &order.order_id; - if !ledger.posted(id, PostKind::Accepted) { - due.push(Due { - order_id: id.clone(), - kind: PostKind::Accepted, - text: accepted_text(order), - }); - } - if ledger.posted(id, PostKind::Result) { - continue; - } - let Some(record) = ledger.outcome_of(id) else { - continue; - }; - let mut line = record.outcome.describe(); - if let Outcome::Submitted { command_id } = &record.outcome { - match deploy_state(command_id) { - Ok((state, detail)) if deploy_finished(state) => { - line = format!("{line}: {state} {}", escape(detail.as_bytes())); - } - _ => continue, - } - } - due.push(Due { - order_id: id.clone(), - kind: PostKind::Result, - text: format!( - "Steward order {id}: {line}\n{}", - escape(order.operation.describe().as_bytes()) - ), - }); - } - due -} - -/// Post what is due, in order, stopping at the first failure. Returns how many -/// went out. -pub(crate) fn post_pass(ctx: &Ctx) -> Result { - let ledger = ctx.store.ledger()?; - let mut sent = 0; - let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); - for due in due_posts(&ledger, &deploy_state) { - let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); - let event_id = ctx.homeserver.post(&transaction, &due.text)?; - ctx.store.put( - &post_key(&due.order_id, due.kind), - POST_SCHEMA, - &PostRecord { - schema_version: POST_SCHEMA.into(), - post_key: post_key(&due.order_id, due.kind), - event_id, - posted_at_unix_millis: now_millis()?, - }, - )?; - sent += 1; - } - Ok(sent) -} - -// ----------------------------------------------------------------- frames -- - -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct OrderRequest { - pub(crate) operation: Operation, - pub(crate) reason: String, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ShowRequest { - pub(crate) id: String, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Request { - Order(OrderRequest), - Show(ShowRequest), - List, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct OrderView { - pub(crate) order: OrderRecord, - pub(crate) start: Option, - pub(crate) outcome: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct OrderLine { - pub(crate) order_id: String, - pub(crate) status: String, - pub(crate) summary: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Response { - Accepted { order_id: String }, - Refused(Refusal), - Show(Box), - List(Vec), -} - -fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { - let bytes = rmp_serde::to_vec_named(value)?; - stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; - stream.write_all(&bytes)?; - Ok(stream.flush()?) -} - -fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { - let mut length = [0u8; 4]; - stream.read_exact(&mut length)?; - let length = u32::from_be_bytes(length) as usize; - ensure!(length <= limit, "a frame exceeds its limit"); - let mut bytes = vec![0u8; length]; - stream.read_exact(&mut bytes)?; - Ok(bytes) -} - -fn status_word(view: &OrderView) -> &'static str { - match (&view.outcome, &view.start) { - (Some(record), _) => match record.outcome { - Outcome::Ran => "ran", - Outcome::Failed { .. } => "failed", - Outcome::Refused(_) => "refused", - Outcome::Lost => "lost", - Outcome::Submitted { .. } => "submitted", - }, - (None, Some(_)) => "running", - (None, None) => "accepted", - } -} - -fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { - OrderView { - order: order.clone(), - start: ledger.start_of(&order.order_id).cloned(), - outcome: ledger.outcome_of(&order.order_id).cloned(), - } -} - -fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { - write_frame(stream, &response) -} - -fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { - answer(stream, Response::Refused(Refusal::new(field, code))) -} - -/// Serve one connection. The peer is authenticated before its frame is read. -pub(crate) fn handle( - ctx: &Ctx, - stream: &mut UnixStream, - queue: &mpsc::Sender, -) -> Result<()> { - stream.set_read_timeout(Some(Duration::from_secs(5)))?; - stream.set_write_timeout(Some(Duration::from_secs(5)))?; - let peer = ctx.peers.peer(stream)?; - // The steward may order and read; root, the operator, may only read. - let steward = peer.uid == ctx.steward_uid; - if !steward && peer.uid != 0 { - eprintln!("steward socket: refused a peer that is not the steward"); - return refuse(stream, "peer", Code::Unauthorized); - } - let request = read_frame(stream, REQUEST_FRAME_LIMIT) - .ok() - .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); - let Some(request) = request else { - eprintln!("steward socket: refused a malformed frame"); - return refuse(stream, "frame", Code::Malformed); - }; - match request { - Request::Order(_) if !steward => { - eprintln!("steward socket: refused an order from a peer that is not the steward"); - refuse(stream, "peer", Code::Unauthorized) - } - Request::Order(request) => { - let checked = - validate_reason(&request.reason).and_then(|()| request.operation.validate()); - if let Err(refusal) = checked { - eprintln!("steward socket: refused an order at the grammar"); - return answer(stream, Response::Refused(refusal)); - } - let order = OrderRecord { - schema_version: ORDER_SCHEMA.into(), - order_id: format!("ord-{}", Uuid::new_v4()), - principal: ctx.principal.clone(), - issuer_uid: peer.uid, - issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), - issuer_unit: peer.issuer_unit, - issued_at_unix_millis: now_millis()?, - operation: request.operation, - reason: request.reason, - }; - match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { - Ok(true) => {} - _ => return refuse(stream, "store", Code::Unavailable), - } - queue - .send(order.order_id.clone()) - .map_err(|_| anyhow!("the executor has stopped"))?; - answer( - stream, - Response::Accepted { - order_id: order.order_id, - }, - ) - } - Request::Show(request) => { - if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) - { - return refuse(stream, "id", Code::BadShape); - } - let ledger = ctx.store.ledger()?; - match ledger.orders.iter().find(|order| order.order_id == request.id) { - Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), - None => refuse(stream, "id", Code::Unknown), - } - } - Request::List => { - let ledger = ctx.store.ledger()?; - let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); - let lines = ledger - .orders - .iter() - .skip(skip) - .map(|order| { - let view = view_of(&ledger, order); - OrderLine { - order_id: order.order_id.clone(), - status: status_word(&view).into(), - summary: order.operation.describe(), - } - }) - .collect(); - answer(stream, Response::List(lines)) - } - } -} - -// ------------------------------------------------------------- real ports -- - -/// `systemctl` and `systemd-run`, found by name. -pub(crate) struct HostSystemd; - -/// The names in one `systemctl show` list value. systemd quotes a name that -/// holds a backslash and doubles the backslash inside the quotes. -fn parse_names(value: &str) -> Vec { - let mut names = Vec::new(); - let mut current = String::new(); - let (mut quoted, mut open, mut escaped) = (false, false, false); - for character in value.chars() { - match character { - _ if escaped => { - current.push(character); - escaped = false; - } - '\\' if quoted => escaped = true, - '"' => { - quoted = !quoted; - open = true; - } - _ if character.is_whitespace() && !quoted => { - if open { - names.push(std::mem::take(&mut current)); - open = false; - } - } - _ => { - current.push(character); - open = true; - } - } - } - if open { - names.push(current); - } - names -} - -/// The properties one `systemctl show` is asked for: identity, what fires at -/// the unit's end, who is bound to it, and what it upholds or groups. -fn show_properties() -> String { - ["Id", "Names", "LoadState"] - .into_iter() - .chain(ACTIONS) - .chain(ON_END) - .chain(["BoundBy", "Upholds", "ConsistsOf"]) - .collect::>() - .join(",") -} - -fn parse_show(output: &str) -> Result { - let mut facts = UnitFacts::default(); - for line in output.lines() { - let Some((key, value)) = line.split_once('=') else { - continue; - }; - match key { - "Id" => facts.id = value.to_owned(), - "Names" => facts.names = parse_names(value), - "LoadState" => facts.not_found = value == "not-found", - "BoundBy" => facts.bound_by = parse_names(value), - "Upholds" => facts.upholds = parse_names(value), - "ConsistsOf" => facts.consists_of = parse_names(value), - _ if ACTIONS.contains(&key) => { - if !matches!(value, "" | "none") { - facts.on_end.push(key.to_owned()); - } - } - _ if ON_END.contains(&key) => { - facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); - } - _ => {} - } - } - ensure!(!facts.id.is_empty(), "systemctl named no unit"); - Ok(facts) -} - -impl Systemd for HostSystemd { - fn show(&self, unit: &str) -> Result { - let output = Command::new("systemctl") - .args(["show", "--no-pager"]) - .arg(format!("--property={}", show_properties())) - .args(["--", unit]) - .stdin(Stdio::null()) - .output() - .context("running systemctl show")?; - ensure!(output.status.success(), "systemctl show failed"); - parse_show(&String::from_utf8_lossy(&output.stdout)) - } - - fn invocation(&self, unit: &str) -> Option { - let output = Command::new("systemctl") - .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) - .arg(unit) - .stdin(Stdio::null()) - .output() - .ok()?; - let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - (output.status.success() && !value.is_empty()).then_some(value) - } - - fn run(&self, unit: &str, step: &Step) -> Result { - let mut command = Command::new("systemd-run"); - command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); - command.args(["-p", "RuntimeMaxSec=15min"]); - for (name, value) in &step.environment { - command.arg(format!("--setenv={name}={value}")); - } - let output = command - .arg("--") - .args(&step.argv) - .stdin(Stdio::null()) - .output() - .context("running systemd-run")?; - Ok(RunReport { - exit_code: output.status.code(), - signal: output.status.signal(), - stdout_tail: tail(&output.stdout), - stderr_tail: tail(&output.stderr), - }) - } -} - -/// A system service or scope names the issuer: the first unit under -/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices -/// above it are the manager's, and what a delegated subtree adds below it is -/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. -fn unit_from_cgroup(text: &str) -> String { - let line = text - .lines() - .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) - .unwrap_or_default(); - let path = line.rsplit_once(':').map_or("", |(_, path)| path); - let unit = path.strip_prefix("/system.slice/").and_then(|under| { - for segment in under.split('/') { - if segment.ends_with(".slice") { - continue; - } - let unit = segment.ends_with(".service") || segment.ends_with(".scope"); - return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); - } - None - }); - unit.map_or_else(|| "unknown".to_owned(), str::to_owned) -} - -/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. -pub(crate) struct KernelPeers; - -impl PeerCredentials for KernelPeers { - fn peer(&self, stream: &UnixStream) -> Result { - use std::os::fd::AsRawFd; - let mut credentials = libc::ucred { - pid: 0, - uid: 0, - gid: 0, - }; - let mut length = std::mem::size_of::() as libc::socklen_t; - // SAFETY: the descriptor is a live socket and the pointer and length - // describe `credentials`, which outlives the call. - let status = unsafe { - libc::getsockopt( - stream.as_raw_fd(), - libc::SOL_SOCKET, - libc::SO_PEERCRED, - (&mut credentials as *mut libc::ucred).cast(), - &mut length, - ) - }; - ensure!( - status == 0 && length as usize == std::mem::size_of::(), - "peer credentials are unavailable" - ); - let cgroup = - std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); - Ok(Peer { - uid: credentials.uid, - issuer_unit: unit_from_cgroup(&cgroup), - }) - } -} - -/// `mind-control --instance check`; a missing binary is a held brake. -pub(crate) struct MindControl { - pub(crate) program: PathBuf, -} - -impl BrakeCheck for MindControl { - fn clear(&self, principal: &str) -> bool { - Command::new(&self.program) - .args(["--instance", principal, "check"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .is_ok_and(|status| status.success()) - } -} - -/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is -/// never chunked. The token is read from its credential file for each post. -pub(crate) struct Synapse { - pub(crate) address: std::net::SocketAddr, - pub(crate) room: String, - pub(crate) token_file: PathBuf, - /// The whole wait for a reply, however slowly it drips. - pub(crate) reply_within: Duration, -} - -fn percent_encode(text: &str) -> String { - let mut encoded = String::new(); - for byte in text.bytes() { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { - encoded.push(char::from(byte)); - } else { - encoded.push_str(&format!("%{byte:02X}")); - } - } - encoded -} - -impl Homeserver for Synapse { - fn post(&self, txn_id: &str, text: &str) -> Result { - let token = std::fs::read_to_string(&self.token_file) - .context("the homeserver token is unreadable")?; - let token = token.trim(); - ensure!( - !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), - "the homeserver token is not a bearer token" - ); - let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; - let head = format!( - "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ - Host: {}\r\nAuthorization: Bearer {token}\r\n\ - Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", - percent_encode(&self.room), - self.address, - body.len() - ); - let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) - .context("the homeserver is unreachable")?; - socket.set_write_timeout(Some(self.reply_within))?; - socket.write_all(head.as_bytes())?; - socket.write_all(&body)?; - let deadline = std::time::Instant::now() + self.reply_within; - let mut reply = Vec::new(); - let mut chunk = [0u8; 4096]; - while reply.len() < RESPONSE_FRAME_LIMIT { - let remaining = deadline - .checked_duration_since(std::time::Instant::now()) - .filter(|left| !left.is_zero()) - .context("the homeserver reply was too slow")?; - socket.set_read_timeout(Some(remaining))?; - let count = socket - .read(&mut chunk) - .context("the homeserver reply was cut short")?; - if count == 0 { - break; - } - reply.extend_from_slice(&chunk[..count]); - } - let split = reply - .windows(4) - .position(|window| window == b"\r\n\r\n") - .context("the homeserver reply has no body")?; - let status = String::from_utf8_lossy(&reply[..split]); - ensure!( - status.split_whitespace().nth(1) == Some("200"), - "the homeserver refused the post" - ); - #[derive(Deserialize)] - struct Sent { - event_id: String, - } - let sent: Sent = serde_json::from_slice(&reply[split + 4..]) - .map_err(|_| anyhow!("the homeserver reply has no event id"))?; - Ok(sent.event_id) - } -} - -// ---------------------------------------------------------------- binding -- - -/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. -/// It lists no unit of any kind, and any field outside this set is refused. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct StewardBinding { - pub(crate) principal: String, - pub(crate) user: String, - pub(crate) mind_control: PathBuf, - pub(crate) room: String, - pub(crate) homeserver: String, - pub(crate) token_credential: String, -} - -impl StewardBinding { - pub(crate) fn parse(text: &str) -> Result { - let binding: Self = toml::from_str(text) - .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; - binding.validate()?; - Ok(binding) - } - - fn validate(&self) -> Result<()> { - require_id(&self.principal, "steward principal")?; - ensure!( - !self.principal.starts_with('-') && !self.principal.contains('/'), - "steward principal is invalid" - ); - let user_ok = !self.user.is_empty() - && self.user.len() <= 32 - && self.user.bytes().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') - }) - && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); - ensure!(user_ok && self.user != "root", "steward user is invalid or root"); - ensure!( - self.mind_control.is_absolute(), - "mind_control must be an absolute path" - ); - let room_ok = self.room.starts_with('!') - && self.room.len() <= 255 - && self.room.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') - }); - ensure!(room_ok, "room must be a Matrix room id"); - self.address()?; - let credential_ok = !self.token_credential.is_empty() - && !self.token_credential.starts_with('.') - && self.token_credential.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') - }); - ensure!(credential_ok, "token_credential is not a credential name"); - Ok(()) - } - - fn address(&self) -> Result { - let address: std::net::SocketAddr = self - .homeserver - .strip_prefix("http://") - .and_then(|rest| rest.parse().ok()) - .context("homeserver must be http://:")?; - ensure!(address.ip().is_loopback(), "homeserver must be loopback"); - Ok(address) - } -} - -/// Root-admitted: owned by root and writable by no one else. -fn admitted(owner: u32, mode: u32) -> bool { - owner == 0 && mode & 0o022 == 0 -} - -fn load_binding(path: &Path) -> Result { - let metadata = std::fs::metadata(path).context("reading the steward binding")?; - ensure!( - metadata.is_file() && admitted(metadata.uid(), metadata.mode()), - "the steward binding is not a root-owned file closed to writes" - ); - let text = std::fs::read_to_string(path).context("reading the steward binding")?; - StewardBinding::parse(&text) -} - -fn user_uid(name: &str) -> Result { - let name = std::ffi::CString::new(name)?; - // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in - // `buffer`, which outlives every read of them below (none are read). - let (status, found, uid) = unsafe { - let mut passwd: libc::passwd = std::mem::zeroed(); - let mut buffer = vec![0 as libc::c_char; 4096]; - let mut result: *mut libc::passwd = std::ptr::null_mut(); - let status = libc::getpwnam_r( - name.as_ptr(), - &mut passwd, - buffer.as_mut_ptr(), - buffer.len(), - &mut result, - ); - (status, !result.is_null(), passwd.pw_uid) - }; - ensure!(status == 0 && found, "the steward user is unknown"); - ensure!(uid != 0, "the steward user resolves to root"); - Ok(uid) -} - -// ------------------------------------------------------------------ serve -- - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct ServeOptions { - pub(crate) state_store: PathBuf, - pub(crate) store: PathBuf, - pub(crate) steward_binding: PathBuf, - pub(crate) socket: PathBuf, -} - -const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; -const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; -const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; - -pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { - ensure!( - args.next().as_deref() == Some("serve"), - "idunn orders takes the subcommand serve" - ); - let mut options = ServeOptions { - state_store: default_state_store(), - store: DEFAULT_STORE.into(), - steward_binding: DEFAULT_BINDING.into(), - socket: DEFAULT_SOCKET.into(), - }; - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, - "--store" => options.store = path_value(&mut args, "--store")?, - "--steward-binding" => { - options.steward_binding = path_value(&mut args, "--steward-binding")? - } - "--socket" => options.socket = path_value(&mut args, "--socket")?, - _ => bail!("an orders serve option is not part of the grammar"), - } - } - Ok(options) -} - -/// Seconds between post passes: two when the last pass went through, and -/// doubling for each failed pass in a row up to a minute. -fn post_pause(failures: u32) -> u64 { - if failures == 0 { - 2 - } else { - (2u64 << failures.min(5)).min(60) - } -} - -/// Run a thread whose end ends the service: systemd restarts it, and the -/// start recovery settles what it left. -fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { - struct ExitOnDrop(&'static str); - impl Drop for ExitOnDrop { - fn drop(&mut self) { - eprintln!("steward orders: the {} thread ended", self.0); - std::process::exit(1); - } - } - thread::Builder::new() - .name(name.into()) - .spawn(move || { - let _guard = ExitOnDrop(name); - work(); - }) - .expect("a thread can be spawned"); -} - -/// The socket belongs to the steward and no one else may open it; the peer -/// credential check on every connection stays the lock. -fn bind_socket(path: &Path, steward_uid: u32) -> Result { - if let Ok(metadata) = std::fs::symlink_metadata(path) { - use std::os::unix::fs::FileTypeExt; - ensure!( - metadata.file_type().is_socket(), - "the socket path holds something that is not a socket" - ); - std::fs::remove_file(path).context("removing a stale socket")?; - } - let listener = UnixListener::bind(path).context("binding the steward socket")?; - std::os::unix::fs::chown(path, Some(steward_uid), None) - .context("giving the steward socket to the steward")?; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) - .context("setting the steward socket mode")?; - Ok(listener) -} - -/// Files this service creates in Idunn's own stores (a deploy order writes -/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. -const IDUNN_UMASK: libc::mode_t = 0o027; - -pub(crate) fn serve(options: ServeOptions) -> Result<()> { - // SAFETY: umask only sets the process file-mode mask; it cannot fail. - unsafe { libc::umask(IDUNN_UMASK) }; - let binding = load_binding(&options.steward_binding)?; - let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") - .map(PathBuf::from) - .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; - let ctx = Arc::new(Ctx { - store: OrderStore::new(&options.store), - state_store: options.state_store, - steward_uid: user_uid(&binding.user)?, - principal: binding.principal.clone(), - systemd: Arc::new(HostSystemd), - peers: Arc::new(KernelPeers), - brake: Arc::new(MindControl { - program: binding.mind_control.clone(), - }), - homeserver: Arc::new(Synapse { - address: binding.address()?, - room: binding.room.clone(), - token_file: credentials.join(&binding.token_credential), - reply_within: Duration::from_secs(10), - }), - }); - ctx.store.prepare()?; - let pending = recover(&ctx)?; - let listener = bind_socket(&options.socket, ctx.steward_uid)?; - let (queue, orders) = mpsc::channel::(); - for order_id in pending { - queue.send(order_id)?; - } - let executor = Arc::clone(&ctx); - vital("executor", move || { - for order_id in orders { - if let Err(error) = execute(&executor, &order_id) { - eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); - } - } - }); - let poster = Arc::clone(&ctx); - vital("poster", move || { - let mut failures = 0u32; - loop { - match post_pass(&poster) { - Ok(_) => failures = 0, - Err(error) => { - failures = failures.saturating_add(1); - eprintln!("steward orders: posting is delayed: {error:#}"); - } - } - thread::sleep(Duration::from_secs(post_pause(failures))); - } - }); - for stream in listener.incoming() { - match stream { - Ok(mut stream) => { - if let Err(error) = handle(&ctx, &mut stream, &queue) { - eprintln!("steward socket: a connection ended in error: {error:#}"); - } - } - Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), - } - } - bail!("the steward socket stopped accepting") -} - -// ----------------------------------------------------------------- client -- - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum OrderAction { - Issue { operation: Operation, reason: String }, - Show { id: String }, - List, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct OrderCommand { - pub(crate) socket: PathBuf, - pub(crate) action: OrderAction, -} - -pub(crate) fn parse_order(mut args: impl Iterator) -> Result { - let kind = args.next().context("idunn order requires an operation")?; - let mut socket = PathBuf::from(DEFAULT_SOCKET); - let mut reason = None; - let mut positional = Vec::new(); - while let Some(argument) = args.next() { - match argument.as_str() { - "--socket" => socket = path_value(&mut args, "--socket")?, - "--reason" => reason = Some(string_value(&mut args, "--reason")?), - word if word.starts_with('-') => bail!("an order option is not part of the grammar"), - _ => positional.push(argument), - } - } - let action = match kind.as_str() { - "list" => { - ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); - OrderAction::List - } - "show" => { - ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); - OrderAction::Show { - id: positional.remove(0), - } - } - "package-install" | "unit" | "deploy" => { - let operation = match kind.as_str() { - "package-install" => Operation::PackageInstall { - packages: positional, - }, - "unit" => { - ensure!(positional.len() == 2, "unit takes a verb and a unit"); - let verb = Verb::parse(&positional[0]) - .context("the unit verb is not start, stop, restart or reload")?; - Operation::UnitAction { - verb, - unit: positional.remove(1), - } - } - _ => { - ensure!(positional.len() == 1, "deploy takes one selector"); - Operation::Deploy { - selector: positional.remove(0), - } - } - }; - let reason = reason.context("an order requires --reason")?; - validate_reason(&reason)?; - operation.validate()?; - OrderAction::Issue { operation, reason } - } - _ => bail!("the order operation is not part of the grammar"), - }; - Ok(OrderCommand { socket, action }) -} - -fn call(socket: &Path, request: &Request) -> Result { - let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; - stream.set_read_timeout(Some(Duration::from_secs(30)))?; - write_frame(&mut stream, request)?; - let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; - rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) -} - -fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { - if bytes.is_empty() { - return; - } - lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); - let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); - lines.extend(text.split(|byte| *byte == b'\n').map(|line| { - let mut shown = b" | ".to_vec(); - shown.extend_from_slice(line); - shown - })); -} - -/// Raw lines for the terminal; `emit` is their only way out. -fn render_view(view: &OrderView) -> Vec> { - let order = &view.order; - let mut lines: Vec> = vec![ - format!("order {}", order.order_id).into_bytes(), - format!("status {}", status_word(view)).into_bytes(), - format!("operation {}", order.operation.describe()).into_bytes(), - format!( - "issuer {} (invocation {}, uid {})", - order.issuer_unit, order.issuer_invocation, order.issuer_uid - ) - .into_bytes(), - format!("reason {}", quoted(&order.reason)).into_bytes(), - ]; - if let Some(start) = &view.start { - lines.push(format!("started {}", start.actuation).into_bytes()); - } - if let Some(record) = &view.outcome { - lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); - render_tail(&mut lines, "stdout", &record.stdout_tail); - render_tail(&mut lines, "stderr", &record.stderr_tail); - } - lines -} - -pub(crate) fn run_order(command: OrderCommand) -> Result<()> { - let request = match command.action { - OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), - OrderAction::Show { id } => Request::Show(ShowRequest { id }), - OrderAction::List => Request::List, - }; - let mut out = std::io::stdout().lock(); - match call(&command.socket, &request)? { - Response::Accepted { order_id } => writeln!(out, "{order_id}")?, - Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), - Response::Show(view) => emit(&mut out, &render_view(&view))?, - Response::List(lines) => { - let lines: Vec> = lines - .iter() - .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) - .collect(); - emit(&mut out, &lines)? - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - use std::sync::Mutex; - use std::sync::atomic::{AtomicBool, Ordering}; - - use tempfile::TempDir; - - use super::*; - - const STEWARD: u32 = 1000; - const CANARY: &str = "CANARY-7f3a91"; - - /// What `systemctl show` prints for a loaded unit that acts on nothing, - /// plus the given properties (a given key replaces its default). - fn show_text(id: &str, properties: &[(&str, &str)]) -> String { - let mut text = format!("Id={id}\n"); - for (key, default) in [("Names", id), ("LoadState", "loaded")] { - if !properties.iter().any(|(given, _)| *given == key) { - text += &format!("{key}={default}\n"); - } - } - for (key, value) in properties { - text += &format!("{key}={value}\n"); - } - text - } - - struct FakeSystemd { - units: Mutex>, - runs: Mutex>, - shows: Mutex>, - report: Mutex, - } - - impl Systemd for FakeSystemd { - fn show(&self, unit: &str) -> Result { - self.shows.lock().unwrap().push(unit.into()); - let known = self.units.lock().unwrap().get(unit).cloned(); - parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) - } - - fn invocation(&self, _unit: &str) -> Option { - Some("invocation-1".into()) - } - - fn run(&self, unit: &str, step: &Step) -> Result { - self.runs.lock().unwrap().push((unit.into(), step.clone())); - Ok(self.report.lock().unwrap().clone()) - } - } - - struct FakePeers(u32); - - impl PeerCredentials for FakePeers { - fn peer(&self, _stream: &UnixStream) -> Result { - Ok(Peer { - uid: self.0, - issuer_unit: "mind-eureka-self.service".into(), - }) - } - } - - /// Whether the brake is clear, and the principals it was asked about. - struct FakeBrake(AtomicBool, Mutex>); - - impl BrakeCheck for FakeBrake { - fn clear(&self, principal: &str) -> bool { - self.1.lock().unwrap().push(principal.into()); - self.0.load(Ordering::SeqCst) - } - } - - #[derive(Default)] - struct FakeHome { - failing: AtomicBool, - sent: Mutex>, - } - - impl Homeserver for FakeHome { - fn post(&self, txn_id: &str, text: &str) -> Result { - ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); - let mut sent = self.sent.lock().unwrap(); - sent.push((txn_id.into(), text.into())); - Ok(format!("$event{}", sent.len())) - } - } - - struct Harness { - directory: TempDir, - ctx: Ctx, - systemd: Arc, - brake: Arc, - home: Arc, - queue: mpsc::Sender, - _orders: mpsc::Receiver, - } - - fn harness_for(peer_uid: u32) -> Harness { - let directory = TempDir::new().unwrap(); - let systemd = Arc::new(FakeSystemd { - units: Mutex::new(BTreeMap::new()), - runs: Mutex::new(Vec::new()), - shows: Mutex::new(Vec::new()), - report: Mutex::new(RunReport { - exit_code: Some(0), - ..RunReport::default() - }), - }); - let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); - let home = Arc::new(FakeHome::default()); - let (queue, orders) = mpsc::channel(); - let ctx = Ctx { - store: OrderStore::new(&directory.path().join("orders")), - state_store: directory.path().join("state").join("control.cc"), - steward_uid: STEWARD, - principal: "eureka".into(), - systemd: systemd.clone(), - peers: Arc::new(FakePeers(peer_uid)), - brake: brake.clone(), - homeserver: home.clone(), - }; - Harness { - directory, - ctx, - systemd, - brake, - home, - queue, - _orders: orders, - } - } - - fn harness() -> Harness { - harness_for(STEWARD) - } - - impl Harness { - fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { - let (mut client, mut server) = UnixStream::pair().unwrap(); - client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); - write(&mut client); - handle(&self.ctx, &mut server, &self.queue).unwrap(); - let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); - rmp_serde::from_slice(&bytes).unwrap() - } - - fn ask(&self, request: &Request) -> Response { - self.exchange(|client| write_frame(client, request).unwrap()) - } - - fn issue(&self, operation: Operation) -> Response { - self.ask(&Request::Order(OrderRequest { - operation, - reason: "probe".into(), - })) - } - - fn issue_unit(&self, verb: Verb, unit: &str) -> Response { - self.issue(Operation::UnitAction { - verb, - unit: unit.into(), - }) - } - - fn accept(&self, response: Response) -> String { - match response { - Response::Accepted { order_id } => order_id, - other => panic!("expected an accepted order, got {other:?}"), - } - } - - fn outcome(&self, order_id: &str) -> Outcome { - self.ctx - .store - .ledger() - .unwrap() - .outcome_of(order_id) - .expect("the order has an outcome") - .outcome - .clone() - } - - fn runs(&self) -> Vec<(String, Step)> { - self.systemd.runs.lock().unwrap().clone() - } - - /// Tell the fake `systemctl show` what it prints for `name`. - fn describe_unit(&self, name: &str, text: String) { - self.systemd.units.lock().unwrap().insert(name.into(), text); - } - - fn describe(&self, name: &str, properties: &[(&str, &str)]) { - self.describe_unit(name, show_text(name, properties)); - } - } - - fn all_verbs() -> [Verb; 4] { - [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] - } - - #[test] - fn foreign_uid_is_refused_at_the_door() { - let h = harness_for(STEWARD + 1); - let response = h.issue_unit(Verb::Restart, "nginx.service"); - assert_eq!( - response, - Response::Refused(Refusal::new("peer", Code::Unauthorized)) - ); - let response = h.ask(&Request::List); - assert_eq!( - response, - Response::Refused(Refusal::new("peer", Code::Unauthorized)) - ); - assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); - assert!(h.runs().is_empty()); - } - - #[derive(Serialize)] - struct ForgedBody { - operation: Operation, - reason: String, - #[serde(skip_serializing_if = "Option::is_none")] - order_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - requested_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - command: Option, - } - - #[derive(Serialize)] - struct ForgedFrame { - order: ForgedBody, - } - - #[test] - fn order_id_and_requester_are_minted_not_declared() { - let h = harness(); - let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { - let frame = ForgedFrame { - order: ForgedBody { - operation: Operation::UnitAction { - verb: Verb::Restart, - unit: "nginx.service".into(), - }, - reason: "probe".into(), - order_id: order_id.map(str::to_owned), - requested_by: requested_by.map(str::to_owned), - command: command.map(str::to_owned), - }, - }; - h.exchange(|client| write_frame(client, &frame).unwrap()) - }; - // The same frame without the extra field is a good order. - let honest = h.accept(forge(None, None, None)); - for forged in [ - forge(Some("ord-declared"), None, None), - forge(None, Some("steward:someone-else order x"), None), - forge(None, None, Some("reboot-now")), - ] { - assert_eq!( - forged, - Response::Refused(Refusal::new("frame", Code::Malformed)) - ); - } - let ledger = h.ctx.store.ledger().unwrap(); - assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); - let order = &ledger.orders[0]; - assert_eq!(order.order_id, honest); - let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); - assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); - assert_eq!(order.issuer_unit, "mind-eureka-self.service"); - assert_eq!(order.issuer_invocation, "invocation-1"); - assert_eq!(order.principal, "eureka"); - assert_eq!(order.issuer_uid, STEWARD); - assert_eq!( - requester(&order.principal, &order.order_id), - format!("steward:eureka order {}", order.order_id) - ); - } - - #[test] - fn package_names_outside_grammar_are_refused() { - let h = harness(); - let bad = [ - "-y", - "--allow-unauthenticated", - "-", - "a", - "", - "Foo", - "a/b", - "/tmp/x", - "./x", - "x.deb", - "foo.deb", - "foo bar", - "foo;reboot", - "foo$(id)", - "foo`id`", - "foo|bar", - "foo&bar", - "foo\nbar", - "foo*", - "foo?", - "pkg:amd64", - "pkg/stable", - "libc6-", - "foo=", - "foo=1 2", - "foo=1;2", - "foo=1=2", - "fo\u{ff0f}o", - "f\u{43e}o", - ]; - for name in bad { - let response = h.issue(Operation::PackageInstall { - packages: vec!["git-lfs".into(), name.into()], - }); - assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); - } - let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); - assert_eq!( - h.issue(Operation::PackageInstall { packages: many }), - Response::Refused(Refusal::new("package", Code::TooMany)) - ); - assert_eq!( - h.issue(Operation::PackageInstall { packages: vec![] }), - Response::Refused(Refusal::new("package", Code::Empty)) - ); - assert!(!h.ctx.store.path().exists(), "no refusal left a record"); - - let good = [ - "git-lfs", - "matrix-synapse-py3", - "libnss-systemd", - "bubblewrap", - "socat", - "g++", - "libstdc++6", - "python3.11", - "nginx=1.24.0-2ubuntu7", - "foo=1:2.3-4+b1~rc1", - ]; - let id = h.accept(h.issue(Operation::PackageInstall { - packages: good.iter().map(|name| (*name).to_owned()).collect(), - })); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran); - let runs = h.runs(); - assert_eq!(runs.len(), 3); - assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); - // Each name selects exactly itself and apt may remove nothing. - let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; - install.extend(["--no-remove".to_owned(), "--".into()]); - install.extend(good.iter().map(|spec| match spec.split_once('=') { - Some((name, version)) => format!("?exact-name({name})={version}"), - None => format!("?exact-name({spec})"), - })); - assert_eq!(runs[1].1.argv, install); - assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); - assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); - // The last step fails unless every named package is installed. - let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; - present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); - assert_eq!(runs[2].1.argv, present); - for (unit, step) in &runs { - assert!(unit.starts_with("idunn-order-")); - let apt = step.argv[0] == "apt-get"; - assert_eq!( - step.environment, - if apt { - vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] - } else { - Vec::new() - } - ); - } - assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); - assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); - - // A package whose name apt would read as a regex or a conflict is - // exactly itself and never a removal. - for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { - let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); - assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); - } - // A step that failed ends the order: the absent package is a failure. - *h.systemd.report.lock().unwrap() = RunReport { - exit_code: Some(1), - ..RunReport::default() - }; - let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); - execute(&h.ctx, &absent).unwrap(); - assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); - assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); - } - - #[test] - fn any_unit_outside_the_refused_set_runs() { - let h = harness(); - let units = [ - "wg-quick@wg0.service", - "nginx.service", - "matrix-synapse.service", - "ssh.socket", - "huginn@eureka.service", - "claude-connector.service", - "persona-matrix-mouth.service", - "mind-heartbeat.service", - "apt-daily.timer", - // Near misses: the prefix is the rule, not a substring or a stem. - "midunn-x.service", - "not-idunn-x.service", - "idunn.service", - "mind-self.service", - "mind-selfx@a.service", - "mind-persona.service", - "mind-personal@a.service", - ]; - for verb in all_verbs() { - for unit in units { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); - let (transient, step) = h.runs().pop().unwrap(); - assert!(transient.starts_with("idunn-order-")); - assert_eq!( - step.argv, - ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] - ); - } - } - assert_eq!(h.runs().len(), units.len() * 4); - } - - #[test] - fn refused_set_is_refused_by_name_and_alias() { - let h = harness(); - h.describe_unit( - "backup-name.service", - show_text( - "idunn-yggdrasil.service", - &[("Names", "idunn-yggdrasil.service backup-name.service")], - ), - ); - h.describe( - "quiet-alias.service", - &[("Names", "quiet-alias.service mind-persona@void.service")], - ); - let direct = [ - "idunn-yggdrasil.service", - "idunn-orders.service", - "idunn-vili-x.service", - "idunn-order-3f2c.service", - "idunn-yggdrasil.timer", - "idunn-x.socket", - "idunn-.service", - "idunn-yggdrasil.service.service", - "idunn-x@y.service", - "mind-self@eureka.service", - "mind-persona@void.service", - "mind-self@.service", - "mind-persona@a@b.service", - // Case folds: a lookalike is still refused, never run. - "IDUNN-yggdrasil.service", - "Idunn-Orders.service", - "MIND-SELF@eureka.service", - "Mind-Persona@Void.service", - ]; - for verb in all_verbs() { - for unit in direct { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - match h.outcome(&id) { - Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { - assert_eq!(hit, unit, "{verb:?} {unit}"); - assert_eq!(via, Via::Name); - } - other => panic!("{verb:?} {unit} was {other:?}"), - } - } - } - for verb in all_verbs() { - for (unit, hit) in [ - ("backup-name.service", "idunn-yggdrasil.service"), - ("quiet-alias.service", "mind-persona@void.service"), - ] { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - assert_eq!( - h.outcome(&id), - Outcome::Refused(Refused::AlwaysRefused { - unit: hit.into(), - via: Via::Alias - }), - "{verb:?} {unit}" - ); - } - } - assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); - // Every refusal is recorded and has a result post due. - let ledger = h.ctx.store.ledger().unwrap(); - let refusals = ledger - .outcomes - .iter() - .filter(|record| matches!(record.outcome, Outcome::Refused(_))) - .count(); - assert_eq!(refusals, direct.len() * 4 + 8); - let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); - let results = due - .iter() - .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) - .count(); - assert_eq!(results, refusals); - // A refused order never started. - assert!(ledger.starts.is_empty()); - } - - fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - h.outcome(&id) - } - - fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { - match outcome { - Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { - Some(unit.as_str()) - } - _ => None, - } - } - - #[test] - fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { - let h = harness(); - // systemctl show quotes a name with a backslash and doubles the backslash. - h.describe( - "huginn.service", - &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], - ); - let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); - assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); - h.describe( - "quiet.service", - &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], - ); - let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); - assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); - assert!(h.runs().is_empty()); - } - - #[test] - fn unit_orders_run_as_one_job() { - let h = harness(); - for verb in all_verbs() { - let id = h.accept(h.issue_unit(verb, "nginx.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); - let (_, step) = h.runs().pop().unwrap(); - assert_eq!( - step.argv, - ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] - ); - } - } - - #[test] - fn the_rule_reads_only_the_named_unit() { - let h = harness(); - // Whatever the unit lists, no other unit is looked at. - h.describe( - "plain.service", - &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], - ); - h.describe("other.service", &[("LoadState", "not-found")]); - for verb in all_verbs() { - assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); - } - assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); - } - - #[test] - fn a_unit_that_acts_when_it_ends_is_refused() { - let h = harness(); - let properties = [ - ("SuccessAction", "reboot"), - ("FailureAction", "poweroff-force"), - ("StartLimitAction", "soft-reboot-immediate"), - ("JobTimeoutAction", "exit"), - ("FailureAction", "a-spelling-nobody-knows"), - ("OnFailure", "rescue.service"), - ("OnSuccess", "cleanup.service"), - ]; - for (property, value) in properties { - h.describe("ender.service", &[(property, value)]); - for verb in [Verb::Start, Verb::Stop, Verb::Restart] { - assert_eq!( - outcome_of_unit(&h, verb, "ender.service"), - Outcome::Refused(Refused::AlwaysRefused { - unit: "ender.service".into(), - via: Via::OnEnd - }), - "{verb:?} {property}={value}" - ); - } - assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); - } - // `none` and an empty list say the unit does nothing at its end. - h.describe( - "calm.service", - &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], - ); - for verb in all_verbs() { - assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); - } - assert_eq!(h.runs().len(), 11); - } - - #[test] - fn host_names_are_refused() { - let h = harness(); - let names = [ - "systemd-suspend.service", - "systemd-hibernate.service", - "systemd-hybrid-sleep.service", - "systemd-suspend-then-hibernate.service", - "rescue.service", - "emergency.service", - ]; - h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); - for verb in all_verbs() { - for name in names { - assert_eq!( - outcome_of_unit(&h, verb, name), - Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), - "{verb:?} {name}" - ); - } - assert_eq!( - outcome_of_unit(&h, verb, "sleepy.service"), - Outcome::Refused(Refused::AlwaysRefused { - unit: "systemd-hibernate.service".into(), - via: Via::Host - }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); - } - - #[test] - fn a_unit_others_are_bound_to_only_starts() { - let h = harness(); - h.describe("base.service", &[("BoundBy", "first.service second.service")]); - for verb in [Verb::Stop, Verb::Restart] { - assert_eq!( - outcome_of_unit(&h, verb, "base.service"), - Outcome::Refused(Refused::AlwaysRefused { - unit: "first.service".into(), - via: Via::Bound - }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - for verb in [Verb::Start, Verb::Reload] { - assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); - } - } - - #[test] - fn a_unit_that_upholds_others_is_refused_where_it_would_start() { - let h = harness(); - h.describe("holder.service", &[("Upholds", "held.service")]); - for verb in [Verb::Start, Verb::Restart] { - assert_eq!( - outcome_of_unit(&h, verb, "holder.service"), - Outcome::Refused(Refused::AlwaysRefused { - unit: "holder.service".into(), - via: Via::Upholds - }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - for verb in [Verb::Stop, Verb::Reload] { - assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); - } - } - - #[test] - fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { - let h = harness(); - h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); - for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { - assert_eq!( - outcome_of_unit(&h, verb, "wrapper.service"), - Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); - } - - #[test] - fn empty_dependency_lists_name_no_one() { - let h = harness(); - // systemctl prints every property it is asked for, empty or not. - h.describe( - "alone.service", - &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], - ); - for verb in all_verbs() { - assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); - } - } - - #[test] - fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { - let h = harness(); - h.describe("base.service", &[("BoundBy", "first.service")]); - h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); - assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); - assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); - h.systemd.shows.lock().unwrap().clear(); - assert_eq!( - hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), - Some("idunn-ghost.service") - ); - assert!(h.systemd.shows.lock().unwrap().is_empty()); - } - - #[test] - fn each_refusal_is_posted_under_its_own_reason() { - for (via, word) in [ - (Via::Name, "name"), - (Via::Alias, "alias"), - (Via::Host, "host"), - (Via::OnEnd, "on-end"), - (Via::Bound, "bound"), - (Via::Upholds, "upholds"), - ] { - assert_eq!( - Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) - .describe(), - format!("refused: always-refused unit u.service reached by {word}") - ); - } - assert_eq!( - Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), - "refused: wrapper unit, it only wraps m.service" - ); - } - - #[test] - fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { - let h = harness(); - h.describe("ghost.service", &[("LoadState", "not-found")]); - for verb in all_verbs() { - let id = h.accept(h.issue_unit(verb, "ghost.service")); - execute(&h.ctx, &id).unwrap(); - assert!( - matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - } - - #[test] - fn globs_slices_scopes_and_targets_are_not_units() { - let h = harness(); - let bad = [ - "idunn*", - "idunn-*.service", - "idunn-?ggdrasil.service", - "mind-?elf@eureka.service", - "mind-self@*.service", - "mind-[s]elf@eureka.service", - "system.slice", - "poweroff.target", - "multi-user.target", - "user.slice", - "session-1.scope", - "init.scope", - "idunn-yggdrasil", - "mind-self@eureka", - "nginx", - ".service", - "", - "-idunn-x.service", - "-x.service", - "/etc/systemd/system/idunn-yggdrasil.service", - "../idunn-yggdrasil.service", - "idunn-yggdrasil.service\n", - "idunn-yggdrasil.service ", - " idunn-yggdrasil.service", - "idunn-yggdrasil.service\0", - "idunn-yggdrasil.service;reboot", - "idunn\\x2dyggdrasil.service", - "idunn%2dyggdrasil.service", - // Unicode lookalikes of a refused name. - "\u{131}dunn-yggdrasil.service", - "idunn\u{2010}yggdrasil.service", - "idunn\u{ff0d}yggdrasil.service", - "idunn-yggdrasil.service\u{200b}", - "\u{200b}idunn-yggdrasil.service", - "idunn-yggdrasil\u{2024}service", - "mind-self\u{ff20}eureka.service", - ]; - for unit in bad { - for verb in all_verbs() { - assert!( - matches!(h.issue_unit(verb, unit), Response::Refused(_)), - "{unit:?} was not refused by the grammar" - ); - } - } - let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); - assert_eq!( - h.issue_unit(Verb::Start, &long), - Response::Refused(Refusal::new("unit", Code::TooLong)) - ); - assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); - } - - #[test] - fn binding_with_a_unit_list_is_refused() { - let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ - mind_control = \"/usr/local/bin/mind-control\"\n\ - room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ - token_credential = \"idunn-matrix-token\"\n"; - let binding = StewardBinding::parse(valid).unwrap(); - assert_eq!(binding.principal, "eureka"); - assert_eq!(binding.address().unwrap().port(), 8008); - for extra in [ - "units = [\"nginx.service\"]", - "allow = [\"nginx.service\"]", - "deny = [\"idunn-*\"]", - "denylist = [\"idunn-*\"]", - "refused = [\"idunn-*\"]", - "unit = \"nginx.service\"", - "[units]\nnginx = true", - ] { - assert!( - StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), - "{extra} loaded" - ); - } - let swap = |from: &str, to: &str| valid.replace(from, to); - for broken in [ - swap("user = \"mind-eureka-self\"", "user = \"root\""), - swap("user = \"mind-eureka-self\"", "user = \"\""), - swap("/usr/local/bin/mind-control", "mind-control"), - swap("!abc:example.org", "#alias:example.org"), - swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), - swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), - swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), - swap("idunn-matrix-token", "../token"), - swap("principal = \"eureka\"\n", ""), - ] { - assert!(StewardBinding::parse(&broken).is_err()); - } - assert!(admitted(0, 0o100600)); - assert!(admitted(0, 0o100644)); - assert!(!admitted(0, 0o100664)); - assert!(!admitted(0, 0o100666)); - assert!(!admitted(0, 0o100602)); - assert!(!admitted(1000, 0o100600)); - } - - #[test] - fn held_brake_refuses_at_start() { - let h = harness(); - h.brake.0.store(false, Ordering::SeqCst); - let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); - assert!(h.runs().is_empty()); - // The deploy path is gated the same way and writes no command. - let deploy = h.accept(h.issue(Operation::Deploy { - selector: "vili".into(), - })); - execute(&h.ctx, &deploy).unwrap(); - assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); - assert!(!h.ctx.state_store.exists()); - // A brake that cannot be asked is a held brake. - let directory = TempDir::new().unwrap(); - assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); - } - - #[test] - fn brake_check_is_the_exit_status_of_mind_control_check() { - let directory = TempDir::new().unwrap(); - let program = |name: &str, body: &str| { - let path = directory.path().join(name); - std::fs::write(&path, body).unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); - MindControl { program: path } - }; - let recorder = directory.path().join("argv"); - let clear = program( - "clear", - &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), - ); - assert!(clear.clear("eureka")); - assert_eq!( - std::fs::read_to_string(&recorder).unwrap().trim(), - "--instance eureka check" - ); - assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); - } - - #[test] - fn started_order_is_never_rerun() { - let h = harness(); - let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); - // The first order began and its service died: a start with no outcome. - h.ctx - .store - .put( - &id, - START_SCHEMA, - &StartRecord { - schema_version: START_SCHEMA.into(), - order_id: id.clone(), - started_at_unix_millis: 1, - actuation: format!("idunn-order-{}", order_uuid(&id)), - }, - ) - .unwrap(); - let pending = recover(&h.ctx).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Lost); - assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); - execute(&h.ctx, &id).unwrap(); - assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); - // Recovery is idempotent, and a finished order is not run again. - assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); - execute(&h.ctx, &waiting).unwrap(); - execute(&h.ctx, &waiting).unwrap(); - assert_eq!(h.runs().len(), 1); - assert!(recover(&h.ctx).unwrap().is_empty()); - } - - #[test] - fn deploy_order_uses_the_up_primitive() { - let h = harness(); - let id = h.accept(h.issue(Operation::Deploy { - selector: "vili".into(), - })); - execute(&h.ctx, &id).unwrap(); - let command_id = format!("up-{}", order_uuid(&id)); - assert_eq!( - h.outcome(&id), - Outcome::Submitted { - command_id: command_id.clone() - } - ); - let ledger = h.ctx.store.ledger().unwrap(); - assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); - assert!(h.runs().is_empty(), "a deploy runs no transient unit"); - let mut shown = Vec::new(); - crate::control_plane::status( - &mut shown, - &h.ctx.state_store, - &h.directory.path().join("incidents.cc"), - Some(&command_id), - ) - .unwrap(); - let shown = String::from_utf8(shown).unwrap(); - assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); - assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); - // Re-executing writes no second command. - execute(&h.ctx, &id).unwrap(); - let mut again = Vec::new(); - crate::control_plane::status( - &mut again, - &h.ctx.state_store, - &h.directory.path().join("incidents.cc"), - None, - ) - .unwrap(); - assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); - // The result post waits for the transactions: queued is not finished. - let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); - assert_eq!(due.len(), 1); - assert_eq!(due[0].kind, PostKind::Accepted); - assert!(!deploy_finished("queued") && !deploy_finished("running")); - assert!(deploy_finished("succeeded") && deploy_finished("failed")); - assert!(deploy_finished("complete")); - } - - #[test] - fn chat_outage_delays_posts_not_orders() { - let h = harness(); - h.home.failing.store(true, Ordering::SeqCst); - let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran); - assert_eq!(h.runs().len(), 1); - assert!(post_pass(&h.ctx).is_err()); - assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); - h.home.failing.store(false, Ordering::SeqCst); - assert_eq!(post_pass(&h.ctx).unwrap(), 2); - assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); - let sent = h.home.sent.lock().unwrap().clone(); - assert_eq!(sent.len(), 2); - assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); - assert_eq!(sent[1].0, format!("idunn-{id}-result")); - assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); - assert!(sent[0].1.contains("mind-eureka-self.service")); - assert!(sent[0].1.contains("reason: \"probe\"")); - assert!(sent[1].1.contains("ran (exit 0)")); - } - - #[test] - fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { - let h = harness(); - *h.systemd.report.lock().unwrap() = RunReport { - exit_code: Some(100), - signal: None, - stdout_tail: vec![b'o'; 3 * TAIL_BYTES], - stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), - }; - let id = h.accept(h.issue(Operation::PackageInstall { - packages: vec!["git-lfs".into()], - })); - execute(&h.ctx, &id).unwrap(); - assert_eq!( - h.outcome(&id), - Outcome::Failed { - exit_code: Some(100), - signal: None - } - ); - assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); - let ledger = h.ctx.store.ledger().unwrap(); - let record = ledger.outcome_of(&id).unwrap(); - assert_eq!(record.stdout_tail.len(), TAIL_BYTES); - assert!(!format!("{record:?}").contains("secret-looking")); - // Neither the post nor the journal text carries a tail. - let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) - .into_iter() - .filter(|due| due.kind == PostKind::Result) - .collect(); - assert_eq!(results.len(), 1); - assert!(results[0].text.contains("failed (exit 100)")); - assert!(!results[0].text.contains("secret-looking")); - // The steward reads them through the escape sink and nothing raw. - let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { - panic!("expected a show") - }; - let mut shown = Vec::new(); - emit(&mut shown, &render_view(&view)).unwrap(); - let shown = String::from_utf8(shown).unwrap(); - assert!(shown.contains("secret-looking")); - assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); - assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); - assert!(shown.contains("status failed")); - // A show of an unknown or malformed id names the field only. - assert_eq!( - h.ask(&Request::Show(ShowRequest { - id: format!("ord-{}", Uuid::new_v4()) - })), - Response::Refused(Refusal::new("id", Code::Unknown)) - ); - assert_eq!( - h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), - Response::Refused(Refusal::new("id", Code::BadShape)) - ); - let Response::List(lines) = h.ask(&Request::List) else { - panic!("expected a list") - }; - assert_eq!(lines.len(), 1); - assert_eq!(lines[0].status, "failed"); - } - - #[test] - fn a_refusal_or_error_never_echoes_an_input_value() { - let h = harness(); - let mut seen: Vec> = Vec::new(); - let mut note = |response: Response| { - let bytes = rmp_serde::to_vec_named(&response).unwrap(); - if let Response::Refused(refusal) = &response { - seen.push(refusal.to_string().into_bytes()); - } - seen.push(bytes); - }; - for unit in [ - format!("{CANARY}*.service"), - format!("{CANARY}/../idunn-orders.service"), - format!("idunn-{CANARY};x.service"), - CANARY.to_string(), - ] { - note(h.issue_unit(Verb::Restart, &unit)); - } - note(h.issue(Operation::PackageInstall { - packages: vec![format!("{CANARY}/x")], - })); - note(h.issue(Operation::PackageInstall { - packages: vec![format!("pkg={CANARY} 1")], - })); - note(h.issue(Operation::Deploy { - selector: format!("{CANARY} bad"), - })); - note(h.ask(&Request::Order(OrderRequest { - operation: Operation::UnitAction { - verb: Verb::Start, - unit: "nginx.service".into(), - }, - reason: format!("{CANARY}\n"), - }))); - note(h.ask(&Request::Order(OrderRequest { - operation: Operation::UnitAction { - verb: Verb::Start, - unit: "nginx.service".into(), - }, - reason: CANARY.repeat(40), - }))); - note(h.ask(&Request::Show(ShowRequest { - id: format!("ord-{CANARY}"), - }))); - // A frame whose own keys and values are the canary, and plain garbage. - let named: std::collections::BTreeMap> = - [( - CANARY.to_owned(), - [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), - )] - .into_iter() - .collect(); - note(h.exchange(|client| write_frame(client, &named).unwrap())); - note(h.exchange(|client| { - let garbage = format!("{CANARY}{CANARY}"); - client - .write_all(&(garbage.len() as u32).to_be_bytes()) - .unwrap(); - client.write_all(garbage.as_bytes()).unwrap(); - })); - note(h.exchange(|client| { - client.write_all(&u32::MAX.to_be_bytes()).unwrap(); - })); - // The client grammar fails the same way. - let mut messages = Vec::new(); - let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); - for line in [ - words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), - words(&["unit", CANARY, "nginx.service", "--reason", "r"]), - words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), - words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), - words(&[CANARY]), - words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), - words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), - words(&["show"]), - ] { - messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); - } - for line in [ - words(&[CANARY]), - words(&["serve", &format!("--{CANARY}")]), - words(&["serve", "--store"]), - ] { - messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); - } - for bytes in &seen { - assert!( - !String::from_utf8_lossy(bytes).contains(CANARY), - "a response carried an input value" - ); - } - for message in &messages { - assert!(!message.contains(CANARY), "an error carried an input value: {message}"); - } - assert!(!h.ctx.store.path().exists()); - } - - #[test] - fn cgroup_names_the_issuer_unit() { - assert_eq!( - unit_from_cgroup("0::/system.slice/claude-connector.service\n"), - "claude-connector.service" - ); - // What a delegated subtree adds below the unit is the peer's to name. - assert_eq!( - unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), - "huginn@eureka.service" - ); - assert_eq!( - unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), - "getty@tty1.service" - ); - assert_eq!( - unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), - "a.scope" - ); - // A user manager's units, and paths no manager made, name no one. - for path in [ - "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", - "0::/user.slice/user-1000.slice/session-3.scope\n", - "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", - "0::/system.slice/\n", - "0::/\n", - "", - ] { - assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); - } - } - - #[test] - fn systemctl_show_output_is_read_into_facts() { - let facts = parse_show( - "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ - SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ - OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ - Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", - ) - .unwrap(); - assert_eq!(facts.id, "a.service"); - assert_eq!(facts.names, ["a.service", "b.service"]); - assert!(!facts.not_found); - assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); - // A quoted name holds one backslash, not two and not a quote mark. - assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); - assert_eq!(facts.upholds, ["u.service"]); - assert_eq!(facts.consists_of, ["m.service", "n.service"]); - assert!(parse_show("Names=a.service\n").is_err()); - // The request asks for exactly the properties the parse reads. - assert_eq!( - show_properties(), - "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ - OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" - ); - // A backslash outside quotes is a backslash; inside, it escapes one character. - assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); - assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); - assert!(parse_names(" ").is_empty()); - let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); - assert!(gone.not_found); - } - - #[test] - fn the_store_refuses_a_second_record_and_a_foreign_document() { - let h = harness(); - let order = OrderRecord { - schema_version: ORDER_SCHEMA.into(), - order_id: "ord-x".into(), - principal: "eureka".into(), - issuer_uid: 1, - issuer_unit: "u.service".into(), - issuer_invocation: String::new(), - issued_at_unix_millis: 1, - operation: Operation::Deploy { - selector: "vili".into(), - }, - reason: "r".into(), - }; - assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); - assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); - assert_eq!( - std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, - 0o600 - ); - let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); - SingleFileMessagePackBackingStore::new(h.ctx.store.path()) - .compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: "idunn.other".into(), - key: "k".into(), - current: None, - }], - &[foreign], - ) - .unwrap(); - assert!(h.ctx.store.ledger().is_err()); - } - - #[test] - fn synapse_post_is_one_idempotent_put_on_loopback() { - let directory = TempDir::new().unwrap(); - let token_file = directory.path().join("token"); - std::fs::write(&token_file, "tok-123\n").unwrap(); - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let (sent, requests) = mpsc::channel(); - let server = thread::spawn(move || { - for reply in [ - "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", - "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", - ] { - let (mut socket, _) = listener.accept().unwrap(); - let mut received = Vec::new(); - let mut chunk = [0u8; 1024]; - while !received.windows(4).any(|window| window == b"\r\n\r\n") - || !String::from_utf8_lossy(&received).contains("\"body\"") - { - let count = socket.read(&mut chunk).unwrap(); - if count == 0 { - break; - } - received.extend_from_slice(&chunk[..count]); - } - sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); - socket.write_all(reply.as_bytes()).unwrap(); - } - }); - let synapse = Synapse { - address, - room: "!room:example.org".into(), - token_file, - reply_within: Duration::from_secs(10), - }; - assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); - let request = requests.recv().unwrap(); - assert!(request.starts_with( - "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" - )); - assert!(request.contains("Authorization: Bearer tok-123\r\n")); - assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); - let error = synapse.post("txn-2", "x").unwrap_err(); - assert!(!format!("{error:#}").contains("tok-123")); - server.join().unwrap(); - assert!( - Synapse { - address, - room: "!r:e".into(), - token_file: directory.path().join("absent"), - reply_within: Duration::from_secs(10), - } - .post("t", "x") - .is_err() - ); - } - - #[test] - fn grammar_limits_are_exact_at_their_edges() { - let h = harness(); - let order = |operation: Operation, reason: &str| { - h.ask(&Request::Order(OrderRequest { - operation, - reason: reason.into(), - })) - }; - let packages = |names: Vec| Operation::PackageInstall { packages: names }; - let unit = |name: String| Operation::UnitAction { - verb: Verb::Start, - unit: name, - }; - // Package: two characters is the shortest name, 200 bytes the longest. - assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); - assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); - assert_eq!( - order(packages(vec!["a".repeat(201)]), "r"), - Response::Refused(Refusal::new("package", Code::TooLong)) - ); - assert_eq!( - order(packages(vec!["a".into()]), "r"), - Response::Refused(Refusal::new("package", Code::BadShape)) - ); - let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); - assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); - // Unit: 200 bytes with its suffix is the longest. - let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); - assert_eq!(longest.len(), UNIT_LIMIT); - assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); - let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); - assert_eq!( - order(unit(too_long), "r"), - Response::Refused(Refusal::new("unit", Code::TooLong)) - ); - // Reason: 300 bytes is the longest, and it must be there. - assert!(matches!( - order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), - Response::Accepted { .. } - )); - assert_eq!( - order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), - Response::Refused(Refusal::new("reason", Code::TooLong)) - ); - assert_eq!( - order(unit("nginx.service".into()), ""), - Response::Refused(Refusal::new("reason", Code::Empty)) - ); - assert_eq!( - order(unit("nginx.service".into()), "caf\u{e9}"), - Response::Refused(Refusal::new("reason", Code::BadCharacter)) - ); - } - - #[test] - fn a_concluded_order_is_never_run_later() { - let h = harness(); - h.brake.0.store(false, Ordering::SeqCst); - let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - // Someone else clears the brake; the refused order stays refused. - h.brake.0.store(true, Ordering::SeqCst); - execute(&h.ctx, &id).unwrap(); - assert!(h.runs().is_empty()); - let ledger = h.ctx.store.ledger().unwrap(); - assert!(ledger.start_of(&id).is_none()); - assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - } - - #[test] - fn the_orders_cli_has_no_command_string() { - let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); - let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); - assert_eq!( - parsed.action, - OrderAction::Issue { - operation: Operation::UnitAction { - verb: Verb::Restart, - unit: "nginx.service".into() - }, - reason: "why".into() - } - ); - assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); - assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); - assert!(parse(&["show", "ord-x"]).is_ok()); - assert!(parse(&["list"]).is_ok()); - for refused in [ - vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], - vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], - vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], - vec!["run", "sh", "--reason", "r"], - vec!["exec", "--reason", "r"], - vec!["unit", "enable", "nginx.service", "--reason", "r"], - vec!["unit", "restart", "--reason", "r"], - vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], - vec!["unit", "restart", "nginx.service"], - vec!["unit", "restart", "nginx*", "--reason", "r"], - vec!["package-install", "--reason", "r"], - vec!["package-install", "-y", "x", "--reason", "r"], - vec!["deploy", "--reason", "r"], - vec!["deploy", "a", "b", "--reason", "r"], - vec!["list", "extra"], - vec!["show"], - vec![], - ] { - assert!(parse(&refused).is_err(), "{refused:?} parsed"); - } - } - - #[test] - fn an_order_runs_only_under_the_steward_bound_now() { - let mut h = harness(); - let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); - // The binding changes while both wait: another uid, then another - // principal with the old uid. Each is refused with a record, before - // the brake is asked and before anything runs. - h.ctx.steward_uid = STEWARD + 7; - execute(&h.ctx, &first).unwrap(); - h.ctx.steward_uid = STEWARD; - h.ctx.principal = "void".into(); - execute(&h.ctx, &second).unwrap(); - for id in [&first, &second] { - assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); - assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); - } - assert!(h.runs().is_empty()); - assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); - // The refusal is posted like any other. - let ledger = h.ctx.store.ledger().unwrap(); - let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); - assert!(due.iter().any(|due| due.kind == PostKind::Result - && due.text.contains("refused: the issuer is not the current steward"))); - // The other direction: an order the bound steward issued after the - // rebinding, and a recovered one under an unchanged binding, run and - // are gated by the bound principal's brake. - let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); - let resumed = harness(); - let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); - assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); - execute(&resumed.ctx, &waiting).unwrap(); - execute(&h.ctx, &third).unwrap(); - assert_eq!(resumed.outcome(&waiting), Outcome::Ran); - assert_eq!(h.outcome(&third), Outcome::Ran); - assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); - assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); - } - - #[test] - fn root_reads_orders_and_never_issues_one() { - let mut h = harness(); - let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - h.ctx.peers = Arc::new(FakePeers(0)); - let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); - assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); - let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { - panic!("root could not read an order"); - }; - assert_eq!(view.order.order_id, id); - let Response::List(lines) = h.ask(&Request::List) else { - panic!("root could not list orders"); - }; - assert_eq!(lines.len(), 1, "root's refused order left no record"); - h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); - assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); - } - - #[test] - fn every_order_posts_once_in_its_turn() { - let h = harness(); - let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] - .into_iter() - .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) - .collect(); - for id in &ids { - execute(&h.ctx, id).unwrap(); - } - assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); - assert_eq!(post_pass(&h.ctx).unwrap(), 0); - let sent = h.home.sent.lock().unwrap().clone(); - let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); - keys.sort(); - let mut expected: Vec = ids - .iter() - .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) - .collect(); - expected.sort(); - assert_eq!(keys, expected); - // An order accepted later is posted by the next pass alone. - let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); - assert_eq!(post_pass(&h.ctx).unwrap(), 1); - execute(&h.ctx, &late).unwrap(); - assert_eq!(post_pass(&h.ctx).unwrap(), 1); - } - - #[test] - fn a_deploys_result_is_posted_once_its_command_has_finished() { - let h = harness(); - let id = h.accept(h.issue(Operation::Deploy { - selector: "vili".into(), - })); - execute(&h.ctx, &id).unwrap(); - let ledger = h.ctx.store.ledger().unwrap(); - let results = |state: Option<(&'static str, &str)>| { - let owed = due_posts(&ledger, &|_| match state { - Some((state, detail)) => Ok((state, detail.to_owned())), - None => Err(anyhow!("the command cannot be read")), - }); - owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() - }; - assert!(results(Some(("queued", ""))).is_empty()); - assert!(results(Some(("running", "vili:Sealing"))).is_empty()); - assert!(results(None).is_empty()); - for state in ["succeeded", "failed"] { - let owed = results(Some((state, "vili"))); - assert_eq!(owed.len(), 1, "{state}"); - assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); - assert!(owed[0].text.contains("submitted deployment up-")); - } - } - - #[test] - fn intake_gives_up_on_a_peer_that_sends_no_frame() { - let h = harness(); - let (mut client, mut server) = UnixStream::pair().unwrap(); - client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); - // Half a length prefix, then nothing: the connection must end by itself. - client.write_all(&[0, 0]).unwrap(); - let (done, ended) = mpsc::channel(); - let worker = thread::spawn(move || { - let result = handle(&h.ctx, &mut server, &h.queue); - let _ = done.send(result.is_ok()); - h - }); - let answered = ended - .recv_timeout(Duration::from_secs(20)) - .expect("intake never gave up on a silent peer"); - assert!(answered, "a silent peer is refused as malformed, not left as an error"); - let h = worker.join().unwrap(); - assert_eq!( - rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), - Response::Refused(Refusal::new("frame", Code::Malformed)) - ); - assert!(!h.ctx.store.path().exists()); - } - - #[test] - fn a_record_under_another_schema_is_not_read() { - let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { - "order" => h.ctx.store.put( - key, - schema, - &OrderRecord { - schema_version: schema.into(), - order_id: key.into(), - principal: "eureka".into(), - issuer_uid: 1, - issuer_unit: "u.service".into(), - issuer_invocation: String::new(), - issued_at_unix_millis: 1, - operation: Operation::Deploy { - selector: "vili".into(), - }, - reason: "r".into(), - }, - ), - "start" => h.ctx.store.put( - key, - schema, - &StartRecord { - schema_version: schema.into(), - order_id: key.into(), - started_at_unix_millis: 1, - actuation: "idunn-order-x".into(), - }, - ), - "outcome" => h.ctx.store.put( - key, - schema, - &OutcomeRecord { - schema_version: schema.into(), - order_id: key.into(), - ended_at_unix_millis: 1, - outcome: Outcome::Lost, - stdout_tail: Vec::new(), - stderr_tail: Vec::new(), - }, - ), - _ => h.ctx.store.put( - key, - schema, - &PostRecord { - schema_version: schema.into(), - post_key: key.into(), - event_id: "$e".into(), - posted_at_unix_millis: 1, - }, - ), - }; - for (kind, current, key) in [ - ("order", ORDER_SCHEMA, "ord-a"), - ("start", START_SCHEMA, "ord-a"), - ("outcome", OUTCOME_SCHEMA, "ord-a"), - ("post", POST_SCHEMA, "ord-a:accepted"), - ] { - let h = harness(); - assert!(put(&h, kind, current, key).unwrap()); - assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); - let h = harness(); - let other = current.replace(".v1", ".v0"); - assert!(put(&h, kind, &other, key).unwrap()); - assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); - } - } - - #[test] - fn posts_are_retried_after_a_widening_pause() { - let pauses: Vec = (0..9).map(post_pause).collect(); - assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); - } - - #[test] - fn serve_options_are_read_from_every_flag() { - let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); - let defaults = parse(&["serve"]).unwrap(); - assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); - assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); - assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); - let given = parse(&[ - "serve", - "--state-store", - "/a/control.cc", - "--store", - "/b", - "--steward-binding", - "/c.toml", - "--socket", - "/d.sock", - ]) - .unwrap(); - assert_eq!( - given, - ServeOptions { - state_store: "/a/control.cc".into(), - store: "/b".into(), - steward_binding: "/c.toml".into(), - socket: "/d.sock".into(), - } - ); - assert!(parse(&["serve", "--store"]).is_err()); - assert!(parse(&["serve", "--other", "/x"]).is_err()); - assert!(parse(&["start"]).is_err()); - } - - #[test] - fn the_steward_user_is_a_real_user_and_never_root() { - assert!(user_uid("root").is_err()); - assert!(user_uid("no-such-user-here").is_err()); - assert!(user_uid("bad\0name").is_err()); - assert_eq!(user_uid("nobody").unwrap(), 65534); - } - - #[test] - fn the_orders_directory_is_made_private() { - let directory = TempDir::new().unwrap(); - let store = OrderStore::new(&directory.path().join("orders")); - store.prepare().unwrap(); - let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; - assert_eq!(mode, 0o700); - } - - #[test] - fn the_socket_belongs_to_the_steward_alone() { - let directory = TempDir::new().unwrap(); - let path = directory.path().join("steward.sock"); - let me = std::fs::metadata(directory.path()).unwrap().uid(); - let _listener = bind_socket(&path, me).unwrap(); - let metadata = std::fs::metadata(&path).unwrap(); - assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); - // A stale socket is replaced; anything else in the way is not. - let _again = bind_socket(&path, me).unwrap(); - let other = directory.path().join("other"); - std::fs::write(&other, "x").unwrap(); - assert!(bind_socket(&other, me).is_err()); - } - - #[test] - fn synapse_gives_up_on_a_reply_that_never_ends() { - let directory = TempDir::new().unwrap(); - let token_file = directory.path().join("token"); - std::fs::write(&token_file, "tok-123\n").unwrap(); - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - thread::spawn(move || { - // The first connection stays silent; the second drips a byte at a time. - for (index, stream) in listener.incoming().enumerate() { - let Ok(mut socket) = stream else { continue }; - thread::spawn(move || { - let mut chunk = [0u8; 4096]; - let _ = socket.read(&mut chunk); - for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { - if index == 0 || socket.write_all(&[*byte]).is_err() { - break; - } - thread::sleep(Duration::from_millis(40)); - } - thread::sleep(Duration::from_secs(5)); - }); - } - }); - let synapse = Synapse { - address, - room: "!r:e".into(), - token_file, - reply_within: Duration::from_millis(600), - }; - let synapse = Arc::new(synapse); - for _ in 0..2 { - let started = std::time::Instant::now(); - let (done, ended) = mpsc::channel(); - let poster = Arc::clone(&synapse); - thread::spawn(move || { - let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); - }); - let error = ended - .recv_timeout(Duration::from_secs(4)) - .expect("the post never gave up") - .unwrap_err(); - assert!(started.elapsed() < Duration::from_secs(3), "{error}"); - assert!(!error.contains("tok-123")); - } - } - - /// The whole service, as root, against fake `systemctl`, `systemd-run` - /// and a loopback homeserver, with the steward connecting from another - /// uid. It runs only where the test process is root (the verify container). - #[test] - fn serve_end_to_end_as_root() { - // SAFETY: geteuid reads the process credentials. - if unsafe { libc::geteuid() } != 0 { - eprintln!("serve_end_to_end_as_root: skipped, not root"); - return; - } - let directory = TempDir::new().unwrap(); - let root = directory.path().to_owned(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); - let bin = root.join("bin"); - std::fs::create_dir(&bin).unwrap(); - let systemctl = format!( - "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ - case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", - root.display() - ); - let run = format!( - "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", - root.display() - ); - for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { - let path = bin.join(name); - std::fs::write(&path, text).unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); - } - let credentials = root.join("cred"); - std::fs::create_dir(&credentials).unwrap(); - std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let (posted, posts) = mpsc::channel::(); - thread::spawn(move || { - for (index, stream) in listener.incoming().enumerate() { - let Ok(mut socket) = stream else { continue }; - let mut received = Vec::new(); - let mut chunk = [0u8; 4096]; - loop { - let text = String::from_utf8_lossy(&received).into_owned(); - if let Some(split) = text.find("\r\n\r\n") { - let length = text - .lines() - .find_map(|line| line.strip_prefix("Content-Length: ")) - .and_then(|value| value.trim().parse::().ok()) - .unwrap_or(0); - if received.len() >= split + 4 + length { - break; - } - } - let count = socket.read(&mut chunk).unwrap_or(0); - if count == 0 { - break; - } - received.extend_from_slice(&chunk[..count]); - } - let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); - let _ = socket.write_all( - format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), - ); - } - }); - let binding = root.join("steward.toml"); - std::fs::write( - &binding, - format!( - "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ - room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ - token_credential = \"idunn-matrix-token\"\n" - ), - ) - .unwrap(); - std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); - let run_directory = root.join("run"); - std::fs::create_dir(&run_directory).unwrap(); - std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); - let socket = run_directory.join("steward.sock"); - let store = root.join("orders"); - let control = root.join("state").join("control.cc"); - // SAFETY: the only test that reads these variables is this one; PATH - // keeps its old entries. - unsafe { - std::env::set_var( - "PATH", - format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), - ); - std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); - } - let options = ServeOptions { - state_store: control.clone(), - store: store.clone(), - steward_binding: binding, - socket: socket.clone(), - }; - // The unit's umask must not decide the modes of what the service writes. - // SAFETY: umask only sets the process file-mode mask. - let before = unsafe { libc::umask(0o117) }; - let (ended, serve_end) = mpsc::channel::(); - thread::spawn(move || { - let result = serve(options); - let _ = ended.send(format!("{result:?}")); - }); - for _ in 0..100 { - if socket.exists() || serve_end.try_recv().is_ok() { - break; - } - thread::sleep(Duration::from_millis(100)); - } - assert!(socket.exists(), "serve never bound its socket"); - let metadata = std::fs::metadata(&socket).unwrap(); - assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); - // Root is not the steward: it may read, it may not order. - let unit = |name: &str| { - Request::Order(OrderRequest { - operation: Operation::UnitAction { - verb: Verb::Restart, - unit: name.into(), - }, - reason: "end to end".into(), - }) - }; - let as_root = |request: &Request| { - let mut stream = UnixStream::connect(&socket).unwrap(); - stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); - write_frame(&mut stream, request).unwrap(); - rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() - }; - assert_eq!( - as_root(&unit("nginx.service")), - Response::Refused(Refusal::new("peer", Code::Unauthorized)) - ); - assert_eq!(as_root(&Request::List), Response::List(Vec::new())); - let orders = OrderStore::new(&store); - let finished = |order_id: &str| { - for _ in 0..200 { - if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { - return record.clone(); - } - thread::sleep(Duration::from_millis(100)); - } - panic!("{order_id} never concluded"); - }; - // The steward is uid 65534 with gid 0: a service that read the gid for - // the uid would take it for root. - let first = steward_asks(&socket, &unit("nginx.service")); - let Response::Accepted { order_id: first } = first else { - panic!("the steward was not accepted: {first:?}"); - }; - assert_eq!(finished(&first).outcome, Outcome::Ran); - std::fs::write(root.join("fail"), "").unwrap(); - let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { - panic!("the second order was not accepted"); - }; - let failed = finished(&second); - assert_eq!( - failed.outcome, - Outcome::Failed { - exit_code: Some(3), - signal: None - } - ); - assert_eq!(failed.stderr_tail, b"boom\n"); - let deploy = Request::Order(OrderRequest { - operation: Operation::Deploy { - selector: "vili".into(), - }, - reason: "end to end".into(), - }); - let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { - panic!("the deploy was not accepted"); - }; - assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); - let ledger = orders.ledger().unwrap(); - let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); - assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); - // An invocation id is what a succeeding systemctl prints, if anything. - assert_eq!(HostSystemd.invocation("quiet.service"), None); - assert_eq!(HostSystemd.invocation("broken.service"), None); - assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); - // What the service called, exactly. - let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); - assert!( - run_log.contains(&format!( - "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", - order_uuid(&first) - )), - "{run_log}" - ); - let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); - assert!( - systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), - "{systemctl_log}" - ); - // Every record is posted; the deploy's result waits for its command. - let mut seen = Vec::new(); - while seen.len() < 5 { - seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); - } - for (id, kinds) in [ - (&first, ["accepted", "result"].as_slice()), - (&second, &["accepted", "result"]), - (&third, &["accepted"]), - ] { - for kind in kinds { - let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); - assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); - } - } - // Modes: the store is private, and the deploy wrote control.cc under - // Idunn's umask, not the 0117 the process was started with. - let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; - assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); - let control_mode = mode(&control); - // SAFETY: umask only sets the process file-mode mask. - unsafe { libc::umask(before) }; - assert_eq!(control_mode, 0o640); - } - - /// One request as the steward: uid 65534 and gid 0, from a forked child - /// that only makes system calls. - fn steward_asks(socket: &Path, request: &Request) -> Response { - let body = rmp_serde::to_vec_named(request).unwrap(); - let mut frame = (body.len() as u32).to_be_bytes().to_vec(); - frame.extend_from_slice(&body); - // SAFETY: plain data. - let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; - address.sun_family = libc::AF_UNIX as libc::sa_family_t; - for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { - address.sun_path[index] = *byte as libc::c_char; - } - let mut fds = [0i32; 2]; - // SAFETY: `fds` holds two descriptors. - assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); - // SAFETY: the child allocates nothing and ends with _exit. - let pid = unsafe { libc::fork() }; - if pid == 0 { - // SAFETY: only system calls on buffers made before the fork. - unsafe { - libc::close(fds[0]); - libc::alarm(10); - if libc::setgroups(0, std::ptr::null()) != 0 - || libc::setgid(0) != 0 - || libc::setuid(65534) != 0 - { - libc::_exit(2); - } - let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); - if libc::connect( - fd, - (&address as *const libc::sockaddr_un).cast(), - std::mem::size_of::() as libc::socklen_t, - ) != 0 - { - libc::_exit(3); - } - let mut offset = 0usize; - while offset < frame.len() { - let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); - if count <= 0 { - libc::_exit(4); - } - offset += count as usize; - } - let mut buffer = [0u8; 4096]; - loop { - let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); - if count <= 0 { - break; - } - libc::write(fds[1], buffer.as_ptr().cast(), count as usize); - } - libc::_exit(0); - } - } - // SAFETY: the parent's copy of the write end. - unsafe { libc::close(fds[1]) }; - let mut reply = Vec::new(); - // SAFETY: `fds[0]` is an open descriptor this function now owns. - let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; - pipe.read_to_end(&mut reply).unwrap(); - let mut status = 0; - // SAFETY: `pid` is our child. - unsafe { libc::waitpid(pid, &mut status, 0) }; - assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); - rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() - } - - fn host_refusal(verb: Verb, unit: &str) -> Option { - refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) - } - - #[test] - #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] - fn real_host_unit_set() { - for name in [ - "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", - "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", - "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", - "wg-quick@wg-gamecult", "cron", "apt-daily", - ] { - let unit = format!("{name}.service"); - for verb in all_verbs() { - assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); - } - } - for name in [ - "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", - "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", - "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", - "systemd-storagetm", - ] { - let unit = format!("{name}.service"); - for verb in [Verb::Start, Verb::Stop, Verb::Restart] { - assert_eq!( - host_refusal(verb, &unit), - Some(refused(&unit, Via::OnEnd)), - "{verb:?} {unit}" - ); - } - } - for unit in HOST_NAMES { - for verb in all_verbs() { - assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); - } - } - for unit in [ - "systemd-networkd.service", - "user-runtime-dir@1000.service", - "user-runtime-dir@105.service", - "user-runtime-dir@978.service", - ] { - assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); - assert!( - matches!( - host_refusal(Verb::Stop, unit), - Some(Refused::AlwaysRefused { via: Via::Bound, .. }) - ), - "{unit}" - ); - } - let listing = Command::new("systemctl") - .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) - .output() - .unwrap(); - let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); - let units: Vec<&str> = listing - .lines() - .filter_map(|line| line.split_whitespace().next()) - .filter(|unit| validate_unit(unit).is_ok()) - .collect(); - for verb in all_verbs() { - let mut counts = BTreeMap::new(); - for unit in &units { - let key = match refusal_for(&HostSystemd, unit, verb) { - Ok(None) => "ok".to_owned(), - Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), - Ok(Some(other)) => format!("{other:?}"), - Err(_) => "unresolved".to_owned(), - }; - *counts.entry(key).or_insert(0usize) += 1; - } - println!("{verb:?} over {} units: {counts:?}", units.len()); - } - } - - #[test] - #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] - fn one_job_argv_against_a_real_manager() { - let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); - std::fs::create_dir_all(&directory).unwrap(); - let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); - let body = "[Service]\nExecStart=/bin/sleep 600\n"; - std::fs::write(directory.join(a), body).unwrap(); - std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); - let user = |args: &[&str]| { - let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); - (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) - }; - let outcome = std::panic::catch_unwind(|| { - let one_job = |verb: Verb, unit: &str| { - let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); - let mut argv = step.argv.clone(); - argv.insert(1, "--user".into()); - let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); - assert!(status.success(), "{verb:?} {unit}"); - }; - assert!(user(&["daemon-reload"]).0); - // A start of b leaves a running, though b conflicts with it. - assert!(user(&["start", a]).0); - one_job(Verb::Start, b); - assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); - assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); - // A restart of b, with both running, leaves a running. - one_job(Verb::Restart, b); - assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); - assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); - }); - user(&["stop", a, b]); - std::fs::remove_file(directory.join(a)).unwrap(); - std::fs::remove_file(directory.join(b)).unwrap(); - user(&["daemon-reload"]); - outcome.unwrap(); - } -}