diff --git a/Cargo.lock b/Cargo.lock index dfc3087..44c9457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "libc", "rmp-serde", "serde", + "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index dcaca2e..ee797ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } +serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/README.md b/README.md index a5c48bc..8c6935c 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,64 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. +### Steward orders + +A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), +takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. +It has its own store, `orders.cc`, and its own binding, `steward.toml`; the +survival daemon takes no order. There is still no command string: + +```bash +idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them +idunn order unit --reason TEXT +idunn order deploy --reason TEXT # the same commit as `idunn up` +idunn order show # status and output tails +idunn order list +``` + +A unit order submits exactly one job, for the unit it names: every verb runs +`systemctl --job-mode=ignore-requirements -- `, so systemd builds a +transaction of one job. It does not start what the unit requires or restart what +is part of it: order the dependency first, and order `postgresql@16-main.service` +rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is +not: systemd still runs full transactions of its own for `Restart=` after a +failure, for a `StopWhenUnneeded=` dependency left without a user, and for a +`Conflicts=` partner of a unit that fails, so an ordered stop of a unit +that something restarts does not last, and a start of a `Restart=always` +unit can report failure and be active seconds later. A unit +order may act on any unit except Idunn's own (`idunn-*`) and the minds' own +(`mind-self@*`, `mind-persona@*`), and the units that end or take the host +(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, +`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled +into the binary and read from no configuration. An order is refused, recorded +and posted when the ordered unit, or an alias it resolves from, is one of them; +when it has an emergency action (`SuccessAction=`, `FailureAction=`, +`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` +or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another +unit is `BoundBy=` it, because systemd stops that unit with it whatever the job +mode; for a start or restart, when it has an `Upholds=` unit, which systemd +starts in a full transaction whatever the job mode; or, for a stop, restart or +reload, when it has a `ConsistsOf=` member, a wrapper such as +`postgresql.service` whose verbs would exit 0 and touch nothing. A unit +`systemctl show` cannot resolve fails the order. Slices, scopes, +targets and globs are not units an order can name. + +A package order installs the packages it names and nothing else: apt runs with +`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` +match themselves, and a last step fails the order unless every named package +is installed. + +An order runs once, after the steward's uid is checked and the instance's +brake check (`mind-control --instance check`) exits 0. The check +that the order's principal and uid are the bound steward's runs again at every +execution, so an order accepted before `steward.toml` changed is refused and +recorded, not run. The socket belongs to the steward user alone (mode 0600); +root may `show` and `list` orders but never issue one. An order that had +started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to +`#operator` as a retried projection of `orders.cc`: a chat outage delays the +posts and not the orders. A refusal names the field and a code, never the value +it refused. + ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service new file mode 100644 index 0000000..f2681c2 --- /dev/null +++ b/deploy/idunn-orders.service @@ -0,0 +1,47 @@ +[Unit] +Description=Idunn steward orders: typed package, unit and deploy orders from the steward +# Deliberately no After= or Requires= on any managed target, Synapse or +# idunn-yggdrasil: orders are how an operator repairs those, so none of them +# may be a condition of this unit starting. Posts are a retried projection and +# wait for Synapse on their own. + +[Service] +Type=simple + +# Root, from the same binary as idunn-yggdrasil, with its own store and socket. +# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: +# no order code runs in `idunn serve`. +# +# The unit name matches idunn-*, which is in the always-refused set, so no +# order can stop or restart this service. +User=root +# No Group= and no UMask=: the service gives its socket to the steward user +# (mode 0600) itself and sets Idunn's own umask for the files it writes, so +# nothing here reaches control.cc. The steward user is the only peer the +# service answers: it checks SO_PEERCRED on every connection. +ExecStart=/usr/local/bin/idunn orders serve \ + --store /var/lib/gamecult/idunn-orders \ + --state-store /var/lib/gamecult/idunn/control.cc \ + --steward-binding /etc/gamecult/idunn/steward.toml \ + --socket /run/idunn-orders/steward.sock +Restart=always +RestartSec=5 +TimeoutStopSec=20 + +RuntimeDirectory=idunn-orders +RuntimeDirectoryMode=0755 + +# The Synapse access token of @idunn, copied from a root-only file, and read +# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. +LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token + +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +# Its own order store, and the control store a deploy order commits a command +# to. Transient units started with systemd-run run in the manager's namespace, +# so packages and units are not limited by this list. +ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn + +[Install] +WantedBy=multi-user.target diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index da82bd2..1c45594 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -104,6 +104,11 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web +# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a +# target whose roots are not created yet cannot stop Idunn itself from starting +# (Idunn startup never depends on a managed target). A root created after Idunn +# starts is writable only from Idunn's next restart. +ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases [Install] WantedBy=multi-user.target diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md new file mode 100644 index 0000000..95c0eee --- /dev/null +++ b/docs/deploy-provenance-map.md @@ -0,0 +1,394 @@ +# Deployment provenance: map + +Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, +session `self-2026-10-23-eureka-b`). The ends are in +[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, +rulings, cut specs and follow-ups live in Eureka's mind; this page holds what +has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are +at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`, +the head of Hands attempt 1, which revision 2 of the spec builds on. + +## Cut `tree-executable-members` + +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2` +(revision 2, after Soul pass 1 found the real interpreter tree refused). It +serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's +admission waits on it) and will serve gamecult-chat C3 too: the presence +launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact +(README, launcher fragment), and Synapse needs a Python interpreter there. + +### Body facts + +- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable` + bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and + `validate` (`:740-760`) requires it to equal the declaration. +- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its + executable set from `root.join(receipt.destination)` only. `harden_root_tree` + (`:6858-6892`) seals every directory 0555 and every file 0444 unless that + exact path is in the set. A directory artifact with `executable = true` is + therefore silently meaningless today: its files are all 0444. Nothing refuses + it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin, + StreamPixels service and web surveyed): `executable = true` is on files only. +- T3. A tree output is copied by `copy_artifact`/`copy_tree` + (`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest + (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not + mode bits. So runner or git exec bits never reach the release: harden is the + only mode authority. +- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink + only if its target is relative with no `..`. It runs only inside + `harden_root_tree`, at install, so a tree with a `..` or absolute link copies, + digests and seals, then fails at `install_release` (Soul s1: copying and + digesting the real tree twice took 160 s before harden refused it). The rule + is sound but too narrow: it refuses the real interpreter tree (T13). The + frozen-source path has a physical resolver (`resolve_frozen_source_symlink`, + `:6617`) that walks the filesystem with a 40-link budget and a memo. Its + first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs + the tree on disk to answer. Revision 2 does not reuse it (T14). +- T5. The sealed release id hashes `(schema, plan_id, artifacts, + external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes + the whole plan including `recipe_blob`. A changed member list is therefore + already a new plan and a new release. The receipt must still carry the list, + because `harden_installed_release` reads receipts only; its two callers + (`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden + every installed release on each observation. +- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with + `#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every + existing receipt's bytes and release id: the plan's `deadlines` field + (`:574-577`) is the precedent. +- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file + whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB) + is held in Idunn's memory three times per install (seal, install, every + validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB + chunks (S9). The cut collapses both onto one streaming helper. +- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the + artifact destination a single filename. Members are paths inside that + destination, so it does not change. `require_relative_path` (`:1936-1949`) + with `allow_current = false` already refuses absolute, `..`, root and prefix + components. +- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy, + mode check, digest, pinned-digest check and receipt, the same steps as + `collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode + check survives every test. Revision 2 collapses both onto one `seal_artifact` + in `drivers.rs`. Each caller keeps only how it finds the source path. +- T10. The transient unit runs with `ProtectSystem=strict` and the release in + `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no + `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. + Shared libraries need no exec bit. +- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`, + installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is + identical between `8760a4f` and `6e2a31c`, so the live code is the + reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in + `src/`: the presence launcher and member route). Installing this cut ships all + of them. Idunn's own body changes only through gamecult-ops + `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, + bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T13. Link census of the real interpreter tree (Yggdrasil probe of + `cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`, + sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049 + after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all + their `..` components before the first name (every one is + `share/terminfo// -> ..//`, one `..` each). 0 have a `..` + after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it. + Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1). +- T14. Lexically normalising `/` does not check + containment. The kernel resolves a target one component at a time, so a `..` + after a symlinked name climbs from where that link points, not from where its + name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x` + normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x` + beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same + way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`, + `L -> D/D/../../../etc/passwd` passed a lexical check). + The sound lexical rule constrains the target's shape: all its `..` components + come before its first name, and they number no more than the link's depth + below the artifact's top. The climb then starts from the link's own + directory, which the walk reached through real directories only (copy and + digest never descend through a link), so it lands on a real ancestor inside + the artifact. After that the target only descends, and any link it passes + through obeys the same rule. By induction on the kernel's hops (capped at 40, + then ELOOP), every hop stays inside. The check is one pass over the target's + components with no filesystem reads, so its cost is linear in the tree and no + resolver is needed. The shapes it refuses (`s/../x`) occur in neither real + tree (T13). + Prior art, from documentation as known to Imagination (high confidence for Go + and securejoin, medium for the tar details): + - Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account + for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug. + - Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc, + containerd and umoci) resolve each component inside the root when a path is + opened, refusing or clamping an escape. They guard one later access, need + the tree on disk, and check again on every open. Idunn instead certifies a + sealed tree once, so that the kernel's own resolution stays inside on every + later access. + - Zip-slip defences and archive extractors check entry names lexically and + protect against link targets by refusing to write through links + (libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for + names; GNU tar creates such links last). They protect the extraction, not + what a kept link points at afterwards. + - OCI layer extraction keeps link targets verbatim, absolute ones included, + because the runtime resolves them inside the container's root. Idunn runs a + release in the host's mount namespace, so absolute targets stay refused. + - Nix keeps link targets verbatim and tracks dependencies by scanning for + references. The whole store is its root, so it has no per-artifact + containment to compare with. + None of these is a static per-link rule for a tree that runs unconfined. The + T14 shape rule fits that case. +- T15. Host targets: `idunn-host` observes, starts and stops processes on + Windows only (the non-Windows `platform` module in `src/host.rs` bails), and + `host.rs` sets no file modes. On Windows a file runs by its type, not by a + mode bit, so `executable` and `executable_members` have nothing to set there. + The file `executable` flag has always been carried and never applied on host + targets. A recipe does not know which workload kind its binding will choose. +- T16. The release root holds only artifact destinations. Runner workspaces, + where external inputs land, are removed before `SealedRelease::new` + (`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and + `validate_installed_release` (`:2815-2822`) digest every receipt's + destination before calling harden. So a rule in the digest walk runs at seal, + at install and at every validate, over every link harden would see. +- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three + copies (runner workspace, staging, installed). Nothing collects old releases or + staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every + Vili deploy leaves about 7 GiB behind. + +### Contract, corrected against the body + +- Any directory artifact may declare `executable_members` (a set of relative + paths). The follow-up limited it to runner-output; that limit protects no + invariant, and the launcher's worktree or fetched trees need the same feature. +- `executable = true` names a file artifact. On a directory output the release + is refused, and `executable_members` on a file output is refused (T2). +- A member is refused at recipe parse if it is empty, absolute, or holds `.`, + `..`, a root or a prefix. At seal it is refused if it is missing, is a + symlink, passes through a symlink or non-directory, or is not a regular file. + Refusal happens before `SealedRelease::new`, so no sealed release can name a + member it cannot seal. +- Harden seals each member 0555 and fails if any declared executable path was + not visited as a regular file. "Every declared executable is sealed 0555, or + the install fails" holds structurally, not by a lookup that can miss. +- Symlinks: a link in a tree artifact is kept when its target is relative, its + `..` components all come before its first name, and they number no more than + the link's depth below the artifact's top (T14). One function, + `check_release_link`, decides this. Only the digest walk calls it, so a bad + link refuses the release at seal, before `SealedRelease::new`, and again at + install and validate (T16). Harden no longer judges links. The climb is + bounded by the artifact's top, not the release root: no live release has a + `..` link (the old rule refused them all), so nothing installed is newly + refused, and each artifact's digest describes a closed tree. +- Host targets carry `executable` and `executable_members` in the receipt, + shape-checked at seal by the shared `seal_artifact`, and do not apply them + (T15). Refusing members there would make a recipe's validity depend on its + binding. +- Vili's trees must + not come from `python -m venv`: its `bin/python` is an absolute link and its + `pyvenv.cfg` names the build workspace. Install packages into the relocatable + interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). + +### Authority + +Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd +targets and by `materialize` in `host.rs` for host targets) decides +membership from the recipe and writes it into the receipt. Harden enforces the +receipt and decides nothing. Runner exec bits and git `100755` modes are dead +inputs for release modes. Link containment belongs to `check_release_link`, +which only `digest_tree` calls. `validate_release_symlink` is deleted, and +harden's symlink arm leaves links as they are. + +### Acceptance sizing + +The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch +on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It +removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 +terminfo links (T13). If a tree holds a link the T14 rule refuses, the job +prints it and stops, and the link is reported to Self, not deleted. + +## Cut `isolation-credential-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, +at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It +serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, +first deviation): every `idunn up vili` is refused before fencing. + +### Body facts (probed read-only on Yggdrasil, 2026-10-24) + +- I1. `idunn up vili` runs 6 and 7 failed with + `systemd workload isolation properties differ from the Idunn contract`, the + message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, + only one differs on the failed Vili unit: `LoadCredential`. Idunn expects + `""` because Vili has no service credential besides the presence identity + (`service_credential_names` empty); systemd shows `[unprintable]`. Every + other conjunct matches, InaccessiblePaths included (one value across all + five idunn-vili/odin units). +- I2. systemd 257.13 `systemctl show -p LoadCredential` prints + `LoadCredential=[unprintable]` for every unit, whatever the credentials: + `ssh.service` and `systemd-journald.service` (not DynamicUser, zero + LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin + unit (one). The display form carries no information, so the conjunct was + vacuous whenever credentials existed and always false when none did. Odin + and StreamPixels passed only because each carries a regular credential. +- I3. The exact value is readable: D-Bus + `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) + answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 + "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. +- I4. What the workload can actually read is the delivered directory. On the + host, `/run/credentials/` holds a directory only for units with credentials + (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID + root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` + (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` + descriptor, never a credential. +- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn + deployment transaction authority"), with its twin clause in + `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit + touched it. No ruling or finding in any campaign concerns workload isolation + verification; gamecult-chat's foreign-workloads cut touched + `CREDENTIALS_DIRECTORY` reservation, not this check. +- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both + MessagePack encodings, compact (positional) included + (`persisted_systemd_observations_decode_as_the_systemd_variant`, + `:9915-9930`), so deleting the `load_credential` field shifts every later + slot of a live incumbent's record. The field is also in the drift list + (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed + workload: changing what the field holds without leaving that list would make + the next Idunn refuse to stop the live Odin and StreamPixels incumbents. +- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 + properties for a Vili unit (presence identity only) and the Odin unit + (presence identity plus one service credential), captured 2026-10-24 into + this Imagination's scratchpad `fixtures/`. +- I8. The new rule holds for every live systemd incumbent: inside each + MainPID root the delivered entries equal its binding's `secret_files` keys + minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, + StreamPixels web 1; key names compared, no values read). An Idunn restart onto + this cut re-observes them (`:3955-3970`) and passes. + +### Decision + +A property systemd cannot display does not belong in the comparison. The +invariant it reached for, that the workload holds exactly the credentials Idunn +delivered, is observed where it is decided: the entries of +`/run/credentials/` in the MainPID's root equal Idunn's service +credential names, absent directory equal to none. That covers +`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and +needs no new program. The D-Bus read (I3) is exact but sees only one of those +properties and adds a second systemd client and parser; rejected. + +### Authority + +Owner: `observe_unit` decides the credential rule, from the delivered +directory. `LoadCredential`'s display form is no longer read or compared; the +`load_credential` field is a retired wire slot, written empty, out of the drift +list, kept only so persisted records decode. Forbidden writers: the +`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping +the slot waits on the next schema revision of the observation. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). After it, Vili's Hands reruns from step 4 of +`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. + +## Cut `runner-failure-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at +`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area +Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, +transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 +UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and +nobody could tell which recipe step failed. + +### Body facts (source read at `804ed75`; host read-only, 2026-10-25) + +- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by + `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and + `Command::output()`: stdout and stderr are each buffered whole in the + daemon's memory, unbounded. On a non-zero exit the error is + `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The + container is gone the moment it exits, so `docker ps -a` and `docker logs` + hold nothing. +- R2. The empty message: the failing process wrote nothing to stderr, and + stdout, the only stream that carried its report, is discarded at + `:2182-2193`. The same interpolation echoes whatever a step does write to + stderr into `last_error`, `pre_fencing_abort.error` and + `completion.error` (`bounded_detail`, 2048 bytes), and from there into + `idunn up`'s `deployment failed:` line and `idunn status`. +- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops + over `declaration.steps` and calls `run_in_workspace` with argv only; + `step.id` is used only in the environment precondition. External inputs + (`curl` in the runner, `:2247-2258`) fail through the same helper with the + same anonymous message. +- R4. Which step failed on 2026-10-25, inferred from the failed transaction's + staging root, which survives (926 MiB, + `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, + `node_modules/.package-lock.json` was written at 00:51:24 (so + `install-vili-dependencies`, `npm ci`, finished), nothing outside + `node_modules` is newer, and `release/vili` (the next step's output) is + absent; the python workspace holds only fetched inputs. The failing step was + `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test + reporter writes to stdout: exactly the stream R1 discards. Inference from + residue, not from output. +- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) + sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which + persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal + on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only + provider-exclusion lines. +- R6. Stores. `control.cc` (root 0640) holds commands, transactions + (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and + expiry requests; `ControlSnapshot::read` refuses any other type + (`:3420`). Records must re-encode byte-exactly (`decode_record`, + `:3717-3727`), so a new transaction field is a schema bump and a v4 lift + (`:4100-4110`). Terminal transactions move verbatim to `history.cc` + (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. + `incidents.cc` lives in the published projection directory and by its own + doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, + because a type added there stops continuity for every target on a rollback. +- R7. The daemon runs as root (`User=root`). Runner secrets reach the + container as read-only file mounts named by env (`:2041-2047`), so any text + a step derives from them can land in its output. +- R8. Staging residue: 475 transaction roots, 55 GiB under + `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces + because `materialize` returns before `remove_tree_inside`. + +### Decision + +The failure becomes a typed value at the runner, `RunnerStepFailure`: the +subject (recipe step or external input, by id), the runner id, the exit code or +signal, and the last 8 KiB of stdout and of stderr with each stream's total +byte count. Its `Display` carries the subject and status, never output. +Capture streams both pipes through bounded tails, so a verbose build no longer +buffers whole in the daemon. + +The tail lives in a new private store, `runner-evidence.cc`, beside +`control.cc` (mode 0600), one record per transaction +(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the +transaction record: that is a v5 bump whose rollback makes every older Idunn +refuse its control store (R6), the scar `incident.rs` already names. Not in +`incidents.cc`: published, and free text is barred there. Not in the journal: +journal readers are a wider set than root, so the journal line names the +transaction, the subject, the status and the store, and no output. + +No redaction. Matching secret bytes would make the daemon read every secret, +and still misses any encoding, prefix or transform of one; a claimed redaction +that leaks is worse than a stated rule. The rule instead is the reader set: the +store's only readers (root) can already read every secret the runner was +given. + +Keeping the failed container (dropping `--rm`) is worse: its output then lives +in Docker's json-file log, unbounded, untyped, outside Idunn's state and +readable by the docker group; containers accumulate with no owner to reap them; +`docker system prune` erases the evidence; and nothing ties a container to the +transaction in `idunn status`. + +### Authority + +Owner: the runner decides what failed and keeps the bounded tail; the engine's +pre-fence failure path is the one writer of `runner-evidence.cc`. +`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the +failure's `Display`. Forbidden writers: the `stderr` interpolation in +`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the +store needs no external owner; staging roots (R8) stay with +`ghostlight-verse:follow_up:idunn-generation-retention`. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). The next failed Sealing then shows its step and tail on +`sudo idunn status --command `. diff --git a/src/control_plane.rs b/src/control_plane.rs index 9597d43..cd49073 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -42,11 +42,14 @@ use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, RunnerStepFailure, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; +use crate::runner_evidence::{ + self, Kept, RunnerFailureRecord, runner_evidence_store_path, unreadable_store_path, +}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -2931,6 +2934,10 @@ enum Command { recipe: PathBuf, binding: Option, }, + #[cfg(unix)] + OrdersServe(crate::steward_orders::ServeOptions), + #[cfg(unix)] + Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { @@ -2970,6 +2977,10 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + #[cfg(unix)] + Command::OrdersServe(options) => crate::steward_orders::serve(options), + #[cfg(unix)] + Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -3022,6 +3033,10 @@ fn parse(args: impl Iterator) -> Result { "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), + #[cfg(unix)] + "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), + #[cfg(unix)] + "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -4727,20 +4742,21 @@ fn resolve_selector( Ok(targets) } -fn submit( +/// The one commit primitive for a deployment command: `idunn up` and a steward +/// deploy order both write theirs here. Returns the commit time. +pub(crate) fn commit_deployment_command( store_path: &Path, + command_id: &str, selector: &str, requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { +) -> Result { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), + command_id: command_id.into(), kind: CommandKind::Deploy, selector: selector.into(), requested_by: requested_by.into(), @@ -4758,7 +4774,33 @@ fn submit( )?, "deployment command id collided" ); - println!("{}", command.command_id); + Ok(now) +} + +/// A command's present state as `idunn status --command` derives it, for the +/// steward order poster: `queued` and `running` are not terminal. +#[cfg(unix)] +pub(crate) fn command_state( + state_store: &Path, + command_id: &str, +) -> Result<(&'static str, String)> { + let snapshot = ControlSnapshot::read(state_store)?; + let archived = read_history_transactions(state_store); + let mut transactions = snapshot.transaction_for_command(command_id); + transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); + Ok(derived_command_status(&transactions)) +} + +fn submit( + store_path: &Path, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + let command_id = format!("up-{}", Uuid::new_v4()); + let now = commit_deployment_command(store_path, &command_id, selector, requested_by)?; + println!("{command_id}"); if !wait { return Ok(()); } @@ -4769,10 +4811,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command.command_id), + .any(|stored| stored.value.command_id == command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command.command_id); + let transactions = snapshot.transaction_for_command(&command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4797,7 +4839,7 @@ fn submit( } } -fn status( +pub(crate) fn status( out: &mut impl Write, store_path: &Path, incident_store: &Path, @@ -4877,6 +4919,13 @@ fn status( } if let Some(abort) = &transaction.pre_fencing_abort { writeln!(out, " pre-fencing abort {abort:?}")?; + // The step that failed and its output tail live in their + // own private store. An unreadable store costs its lines. + runner_evidence::show_status( + out, + &runner_evidence_store_path(store_path), + &transaction.transaction_id, + )?; } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -5374,6 +5423,32 @@ impl Engine { IncidentStore::new(&self.options.incident_store) } + /// A failed runner step keeps its output tail in the runner evidence + /// store and says so on the journal, naming no output. The store write is + /// best effort: its failure is named in the line and never stops the + /// abort that follows. + fn keep_runner_failure( + &self, + transaction: &DeploymentTransaction, + error: &anyhow::Error, + ) -> Option { + let failure = error.downcast_ref::()?; + let path = runner_evidence_store_path(&self.options.state_store); + let recorded = now_millis().and_then(|now| { + runner_evidence::record( + &path, + &RunnerFailureRecord::from_failure( + &transaction.transaction_id, + &transaction.command_id, + &transaction.target, + failure, + now, + ), + ) + }); + Some(runner_failure_journal_line(transaction, failure, &path, &recorded)) + } + /// Settle the outcome of one incident operation. A failure never leaves /// this function: it is traced once per episode, under the operation and /// the subject it was decided for (`*` for a pass over the whole store), @@ -6315,6 +6390,9 @@ impl Engine { if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() { + if let Some(line) = self.keep_runner_failure(&latest.value, &error) { + eprintln!("{line}"); + } self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -10553,7 +10631,13 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -fn now_millis() -> Result { +/// The control store `idunn up` and `idunn serve` use unless told otherwise. +#[cfg(unix)] +pub(crate) fn default_state_store() -> PathBuf { + RuntimeOptions::default().state_store +} + +pub(crate) fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -10573,12 +10657,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -fn string_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -fn path_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -10594,7 +10678,7 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -fn require_selector(value: &str) -> Result<()> { +pub(crate) fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } @@ -10641,6 +10725,43 @@ const DETAIL_EMPTY: &str = "(no detail)"; /// only judge, and this output always satisfies it. NUL is dropped, empty /// input gets a fixed placeholder, and an over-long value keeps its head and /// its tail (where a build log's error is) cut on char boundaries. +/// The journal line for a failed runner step: the transaction, the subject and +/// status (the failure's `Display`), and where the evidence is. Never output. +fn runner_failure_journal_line( + transaction: &DeploymentTransaction, + failure: &RunnerStepFailure, + store_path: &Path, + recorded: &Result, +) -> String { + let head = format!( + "Idunn transaction {} (command {}, target {}) {failure}", + transaction.transaction_id, transaction.command_id, transaction.target + ); + match recorded { + Ok(kept) => { + let aside = match kept { + Kept::Stored => String::new(), + Kept::SetAside { replaced } => format!( + "; an undecodable evidence store was set aside as {}{}", + unreadable_store_path(store_path).display(), + match replaced { + Some(last_written) => format!( + ", replacing an earlier set-aside store last written at unix millis {last_written}" + ), + None => String::new(), + } + ), + }; + format!( + "{head}; evidence in {}{aside}; read it with idunn status --command {}", + store_path.display(), + transaction.command_id + ) + } + Err(error) => format!("{head}; evidence was not kept: {error:#}"), + } +} + fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -10681,7 +10802,10 @@ fn usage() -> &'static str { idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\ + idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ + idunn order --reason TEXT\n\ + idunn order show ID | idunn order list\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -11893,9 +12017,45 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["order", "run", "sh", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], + vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], + vec!["orders", "serve", "--allow-units", "nginx.service"], + vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } + // The declarative order forms parse, and only as order commands. + #[cfg(unix)] + for arguments in [ + vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r"], + vec!["order", "list"], + vec!["orders", "serve"], + ] { + let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); + assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); + } + } + + #[test] + fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + commit_deployment_command(&store, "up-one", "vili", "steward:eureka order ord-1")?; + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.commands.len(), 1); + let command = &snapshot.commands[0].value; + assert_eq!(command.command_id, "up-one"); + assert_eq!(command.selector, "vili"); + assert_eq!(command.requested_by, "steward:eureka order ord-1"); + assert!(commit_deployment_command(&store, "up-one", "other", "x").is_err()); + assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); + Ok(()) } #[test] @@ -24657,4 +24817,435 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } } + + // BEGIN runner-failure-evidence tests + // A failed runner step is named, its output tail is kept in + // runner-evidence.cc, and no output reaches an error, control.cc or the + // journal line. The runner is the real DockerRunnerDriver driving a fake + // docker program, through the engine's Sealing path. + + const RUNNER_CANARY: &str = "CANARY-7c41e0d9"; + + /// A source that freezes a fixed tree, so a deployment reaches the runner. + struct TreeSource { + recipe: String, + root: PathBuf, + } + + impl SourcePort for TreeSource { + fn resolve( + &self, + _: &OperatorBinding, + _: &str, + _: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result { + Ok(FrozenSourceReceipt { + transaction_id: transaction_id.to_owned(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: format!("sha256-{}", "a".repeat(64)), + }) + } + + fn observe_frozen( + &self, + plan: &CompiledDeploymentPlan, + receipt: &FrozenSourceReceipt, + ) -> Result { + Ok(crate::drivers::FrozenSource::fixture( + receipt.clone(), + plan.source.clone(), + plan.recipe_blob.clone(), + self.root.clone(), + )) + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// The unrouted recipe with two steps ahead of its build, and no external + /// input: the second step, `test-daemon`, is the one the fake docker fails. + fn two_step_recipe() -> String { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let (head, rest) = recipe + .split_once("[[external_inputs]]") + .expect("the recipe declares an external input"); + let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts"); + let head = head.replacen( + "[[steps]]\nid = \"build\"", + "[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\ + [[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\ + [[steps]]\nid = \"build\"", + 1, + ); + format!("{head}[[artifacts]]{tail}") + } + + /// A Sealing world whose runner is the real driver over a fake docker: the + /// first call succeeds, the second prints the canary and an escape + /// sequence on both streams and exits 3. + #[cfg(unix)] + fn failing_runner_world() -> Result<(EngineFixture, TempDir)> { + use std::os::unix::fs::PermissionsExt; + let tools = TempDir::new()?; + let docker = tools.path().join("docker"); + std::fs::write( + &docker, + format!( + "#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n" + ), + )?; + std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?; + let recipe = two_step_recipe(); + let mut world = sealing_world(&recipe)?; + let tree = tools.path().join("tree"); + std::fs::create_dir_all(&tree)?; + std::fs::write(tree.join("README"), b"source")?; + world.engine.source = Arc::new(TreeSource { recipe, root: tree }); + world.engine.docker_runner = DockerRunnerDriver { docker_program: docker }; + Ok((world, tools)) + } + + /// Scheduler ticks until the deployment ends (or the budget is spent). + fn tick_until_ended(world: &EngineFixture) -> Result { + for _ in 0..24 { + world.engine.resume_backoff.lock().unwrap().clear(); + let _ = world.engine.run_scheduler_tick(); + if latest(world)?.completion.is_some() { + break; + } + } + latest(world) + } + + fn printed_status(world: &EngineFixture) -> Result { + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?; + Ok(String::from_utf8(printed)?) + } + + fn canary_failure(stdout: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject: crate::drivers::RunnerSubject::Step("test-daemon".into()), + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stdout.to_vec(), + stderr_total_bytes: stdout.len() as u64, + } + } + + #[cfg(unix)] + #[test] + fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error); + }; + // Named: the second step, on its runner, with the exit status. + assert!( + error.contains("runner step test-daemon on runner rust exited with exit status 3"), + "{error}" + ); + // Never echoed: no error text, no control.cc byte, carries the output. + let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; + for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); + } + // The terminal transaction has moved to history.cc: both stores. + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + let bytes = std::fs::read(&store)?; + assert!(!bytes.is_empty(), "{} is empty", store.display()); + assert!( + !bytes.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), + "runner output reached {}", + store.display() + ); + } + // Kept: in the private store, tails of both streams, and on status. + let evidence = runner_evidence_store_path(&world.state_store); + assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("no evidence for the failed transaction")?; + assert_eq!(record.subject_id, "test-daemon"); + assert_eq!(record.exit_code, Some(3)); + assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY)); + let printed = printed_status(&world)?; + assert!(printed.contains("failed step test-daemon"), "{printed}"); + assert!(printed.contains(RUNNER_CANARY), "{printed}"); + assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte"); + Ok(()) + } + + #[test] + fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> { + let world = EngineFixture::new()?; + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let line = world + .engine + .keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing")) + .context("a runner failure produced no journal line")?; + assert!(!line.contains(RUNNER_CANARY), "{line}"); + for needle in [ + transaction.transaction_id.as_str(), + "up-ghostlight", + "ghostlight", + "runner step test-daemon on runner rust exited with exit status 3", + "runner-evidence.cc", + "idunn status --command up-ghostlight", + ] { + assert!(line.contains(needle), "{needle} missing from {line}"); + } + // Not a runner failure: nothing is kept and nothing is said. + assert!( + world + .engine + .keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure")) + .is_none() + ); + // The line a failed write produces says so, with no output. + let unwritable = runner_failure_journal_line( + &transaction, + &failure, + Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"), + &Err(anyhow::anyhow!("disk full")), + ); + assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}"); + assert!(!unwritable.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + // A directory where the store should be: every write to it fails. + std::fs::create_dir(runner_evidence_store_path(&world.state_store))?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b""))) + .context("no journal line")?; + assert!(line.contains("evidence was not kept"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_undecodable_evidence_store_does_not_stop_the_next_failure_keeping_its_evidence() -> Result<()> { + let damage = format!("\u{c1} not a store {RUNNER_CANARY}-in-the-damage"); + let (world, _tools) = failing_runner_world()?; + let evidence = runner_evidence_store_path(&world.state_store); + std::fs::write(&evidence, damage.as_bytes())?; + let ended = tick_until_ended(&world)?; + // The transaction's own outcome is the runner failure, not the store. + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the damaged store changed the outcome: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + assert!(!error.contains("damage"), "{error}"); + // The damage is kept aside; the failure's evidence is in a fresh store. + assert_eq!(std::fs::read(unreadable_store_path(&evidence))?, damage.as_bytes()); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("the failure kept no evidence beside a damaged store")?; + assert_eq!(record.subject_id, "test-daemon"); + assert!(printed_status(&world)?.contains("failed step test-daemon")); + // The journal says it, naming the file and none of its content. + std::fs::write(&evidence, damage.as_bytes())?; + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"x"))) + .context("no journal line")?; + assert!(line.contains("an undecodable evidence store was set aside as"), "{line}"); + assert!(line.contains("runner-evidence.unreadable.cc"), "{line}"); + assert!(!line.contains("damage") && !line.contains("not a store"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?; + let printed = printed_status(&world)?; + assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}"); + assert!(printed.contains("target service phase"), "{printed}"); + assert!(printed.contains("pre-fencing abort"), "{printed}"); + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + Ok(()) + } + + /// Everything status prints from the evidence store goes through one + /// allowlist, so a hostile build log reaches the terminal as visible + /// escapes whatever its class. + #[cfg(unix)] + #[test] + fn status_prints_hostile_runner_output_as_visible_escapes_only() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let hostile: Vec<(&str, &str)> = vec![ + ("\u{1b}[2J", "\\u{1b}[2J"), + ("\u{7}", "\\u{7}"), + ("\r", "\\u{d}"), + ("\u{202e}", "\\u{202e}"), + ("\u{2066}", "\\u{2066}"), + ("\u{85}", "\\u{85}"), + ("\u{e0041}", "\\u{e0041}"), + ("\u{200b}", "\\u{200b}"), + ("\u{feff}", "\\u{feff}"), + ]; + let mut tail: Vec = Vec::new(); + for &(raw, _) in &hostile { + tail.extend_from_slice(b"a"); + tail.extend_from_slice(raw.as_bytes()); + tail.extend_from_slice(b"b\n"); + } + tail.extend_from_slice(b"bad \xff\xc1 bytes\n"); + runner_evidence::record( + &runner_evidence_store_path(&world.state_store), + &RunnerFailureRecord::from_failure( + &ended.transaction_id, + &ended.command_id, + &ended.target, + &canary_failure(&tail), + 9, + ), + )?; + let printed = printed_status(&world)?; + for &(raw, escaped) in &hostile { + assert!(!printed.contains(raw), "{escaped} reached the terminal raw"); + assert!(printed.contains(&format!("a{escaped}b")), "{escaped} missing: {printed}"); + } + assert!(printed.contains("bad \\xff\\xc1 bytes"), "{printed}"); + let section = printed.find(" failed step").context("no evidence section")?; + assert!(printed[section..].is_ascii(), "non-ASCII reached the terminal: {printed}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn status_prints_the_unreadable_store_line_escaped() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + let evidence = runner_evidence_store_path(&world.state_store); + // The decode error of this store quotes the stored subject kind. + let mut bytes = std::fs::read(&evidence)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker in the evidence store")?; + bytes[at..at + 4].copy_from_slice(b"\x1b[2J"); + std::fs::write(&evidence, bytes)?; + let printed = printed_status(&world)?; + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + assert!(!printed.contains('\u{1b}'), "an escape byte reached the terminal"); + assert!(printed.contains("\\u{1b}[2J"), "{printed}"); + Ok(()) + } + + #[test] + fn the_journal_line_names_a_displaced_set_aside_store_by_time_only() -> Result<()> { + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let path = Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"); + let first = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: None }), + ); + assert!(first.contains("was set aside as"), "{first}"); + assert!(!first.contains("replacing"), "{first}"); + let second = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: Some(1234) }), + ); + assert!( + second.contains("replacing an earlier set-aside store last written at unix millis 1234"), + "{second}" + ); + assert!(!second.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + // The store a rolled-back Idunn reads: it refuses any foreign type. + ControlSnapshot::read(&world.state_store)?; + let mut transactions = 0; + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + for envelope in SingleFileMessagePackBackingStore::new(&store).pull_all_read_only_snapshot()? { + assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); + if envelope.r#type == DeploymentTransaction::TYPE { + transactions += 1; + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + } + } + } + assert_eq!(transactions, 1, "the failed transaction is in neither store"); + Ok(()) + } + + /// The records control.cc holds re-encode byte for byte. The hex was + /// captured at base 804ed75, before this cut touched anything: a runner + /// failure adds no field, no type and no schema to the control store. + #[test] + fn control_records_encode_exactly_as_they_did_at_base() -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?; + transaction.transaction_id = "tx-pinned".into(); + transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into()); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "runner step test-daemon on runner rust exited with exit status 3".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); + let command_hex = hex(&command_envelope(&command, 100)?.payload); + let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload); + assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed"); + assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed"); + Ok(()) + } + + const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; + const PINNED_TRANSACTION_HEX: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; + // END runner-failure-evidence tests } diff --git a/src/drivers.rs b/src/drivers.rs index 0eb0928..3ea1145 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -157,6 +157,24 @@ pub struct FrozenSourceReceipt { pub snapshot_sha256: String, } +#[cfg(test)] +impl FrozenSource { + /// A frozen source over an existing tree, for tests that drive a runner. + pub(crate) fn fixture( + receipt: FrozenSourceReceipt, + facts: SourceSelectionFacts, + recipe_bytes: Vec, + root: PathBuf, + ) -> Self { + Self { + receipt, + facts, + recipe_bytes, + root, + } + } +} + impl FrozenSourceReceipt { pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> { plan.validate()?; @@ -470,6 +488,9 @@ pub struct SystemdWorkloadObservation { pub no_new_privileges: bool, pub umask: String, pub inaccessible_paths: String, + /// Retired wire slot. systemd 257 displays `LoadCredential` as `[unprintable]` for + /// every unit, so it was never evidence: written empty, never compared. Kept so + /// compact (positional) MessagePack records of earlier Idunns still decode. pub load_credential: String, pub main_pid: u32, pub process_start_time: u64, @@ -584,7 +605,9 @@ pub trait WorkloadPort { /// /// Idunn launches candidates as transient units with `Restart=no`, so a /// unit that has failed, or that systemd no longer knows, will not come - /// back on its own. Past the fence every error is otherwise treated as + /// back on its own. Those units are collected once they fail + /// (`INCARNATION_UNIT_LIFECYCLE`), so a failed candidate usually reads as + /// forgotten rather than failed. Past the fence every error is otherwise treated as /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; @@ -2149,6 +2172,115 @@ pub(crate) fn docker_run_args( Ok(args) } +/// Bytes of each output stream that a failed runner step keeps. The daemon +/// never holds more than this per stream while the step runs. +pub(crate) const RUNNER_OUTPUT_TAIL_BYTES: usize = 8192; + +/// What a runner container was running: a recipe step or an external input, +/// by the id the recipe gave it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RunnerSubject { + Step(String), + ExternalInput(String), +} + +impl RunnerSubject { + pub(crate) fn id(&self) -> &str { + match self { + Self::Step(id) | Self::ExternalInput(id) => id, + } + } + + fn noun(&self) -> &'static str { + match self { + Self::Step(_) => "step", + Self::ExternalInput(_) => "external input", + } + } +} + +/// A runner container that did not exit successfully: what failed, how, and +/// the last `RUNNER_OUTPUT_TAIL_BYTES` of each stream with its total size. +/// The tails are evidence for `runner-evidence.cc`. `Display` and `Debug` +/// carry none of them, so no error chain, record or log line renders output. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RunnerStepFailure { + pub(crate) subject: RunnerSubject, + pub(crate) runner: String, + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stdout_total_bytes: u64, + pub(crate) stderr_tail: Vec, + pub(crate) stderr_total_bytes: u64, +} + +impl RunnerStepFailure { + /// The status as an operator reads it. + pub(crate) fn status(&self) -> String { + runner_status(self.exit_code, self.signal) + } +} + +/// How a container ended, as an operator reads it. +pub(crate) fn runner_status(exit_code: Option, signal: Option) -> String { + match (exit_code, signal) { + (Some(code), _) => format!("exit status {code}"), + (None, Some(signal)) => format!("signal {signal}"), + (None, None) => "an unknown status".to_owned(), + } +} + +impl std::fmt::Display for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "runner {} {} on runner {} exited with {}; its output tail is kept in runner-evidence.cc", + self.subject.noun(), + self.subject.id(), + self.runner, + self.status() + ) + } +} + +impl std::fmt::Debug for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerStepFailure") + .field("subject", &self.subject) + .field("runner", &self.runner) + .field("exit_code", &self.exit_code) + .field("signal", &self.signal) + .field("stdout_total_bytes", &self.stdout_total_bytes) + .field("stderr_total_bytes", &self.stderr_total_bytes) + .finish_non_exhaustive() + } +} + +impl std::error::Error for RunnerStepFailure {} + +/// Read `reader` to its end keeping only the last `cap` bytes and counting +/// every byte, so a verbose step never buffers whole in the daemon. +fn drain_tail(mut reader: impl Read, cap: usize) -> std::io::Result<(Vec, u64)> { + let mut tail = Vec::new(); + let mut total = 0u64; + let mut chunk = [0u8; 8192]; + loop { + let read = match reader.read(&mut chunk) { + Ok(0) => return Ok((tail, total)), + Ok(read) => read, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + total += read as u64; + tail.extend_from_slice(&chunk[..read]); + if tail.len() > cap { + tail.drain(..tail.len() - cap); + } + } +} + /// Docker is only a runner substrate. Recipe argv stays an argv vector, the /// operator binding selects the exact image/network/mount affordances, and the /// driver returns complete materialization receipts rather than build truth by @@ -2167,7 +2299,7 @@ impl Default for DockerRunnerDriver { } impl DockerRunnerDriver { - fn docker(&self, args: I) -> Result + fn docker(&self, subject: RunnerSubject, runner_id: &str, args: I) -> Result<()> where I: IntoIterator, S: AsRef, @@ -2176,23 +2308,50 @@ impl DockerRunnerDriver { self.docker_program.is_absolute(), "Docker runner program is not absolute" ); - let output = Command::new(&self.docker_program) + let mut child = Command::new(&self.docker_program) .args(args) .stdin(Stdio::null()) - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .with_context(|| format!("starting Docker runner {}", self.docker_program.display()))?; - if !output.status.success() { - bail!( - "Docker runner exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); + let stdout = child.stdout.take().context("Docker runner has no stdout pipe")?; + let stderr = child.stderr.take().context("Docker runner has no stderr pipe")?; + let stdout = thread::spawn(move || drain_tail(stdout, RUNNER_OUTPUT_TAIL_BYTES)); + let stderr = thread::spawn(move || drain_tail(stderr, RUNNER_OUTPUT_TAIL_BYTES)); + let status = child.wait().context("waiting for the Docker runner")?; + let (stdout_tail, stdout_total_bytes) = stdout + .join() + .map_err(|_| anyhow!("the runner stdout reader panicked"))? + .context("reading the runner stdout")?; + let (stderr_tail, stderr_total_bytes) = stderr + .join() + .map_err(|_| anyhow!("the runner stderr reader panicked"))? + .context("reading the runner stderr")?; + if status.success() { + return Ok(()); } - Ok(output) + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal = None; + Err(RunnerStepFailure { + subject, + runner: runner_id.to_owned(), + exit_code: status.code(), + signal, + stdout_tail, + stdout_total_bytes, + stderr_tail, + stderr_total_bytes, + } + .into()) } fn run_in_workspace( &self, + step_id: &str, + runner_id: &str, runner: &DockerRunnerBinding, workspace: &Path, working_directory: &Path, @@ -2209,8 +2368,7 @@ impl DockerRunnerDriver { build_machine_id_file(workspace)?; let spec = ContainerSpec::for_step(runner, required_environment, source_stamp)?; let args = docker_run_args(&spec, workspace, working_directory, argv)?; - self.docker(args)?; - Ok(()) + self.docker(RunnerSubject::Step(step_id.to_owned()), runner_id, args) } fn materialize_external_input( @@ -2252,7 +2410,11 @@ impl DockerRunnerDriver { input.url.clone(), ]; let args = docker_run_args(&spec, workspace, Path::new("."), &curl_argv)?; - self.docker(args)?; + self.docker( + RunnerSubject::ExternalInput(input.id.clone()), + &input.runner, + args, + )?; let bytes = fs::read(&destination).with_context(|| { format!( "reading materialized external input {}", @@ -2374,6 +2536,8 @@ impl RunnerPort for DockerRunnerDriver { ); } self.run_in_workspace( + &step.id, + &step.runner, runner, workspace, &step.working_directory, @@ -2412,6 +2576,13 @@ impl RunnerPort for DockerRunnerDriver { } } +/// How every incarnation unit lives and dies. It runs once (`Restart=no`) +/// and systemd forgets it once it is inactive or failed (`--collect`): the +/// outcome is Idunn's to record in its own store, and a failed transient unit +/// left loaded is only clutter that marks the host degraded. +const INCARNATION_UNIT_LIFECYCLE: [&str; 3] = + ["--collect", "--property=Type=exec", "--property=Restart=no"]; + /// systemd owns process execution. This driver lowers one validated launch /// contract into a transient unit and then proves the native process and /// executable that systemd actually started. It does not decide admission, @@ -2459,7 +2630,6 @@ fn changed_systemd_workload_identity_fields( no_new_privileges, umask, inaccessible_paths, - load_credential, main_pid, process_start_time, process_uids, @@ -2859,7 +3029,6 @@ impl SystemdTransientWorkloadDriver { OsString::from("--property=NoNewPrivileges"), OsString::from("--property=UMask"), OsString::from("--property=InaccessiblePaths"), - OsString::from("--property=LoadCredential"), OsString::from("--property=OpenFile"), OsString::from("--property=WorkingDirectory"), OsString::from("--property=ControlGroup"), @@ -2971,38 +3140,7 @@ impl SystemdTransientWorkloadDriver { let umask = required_systemd_property(&values, "UMask")?.to_owned(); let inaccessible_paths = required_systemd_property(&values, "InaccessiblePaths")?.to_owned(); - let load_credential = required_systemd_property(&values, "LoadCredential")?.to_owned(); - ensure!( - service_type == "exec" - && restart_policy == "no" - && kill_mode == "mixed" - // `User` is deliberately not asserted here. Idunn never emits a - // User property, and systemd reports the identity it *allocated* - // for a DynamicUser unit -- the unit name, or a generated `_du…` - // when that is too long -- so the property is never empty for a - // running workload and cannot tell an allocated identity from a - // pinned one. The invariant it was reaching for, that the workload - // does not run as a persistent named account, is carried by - // `dynamic_user` below and by Idunn never passing User at all. - && dynamic_user - && supplementary_groups.is_empty() - && capability_bounding_set.is_empty() - && ambient_capabilities.is_empty() - && private_mounts - && private_pids - && protect_proc == "invisible" - && proc_subset == "all" - && no_new_privileges - && umask == "0007" - && inaccessible_paths == self.credential_root.display().to_string() - && load_credential - == if service_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }, - "systemd workload isolation properties differ from the Idunn contract" - ); + require_systemd_isolation_contract(&values, &self.credential_root)?; let working_directory = PathBuf::from(required_systemd_property(&values, "WorkingDirectory")?); ensure!( @@ -3144,6 +3282,11 @@ impl SystemdTransientWorkloadDriver { ); Some(path) }; + require_delivered_credentials( + &process_root.join("root"), + unit, + service_credential_names, + )?; let activation_descriptor = &parent_only_file_descriptors[0]; ensure!( activation_descriptor.fd_name == IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME @@ -3251,7 +3394,7 @@ impl SystemdTransientWorkloadDriver { no_new_privileges, umask, inaccessible_paths, - load_credential, + load_credential: String::new(), main_pid, process_start_time, process_uids: process_security.uids, @@ -3465,12 +3608,7 @@ impl SystemdTransientWorkloadDriver { // workload. The same applies to `Group` when no fixed state group is // bound. In that case the numeric primary GID must match the allocated // UID; a fixed state group still requires both its exact name and GID. - let expected_load_credential = if regular_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }; - let launch_contract: [(bool, &str); 27] = [ + let launch_contract: [(bool, &str); 26] = [ ( observation.unit_description == expected_description, "unit description", @@ -3510,10 +3648,6 @@ impl SystemdTransientWorkloadDriver { observation.inaccessible_paths == self.credential_root.display().to_string(), "inaccessible paths", ), - ( - observation.load_credential == expected_load_credential, - "load credential", - ), ( observation.working_directory == installed, "working directory", @@ -3639,8 +3773,9 @@ impl SystemdTransientWorkloadDriver { OsString::from("--no-ask-password"), OsString::from("--expand-environment=no"), OsString::from(format!("--unit={unit}")), - OsString::from("--property=Type=exec"), - OsString::from("--property=Restart=no"), + ]; + args.extend(INCARNATION_UNIT_LIFECYCLE.map(OsString::from)); + args.extend([ OsString::from("--property=KillMode=mixed"), OsString::from("--property=DynamicUser=yes"), OsString::from("--property=SupplementaryGroups="), @@ -3676,7 +3811,7 @@ impl SystemdTransientWorkloadDriver { OsString::from(format!("--working-directory={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", bundle.display())), - ]; + ]); for open_file in parent_only_open_files { args.push(OsString::from(format!("--property=OpenFile={open_file}"))); } @@ -7269,6 +7404,44 @@ fn path_inside_process_root(process_root: &Path, absolute_path: &Path) -> Result Ok(process_root.join(absolute_path.strip_prefix(Path::new("/"))?)) } +/// The workload holds exactly the credentials Idunn delivered: the entries of +/// `/run/credentials/` in the MainPID's root (`process_root` is that root) +/// equal `expected`, and an absent directory is the empty set. Only entry names +/// are read; a credential is never opened here. The message names no credential. +fn require_delivered_credentials( + process_root: &Path, + unit: &str, + expected: &[String], +) -> Result<()> { + let directory = path_inside_process_root( + process_root, + &Path::new("/run/credentials").join(unit), + )?; + let mut delivered = Vec::new(); + match fs::read_dir(&directory) { + Ok(entries) => { + for entry in entries { + let name = entry + .context("reading a delivered credential entry")? + .file_name() + .into_string() + .map_err(|_| anyhow!("delivered credential name is not UTF-8"))?; + delivered.push(name); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).context("reading the delivered credential directory"); + } + } + delivered.sort(); + ensure!( + delivered == expected, + "systemd delivered credentials that differ from the Idunn launch" + ); + Ok(()) +} + fn linux_process_security( status_path: &Path, expected_host_pid: u32, @@ -7484,6 +7657,56 @@ fn parse_systemd_boolean(values: &BTreeMap, name: &str) -> Resul } } +/// The isolation properties of a transient workload unit, judged from `systemctl +/// show`. `LoadCredential` is absent on purpose: systemd 257 displays it as +/// `[unprintable]` whatever the unit holds, so what the workload was given is +/// judged by `require_delivered_credentials` from the delivered directory. +fn require_systemd_isolation_contract( + values: &BTreeMap, + credential_root: &Path, +) -> Result<()> { + let service_type = required_systemd_property(values, "Type")?; + let restart_policy = required_systemd_property(values, "Restart")?; + let kill_mode = required_systemd_property(values, "KillMode")?; + let dynamic_user = parse_systemd_boolean(values, "DynamicUser")?; + let supplementary_groups = systemd_property(values, "SupplementaryGroups")?; + let capability_bounding_set = systemd_property(values, "CapabilityBoundingSet")?; + let ambient_capabilities = systemd_property(values, "AmbientCapabilities")?; + let private_mounts = parse_systemd_boolean(values, "PrivateMounts")?; + let private_pids = parse_systemd_boolean(values, "PrivatePIDs")?; + let protect_proc = required_systemd_property(values, "ProtectProc")?; + let proc_subset = required_systemd_property(values, "ProcSubset")?; + let no_new_privileges = parse_systemd_boolean(values, "NoNewPrivileges")?; + let umask = required_systemd_property(values, "UMask")?; + let inaccessible_paths = required_systemd_property(values, "InaccessiblePaths")?; + ensure!( + service_type == "exec" + && restart_policy == "no" + && kill_mode == "mixed" + // `User` is deliberately not asserted here. Idunn never emits a + // User property, and systemd reports the identity it *allocated* + // for a DynamicUser unit -- the unit name, or a generated `_du…` + // when that is too long -- so the property is never empty for a + // running workload and cannot tell an allocated identity from a + // pinned one. The invariant it was reaching for, that the workload + // does not run as a persistent named account, is carried by + // `dynamic_user` below and by Idunn never passing User at all. + && dynamic_user + && supplementary_groups.is_empty() + && capability_bounding_set.is_empty() + && ambient_capabilities.is_empty() + && private_mounts + && private_pids + && protect_proc == "invisible" + && proc_subset == "all" + && no_new_privileges + && umask == "0007" + && inaccessible_paths == credential_root.display().to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + Ok(()) +} + fn read_process_environment(path: &Path) -> Result>> { let bytes = fs::read(path) .with_context(|| format!("reading process environment {}", path.display()))?; @@ -9729,6 +9952,16 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// An incarnation unit runs once and is collected when it ends, so a + /// failed candidate does not stay loaded and mark the host degraded. + /// Collection is safe because a forgotten unit already reads as + /// permanently stopped (the test below). + #[test] + fn incarnation_units_run_once_and_are_collected() { + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--collect")); + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--property=Restart=no")); + } + /// Admission audit, claim 3: `is_permanently_stopped` now lets the /// Isolation gate skip `prove_isolation` against the incumbent. Enumerate /// what it answers `true` for, using a fake `systemctl show` that replays @@ -9968,6 +10201,168 @@ processes 9 ); } + #[cfg(unix)] + fn fixture_show_properties(text: &str) -> Result> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let shown = temp.path().join("shown"); + std::fs::write(&shown, text)?; + let systemctl = temp.path().join("systemctl"); + std::fs::write( + &systemctl, + format!("#!/bin/sh\ncat '{}'\nexit 0\n", shown.display()), + )?; + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; + let driver = SystemdTransientWorkloadDriver { + systemctl_program: systemctl, + ..Default::default() + }; + Ok(driver + .show_unit("idunn-test.service")? + .context("fixture unit was not found")? + .properties) + } + + /// systemd 257 prints `LoadCredential=[unprintable]` for every unit, so the + /// isolation contract must admit the real output of a unit with no service + /// credential as well as one with one. The inputs are real captures, not a + /// re-spelling. + #[cfg(unix)] + #[test] + fn systemd_257_isolation_contract_admits_a_presence_only_target_and_one_with_service_credentials( + ) -> Result<()> { + let root = Path::new("/run/idunn/activation-credentials"); + let vili = include_str!("../tests/fixtures/systemd-257/vili-presence-identity-only.show"); + let odin = include_str!( + "../tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show" + ); + for text in [vili, odin] { + assert!(text.contains("LoadCredential=[unprintable]")); + require_systemd_isolation_contract(&fixture_show_properties(text)?, root)?; + } + let default_proc = vili.replace("ProtectProc=invisible", "ProtectProc=default"); + assert_ne!(default_proc, vili); + let other_path = vili.replace( + "/run/idunn/activation-credentials", + "/run/idunn/elsewhere", + ); + assert_ne!(other_path, vili); + for text in [default_proc, other_path] { + let error = + require_systemd_isolation_contract(&fixture_show_properties(&text)?, root) + .expect_err("a drifted isolation property must be refused"); + assert_eq!( + error.to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn delivered_credentials_are_exactly_the_launch_names() -> Result<()> { + let unit = "idunn-test.service"; + let names = |list: &[&str]| list.iter().map(|name| name.to_string()).collect::>(); + // `Some(entries)` creates the unit's directory in a fresh process root. + let check = |delivered: Option<&[&str]>, expected: &[&str]| -> Result<()> { + let temp = tempfile::tempdir()?; + if let Some(entries) = delivered { + let directory = temp.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + for entry in entries { + std::fs::write(directory.join(entry), "canary-credential-contents")?; + } + } + let result = require_delivered_credentials(temp.path(), unit, &names(expected)); + if let Err(error) = &result { + let text = format!("{error:#}"); + for canary in ["STOWAWAY", "ODIN_TOPOLOGY_IDENTITY", "canary-credential"] { + assert!(!text.contains(canary), "error echoed {canary}"); + } + assert!(!text.contains(&*temp.path().to_string_lossy())); + } + result + }; + check(None, &[])?; + check(Some(&[]), &[])?; + check(Some(&["ODIN_TOPOLOGY_IDENTITY"]), &["ODIN_TOPOLOGY_IDENTITY"])?; + check(Some(&["B", "A"]), &["A", "B"])?; + for (delivered, expected) in [ + (None, &["ODIN_TOPOLOGY_IDENTITY"][..]), + (Some(&[][..]), &["ODIN_TOPOLOGY_IDENTITY"][..]), + ( + Some(&["ODIN_TOPOLOGY_IDENTITY", "STOWAWAY"][..]), + &["ODIN_TOPOLOGY_IDENTITY"][..], + ), + (Some(&["STOWAWAY"][..]), &[][..]), + (Some(&["A"][..]), &["B"][..]), + (Some(&["A"][..]), &["A", "B"][..]), + ] { + let error = check(delivered, expected) + .expect_err("delivered credentials differing from the launch must be refused"); + assert_eq!( + error.to_string(), + "systemd delivered credentials that differ from the Idunn launch" + ); + } + // A directory that cannot be listed is a refusal, never the empty set. + let temp = tempfile::tempdir()?; + let parent = temp.path().join("run/credentials"); + std::fs::create_dir_all(&parent)?; + std::fs::write(parent.join(unit), "")?; + let error = require_delivered_credentials(temp.path(), unit, &[]) + .expect_err("an unreadable credential directory must be refused"); + assert!(!format!("{error:#}").contains(&*temp.path().to_string_lossy())); + // The rule reads the process root it is given, never the host's /run. + let temp = tempfile::tempdir()?; + let outside = tempfile::tempdir()?; + let directory = outside.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + std::fs::write(directory.join("A"), "")?; + require_delivered_credentials(temp.path(), unit, &[])?; + assert!(require_delivered_credentials(temp.path(), unit, &names(&["A"])).is_err()); + Ok(()) + } + + /// Live Odin and StreamPixels hold records persisted by the Idunn that + /// compared `LoadCredential`: `[unprintable]` in the retired slot. The bytes + /// are that Idunn's own output (tests/fixtures/pre-cut-*); they must decode + /// under both encodings and read as no drift against a new observation, or + /// `stop` refuses to stop the incumbents. + #[test] + fn an_incumbent_recorded_with_the_display_form_is_not_identity_drift() -> Result<()> { + let decode_hex = |text: &str| -> Vec { + (0..text.trim().len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text.trim()[at..at + 2], 16).unwrap()) + .collect() + }; + let mut recorded = systemd_audit_observation("no", 61_000); + recorded.load_credential = "[unprintable]".into(); + let observed = systemd_audit_observation("no", 61_000); + assert!(observed.load_credential.is_empty()); + for hex in [ + include_str!("../tests/fixtures/pre-cut-systemd-observation.compact.hex"), + include_str!("../tests/fixtures/pre-cut-systemd-observation.named.hex"), + ] { + let decoded: WorkloadObservation = rmp_serde::from_slice(&decode_hex(hex))?; + assert_eq!(decoded, WorkloadObservation::Systemd(recorded.clone())); + let WorkloadObservation::Systemd(prior) = decoded else { + unreachable!() + }; + assert!(changed_systemd_workload_identity_fields(&observed, &prior).is_empty()); + } + let mut drifted = observed.clone(); + drifted.main_pid += 1; + assert_eq!( + changed_systemd_workload_identity_fields(&drifted, &recorded), + vec!["main_pid"] + ); + Ok(()) + } + #[test] fn host_isolation_is_two_live_distinct_processes_on_one_host() { let live = |pid: u32, created: u64| { @@ -16015,4 +16410,292 @@ processes 9 ); Ok(()) } + + // BEGIN runner-failure-evidence tests + // The runner decides what failed and what it kept: these drive the real + // DockerRunnerDriver over a fake docker program, never the capture alone. + + const RUNNER_CANARY: &str = "CANARY-5d2e91b7"; + + #[cfg(unix)] + fn fake_docker(dir: &Path, body: &str) -> Result { + use std::os::unix::fs::PermissionsExt; + let program = dir.join("docker"); + fs::write(&program, format!("#!/bin/sh\n{body}\n"))?; + fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?; + Ok(DockerRunnerDriver { + docker_program: program, + }) + } + + /// One recipe step through the real `run_in_workspace` path. + #[cfg(unix)] + fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> { + let workspace = dir.join("workspace"); + fs::create_dir_all(&workspace)?; + driver.run_in_workspace( + step_id, + "rust", + &fixture_docker_runner_binding(), + &workspace, + Path::new("."), + &["cargo".to_owned(), "test".to_owned()], + &std::collections::BTreeSet::new(), + ("IDUNN_STAMP", "rev"), + ) + } + + #[cfg(unix)] + fn failure_of(result: Result<()>) -> Result { + let error = result.expect_err("the step did not fail"); + error + .downcast_ref::() + .cloned() + .with_context(|| format!("the failure is not typed: {error:#}")) + } + + #[cfg(unix)] + #[test] + fn a_failed_step_is_named_with_both_tails() -> Result<()> { + let temp = tempfile::tempdir()?; + // The first step passes, the second fails: the count is the fake's. + let driver = fake_docker( + temp.path(), + &format!( + "n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\ + echo 'stderr report' >&2\nexit 3" + ), + )?; + run_step(&driver, temp.path(), "first")?; + let failure = failure_of(run_step(&driver, temp.path(), "second"))?; + assert_eq!(failure.subject, RunnerSubject::Step("second".into())); + assert_eq!(failure.runner, "rust"); + assert_eq!(failure.exit_code, Some(3)); + assert_eq!(failure.signal, None); + assert!( + failure + .stdout_tail + .ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()), + "stdout was dropped: {:?}", + String::from_utf8_lossy(&failure.stdout_tail) + ); + assert_eq!(failure.stderr_tail, b"stderr report\n"); + assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "kill -9 $$")?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert_eq!((failure.exit_code, failure.signal), (None, Some(9))); + assert!(failure.to_string().contains("exited with signal 9"), "{failure}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_external_input_is_named_as_one() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?; + let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?; + let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?; + let input_id = declaration.external_inputs[0].id.clone(); + let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]); + fs::create_dir_all(&workspaces["rust"])?; + let error = driver + .materialize_external_input(&declaration, &binding, &workspaces, &input_id) + .expect_err("the download did not fail"); + let failure = error + .downcast_ref::() + .with_context(|| format!("the failure is not typed: {error:#}"))?; + assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone())); + assert_eq!(failure.exit_code, Some(22)); + assert!( + failure.to_string().contains(&format!("runner external input {input_id} on runner rust")), + "{failure}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn runner_output_never_reaches_the_error_text() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY)); + let error = anyhow::Error::new(failure.clone()).context("sealing"); + for text in [ + format!("{error:#}"), + format!("{error:?}"), + format!("{failure}"), + format!("{failure:?}"), + ] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}"); + } + Ok(()) + } + + /// A reader that answers from a script, so a read error the OS rarely + /// produces (an interrupted call) is produced on demand. + struct ScriptedReader(Vec>>); + + impl Read for ScriptedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.0.is_empty() { + return Ok(0); + } + let bytes = self.0.remove(0)?; + buffer[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + + struct BrokenReader; + + impl Read for BrokenReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("persistent read failure")) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_loses_no_byte() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + Ok(b"cd".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + ]); + let drained = drain_tail(reader, 3).expect("an interrupted read ended the drain"); + assert_eq!(drained, (b"bcd".to_vec(), 4)); + } + + #[test] + fn a_persistent_read_error_ends_the_drain_instead_of_spinning() { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(drain_tail(BrokenReader, 8).is_err()); + }); + assert_eq!( + receiver.recv_timeout(Duration::from_secs(5)), + Ok(true), + "a persistent read error spun or was swallowed" + ); + } + + #[test] + fn a_non_interrupted_read_error_after_data_is_returned_not_retried() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::BrokenPipe)), + Ok(b"cd".to_vec()), + ]); + let error = drain_tail(reader, 8).expect_err("a broken pipe was retried past"); + assert_eq!(error.kind(), ErrorKind::BrokenPipe); + } + + #[cfg(unix)] + #[test] + fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { + let temp = tempfile::tempdir()?; + let data = temp.path().join("data"); + for size in [ + 0, + 1, + RUNNER_OUTPUT_TAIL_BYTES - 1, + RUNNER_OUTPUT_TAIL_BYTES, + RUNNER_OUTPUT_TAIL_BYTES + 1, + 1024 * 1024, + ] { + // Distinct neighbours, so a shifted or head-keeping tail differs. + let bytes: Vec = (0..size).map(|index| (index % 251) as u8).collect(); + fs::write(&data, &bytes)?; + let driver = fake_docker( + temp.path(), + &format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..]; + assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes"); + assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes"); + assert_eq!(failure.stdout_total_bytes, size as u64); + assert_eq!(failure.stderr_total_bytes, size as u64); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> { + let temp = tempfile::tempdir()?; + run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?; + let missing = DockerRunnerDriver { + docker_program: temp.path().join("no-such-docker"), + }; + let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran"); + assert!(error.downcast_ref::().is_none()); + assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); + Ok(()) + } + + /// The 2026-10-25 vili failure: `node --test` reported on stdout alone, + /// stderr stayed empty (the runner's /tmp is a noexec tmpfs) and the step + /// exited 1. The old capture dropped stdout and printed an empty message. + #[cfg(unix)] + #[test] + fn a_step_that_reports_only_on_stdout_keeps_that_report_and_its_name() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'not ok 1 - daemon {RUNNER_CANARY}'\necho '# fail 1'\nexit 1"), + )?; + let error = run_step(&driver, temp.path(), "test-daemon").expect_err("the step passed"); + // The error names the step and carries no output. + let text = format!("{error:#}"); + assert!(text.contains("runner step test-daemon on runner rust exited with exit status 1"), "{text}"); + assert!(!text.contains(RUNNER_CANARY) && !text.contains("not ok"), "{text}"); + // The evidence keeps the stdout tail; the empty stderr is a recorded 0. + let failure = error + .downcast_ref::() + .context("the failure is not typed")?; + let store = temp.path().join("runner-evidence.cc"); + let record = crate::runner_evidence::RunnerFailureRecord::from_failure( + "tx-vili", + "up-vili", + "vili", + failure, + 1, + ); + crate::runner_evidence::record(&store, &record)?; + let kept = crate::runner_evidence::for_transaction(&store, "tx-vili")?.context("no evidence kept")?; + assert_eq!(kept.subject_id, "test-daemon"); + assert_eq!(kept.exit_code, Some(1)); + assert_eq!( + String::from_utf8_lossy(&kept.stdout_tail), + format!("not ok 1 - daemon {RUNNER_CANARY}\n# fail 1\n") + ); + assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); + assert!(kept.stderr_tail.is_empty()); + assert_eq!(kept.stderr_total_bytes, 0); + let mut shown = Vec::new(); + crate::runner_evidence::show_status(&mut shown, &store, "tx-vili")?; + let rendered = String::from_utf8(shown)?; + assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); + assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); + // Debug names the failure and prints no tail. + let debug = format!("{failure:?}"); + assert!(debug.starts_with("RunnerStepFailure") && !debug.contains("not ok"), "{debug}"); + Ok(()) + } + // END runner-failure-evidence tests } diff --git a/src/host_actuator.rs b/src/host_actuator.rs index f7bf4c7..8fd776c 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1151,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - while let Some(frame) = client.receive_once()? { + if let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/lib.rs b/src/lib.rs index e71a17f..b5499a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,4 +7,7 @@ pub mod host_actuator; #[cfg(unix)] pub mod launcher; mod incident; +mod runner_evidence; pub mod provisioning; +#[cfg(unix)] +mod steward_orders; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs new file mode 100644 index 0000000..30dd1d2 --- /dev/null +++ b/src/runner_evidence.rs @@ -0,0 +1,805 @@ +//! The output tail of a failed runner step. +//! +//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the +//! last bytes of a failed runner container's stdout and stderr, one record per +//! transaction. The engine's pre-fence failure path is its only writer, and +//! `idunn status --command` its only reader. It is never `control.cc`, which +//! refuses any type it does not know, and never `incidents.cc`, which is +//! published and carries no free text. +//! +//! The tail is raw runner output. It is kept unredacted and readable by root +//! alone (mode 0600): root can already read every secret the runner was given. +//! It never enters an error, a transaction record or the journal. The status +//! reader is a terminal, so everything it prints from this store goes through +//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + TryCompareExchangeSnapshotOutcome, +}; +use serde::{Deserialize, Serialize}; + +use crate::control_plane::{decode_record, require_id, typed_envelope}; +use crate::drivers::{ + LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject, + exchange_behind_private_lock, runner_status, +}; + +const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1"; + +/// Records kept; the writer drops the oldest past this, so the store needs no +/// other owner. +pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32; + +/// `runner-evidence.cc`, beside the control store. +pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf { + state_store.with_file_name("runner-evidence.cc") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum SubjectKind { + Step, + ExternalInput, +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")] +pub(crate) struct RunnerFailureRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) transaction_id: String, + #[cultcache(key = 2)] + pub(crate) command_id: String, + #[cultcache(key = 3)] + pub(crate) target: String, + #[cultcache(key = 4)] + pub(crate) subject_kind: SubjectKind, + #[cultcache(key = 5)] + pub(crate) subject_id: String, + #[cultcache(key = 6)] + pub(crate) runner: String, + #[cultcache(key = 7)] + pub(crate) exit_code: Option, + #[cultcache(key = 8)] + pub(crate) signal: Option, + #[cultcache(key = 9)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 10)] + pub(crate) stdout_total_bytes: u64, + #[cultcache(key = 11)] + pub(crate) stderr_tail: Vec, + #[cultcache(key = 12)] + pub(crate) stderr_total_bytes: u64, + #[cultcache(key = 13)] + pub(crate) recorded_at_unix_millis: u64, +} + +/// Tails stay out of `Debug`, so a failed assertion or a logged record never +/// prints runner output. +impl std::fmt::Debug for RunnerFailureRecord { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerFailureRecord") + .field("transaction_id", &self.transaction_id) + .field("subject_id", &self.subject_id) + .finish_non_exhaustive() + } +} + +impl RunnerFailureRecord { + pub(crate) fn from_failure( + transaction_id: &str, + command_id: &str, + target: &str, + failure: &RunnerStepFailure, + now: u64, + ) -> Self { + let (subject_kind, subject_id) = match &failure.subject { + RunnerSubject::Step(id) => (SubjectKind::Step, id), + RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id), + }; + Self { + schema_version: RUNNER_FAILURE_SCHEMA.to_owned(), + transaction_id: transaction_id.to_owned(), + command_id: command_id.to_owned(), + target: target.to_owned(), + subject_kind, + subject_id: subject_id.clone(), + runner: failure.runner.clone(), + exit_code: failure.exit_code, + signal: failure.signal, + stdout_tail: failure.stdout_tail.clone(), + stdout_total_bytes: failure.stdout_total_bytes, + stderr_tail: failure.stderr_tail.clone(), + stderr_total_bytes: failure.stderr_total_bytes, + recorded_at_unix_millis: now, + } + } + + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == RUNNER_FAILURE_SCHEMA, + "runner failure schema is unsupported" + ); + require_id(&self.transaction_id, "runner failure transaction")?; + require_id(&self.command_id, "runner failure command")?; + require_id(&self.target, "runner failure target")?; + require_id(&self.subject_id, "runner failure subject")?; + require_id(&self.runner, "runner failure runner")?; + for (tail, total, stream) in [ + (&self.stdout_tail, self.stdout_total_bytes, "stdout"), + (&self.stderr_tail, self.stderr_total_bytes, "stderr"), + ] { + ensure!( + tail.len() <= RUNNER_OUTPUT_TAIL_BYTES, + "runner failure {stream} tail exceeds its bound" + ); + ensure!( + tail.len() as u64 <= total, + "runner failure {stream} tail exceeds its stream" + ); + } + Ok(()) + } + + fn envelope(&self) -> Result { + self.validate()?; + typed_envelope( + &self.transaction_id, + Self::TYPE, + RUNNER_FAILURE_SCHEMA, + self, + self.recorded_at_unix_millis, + ) + } +} + +/// One read of the store: the envelopes exactly as stored, and the records +/// decoded from them in the same order. +fn snapshot(path: &Path) -> Result<(Vec, Vec)> { + let envelopes = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading the runner evidence store")?; + let mut records = Vec::new(); + for envelope in &envelopes { + ensure!( + envelope.r#type == RunnerFailureRecord::TYPE + && envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA), + "the runner evidence store holds a document that is not a runner failure" + ); + let record: RunnerFailureRecord = decode_record(envelope)?; + record.validate()?; + ensure!( + envelope.key == record.transaction_id, + "runner evidence store key differs from the record's transaction" + ); + records.push(record); + } + Ok((envelopes, records)) +} + +/// How a write left the store. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kept { + Stored, + /// The store held bytes that do not decode; it was moved to + /// `unreadable_store_path` and the record opened a fresh one. `replaced` + /// is the last-written time (unix milliseconds) of an earlier set-aside + /// store this one displaced, which is gone: one slot, and it says so. + SetAside { replaced: Option }, +} + +/// Where an undecodable store is kept. One slot: the next one replaces it, so +/// the damage a bad store can cost is one extra file. +pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf { + path.with_file_name("runner-evidence.unreadable.cc") +} + +/// Moves the store over the previous set-aside one, root-only, and reports +/// what it displaced. Nothing of the store's content is named: a decode error +/// can quote stored values, and none is carried here. +fn set_aside_unreadable(path: &Path) -> Result> { + let aside = unreadable_store_path(path); + let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| { + earlier + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |since| since.as_millis() as u64) + }); + std::fs::rename(path, &aside) + .context("the runner evidence store is undecodable and could not be set aside")?; + keep_private(&aside)?; + Ok(replaced) +} + +/// How the writer reads the store's bytes; tests inject an I/O error here. +type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result>; + +fn unreadable_fault(error: &std::io::Error) -> anyhow::Error { + anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind()) +} + +/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the +/// oldest records go in the same exchange. The file ends at mode 0600. +/// +/// A store is set aside only when bytes that were read do not decode. An I/O +/// error leaves the store in place, keeps no evidence for that failure, and +/// the error names its kind for the journal. +pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { + record_reading(path, record, &|path| std::fs::read(path)) +} + +fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result { + let first = match read(path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(unreadable_fault(&error)), + }; + let (envelopes, records, how) = match snapshot(path) { + Ok((envelopes, records)) => (envelopes, records, Kept::Stored), + Err(_) => { + let Some(first) = first else { + bail!("the runner evidence store could not be decoded"); + }; + // Undecodable only if the same bytes are still there to read: a + // second read that fails, or finds other bytes, is not a verdict. + match read(path) { + Ok(second) if second == first => {} + Ok(_) => bail!("the runner evidence store changed while it was being read"), + Err(error) => return Err(unreadable_fault(&error)), + } + let replaced = set_aside_unreadable(path)?; + (Vec::new(), Vec::new(), Kept::SetAside { replaced }) + } + }; + let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes + .iter() + .zip(&records) + .filter(|(_, stored)| stored.transaction_id != record.transaction_id) + .map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone())) + .collect(); + kept.push((record.recorded_at_unix_millis, record.envelope()?)); + // Stable: records of one instant keep their stored order, so the record + // being written survives a tie. + kept.sort_by_key(|(recorded_at, _)| *recorded_at); + let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS); + let replacements: Vec<_> = kept + .into_iter() + .skip(excess) + .map(|(_, envelope)| envelope) + .collect(); + match exchange_behind_private_lock(path, &envelopes, &replacements)? { + TryCompareExchangeSnapshotOutcome::Exchanged => {} + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("the runner evidence store changed while it was being written") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + return Err(LockContended { + store: "runner evidence store", + } + .into()); + } + } + keep_private(path)?; + Ok(how) +} + +#[cfg(unix)] +fn keep_private(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("making the runner evidence store private") +} + +#[cfg(not(unix))] +fn keep_private(_path: &Path) -> Result<()> { + Ok(()) +} + +/// The one reader. An absent store holds nothing. +pub(crate) fn for_transaction( + path: &Path, + transaction_id: &str, +) -> Result> { + Ok(snapshot(path)? + .1 + .into_iter() + .find(|stored| stored.transaction_id == transaction_id)) +} + +/// The record as `idunn status --command` shows it, as raw lines. They are +/// not text yet: `show_status` is the only caller, and `emit` the only +/// writer. +fn render(record: &RunnerFailureRecord) -> Vec> { + let noun = match record.subject_kind { + SubjectKind::Step => "step", + SubjectKind::ExternalInput => "external input", + }; + let mut lines: Vec> = vec![ + format!(" failed {noun} {}", record.subject_id).into_bytes(), + format!(" runner {}", record.runner).into_bytes(), + format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(), + ]; + for (stream, tail, total) in [ + ("stdout", &record.stdout_tail, record.stdout_total_bytes), + ("stderr", &record.stderr_tail, record.stderr_total_bytes), + ] { + lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes()); + let text = tail.strip_suffix(b"\n").unwrap_or(tail); + if !text.is_empty() { + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); + } + } + lines +} + +/// What `idunn status --command` prints for a transaction's runner evidence: +/// its record, or one line naming why the store is unreadable. Nothing is +/// written when the transaction has no record. +pub(crate) fn show_status( + out: &mut impl std::io::Write, + store: &Path, + transaction_id: &str, +) -> std::io::Result<()> { + let lines = match for_transaction(store, transaction_id) { + Ok(Some(record)) => render(&record), + Ok(None) => return Ok(()), + Err(error) => { + vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()] + } + }; + emit(out, &lines) +} + +/// The one sink: every byte the status read prints from the evidence store, +/// or from an error about it, is written here and nowhere else. An allowlist: +/// printable ASCII and tab pass; every other code point is a visible +/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline +/// inside a line is escaped too: this function writes the only ones. +pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { + for line in lines { + out.write_all(escape(line).as_bytes())?; + out.write_all(b"\n")?; + } + Ok(()) +} + +pub(crate) fn escape(bytes: &[u8]) -> String { + let mut shown = String::with_capacity(bytes.len()); + for chunk in bytes.utf8_chunks() { + for character in chunk.valid().chars() { + match character { + ' '..='~' | '\t' => shown.push(character), + _ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))), + } + } + for byte in chunk.invalid() { + shown.push_str(&format!("\\x{byte:02x}")); + } + } + shown +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject, + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stderr.to_vec(), + stderr_total_bytes: stderr.len() as u64, + } + } + + /// What status writes for a record, through the one sink. + fn shown(record: &RunnerFailureRecord) -> String { + let mut written = Vec::new(); + emit(&mut written, &render(record)).expect("a Vec takes any write"); + String::from_utf8(written).expect("the sink writes ASCII only") + } + + fn shown_lines(record: &RunnerFailureRecord) -> Vec { + shown(record).lines().map(str::to_owned).collect() + } + + fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { + RunnerFailureRecord::from_failure( + transaction, + "up-service", + "service", + &failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"), + now, + ) + } + + #[test] + fn the_evidence_store_keeps_the_newest_32() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // The literal, not the constant: the spec's bound is 32. + for index in 0..=32 { + record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; + } + let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); + assert_eq!(kept.len(), 32); + assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); + assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); + assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); + // Rewriting a resident transaction replaces it and drops nothing. + let mut again = record_at("tx-05", 5000); + again.stdout_tail = b"rewritten\n".to_vec(); + again.stdout_total_bytes = 10; + record(&path, &again)?; + let kept = snapshot(&path)?.1; + assert_eq!(kept.len(), 32); + assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); + let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; + assert_eq!(stored.stdout_tail, b"rewritten\n"); + assert_eq!(for_transaction(&path, "tx-00")?, None); + Ok(()) + } + + #[test] + fn a_record_round_trips_byte_for_byte() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let written = RunnerFailureRecord::from_failure( + "tx-a", + "up-service", + "service", + &failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""), + 77, + ); + record(&path, &written)?; + assert_eq!(for_transaction(&path, "tx-a")?, Some(written)); + assert_eq!(for_transaction(&path, "tx-b")?, None); + Ok(()) + } + + #[test] + fn the_evidence_store_is_private() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // Whatever the umask, the mode is the store's own rule, on every write. + for index in 0..2 { + record(&path, &record_at(&format!("tx-{index}"), 10 + index))?; + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn a_tail_over_its_bound_or_its_stream_is_refused() { + let mut oversize = record_at("tx-a", 1); + oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1]; + oversize.stdout_total_bytes = u64::MAX; + assert!(oversize.validate().is_err()); + let mut impossible = record_at("tx-a", 1); + impossible.stderr_total_bytes = 1; + assert!(impossible.validate().is_err()); + let mut exact = record_at("tx-a", 1); + exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES]; + exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64; + assert!(exact.validate().is_ok()); + } + + #[test] + fn status_renders_the_tail_with_control_bytes_escaped() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec(); + stored.stdout_total_bytes = 5000; + stored.stderr_tail = b"bad\r\x07".to_vec(); + stored.stderr_total_bytes = 6; + let lines = shown_lines(&stored); + let text = lines.join("\n"); + assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); + assert!(!text.contains('\u{7}')); + assert_eq!( + lines, + vec![ + " failed step build", + " runner rust", + " status exit status 3", + " stdout (last 30 of 5000 bytes)", + " | first", + " | \\u{1b}[2Jsecond\\u{0}end\tTab", + " | last", + " stderr (last 5 of 6 bytes)", + " | bad\\u{d}\\u{7}", + ] + ); + } + + #[test] + fn debug_names_the_record_and_prints_no_tail() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"SECRET-OUTPUT".to_vec(); + stored.stdout_total_bytes = 13; + let debug = format!("{stored:?}"); + assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}"); + } + + #[test] + fn an_empty_tail_renders_only_its_header() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = Vec::new(); + stored.stdout_total_bytes = 0; + let lines = shown_lines(&stored); + assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); + assert!(!lines.iter().any(|line| line == " | ")); + } + + #[test] + fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 not a store")?; + assert!(for_transaction(&path, "tx-a").is_err()); + Ok(()) + } + + fn files_beside(path: &Path) -> Result> { + let mut names: Vec = std::fs::read_dir(path.parent().context("no parent")?)? + .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned())) + .collect::>()?; + names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock")); + names.sort(); + Ok(names) + } + + /// A store a newer Idunn wrote: well-formed, but a schema this one does not know. + fn write_future_store(path: &Path) -> Result<()> { + let mut future = record_at("tx-future", 1); + future.schema_version = "idunn.runner_failure.v2".into(); + let envelope = typed_envelope( + &future.transaction_id, + RunnerFailureRecord::TYPE, + "idunn.runner_failure.v2", + &future, + 1, + )?; + let outcome = exchange_behind_private_lock(path, &[], &[envelope])?; + ensure!( + matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged), + "the future store was not written" + ); + Ok(()) + } + + #[test] + fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damage = b"\xc1 not a store \x00 damage"; + std::fs::write(&path, damage)?; + let kept = record(&path, &record_at("tx-a", 5))?; + assert_eq!(kept, Kept::SetAside { replaced: None }); + // Kept, not deleted: the damage is where the journal says, root-only. + let aside = unreadable_store_path(&path); + assert_eq!(std::fs::read(&aside)?, damage); + assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600); + // The failure's evidence is in a fresh, private store. + assert_eq!( + for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id), + Some("tx-a".to_owned()) + ); + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + // A healthy store is not set aside again. + assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored); + assert_eq!(std::fs::read(&aside)?, damage); + Ok(()) + } + + #[test] + fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + write_future_store(&path)?; + let before = std::fs::read(&path)?; + assert!(for_transaction(&path, "tx-future").is_err()); + assert_eq!( + record(&path, &record_at("tx-a", 5))?, + Kept::SetAside { replaced: None } + ); + assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); + assert!(for_transaction(&path, "tx-a")?.is_some()); + assert_eq!(for_transaction(&path, "tx-future")?, None); + Ok(()) + } + + #[test] + fn at_most_one_unreadable_store_is_kept() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"]; + for (index, damage) in damages.into_iter().enumerate() { + std::fs::write(&path, damage)?; + record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?; + assert_eq!( + std::fs::read(unreadable_store_path(&path))?, + damage, + "the newest damage is the one kept" + ); + } + assert_eq!( + files_beside(&path)?, + vec![ + "runner-evidence.cc".to_owned(), + "runner-evidence.unreadable.cc".to_owned() + ] + ); + Ok(()) + } + + #[test] + fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // A directory where the store should be: unreadable, not undecodable. + std::fs::create_dir(&path)?; + assert!(record(&path, &record_at("tx-a", 1)).is_err()); + assert!(path.is_dir()); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn every_code_point_but_printable_ascii_is_a_visible_escape() { + // The allowlist, exhaustively: no class list decides anything. + for code in 0..=0x10ffffu32 { + let Some(character) = char::from_u32(code) else { + continue; + }; + let text = character.to_string(); + let shown = escape(text.as_bytes()); + if matches!(character, ' '..='~' | '\t') { + assert_eq!(shown, text, "U+{code:04X} should pass"); + } else { + assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}"); + } + } + // Invalid UTF-8 is escaped byte by byte; valid text round it passes. + assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82"); + assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}"); + } + + #[test] + fn a_newline_inside_a_line_cannot_forge_a_line() { + let mut stored = record_at("tx-a", 1); + stored.subject_id = "build\n status exit status 0".into(); + let lines = shown_lines(&stored); + assert_eq!(lines[0], " failed step build\\u{a} status exit status 0"); + assert_eq!( + lines.iter().filter(|line| line.starts_with(" status ")).count(), + 1 + ); + } + + /// A store whose first record is well-formed except that its subject kind + /// holds `marker`: the decode error quotes it. + fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> { + record(path, &record_at("tx-a", 1))?; + let mut bytes = std::fs::read(path)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker")?; + bytes[at..at + 4].copy_from_slice(marker); + std::fs::write(path, bytes)?; + Ok(()) + } + + #[test] + fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"\x1b[2J")?; + let mut written = Vec::new(); + show_status(&mut written, &path, "tx-a")?; + assert!(!written.contains(&0x1b), "an escape byte reached the terminal"); + assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n'))); + let text = String::from_utf8(written)?; + assert!(text.starts_with(" runner evidence unreadable: "), "{text}"); + assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}"); + Ok(()) + } + + #[test] + fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"CNRY")?; + let damage = std::fs::read(&path)?; + // Sanity: this damage decodes to an error that quotes the canary. + assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY")); + // A non-empty directory where the aside goes: rename fails even as root. + let aside = unreadable_store_path(&path); + std::fs::create_dir(&aside)?; + std::fs::write(aside.join("occupied"), b"x")?; + let error = record(&path, &record_at("tx-b", 2)).unwrap_err(); + for text in [format!("{error:#}"), format!("{error:?}")] { + assert!(text.contains("could not be set aside"), "{text}"); + assert!(!text.contains("CNRY"), "stored content reached the error: {text}"); + } + // The failure cost nothing: the store is as it was, with no evidence + // written over it. + assert_eq!(std::fs::read(&path)?, damage); + assert!(aside.join("occupied").exists()); + Ok(()) + } + + #[test] + fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + record(&path, &record_at("tx-a", 1))?; + let before = std::fs::read(&path)?; + let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied); + // The first read fails on a healthy store. + let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, before, "the store changed"); + assert!(!unreadable_store_path(&path).exists()); + assert!(for_transaction(&path, "tx-b")?.is_none()); + // The second read, the one that confirms the bytes do not decode, fails. + std::fs::write(&path, b"\xc1 not a store")?; + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| { + calls.set(calls.get() + 1); + if calls.get() == 2 { + return Err(denied()); + } + std::fs::read(path) + }; + let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + // Bytes that changed between the reads are no verdict either. + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| -> std::io::Result> { + calls.set(calls.get() + 1); + let mut bytes = std::fs::read(path)?; + bytes.push(calls.get() as u8); + Ok(bytes) + }; + assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err()); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 first")?; + assert_eq!( + record(&path, &record_at("tx-a", 1))?, + Kept::SetAside { replaced: None } + ); + std::fs::write(&path, b"\xc1 second")?; + let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else { + bail!("a displaced set-aside store went unreported"); + }; + assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second"); + Ok(()) + } +} diff --git a/src/steward_orders.rs b/src/steward_orders.rs new file mode 100644 index 0000000..d4be1ee --- /dev/null +++ b/src/steward_orders.rs @@ -0,0 +1,4079 @@ +//! Typed orders from Idunn's steward. +//! +//! `idunn orders serve` is a second root unit from the same binary as +//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints +//! order ids, decides acceptance and refusal, actuates, and records outcomes. +//! The survival daemon takes no part: `control.cc` never learns an order type. +//! +//! An order is a closed enum, never a command string: install packages, act on +//! a systemd unit, or deploy a target through the same commit primitive +//! `idunn up` uses. Every unit is allowed except the always-refused set, which +//! is compiled in here and read from nowhere: no binding field, record or flag +//! holds it. The `#operator` post is a retried projection of the store. +//! +//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the +//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and +//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE +//! `mind-control check`; START record before actuation, at most once; RUN; +//! RESULT outcome; POST accepted and result. +//! +//! No refusal or error here carries a value the issuer supplied: it names the +//! field and a code. A refused unit that a record names was resolved by +//! systemd and passed the unit grammar. + +use std::fmt; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::control_plane::{ + command_state, commit_deployment_command, decode_record, default_state_store, now_millis, + path_value, require_id, require_selector, string_value, typed_envelope, +}; +use crate::runner_evidence::{emit, escape}; + +const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; +const START_SCHEMA: &str = "idunn.steward_order_start.v1"; +const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; +const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; + +const REQUEST_FRAME_LIMIT: usize = 16 * 1024; +const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; +const TAIL_BYTES: usize = 8 * 1024; +const REASON_LIMIT: usize = 300; +const PACKAGE_LIMIT: usize = 16; +const UNIT_LIMIT: usize = 200; +const LIST_LIMIT: usize = 50; +const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; + +/// The always-refused set: Idunn's own units and the minds' own. Matched on +/// every name a unit answers to, ASCII case-insensitively (systemd units are +/// case-sensitive, so the fold only widens the refusal). +fn is_refused_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + ["idunn-", "mind-self@", "mind-persona@"] + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +/// Units whose binaries suspend, hibernate or open recovery: no property +/// reveals it, so they are refused by name, on every name they answer to. +const HOST_NAMES: [&str; 6] = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", +]; + +fn is_host_name(name: &str) -> bool { + HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) +} + +// ---------------------------------------------------------------- grammar -- + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Code { + Empty, + TooLong, + TooMany, + BadCharacter, + BadShape, + Malformed, + Unauthorized, + Unknown, + Unavailable, +} + +/// A refusal names the field and a code. It never carries the offending value. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Refusal { + pub(crate) field: String, + pub(crate) code: Code, +} + +impl Refusal { + fn new(field: &'static str, code: Code) -> Self { + Self { + field: field.to_owned(), + code, + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{} refused: {:?}", self.field, self.code) + } +} + +impl std::error::Error for Refusal {} + +fn validate_package(spec: &str) -> Result<(), Refusal> { + const FIELD: &str = "package"; + if spec.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if spec.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let (name, version) = match spec.split_once('=') { + Some((name, version)) => (name, Some(version)), + None => (spec, None), + }; + let bytes = name.as_bytes(); + if bytes.len() < 2 { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); + let rest = bytes[1..].iter().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') + }); + if !head || !rest { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + // apt reads `name-` as a removal and `name.deb` as a local file. + if name.ends_with('-') || name.ends_with(".deb") { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if let Some(version) = version { + if version.is_empty() { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if !version.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') + }) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + } + Ok(()) +} + +fn validate_unit(unit: &str) -> Result<(), Refusal> { + const FIELD: &str = "unit"; + if unit.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if unit.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let allowed = unit.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') + }); + if !allowed || unit.starts_with('-') { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + match UNIT_SUFFIXES + .iter() + .find_map(|suffix| unit.strip_suffix(suffix)) + { + Some(stem) if !stem.is_empty() => Ok(()), + _ => Err(Refusal::new(FIELD, Code::BadShape)), + } +} + +fn validate_reason(reason: &str) -> Result<(), Refusal> { + const FIELD: &str = "reason"; + if reason.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if reason.len() > REASON_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Verb { + Start, + Stop, + Restart, + Reload, +} + +impl Verb { + fn as_str(self) -> &'static str { + match self { + Verb::Start => "start", + Verb::Stop => "stop", + Verb::Restart => "restart", + Verb::Reload => "reload", + } + } + + fn parse(word: &str) -> Option { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + .into_iter() + .find(|verb| verb.as_str() == word) + } +} + +/// The closed set of things an order can ask for. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Operation { + PackageInstall { packages: Vec }, + UnitAction { verb: Verb, unit: String }, + Deploy { selector: String }, +} + +/// One fixed argv for `systemd-run`, built from the typed operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Step { + pub(crate) argv: Vec, + pub(crate) environment: Vec<(String, String)>, +} + +impl Operation { + fn validate(&self) -> Result<(), Refusal> { + match self { + Operation::PackageInstall { packages } => { + if packages.is_empty() { + return Err(Refusal::new("package", Code::Empty)); + } + if packages.len() > PACKAGE_LIMIT { + return Err(Refusal::new("package", Code::TooMany)); + } + packages.iter().try_for_each(|package| validate_package(package)) + } + Operation::UnitAction { unit, .. } => validate_unit(unit), + Operation::Deploy { selector } => require_selector(selector) + .map_err(|_| Refusal::new("selector", Code::BadShape)), + } + } + + fn steps(&self) -> Vec { + let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); + let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; + match self { + Operation::PackageInstall { packages } => { + // Never a removal, never a pattern: each name selects exactly + // itself, and apt aborts rather than displace an installed package. + let mut install: Vec = + words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); + let mut present: Vec = words(&["dpkg", "-s", "--"]); + for package in packages { + let (name, version) = package.split_once('=').unwrap_or((package, "")); + install.push(match version { + "" => format!("?exact-name({name})"), + version => format!("?exact-name({name})={version}"), + }); + present.push(name.to_owned()); + } + // An exact name that matches nothing installs nothing and exits 0, + // so the last step fails unless every named package is installed. + vec![ + Step { + argv: words(&["apt-get", "update", "-q"]), + environment: apt_environment.clone(), + }, + Step { + argv: install, + environment: apt_environment, + }, + Step { + argv: present, + environment: Vec::new(), + }, + ] + } + Operation::UnitAction { verb, unit } => vec![Step { + argv: vec![ + "systemctl".into(), + verb.as_str().into(), + "--job-mode=ignore-requirements".into(), + "--".into(), + unit.clone(), + ], + environment: Vec::new(), + }], + Operation::Deploy { .. } => Vec::new(), + } + } + + fn describe(&self) -> String { + match self { + Operation::PackageInstall { packages } => { + format!("package-install {}", packages.join(" ")) + } + Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), + Operation::Deploy { selector } => format!("deploy {selector}"), + } + } +} + +// ---------------------------------------------------------------- records -- + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] +pub(crate) struct OrderRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) principal: String, + #[cultcache(key = 3)] + pub(crate) issuer_uid: u32, + #[cultcache(key = 4)] + pub(crate) issuer_unit: String, + #[cultcache(key = 5)] + pub(crate) issuer_invocation: String, + #[cultcache(key = 6)] + pub(crate) issued_at_unix_millis: u64, + #[cultcache(key = 7)] + pub(crate) operation: Operation, + #[cultcache(key = 8)] + pub(crate) reason: String, +} + +impl fmt::Debug for OrderRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OrderRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_start", + schema = "idunn.steward_order_start.v1" +)] +pub(crate) struct StartRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) started_at_unix_millis: u64, + /// The transient unit name, or the deployment command id. + #[cultcache(key = 3)] + pub(crate) actuation: String, +} + +impl fmt::Debug for StartRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StartRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Via { + Name, + Alias, + Host, + OnEnd, + Bound, + Upholds, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Refused { + AlwaysRefused { unit: String, via: Via }, + /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload + /// of the wrapper exits 0 and leaves the member alone. + Wrapper { member: String }, + Braked, + /// The order's principal or issuing uid is not the steward bound now. + NotSteward, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Outcome { + Ran, + Failed { + exit_code: Option, + signal: Option, + }, + Refused(Refused), + Lost, + Submitted { command_id: String }, +} + +impl Outcome { + fn describe(&self) -> String { + match self { + Outcome::Ran => "ran (exit 0)".into(), + Outcome::Failed { + exit_code: Some(code), + .. + } => format!("failed (exit {code})"), + Outcome::Failed { + signal: Some(signal), + .. + } => format!("failed (signal {signal})"), + Outcome::Failed { .. } => "failed (not run to an exit)".into(), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( + "refused: always-refused unit {} reached by {}", + escape(unit.as_bytes()), + match via { + Via::Name => "name", + Via::Alias => "alias", + Via::Host => "host", + Via::OnEnd => "on-end", + Via::Bound => "bound", + Via::Upholds => "upholds", + } + ), + Outcome::Refused(Refused::Wrapper { member }) => format!( + "refused: wrapper unit, it only wraps {}", + escape(member.as_bytes()) + ), + Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Refused(Refused::NotSteward) => { + "refused: the issuer is not the current steward".into() + } + Outcome::Lost => "lost: the orders service restarted mid-run".into(), + Outcome::Submitted { command_id } => { + format!("submitted deployment {}", escape(command_id.as_bytes())) + } + } + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_outcome", + schema = "idunn.steward_order_outcome.v1" +)] +pub(crate) struct OutcomeRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) ended_at_unix_millis: u64, + #[cultcache(key = 3)] + pub(crate) outcome: Outcome, + #[cultcache(key = 4)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 5)] + pub(crate) stderr_tail: Vec, +} + +/// Tails stay out of `Debug`: they are raw program output. +impl fmt::Debug for OutcomeRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutcomeRecord") + .field("order_id", &self.order_id) + .field("outcome", &self.outcome) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum PostKind { + Accepted, + Result, +} + +impl PostKind { + fn as_str(self) -> &'static str { + match self { + PostKind::Accepted => "accepted", + PostKind::Result => "result", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_post", + schema = "idunn.steward_order_post.v1" +)] +pub(crate) struct PostRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `:`. + #[cultcache(key = 1)] + pub(crate) post_key: String, + #[cultcache(key = 2)] + pub(crate) event_id: String, + #[cultcache(key = 3)] + pub(crate) posted_at_unix_millis: u64, +} + +// ------------------------------------------------------------------ store -- + +/// `orders.cc`: root-only, written by `idunn orders serve` alone. +pub(crate) struct OrderStore { + directory: PathBuf, +} + +#[derive(Default)] +struct Ledger { + orders: Vec, + starts: Vec, + outcomes: Vec, + posts: Vec, +} + +impl Ledger { + fn start_of(&self, order_id: &str) -> Option<&StartRecord> { + self.starts.iter().find(|start| start.order_id == order_id) + } + + fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { + self.outcomes + .iter() + .find(|outcome| outcome.order_id == order_id) + } + + fn posted(&self, order_id: &str, kind: PostKind) -> bool { + let key = post_key(order_id, kind); + self.posts.iter().any(|post| post.post_key == key) + } +} + +fn post_key(order_id: &str, kind: PostKind) -> String { + format!("{order_id}:{}", kind.as_str()) +} + +impl OrderStore { + pub(crate) fn new(directory: &Path) -> Self { + Self { + directory: directory.to_owned(), + } + } + + fn path(&self) -> PathBuf { + self.directory.join("orders.cc") + } + + fn prepare(&self) -> Result<()> { + std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; + std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) + .context("making the orders directory private") + } + + /// Insert a record whose (type, key) is absent. `false` means it was there. + fn put(&self, key: &str, schema: &str, value: &T) -> Result { + self.prepare()?; + let path = self.path(); + let now = now_millis()?; + let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: T::TYPE.into(), + key: key.into(), + current: None, + }], + &[typed_envelope(key, T::TYPE, schema, value, now)?], + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .context("making the orders store private")?; + Ok(inserted) + } + + fn ledger(&self) -> Result { + let path = self.path(); + let mut ledger = Ledger::default(); + if !path.exists() { + return Ok(ledger); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .context("reading the orders store")?; + for envelope in &envelopes { + let schema = envelope.schema_id.as_deref(); + let key = envelope.key.as_str(); + match envelope.r#type.as_str() { + OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { + let record: OrderRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an order is stored under another key"); + ledger.orders.push(record); + } + StartRecord::TYPE if schema == Some(START_SCHEMA) => { + let record: StartRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "a start is stored under another key"); + ledger.starts.push(record); + } + OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { + let record: OutcomeRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an outcome is stored under another key"); + ledger.outcomes.push(record); + } + PostRecord::TYPE if schema == Some(POST_SCHEMA) => { + let record: PostRecord = decode_record(envelope)?; + ensure!(record.post_key == key, "a post is stored under another key"); + ledger.posts.push(record); + } + _ => bail!("the orders store holds a document that is not an order record"), + } + } + ledger + .orders + .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); + Ok(ledger) + } +} + +// ------------------------------------------------------------------ ports -- + +/// The properties naming what systemd does when a unit ends or its job times +/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). +const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; + +/// The properties naming units systemd starts when a unit ends. +const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; + +/// What `systemctl show` says about one unit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct UnitFacts { + pub(crate) id: String, + pub(crate) names: Vec, + pub(crate) not_found: bool, + /// The properties that make the unit act when it ends: each emergency + /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. + pub(crate) on_end: Vec, + /// The units bound to this one at runtime (`BoundBy=`): systemd stops + /// them when this unit stops, whatever the job mode. + pub(crate) bound_by: Vec, + /// The units this one upholds (`Upholds=`): systemd starts them in a full + /// transaction whenever this unit is active, whatever the job mode. + pub(crate) upholds: Vec, + /// The units this one groups (`ConsistsOf=`): the verbs reach them only + /// through propagation, which the job mode removes. + pub(crate) consists_of: Vec, +} + +impl UnitFacts { + fn identities(&self) -> impl Iterator { + std::iter::once(&self.id).chain(self.names.iter()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RunReport { + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stderr_tail: Vec, +} + +pub(crate) trait Systemd: Send + Sync { + fn show(&self, unit: &str) -> Result; + fn invocation(&self, unit: &str) -> Option; + /// Run one step as the transient unit `unit`, to its end. + fn run(&self, unit: &str, step: &Step) -> Result; +} + +pub(crate) struct Peer { + pub(crate) uid: u32, + pub(crate) issuer_unit: String, +} + +pub(crate) trait PeerCredentials: Send + Sync { + fn peer(&self, stream: &UnixStream) -> Result; +} + +pub(crate) trait BrakeCheck: Send + Sync { + /// `true` only when the instance's brake check exits 0. + fn clear(&self, principal: &str) -> bool; +} + +pub(crate) trait Homeserver: Send + Sync { + /// Idempotent by `txn_id`; returns the event id. + fn post(&self, txn_id: &str, text: &str) -> Result; +} + +pub(crate) struct Ctx { + pub(crate) store: OrderStore, + pub(crate) state_store: PathBuf, + pub(crate) steward_uid: u32, + pub(crate) principal: String, + pub(crate) systemd: Arc, + pub(crate) peers: Arc, + pub(crate) brake: Arc, + pub(crate) homeserver: Arc, +} + +// ------------------------------------------------------------------ scope -- + +fn refused(unit: &str, via: Via) -> Refused { + Refused::AlwaysRefused { + unit: unit.to_owned(), + via, + } +} + +/// Would `verb` on `unit` be refused. systemd builds the transaction and the +/// order tells it to build one job (`--job-mode=ignore-requirements`), so this +/// reads only what that job mode cannot remove from the one named unit: the +/// name it answers to, what fires when it ends, who is bound to it, what it upholds and +/// what it only wraps. +fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { + if is_refused_name(unit) { + return Ok(Some(refused(unit, Via::Name))); + } + let facts = systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + for name in facts.identities() { + if is_refused_name(name) { + return Ok(Some(refused(name, Via::Alias))); + } + if is_host_name(name) { + return Ok(Some(refused(name, Via::Host))); + } + } + if verb != Verb::Reload && !facts.on_end.is_empty() { + return Ok(Some(refused(unit, Via::OnEnd))); + } + if matches!(verb, Verb::Stop | Verb::Restart) + && let Some(bound) = facts.bound_by.first() + { + return Ok(Some(refused(bound, Via::Bound))); + } + if matches!(verb, Verb::Start | Verb::Restart) && !facts.upholds.is_empty() { + return Ok(Some(refused(unit, Via::Upholds))); + } + if verb != Verb::Start + && let Some(member) = facts.consists_of.first() + { + return Ok(Some(Refused::Wrapper { + member: member.clone(), + })); + } + Ok(None) +} + +// -------------------------------------------------------------- execution -- + +fn tail(bytes: &[u8]) -> Vec { + bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() +} + +fn conclude( + ctx: &Ctx, + order_id: &str, + outcome: Outcome, + stdout_tail: &[u8], + stderr_tail: &[u8], +) -> Result<()> { + ctx.store.put( + order_id, + OUTCOME_SCHEMA, + &OutcomeRecord { + schema_version: OUTCOME_SCHEMA.into(), + order_id: order_id.into(), + ended_at_unix_millis: now_millis()?, + outcome, + stdout_tail: tail(stdout_tail), + stderr_tail: tail(stderr_tail), + }, + )?; + Ok(()) +} + +fn order_uuid(order_id: &str) -> &str { + order_id.strip_prefix("ord-").unwrap_or(order_id) +} + +pub(crate) fn requester(principal: &str, order_id: &str) -> String { + format!("steward:{principal} order {order_id}") +} + +/// Run one accepted order. A start record is written before anything is +/// actuated and an order that has one is never run again. +pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { + let ledger = ctx.store.ledger()?; + let order = ledger + .orders + .iter() + .find(|order| order.order_id == order_id) + .context("the order is not in the store")?; + if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { + return Ok(()); + } + // One gate for a fresh order and a recovered one: who may order is what the + // binding says now, not what it said when the order was accepted. + if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { + return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); + } + if order.operation.validate().is_err() { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the stored order no longer satisfies the grammar", + ); + } + if let Operation::UnitAction { verb, unit } = &order.operation { + match refusal_for(ctx.systemd.as_ref(), unit, *verb) { + Ok(None) => {} + Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), + Err(_) => { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the unit could not be resolved, so it was not acted on", + ); + } + } + } + if !ctx.brake.clear(&ctx.principal) { + return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); + } + let uuid = order_uuid(order_id); + let command_id = format!("up-{uuid}"); + let steps = order.operation.steps(); + let actuation = match &order.operation { + Operation::Deploy { .. } => command_id.clone(), + _ => format!("idunn-order-{uuid}"), + }; + let started = StartRecord { + schema_version: START_SCHEMA.into(), + order_id: order_id.into(), + started_at_unix_millis: now_millis()?, + actuation, + }; + if !ctx.store.put(order_id, START_SCHEMA, &started)? { + return Ok(()); + } + if let Operation::Deploy { selector } = &order.operation { + let outcome = match commit_deployment_command( + &ctx.state_store, + &command_id, + selector, + &requester(&order.principal, order_id), + ) { + Ok(_) => Outcome::Submitted { command_id }, + Err(_) => Outcome::Failed { + exit_code: None, + signal: None, + }, + }; + return conclude(ctx, order_id, outcome, b"", b""); + } + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let mut outcome = Outcome::Ran; + for (index, step) in steps.iter().enumerate() { + let unit = if steps.len() == 1 { + format!("idunn-order-{uuid}") + } else { + format!("idunn-order-{uuid}-{}", index + 1) + }; + match ctx.systemd.run(&unit, step) { + Ok(report) => { + stdout = report.stdout_tail; + stderr = report.stderr_tail; + if report.exit_code != Some(0) { + outcome = Outcome::Failed { + exit_code: report.exit_code, + signal: report.signal, + }; + break; + } + } + Err(_) => { + outcome = Outcome::Failed { + exit_code: None, + signal: None, + }; + stderr = b"the transient unit could not be started".to_vec(); + break; + } + } + } + conclude(ctx, order_id, outcome, &stdout, &stderr) +} + +/// On service start: an order with a start record and no outcome ran, or began +/// to, in a life that ended. It is Lost and never run again. Returns the +/// orders that never started, oldest first. +pub(crate) fn recover(ctx: &Ctx) -> Result> { + let ledger = ctx.store.ledger()?; + for start in &ledger.starts { + if ledger.outcome_of(&start.order_id).is_none() { + conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; + } + } + Ok(ledger + .orders + .iter() + .filter(|order| { + ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() + }) + .map(|order| order.order_id.clone()) + .collect()) +} + +// ---------------------------------------------------------------- posting -- + +struct Due { + order_id: String, + kind: PostKind, + text: String, +} + +fn quoted(reason: &str) -> String { + format!("\"{}\"", escape(reason.as_bytes())) +} + +fn accepted_text(order: &OrderRecord) -> String { + format!( + "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", + order.order_id, + escape(order.operation.describe().as_bytes()), + escape(order.issuer_unit.as_bytes()), + escape(order.issuer_invocation.as_bytes()), + quoted(&order.reason) + ) +} + +/// A deploy's command has finished once its state is neither queued nor running. +fn deploy_finished(state: &str) -> bool { + !matches!(state, "queued" | "running") +} + +/// What is owed to the room. `deploy_state` is a deployment command's derived +/// state and detail, read from the control store by the caller. +fn due_posts( + ledger: &Ledger, + deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, +) -> Vec { + let mut due = Vec::new(); + for order in &ledger.orders { + let id = &order.order_id; + if !ledger.posted(id, PostKind::Accepted) { + due.push(Due { + order_id: id.clone(), + kind: PostKind::Accepted, + text: accepted_text(order), + }); + } + if ledger.posted(id, PostKind::Result) { + continue; + } + let Some(record) = ledger.outcome_of(id) else { + continue; + }; + let mut line = record.outcome.describe(); + if let Outcome::Submitted { command_id } = &record.outcome { + match deploy_state(command_id) { + Ok((state, detail)) if deploy_finished(state) => { + line = format!("{line}: {state} {}", escape(detail.as_bytes())); + } + _ => continue, + } + } + due.push(Due { + order_id: id.clone(), + kind: PostKind::Result, + text: format!( + "Steward order {id}: {line}\n{}", + escape(order.operation.describe().as_bytes()) + ), + }); + } + due +} + +/// Post what is due, in order, stopping at the first failure. Returns how many +/// went out. +pub(crate) fn post_pass(ctx: &Ctx) -> Result { + let ledger = ctx.store.ledger()?; + let mut sent = 0; + let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); + for due in due_posts(&ledger, &deploy_state) { + let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); + let event_id = ctx.homeserver.post(&transaction, &due.text)?; + ctx.store.put( + &post_key(&due.order_id, due.kind), + POST_SCHEMA, + &PostRecord { + schema_version: POST_SCHEMA.into(), + post_key: post_key(&due.order_id, due.kind), + event_id, + posted_at_unix_millis: now_millis()?, + }, + )?; + sent += 1; + } + Ok(sent) +} + +// ----------------------------------------------------------------- frames -- + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct OrderRequest { + pub(crate) operation: Operation, + pub(crate) reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ShowRequest { + pub(crate) id: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Request { + Order(OrderRequest), + Show(ShowRequest), + List, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderView { + pub(crate) order: OrderRecord, + pub(crate) start: Option, + pub(crate) outcome: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderLine { + pub(crate) order_id: String, + pub(crate) status: String, + pub(crate) summary: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Response { + Accepted { order_id: String }, + Refused(Refusal), + Show(Box), + List(Vec), +} + +fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { + let bytes = rmp_serde::to_vec_named(value)?; + stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; + stream.write_all(&bytes)?; + Ok(stream.flush()?) +} + +fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { + let mut length = [0u8; 4]; + stream.read_exact(&mut length)?; + let length = u32::from_be_bytes(length) as usize; + ensure!(length <= limit, "a frame exceeds its limit"); + let mut bytes = vec![0u8; length]; + stream.read_exact(&mut bytes)?; + Ok(bytes) +} + +fn status_word(view: &OrderView) -> &'static str { + match (&view.outcome, &view.start) { + (Some(record), _) => match record.outcome { + Outcome::Ran => "ran", + Outcome::Failed { .. } => "failed", + Outcome::Refused(_) => "refused", + Outcome::Lost => "lost", + Outcome::Submitted { .. } => "submitted", + }, + (None, Some(_)) => "running", + (None, None) => "accepted", + } +} + +fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { + OrderView { + order: order.clone(), + start: ledger.start_of(&order.order_id).cloned(), + outcome: ledger.outcome_of(&order.order_id).cloned(), + } +} + +fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { + write_frame(stream, &response) +} + +fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { + answer(stream, Response::Refused(Refusal::new(field, code))) +} + +/// Serve one connection. The peer is authenticated before its frame is read. +pub(crate) fn handle( + ctx: &Ctx, + stream: &mut UnixStream, + queue: &mpsc::Sender, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + let peer = ctx.peers.peer(stream)?; + // The steward may order and read; root, the operator, may only read. + let steward = peer.uid == ctx.steward_uid; + if !steward && peer.uid != 0 { + eprintln!("steward socket: refused a peer that is not the steward"); + return refuse(stream, "peer", Code::Unauthorized); + } + let request = read_frame(stream, REQUEST_FRAME_LIMIT) + .ok() + .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); + let Some(request) = request else { + eprintln!("steward socket: refused a malformed frame"); + return refuse(stream, "frame", Code::Malformed); + }; + match request { + Request::Order(_) if !steward => { + eprintln!("steward socket: refused an order from a peer that is not the steward"); + refuse(stream, "peer", Code::Unauthorized) + } + Request::Order(request) => { + let checked = + validate_reason(&request.reason).and_then(|()| request.operation.validate()); + if let Err(refusal) = checked { + eprintln!("steward socket: refused an order at the grammar"); + return answer(stream, Response::Refused(refusal)); + } + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: format!("ord-{}", Uuid::new_v4()), + principal: ctx.principal.clone(), + issuer_uid: peer.uid, + issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), + issuer_unit: peer.issuer_unit, + issued_at_unix_millis: now_millis()?, + operation: request.operation, + reason: request.reason, + }; + match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { + Ok(true) => {} + _ => return refuse(stream, "store", Code::Unavailable), + } + queue + .send(order.order_id.clone()) + .map_err(|_| anyhow!("the executor has stopped"))?; + answer( + stream, + Response::Accepted { + order_id: order.order_id, + }, + ) + } + Request::Show(request) => { + if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) + { + return refuse(stream, "id", Code::BadShape); + } + let ledger = ctx.store.ledger()?; + match ledger.orders.iter().find(|order| order.order_id == request.id) { + Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), + None => refuse(stream, "id", Code::Unknown), + } + } + Request::List => { + let ledger = ctx.store.ledger()?; + let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); + let lines = ledger + .orders + .iter() + .skip(skip) + .map(|order| { + let view = view_of(&ledger, order); + OrderLine { + order_id: order.order_id.clone(), + status: status_word(&view).into(), + summary: order.operation.describe(), + } + }) + .collect(); + answer(stream, Response::List(lines)) + } + } +} + +// ------------------------------------------------------------- real ports -- + +/// `systemctl` and `systemd-run`, found by name. +pub(crate) struct HostSystemd; + +/// The names in one `systemctl show` list value. systemd quotes a name that +/// holds a backslash and doubles the backslash inside the quotes. +fn parse_names(value: &str) -> Vec { + let mut names = Vec::new(); + let mut current = String::new(); + let (mut quoted, mut open, mut escaped) = (false, false, false); + for character in value.chars() { + match character { + _ if escaped => { + current.push(character); + escaped = false; + } + '\\' if quoted => escaped = true, + '"' => { + quoted = !quoted; + open = true; + } + _ if character.is_whitespace() && !quoted => { + if open { + names.push(std::mem::take(&mut current)); + open = false; + } + } + _ => { + current.push(character); + open = true; + } + } + } + if open { + names.push(current); + } + names +} + +/// The properties one `systemctl show` is asked for: identity, what fires at +/// the unit's end, who is bound to it, and what it upholds or groups. +fn show_properties() -> String { + ["Id", "Names", "LoadState"] + .into_iter() + .chain(ACTIONS) + .chain(ON_END) + .chain(["BoundBy", "Upholds", "ConsistsOf"]) + .collect::>() + .join(",") +} + +fn parse_show(output: &str) -> Result { + let mut facts = UnitFacts::default(); + for line in output.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + match key { + "Id" => facts.id = value.to_owned(), + "Names" => facts.names = parse_names(value), + "LoadState" => facts.not_found = value == "not-found", + "BoundBy" => facts.bound_by = parse_names(value), + "Upholds" => facts.upholds = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), + _ if ACTIONS.contains(&key) => { + if !matches!(value, "" | "none") { + facts.on_end.push(key.to_owned()); + } + } + _ if ON_END.contains(&key) => { + facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); + } + _ => {} + } + } + ensure!(!facts.id.is_empty(), "systemctl named no unit"); + Ok(facts) +} + +impl Systemd for HostSystemd { + fn show(&self, unit: &str) -> Result { + let output = Command::new("systemctl") + .args(["show", "--no-pager"]) + .arg(format!("--property={}", show_properties())) + .args(["--", unit]) + .stdin(Stdio::null()) + .output() + .context("running systemctl show")?; + ensure!(output.status.success(), "systemctl show failed"); + parse_show(&String::from_utf8_lossy(&output.stdout)) + } + + fn invocation(&self, unit: &str) -> Option { + let output = Command::new("systemctl") + .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) + .arg(unit) + .stdin(Stdio::null()) + .output() + .ok()?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + (output.status.success() && !value.is_empty()).then_some(value) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + let mut command = Command::new("systemd-run"); + command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); + command.args(["-p", "RuntimeMaxSec=15min"]); + for (name, value) in &step.environment { + command.arg(format!("--setenv={name}={value}")); + } + let output = command + .arg("--") + .args(&step.argv) + .stdin(Stdio::null()) + .output() + .context("running systemd-run")?; + Ok(RunReport { + exit_code: output.status.code(), + signal: output.status.signal(), + stdout_tail: tail(&output.stdout), + stderr_tail: tail(&output.stderr), + }) + } +} + +/// A system service or scope names the issuer: the first unit under +/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices +/// above it are the manager's, and what a delegated subtree adds below it is +/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. +fn unit_from_cgroup(text: &str) -> String { + let line = text + .lines() + .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) + .unwrap_or_default(); + let path = line.rsplit_once(':').map_or("", |(_, path)| path); + let unit = path.strip_prefix("/system.slice/").and_then(|under| { + for segment in under.split('/') { + if segment.ends_with(".slice") { + continue; + } + let unit = segment.ends_with(".service") || segment.ends_with(".scope"); + return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); + } + None + }); + unit.map_or_else(|| "unknown".to_owned(), str::to_owned) +} + +/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. +pub(crate) struct KernelPeers; + +impl PeerCredentials for KernelPeers { + fn peer(&self, stream: &UnixStream) -> Result { + use std::os::fd::AsRawFd; + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = std::mem::size_of::() as libc::socklen_t; + // SAFETY: the descriptor is a live socket and the pointer and length + // describe `credentials`, which outlives the call. + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&mut credentials as *mut libc::ucred).cast(), + &mut length, + ) + }; + ensure!( + status == 0 && length as usize == std::mem::size_of::(), + "peer credentials are unavailable" + ); + let cgroup = + std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); + Ok(Peer { + uid: credentials.uid, + issuer_unit: unit_from_cgroup(&cgroup), + }) + } +} + +/// `mind-control --instance check`; a missing binary is a held brake. +pub(crate) struct MindControl { + pub(crate) program: PathBuf, +} + +impl BrakeCheck for MindControl { + fn clear(&self, principal: &str) -> bool { + Command::new(&self.program) + .args(["--instance", principal, "check"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} + +/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is +/// never chunked. The token is read from its credential file for each post. +pub(crate) struct Synapse { + pub(crate) address: std::net::SocketAddr, + pub(crate) room: String, + pub(crate) token_file: PathBuf, + /// The whole wait for a reply, however slowly it drips. + pub(crate) reply_within: Duration, +} + +fn percent_encode(text: &str) -> String { + let mut encoded = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +impl Homeserver for Synapse { + fn post(&self, txn_id: &str, text: &str) -> Result { + let token = std::fs::read_to_string(&self.token_file) + .context("the homeserver token is unreadable")?; + let token = token.trim(); + ensure!( + !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), + "the homeserver token is not a bearer token" + ); + let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; + let head = format!( + "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ + Host: {}\r\nAuthorization: Bearer {token}\r\n\ + Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", + percent_encode(&self.room), + self.address, + body.len() + ); + let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) + .context("the homeserver is unreachable")?; + socket.set_write_timeout(Some(self.reply_within))?; + socket.write_all(head.as_bytes())?; + socket.write_all(&body)?; + let deadline = std::time::Instant::now() + self.reply_within; + let mut reply = Vec::new(); + let mut chunk = [0u8; 4096]; + while reply.len() < RESPONSE_FRAME_LIMIT { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|left| !left.is_zero()) + .context("the homeserver reply was too slow")?; + socket.set_read_timeout(Some(remaining))?; + let count = socket + .read(&mut chunk) + .context("the homeserver reply was cut short")?; + if count == 0 { + break; + } + reply.extend_from_slice(&chunk[..count]); + } + let split = reply + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("the homeserver reply has no body")?; + let status = String::from_utf8_lossy(&reply[..split]); + ensure!( + status.split_whitespace().nth(1) == Some("200"), + "the homeserver refused the post" + ); + #[derive(Deserialize)] + struct Sent { + event_id: String, + } + let sent: Sent = serde_json::from_slice(&reply[split + 4..]) + .map_err(|_| anyhow!("the homeserver reply has no event id"))?; + Ok(sent.event_id) + } +} + +// ---------------------------------------------------------------- binding -- + +/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. +/// It lists no unit of any kind, and any field outside this set is refused. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct StewardBinding { + pub(crate) principal: String, + pub(crate) user: String, + pub(crate) mind_control: PathBuf, + pub(crate) room: String, + pub(crate) homeserver: String, + pub(crate) token_credential: String, +} + +impl StewardBinding { + pub(crate) fn parse(text: &str) -> Result { + let binding: Self = toml::from_str(text) + .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; + binding.validate()?; + Ok(binding) + } + + fn validate(&self) -> Result<()> { + require_id(&self.principal, "steward principal")?; + ensure!( + !self.principal.starts_with('-') && !self.principal.contains('/'), + "steward principal is invalid" + ); + let user_ok = !self.user.is_empty() + && self.user.len() <= 32 + && self.user.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) + && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); + ensure!(user_ok && self.user != "root", "steward user is invalid or root"); + ensure!( + self.mind_control.is_absolute(), + "mind_control must be an absolute path" + ); + let room_ok = self.room.starts_with('!') + && self.room.len() <= 255 + && self.room.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') + }); + ensure!(room_ok, "room must be a Matrix room id"); + self.address()?; + let credential_ok = !self.token_credential.is_empty() + && !self.token_credential.starts_with('.') + && self.token_credential.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') + }); + ensure!(credential_ok, "token_credential is not a credential name"); + Ok(()) + } + + fn address(&self) -> Result { + let address: std::net::SocketAddr = self + .homeserver + .strip_prefix("http://") + .and_then(|rest| rest.parse().ok()) + .context("homeserver must be http://:")?; + ensure!(address.ip().is_loopback(), "homeserver must be loopback"); + Ok(address) + } +} + +/// Root-admitted: owned by root and writable by no one else. +fn admitted(owner: u32, mode: u32) -> bool { + owner == 0 && mode & 0o022 == 0 +} + +fn load_binding(path: &Path) -> Result { + let metadata = std::fs::metadata(path).context("reading the steward binding")?; + ensure!( + metadata.is_file() && admitted(metadata.uid(), metadata.mode()), + "the steward binding is not a root-owned file closed to writes" + ); + let text = std::fs::read_to_string(path).context("reading the steward binding")?; + StewardBinding::parse(&text) +} + +fn user_uid(name: &str) -> Result { + let name = std::ffi::CString::new(name)?; + // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in + // `buffer`, which outlives every read of them below (none are read). + let (status, found, uid) = unsafe { + let mut passwd: libc::passwd = std::mem::zeroed(); + let mut buffer = vec![0 as libc::c_char; 4096]; + let mut result: *mut libc::passwd = std::ptr::null_mut(); + let status = libc::getpwnam_r( + name.as_ptr(), + &mut passwd, + buffer.as_mut_ptr(), + buffer.len(), + &mut result, + ); + (status, !result.is_null(), passwd.pw_uid) + }; + ensure!(status == 0 && found, "the steward user is unknown"); + ensure!(uid != 0, "the steward user resolves to root"); + Ok(uid) +} + +// ------------------------------------------------------------------ serve -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ServeOptions { + pub(crate) state_store: PathBuf, + pub(crate) store: PathBuf, + pub(crate) steward_binding: PathBuf, + pub(crate) socket: PathBuf, +} + +const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; +const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; +const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; + +pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { + ensure!( + args.next().as_deref() == Some("serve"), + "idunn orders takes the subcommand serve" + ); + let mut options = ServeOptions { + state_store: default_state_store(), + store: DEFAULT_STORE.into(), + steward_binding: DEFAULT_BINDING.into(), + socket: DEFAULT_SOCKET.into(), + }; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, + "--store" => options.store = path_value(&mut args, "--store")?, + "--steward-binding" => { + options.steward_binding = path_value(&mut args, "--steward-binding")? + } + "--socket" => options.socket = path_value(&mut args, "--socket")?, + _ => bail!("an orders serve option is not part of the grammar"), + } + } + Ok(options) +} + +/// Seconds between post passes: two when the last pass went through, and +/// doubling for each failed pass in a row up to a minute. +fn post_pause(failures: u32) -> u64 { + if failures == 0 { + 2 + } else { + (2u64 << failures.min(5)).min(60) + } +} + +/// Run a thread whose end ends the service: systemd restarts it, and the +/// start recovery settles what it left. +fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { + struct ExitOnDrop(&'static str); + impl Drop for ExitOnDrop { + fn drop(&mut self) { + eprintln!("steward orders: the {} thread ended", self.0); + std::process::exit(1); + } + } + thread::Builder::new() + .name(name.into()) + .spawn(move || { + let _guard = ExitOnDrop(name); + work(); + }) + .expect("a thread can be spawned"); +} + +/// The socket belongs to the steward and no one else may open it; the peer +/// credential check on every connection stays the lock. +fn bind_socket(path: &Path, steward_uid: u32) -> Result { + if let Ok(metadata) = std::fs::symlink_metadata(path) { + use std::os::unix::fs::FileTypeExt; + ensure!( + metadata.file_type().is_socket(), + "the socket path holds something that is not a socket" + ); + std::fs::remove_file(path).context("removing a stale socket")?; + } + let listener = UnixListener::bind(path).context("binding the steward socket")?; + std::os::unix::fs::chown(path, Some(steward_uid), None) + .context("giving the steward socket to the steward")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("setting the steward socket mode")?; + Ok(listener) +} + +/// Files this service creates in Idunn's own stores (a deploy order writes +/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. +const IDUNN_UMASK: libc::mode_t = 0o027; + +pub(crate) fn serve(options: ServeOptions) -> Result<()> { + // SAFETY: umask only sets the process file-mode mask; it cannot fail. + unsafe { libc::umask(IDUNN_UMASK) }; + let binding = load_binding(&options.steward_binding)?; + let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") + .map(PathBuf::from) + .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; + let ctx = Arc::new(Ctx { + store: OrderStore::new(&options.store), + state_store: options.state_store, + steward_uid: user_uid(&binding.user)?, + principal: binding.principal.clone(), + systemd: Arc::new(HostSystemd), + peers: Arc::new(KernelPeers), + brake: Arc::new(MindControl { + program: binding.mind_control.clone(), + }), + homeserver: Arc::new(Synapse { + address: binding.address()?, + room: binding.room.clone(), + token_file: credentials.join(&binding.token_credential), + reply_within: Duration::from_secs(10), + }), + }); + ctx.store.prepare()?; + let pending = recover(&ctx)?; + let listener = bind_socket(&options.socket, ctx.steward_uid)?; + let (queue, orders) = mpsc::channel::(); + for order_id in pending { + queue.send(order_id)?; + } + let executor = Arc::clone(&ctx); + vital("executor", move || { + for order_id in orders { + if let Err(error) = execute(&executor, &order_id) { + eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); + } + } + }); + let poster = Arc::clone(&ctx); + vital("poster", move || { + let mut failures = 0u32; + loop { + match post_pass(&poster) { + Ok(_) => failures = 0, + Err(error) => { + failures = failures.saturating_add(1); + eprintln!("steward orders: posting is delayed: {error:#}"); + } + } + thread::sleep(Duration::from_secs(post_pause(failures))); + } + }); + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + if let Err(error) = handle(&ctx, &mut stream, &queue) { + eprintln!("steward socket: a connection ended in error: {error:#}"); + } + } + Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), + } + } + bail!("the steward socket stopped accepting") +} + +// ----------------------------------------------------------------- client -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OrderAction { + Issue { operation: Operation, reason: String }, + Show { id: String }, + List, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct OrderCommand { + pub(crate) socket: PathBuf, + pub(crate) action: OrderAction, +} + +pub(crate) fn parse_order(mut args: impl Iterator) -> Result { + let kind = args.next().context("idunn order requires an operation")?; + let mut socket = PathBuf::from(DEFAULT_SOCKET); + let mut reason = None; + let mut positional = Vec::new(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--socket" => socket = path_value(&mut args, "--socket")?, + "--reason" => reason = Some(string_value(&mut args, "--reason")?), + word if word.starts_with('-') => bail!("an order option is not part of the grammar"), + _ => positional.push(argument), + } + } + let action = match kind.as_str() { + "list" => { + ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); + OrderAction::List + } + "show" => { + ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); + OrderAction::Show { + id: positional.remove(0), + } + } + "package-install" | "unit" | "deploy" => { + let operation = match kind.as_str() { + "package-install" => Operation::PackageInstall { + packages: positional, + }, + "unit" => { + ensure!(positional.len() == 2, "unit takes a verb and a unit"); + let verb = Verb::parse(&positional[0]) + .context("the unit verb is not start, stop, restart or reload")?; + Operation::UnitAction { + verb, + unit: positional.remove(1), + } + } + _ => { + ensure!(positional.len() == 1, "deploy takes one selector"); + Operation::Deploy { + selector: positional.remove(0), + } + } + }; + let reason = reason.context("an order requires --reason")?; + validate_reason(&reason)?; + operation.validate()?; + OrderAction::Issue { operation, reason } + } + _ => bail!("the order operation is not part of the grammar"), + }; + Ok(OrderCommand { socket, action }) +} + +fn call(socket: &Path, request: &Request) -> Result { + let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + write_frame(&mut stream, request)?; + let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; + rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) +} + +fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); + let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); +} + +/// Raw lines for the terminal; `emit` is their only way out. +fn render_view(view: &OrderView) -> Vec> { + let order = &view.order; + let mut lines: Vec> = vec![ + format!("order {}", order.order_id).into_bytes(), + format!("status {}", status_word(view)).into_bytes(), + format!("operation {}", order.operation.describe()).into_bytes(), + format!( + "issuer {} (invocation {}, uid {})", + order.issuer_unit, order.issuer_invocation, order.issuer_uid + ) + .into_bytes(), + format!("reason {}", quoted(&order.reason)).into_bytes(), + ]; + if let Some(start) = &view.start { + lines.push(format!("started {}", start.actuation).into_bytes()); + } + if let Some(record) = &view.outcome { + lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); + render_tail(&mut lines, "stdout", &record.stdout_tail); + render_tail(&mut lines, "stderr", &record.stderr_tail); + } + lines +} + +pub(crate) fn run_order(command: OrderCommand) -> Result<()> { + let request = match command.action { + OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), + OrderAction::Show { id } => Request::Show(ShowRequest { id }), + OrderAction::List => Request::List, + }; + let mut out = std::io::stdout().lock(); + match call(&command.socket, &request)? { + Response::Accepted { order_id } => writeln!(out, "{order_id}")?, + Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), + Response::Show(view) => emit(&mut out, &render_view(&view))?, + Response::List(lines) => { + let lines: Vec> = lines + .iter() + .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) + .collect(); + emit(&mut out, &lines)? + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tempfile::TempDir; + + use super::*; + + const STEWARD: u32 = 1000; + const CANARY: &str = "CANARY-7f3a91"; + + /// What `systemctl show` prints for a loaded unit that acts on nothing, + /// plus the given properties (a given key replaces its default). + fn show_text(id: &str, properties: &[(&str, &str)]) -> String { + let mut text = format!("Id={id}\n"); + for (key, default) in [("Names", id), ("LoadState", "loaded")] { + if !properties.iter().any(|(given, _)| *given == key) { + text += &format!("{key}={default}\n"); + } + } + for (key, value) in properties { + text += &format!("{key}={value}\n"); + } + text + } + + struct FakeSystemd { + units: Mutex>, + runs: Mutex>, + shows: Mutex>, + report: Mutex, + } + + impl Systemd for FakeSystemd { + fn show(&self, unit: &str) -> Result { + self.shows.lock().unwrap().push(unit.into()); + let known = self.units.lock().unwrap().get(unit).cloned(); + parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) + } + + fn invocation(&self, _unit: &str) -> Option { + Some("invocation-1".into()) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + self.runs.lock().unwrap().push((unit.into(), step.clone())); + Ok(self.report.lock().unwrap().clone()) + } + } + + struct FakePeers(u32); + + impl PeerCredentials for FakePeers { + fn peer(&self, _stream: &UnixStream) -> Result { + Ok(Peer { + uid: self.0, + issuer_unit: "mind-eureka-self.service".into(), + }) + } + } + + /// Whether the brake is clear, and the principals it was asked about. + struct FakeBrake(AtomicBool, Mutex>); + + impl BrakeCheck for FakeBrake { + fn clear(&self, principal: &str) -> bool { + self.1.lock().unwrap().push(principal.into()); + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct FakeHome { + failing: AtomicBool, + sent: Mutex>, + } + + impl Homeserver for FakeHome { + fn post(&self, txn_id: &str, text: &str) -> Result { + ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); + let mut sent = self.sent.lock().unwrap(); + sent.push((txn_id.into(), text.into())); + Ok(format!("$event{}", sent.len())) + } + } + + struct Harness { + directory: TempDir, + ctx: Ctx, + systemd: Arc, + brake: Arc, + home: Arc, + queue: mpsc::Sender, + _orders: mpsc::Receiver, + } + + fn harness_for(peer_uid: u32) -> Harness { + let directory = TempDir::new().unwrap(); + let systemd = Arc::new(FakeSystemd { + units: Mutex::new(BTreeMap::new()), + runs: Mutex::new(Vec::new()), + shows: Mutex::new(Vec::new()), + report: Mutex::new(RunReport { + exit_code: Some(0), + ..RunReport::default() + }), + }); + let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); + let home = Arc::new(FakeHome::default()); + let (queue, orders) = mpsc::channel(); + let ctx = Ctx { + store: OrderStore::new(&directory.path().join("orders")), + state_store: directory.path().join("state").join("control.cc"), + steward_uid: STEWARD, + principal: "eureka".into(), + systemd: systemd.clone(), + peers: Arc::new(FakePeers(peer_uid)), + brake: brake.clone(), + homeserver: home.clone(), + }; + Harness { + directory, + ctx, + systemd, + brake, + home, + queue, + _orders: orders, + } + } + + fn harness() -> Harness { + harness_for(STEWARD) + } + + impl Harness { + fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + write(&mut client); + handle(&self.ctx, &mut server, &self.queue).unwrap(); + let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); + rmp_serde::from_slice(&bytes).unwrap() + } + + fn ask(&self, request: &Request) -> Response { + self.exchange(|client| write_frame(client, request).unwrap()) + } + + fn issue(&self, operation: Operation) -> Response { + self.ask(&Request::Order(OrderRequest { + operation, + reason: "probe".into(), + })) + } + + fn issue_unit(&self, verb: Verb, unit: &str) -> Response { + self.issue(Operation::UnitAction { + verb, + unit: unit.into(), + }) + } + + fn accept(&self, response: Response) -> String { + match response { + Response::Accepted { order_id } => order_id, + other => panic!("expected an accepted order, got {other:?}"), + } + } + + fn outcome(&self, order_id: &str) -> Outcome { + self.ctx + .store + .ledger() + .unwrap() + .outcome_of(order_id) + .expect("the order has an outcome") + .outcome + .clone() + } + + fn runs(&self) -> Vec<(String, Step)> { + self.systemd.runs.lock().unwrap().clone() + } + + /// Tell the fake `systemctl show` what it prints for `name`. + fn describe_unit(&self, name: &str, text: String) { + self.systemd.units.lock().unwrap().insert(name.into(), text); + } + + fn describe(&self, name: &str, properties: &[(&str, &str)]) { + self.describe_unit(name, show_text(name, properties)); + } + } + + fn all_verbs() -> [Verb; 4] { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + } + + #[test] + fn foreign_uid_is_refused_at_the_door() { + let h = harness_for(STEWARD + 1); + let response = h.issue_unit(Verb::Restart, "nginx.service"); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + let response = h.ask(&Request::List); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); + assert!(h.runs().is_empty()); + } + + #[derive(Serialize)] + struct ForgedBody { + operation: Operation, + reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + } + + #[derive(Serialize)] + struct ForgedFrame { + order: ForgedBody, + } + + #[test] + fn order_id_and_requester_are_minted_not_declared() { + let h = harness(); + let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { + let frame = ForgedFrame { + order: ForgedBody { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into(), + }, + reason: "probe".into(), + order_id: order_id.map(str::to_owned), + requested_by: requested_by.map(str::to_owned), + command: command.map(str::to_owned), + }, + }; + h.exchange(|client| write_frame(client, &frame).unwrap()) + }; + // The same frame without the extra field is a good order. + let honest = h.accept(forge(None, None, None)); + for forged in [ + forge(Some("ord-declared"), None, None), + forge(None, Some("steward:someone-else order x"), None), + forge(None, None, Some("reboot-now")), + ] { + assert_eq!( + forged, + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + } + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); + let order = &ledger.orders[0]; + assert_eq!(order.order_id, honest); + let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); + assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); + assert_eq!(order.issuer_unit, "mind-eureka-self.service"); + assert_eq!(order.issuer_invocation, "invocation-1"); + assert_eq!(order.principal, "eureka"); + assert_eq!(order.issuer_uid, STEWARD); + assert_eq!( + requester(&order.principal, &order.order_id), + format!("steward:eureka order {}", order.order_id) + ); + } + + #[test] + fn package_names_outside_grammar_are_refused() { + let h = harness(); + let bad = [ + "-y", + "--allow-unauthenticated", + "-", + "a", + "", + "Foo", + "a/b", + "/tmp/x", + "./x", + "x.deb", + "foo.deb", + "foo bar", + "foo;reboot", + "foo$(id)", + "foo`id`", + "foo|bar", + "foo&bar", + "foo\nbar", + "foo*", + "foo?", + "pkg:amd64", + "pkg/stable", + "libc6-", + "foo=", + "foo=1 2", + "foo=1;2", + "foo=1=2", + "fo\u{ff0f}o", + "f\u{43e}o", + ]; + for name in bad { + let response = h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into(), name.into()], + }); + assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); + } + let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); + assert_eq!( + h.issue(Operation::PackageInstall { packages: many }), + Response::Refused(Refusal::new("package", Code::TooMany)) + ); + assert_eq!( + h.issue(Operation::PackageInstall { packages: vec![] }), + Response::Refused(Refusal::new("package", Code::Empty)) + ); + assert!(!h.ctx.store.path().exists(), "no refusal left a record"); + + let good = [ + "git-lfs", + "matrix-synapse-py3", + "libnss-systemd", + "bubblewrap", + "socat", + "g++", + "libstdc++6", + "python3.11", + "nginx=1.24.0-2ubuntu7", + "foo=1:2.3-4+b1~rc1", + ]; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: good.iter().map(|name| (*name).to_owned()).collect(), + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + let runs = h.runs(); + assert_eq!(runs.len(), 3); + assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); + // Each name selects exactly itself and apt may remove nothing. + let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; + install.extend(["--no-remove".to_owned(), "--".into()]); + install.extend(good.iter().map(|spec| match spec.split_once('=') { + Some((name, version)) => format!("?exact-name({name})={version}"), + None => format!("?exact-name({spec})"), + })); + assert_eq!(runs[1].1.argv, install); + assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); + assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); + // The last step fails unless every named package is installed. + let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; + present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); + assert_eq!(runs[2].1.argv, present); + for (unit, step) in &runs { + assert!(unit.starts_with("idunn-order-")); + let apt = step.argv[0] == "apt-get"; + assert_eq!( + step.environment, + if apt { + vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + } else { + Vec::new() + } + ); + } + assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); + + // A package whose name apt would read as a regex or a conflict is + // exactly itself and never a removal. + for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { + let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); + assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); + } + // A step that failed ends the order: the absent package is a failure. + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(1), + ..RunReport::default() + }; + let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); + execute(&h.ctx, &absent).unwrap(); + assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); + assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); + } + + #[test] + fn any_unit_outside_the_refused_set_runs() { + let h = harness(); + let units = [ + "wg-quick@wg0.service", + "nginx.service", + "matrix-synapse.service", + "ssh.socket", + "huginn@eureka.service", + "claude-connector.service", + "persona-matrix-mouth.service", + "mind-heartbeat.service", + "apt-daily.timer", + // Near misses: the prefix is the rule, not a substring or a stem. + "midunn-x.service", + "not-idunn-x.service", + "idunn.service", + "mind-self.service", + "mind-selfx@a.service", + "mind-persona.service", + "mind-personal@a.service", + ]; + for verb in all_verbs() { + for unit in units { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); + let (transient, step) = h.runs().pop().unwrap(); + assert!(transient.starts_with("idunn-order-")); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] + ); + } + } + assert_eq!(h.runs().len(), units.len() * 4); + } + + #[test] + fn refused_set_is_refused_by_name_and_alias() { + let h = harness(); + h.describe_unit( + "backup-name.service", + show_text( + "idunn-yggdrasil.service", + &[("Names", "idunn-yggdrasil.service backup-name.service")], + ), + ); + h.describe( + "quiet-alias.service", + &[("Names", "quiet-alias.service mind-persona@void.service")], + ); + let direct = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-x.service", + "idunn-order-3f2c.service", + "idunn-yggdrasil.timer", + "idunn-x.socket", + "idunn-.service", + "idunn-yggdrasil.service.service", + "idunn-x@y.service", + "mind-self@eureka.service", + "mind-persona@void.service", + "mind-self@.service", + "mind-persona@a@b.service", + // Case folds: a lookalike is still refused, never run. + "IDUNN-yggdrasil.service", + "Idunn-Orders.service", + "MIND-SELF@eureka.service", + "Mind-Persona@Void.service", + ]; + for verb in all_verbs() { + for unit in direct { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + match h.outcome(&id) { + Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { + assert_eq!(hit, unit, "{verb:?} {unit}"); + assert_eq!(via, Via::Name); + } + other => panic!("{verb:?} {unit} was {other:?}"), + } + } + } + for verb in all_verbs() { + for (unit, hit) in [ + ("backup-name.service", "idunn-yggdrasil.service"), + ("quiet-alias.service", "mind-persona@void.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Alias + }), + "{verb:?} {unit}" + ); + } + } + assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); + // Every refusal is recorded and has a result post due. + let ledger = h.ctx.store.ledger().unwrap(); + let refusals = ledger + .outcomes + .iter() + .filter(|record| matches!(record.outcome, Outcome::Refused(_))) + .count(); + assert_eq!(refusals, direct.len() * 4 + 8); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + let results = due + .iter() + .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) + .count(); + assert_eq!(results, refusals); + // A refused order never started. + assert!(ledger.starts.is_empty()); + } + + fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + h.outcome(&id) + } + + fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { + match outcome { + Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { + Some(unit.as_str()) + } + _ => None, + } + } + + #[test] + fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { + let h = harness(); + // systemctl show quotes a name with a backslash and doubles the backslash. + h.describe( + "huginn.service", + &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], + ); + let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); + assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); + h.describe( + "quiet.service", + &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], + ); + let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); + assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); + assert!(h.runs().is_empty()); + } + + #[test] + fn unit_orders_run_as_one_job() { + let h = harness(); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); + let (_, step) = h.runs().pop().unwrap(); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] + ); + } + } + + #[test] + fn the_rule_reads_only_the_named_unit() { + let h = harness(); + // Whatever the unit lists, no other unit is looked at. + h.describe( + "plain.service", + &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], + ); + h.describe("other.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); + } + + #[test] + fn a_unit_that_acts_when_it_ends_is_refused() { + let h = harness(); + let properties = [ + ("SuccessAction", "reboot"), + ("FailureAction", "poweroff-force"), + ("StartLimitAction", "soft-reboot-immediate"), + ("JobTimeoutAction", "exit"), + ("FailureAction", "a-spelling-nobody-knows"), + ("OnFailure", "rescue.service"), + ("OnSuccess", "cleanup.service"), + ]; + for (property, value) in properties { + h.describe("ender.service", &[(property, value)]); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "ender.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "ender.service".into(), + via: Via::OnEnd + }), + "{verb:?} {property}={value}" + ); + } + assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); + } + // `none` and an empty list say the unit does nothing at its end. + h.describe( + "calm.service", + &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.runs().len(), 11); + } + + #[test] + fn host_names_are_refused() { + let h = harness(); + let names = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", + ]; + h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); + for verb in all_verbs() { + for name in names { + assert_eq!( + outcome_of_unit(&h, verb, name), + Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), + "{verb:?} {name}" + ); + } + assert_eq!( + outcome_of_unit(&h, verb, "sleepy.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "systemd-hibernate.service".into(), + via: Via::Host + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); + } + + #[test] + fn a_unit_others_are_bound_to_only_starts() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service second.service")]); + for verb in [Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "base.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "first.service".into(), + via: Via::Bound + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Start, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_unit_that_upholds_others_is_refused_where_it_would_start() { + let h = harness(); + h.describe("holder.service", &[("Upholds", "held.service")]); + for verb in [Verb::Start, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "holder.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "holder.service".into(), + via: Via::Upholds + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { + let h = harness(); + h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); + for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { + assert_eq!( + outcome_of_unit(&h, verb, "wrapper.service"), + Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); + } + + #[test] + fn empty_dependency_lists_name_no_one() { + let h = harness(); + // systemctl prints every property it is asked for, empty or not. + h.describe( + "alone.service", + &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service")]); + h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); + assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); + h.systemd.shows.lock().unwrap().clear(); + assert_eq!( + hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), + Some("idunn-ghost.service") + ); + assert!(h.systemd.shows.lock().unwrap().is_empty()); + } + + #[test] + fn each_refusal_is_posted_under_its_own_reason() { + for (via, word) in [ + (Via::Name, "name"), + (Via::Alias, "alias"), + (Via::Host, "host"), + (Via::OnEnd, "on-end"), + (Via::Bound, "bound"), + (Via::Upholds, "upholds"), + ] { + assert_eq!( + Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) + .describe(), + format!("refused: always-refused unit u.service reached by {word}") + ); + } + assert_eq!( + Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), + "refused: wrapper unit, it only wraps m.service" + ); + } + + #[test] + fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { + let h = harness(); + h.describe("ghost.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "ghost.service")); + execute(&h.ctx, &id).unwrap(); + assert!( + matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + } + + #[test] + fn globs_slices_scopes_and_targets_are_not_units() { + let h = harness(); + let bad = [ + "idunn*", + "idunn-*.service", + "idunn-?ggdrasil.service", + "mind-?elf@eureka.service", + "mind-self@*.service", + "mind-[s]elf@eureka.service", + "system.slice", + "poweroff.target", + "multi-user.target", + "user.slice", + "session-1.scope", + "init.scope", + "idunn-yggdrasil", + "mind-self@eureka", + "nginx", + ".service", + "", + "-idunn-x.service", + "-x.service", + "/etc/systemd/system/idunn-yggdrasil.service", + "../idunn-yggdrasil.service", + "idunn-yggdrasil.service\n", + "idunn-yggdrasil.service ", + " idunn-yggdrasil.service", + "idunn-yggdrasil.service\0", + "idunn-yggdrasil.service;reboot", + "idunn\\x2dyggdrasil.service", + "idunn%2dyggdrasil.service", + // Unicode lookalikes of a refused name. + "\u{131}dunn-yggdrasil.service", + "idunn\u{2010}yggdrasil.service", + "idunn\u{ff0d}yggdrasil.service", + "idunn-yggdrasil.service\u{200b}", + "\u{200b}idunn-yggdrasil.service", + "idunn-yggdrasil\u{2024}service", + "mind-self\u{ff20}eureka.service", + ]; + for unit in bad { + for verb in all_verbs() { + assert!( + matches!(h.issue_unit(verb, unit), Response::Refused(_)), + "{unit:?} was not refused by the grammar" + ); + } + } + let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); + assert_eq!( + h.issue_unit(Verb::Start, &long), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); + } + + #[test] + fn binding_with_a_unit_list_is_refused() { + let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ + mind_control = \"/usr/local/bin/mind-control\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ + token_credential = \"idunn-matrix-token\"\n"; + let binding = StewardBinding::parse(valid).unwrap(); + assert_eq!(binding.principal, "eureka"); + assert_eq!(binding.address().unwrap().port(), 8008); + for extra in [ + "units = [\"nginx.service\"]", + "allow = [\"nginx.service\"]", + "deny = [\"idunn-*\"]", + "denylist = [\"idunn-*\"]", + "refused = [\"idunn-*\"]", + "unit = \"nginx.service\"", + "[units]\nnginx = true", + ] { + assert!( + StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), + "{extra} loaded" + ); + } + let swap = |from: &str, to: &str| valid.replace(from, to); + for broken in [ + swap("user = \"mind-eureka-self\"", "user = \"root\""), + swap("user = \"mind-eureka-self\"", "user = \"\""), + swap("/usr/local/bin/mind-control", "mind-control"), + swap("!abc:example.org", "#alias:example.org"), + swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), + swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), + swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), + swap("idunn-matrix-token", "../token"), + swap("principal = \"eureka\"\n", ""), + ] { + assert!(StewardBinding::parse(&broken).is_err()); + } + assert!(admitted(0, 0o100600)); + assert!(admitted(0, 0o100644)); + assert!(!admitted(0, 0o100664)); + assert!(!admitted(0, 0o100666)); + assert!(!admitted(0, 0o100602)); + assert!(!admitted(1000, 0o100600)); + } + + #[test] + fn held_brake_refuses_at_start() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); + assert!(h.runs().is_empty()); + // The deploy path is gated the same way and writes no command. + let deploy = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &deploy).unwrap(); + assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); + assert!(!h.ctx.state_store.exists()); + // A brake that cannot be asked is a held brake. + let directory = TempDir::new().unwrap(); + assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); + } + + #[test] + fn brake_check_is_the_exit_status_of_mind_control_check() { + let directory = TempDir::new().unwrap(); + let program = |name: &str, body: &str| { + let path = directory.path().join(name); + std::fs::write(&path, body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + MindControl { program: path } + }; + let recorder = directory.path().join("argv"); + let clear = program( + "clear", + &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), + ); + assert!(clear.clear("eureka")); + assert_eq!( + std::fs::read_to_string(&recorder).unwrap().trim(), + "--instance eureka check" + ); + assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); + } + + #[test] + fn started_order_is_never_rerun() { + let h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The first order began and its service died: a start with no outcome. + h.ctx + .store + .put( + &id, + START_SCHEMA, + &StartRecord { + schema_version: START_SCHEMA.into(), + order_id: id.clone(), + started_at_unix_millis: 1, + actuation: format!("idunn-order-{}", order_uuid(&id)), + }, + ) + .unwrap(); + let pending = recover(&h.ctx).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Lost); + assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); + // Recovery is idempotent, and a finished order is not run again. + assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&h.ctx, &waiting).unwrap(); + execute(&h.ctx, &waiting).unwrap(); + assert_eq!(h.runs().len(), 1); + assert!(recover(&h.ctx).unwrap().is_empty()); + } + + #[test] + fn deploy_order_uses_the_up_primitive() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let command_id = format!("up-{}", order_uuid(&id)); + assert_eq!( + h.outcome(&id), + Outcome::Submitted { + command_id: command_id.clone() + } + ); + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); + assert!(h.runs().is_empty(), "a deploy runs no transient unit"); + let mut shown = Vec::new(); + crate::control_plane::status( + &mut shown, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + Some(&command_id), + ) + .unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); + assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); + // Re-executing writes no second command. + execute(&h.ctx, &id).unwrap(); + let mut again = Vec::new(); + crate::control_plane::status( + &mut again, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + None, + ) + .unwrap(); + assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); + // The result post waits for the transactions: queued is not finished. + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert_eq!(due.len(), 1); + assert_eq!(due[0].kind, PostKind::Accepted); + assert!(!deploy_finished("queued") && !deploy_finished("running")); + assert!(deploy_finished("succeeded") && deploy_finished("failed")); + assert!(deploy_finished("complete")); + } + + #[test] + fn chat_outage_delays_posts_not_orders() { + let h = harness(); + h.home.failing.store(true, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + assert_eq!(h.runs().len(), 1); + assert!(post_pass(&h.ctx).is_err()); + assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); + h.home.failing.store(false, Ordering::SeqCst); + assert_eq!(post_pass(&h.ctx).unwrap(), 2); + assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); + let sent = h.home.sent.lock().unwrap().clone(); + assert_eq!(sent.len(), 2); + assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); + assert_eq!(sent[1].0, format!("idunn-{id}-result")); + assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); + assert!(sent[0].1.contains("mind-eureka-self.service")); + assert!(sent[0].1.contains("reason: \"probe\"")); + assert!(sent[1].1.contains("ran (exit 0)")); + } + + #[test] + fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { + let h = harness(); + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(100), + signal: None, + stdout_tail: vec![b'o'; 3 * TAIL_BYTES], + stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), + }; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into()], + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Failed { + exit_code: Some(100), + signal: None + } + ); + assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); + let ledger = h.ctx.store.ledger().unwrap(); + let record = ledger.outcome_of(&id).unwrap(); + assert_eq!(record.stdout_tail.len(), TAIL_BYTES); + assert!(!format!("{record:?}").contains("secret-looking")); + // Neither the post nor the journal text carries a tail. + let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) + .into_iter() + .filter(|due| due.kind == PostKind::Result) + .collect(); + assert_eq!(results.len(), 1); + assert!(results[0].text.contains("failed (exit 100)")); + assert!(!results[0].text.contains("secret-looking")); + // The steward reads them through the escape sink and nothing raw. + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("expected a show") + }; + let mut shown = Vec::new(); + emit(&mut shown, &render_view(&view)).unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert!(shown.contains("secret-looking")); + assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); + assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); + assert!(shown.contains("status failed")); + // A show of an unknown or malformed id names the field only. + assert_eq!( + h.ask(&Request::Show(ShowRequest { + id: format!("ord-{}", Uuid::new_v4()) + })), + Response::Refused(Refusal::new("id", Code::Unknown)) + ); + assert_eq!( + h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), + Response::Refused(Refusal::new("id", Code::BadShape)) + ); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("expected a list") + }; + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].status, "failed"); + } + + #[test] + fn a_refusal_or_error_never_echoes_an_input_value() { + let h = harness(); + let mut seen: Vec> = Vec::new(); + let mut note = |response: Response| { + let bytes = rmp_serde::to_vec_named(&response).unwrap(); + if let Response::Refused(refusal) = &response { + seen.push(refusal.to_string().into_bytes()); + } + seen.push(bytes); + }; + for unit in [ + format!("{CANARY}*.service"), + format!("{CANARY}/../idunn-orders.service"), + format!("idunn-{CANARY};x.service"), + CANARY.to_string(), + ] { + note(h.issue_unit(Verb::Restart, &unit)); + } + note(h.issue(Operation::PackageInstall { + packages: vec![format!("{CANARY}/x")], + })); + note(h.issue(Operation::PackageInstall { + packages: vec![format!("pkg={CANARY} 1")], + })); + note(h.issue(Operation::Deploy { + selector: format!("{CANARY} bad"), + })); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: format!("{CANARY}\n"), + }))); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: CANARY.repeat(40), + }))); + note(h.ask(&Request::Show(ShowRequest { + id: format!("ord-{CANARY}"), + }))); + // A frame whose own keys and values are the canary, and plain garbage. + let named: std::collections::BTreeMap> = + [( + CANARY.to_owned(), + [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), + )] + .into_iter() + .collect(); + note(h.exchange(|client| write_frame(client, &named).unwrap())); + note(h.exchange(|client| { + let garbage = format!("{CANARY}{CANARY}"); + client + .write_all(&(garbage.len() as u32).to_be_bytes()) + .unwrap(); + client.write_all(garbage.as_bytes()).unwrap(); + })); + note(h.exchange(|client| { + client.write_all(&u32::MAX.to_be_bytes()).unwrap(); + })); + // The client grammar fails the same way. + let mut messages = Vec::new(); + let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); + for line in [ + words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), + words(&["unit", CANARY, "nginx.service", "--reason", "r"]), + words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), + words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), + words(&[CANARY]), + words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), + words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), + words(&["show"]), + ] { + messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); + } + for line in [ + words(&[CANARY]), + words(&["serve", &format!("--{CANARY}")]), + words(&["serve", "--store"]), + ] { + messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); + } + for bytes in &seen { + assert!( + !String::from_utf8_lossy(bytes).contains(CANARY), + "a response carried an input value" + ); + } + for message in &messages { + assert!(!message.contains(CANARY), "an error carried an input value: {message}"); + } + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn cgroup_names_the_issuer_unit() { + assert_eq!( + unit_from_cgroup("0::/system.slice/claude-connector.service\n"), + "claude-connector.service" + ); + // What a delegated subtree adds below the unit is the peer's to name. + assert_eq!( + unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), + "huginn@eureka.service" + ); + assert_eq!( + unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), + "getty@tty1.service" + ); + assert_eq!( + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), + "a.scope" + ); + // A user manager's units, and paths no manager made, name no one. + for path in [ + "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", + "0::/user.slice/user-1000.slice/session-3.scope\n", + "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", + "0::/system.slice/\n", + "0::/\n", + "", + ] { + assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); + } + } + + #[test] + fn systemctl_show_output_is_read_into_facts() { + let facts = parse_show( + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ + SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ + OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ + Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", + ) + .unwrap(); + assert_eq!(facts.id, "a.service"); + assert_eq!(facts.names, ["a.service", "b.service"]); + assert!(!facts.not_found); + assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); + // A quoted name holds one backslash, not two and not a quote mark. + assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); + assert_eq!(facts.upholds, ["u.service"]); + assert_eq!(facts.consists_of, ["m.service", "n.service"]); + assert!(parse_show("Names=a.service\n").is_err()); + // The request asks for exactly the properties the parse reads. + assert_eq!( + show_properties(), + "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ + OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" + ); + // A backslash outside quotes is a backslash; inside, it escapes one character. + assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); + assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); + assert!(parse_names(" ").is_empty()); + let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); + assert!(gone.not_found); + } + + #[test] + fn the_store_refuses_a_second_record_and_a_foreign_document() { + let h = harness(); + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: "ord-x".into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }; + assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert_eq!( + std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); + SingleFileMessagePackBackingStore::new(h.ctx.store.path()) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: "idunn.other".into(), + key: "k".into(), + current: None, + }], + &[foreign], + ) + .unwrap(); + assert!(h.ctx.store.ledger().is_err()); + } + + #[test] + fn synapse_post_is_one_idempotent_put_on_loopback() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (sent, requests) = mpsc::channel(); + let server = thread::spawn(move || { + for reply in [ + "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", + "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", + ] { + let (mut socket, _) = listener.accept().unwrap(); + let mut received = Vec::new(); + let mut chunk = [0u8; 1024]; + while !received.windows(4).any(|window| window == b"\r\n\r\n") + || !String::from_utf8_lossy(&received).contains("\"body\"") + { + let count = socket.read(&mut chunk).unwrap(); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); + socket.write_all(reply.as_bytes()).unwrap(); + } + }); + let synapse = Synapse { + address, + room: "!room:example.org".into(), + token_file, + reply_within: Duration::from_secs(10), + }; + assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); + let request = requests.recv().unwrap(); + assert!(request.starts_with( + "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" + )); + assert!(request.contains("Authorization: Bearer tok-123\r\n")); + assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); + let error = synapse.post("txn-2", "x").unwrap_err(); + assert!(!format!("{error:#}").contains("tok-123")); + server.join().unwrap(); + assert!( + Synapse { + address, + room: "!r:e".into(), + token_file: directory.path().join("absent"), + reply_within: Duration::from_secs(10), + } + .post("t", "x") + .is_err() + ); + } + + #[test] + fn grammar_limits_are_exact_at_their_edges() { + let h = harness(); + let order = |operation: Operation, reason: &str| { + h.ask(&Request::Order(OrderRequest { + operation, + reason: reason.into(), + })) + }; + let packages = |names: Vec| Operation::PackageInstall { packages: names }; + let unit = |name: String| Operation::UnitAction { + verb: Verb::Start, + unit: name, + }; + // Package: two characters is the shortest name, 200 bytes the longest. + assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); + assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); + assert_eq!( + order(packages(vec!["a".repeat(201)]), "r"), + Response::Refused(Refusal::new("package", Code::TooLong)) + ); + assert_eq!( + order(packages(vec!["a".into()]), "r"), + Response::Refused(Refusal::new("package", Code::BadShape)) + ); + let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); + assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); + // Unit: 200 bytes with its suffix is the longest. + let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); + assert_eq!(longest.len(), UNIT_LIMIT); + assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); + let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); + assert_eq!( + order(unit(too_long), "r"), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + // Reason: 300 bytes is the longest, and it must be there. + assert!(matches!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), + Response::Accepted { .. } + )); + assert_eq!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), + Response::Refused(Refusal::new("reason", Code::TooLong)) + ); + assert_eq!( + order(unit("nginx.service".into()), ""), + Response::Refused(Refusal::new("reason", Code::Empty)) + ); + assert_eq!( + order(unit("nginx.service".into()), "caf\u{e9}"), + Response::Refused(Refusal::new("reason", Code::BadCharacter)) + ); + } + + #[test] + fn a_concluded_order_is_never_run_later() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + // Someone else clears the brake; the refused order stays refused. + h.brake.0.store(true, Ordering::SeqCst); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty()); + let ledger = h.ctx.store.ledger().unwrap(); + assert!(ledger.start_of(&id).is_none()); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + } + + #[test] + fn the_orders_cli_has_no_command_string() { + let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); + let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); + assert_eq!( + parsed.action, + OrderAction::Issue { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into() + }, + reason: "why".into() + } + ); + assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); + assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); + assert!(parse(&["show", "ord-x"]).is_ok()); + assert!(parse(&["list"]).is_ok()); + for refused in [ + vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], + vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], + vec!["run", "sh", "--reason", "r"], + vec!["exec", "--reason", "r"], + vec!["unit", "enable", "nginx.service", "--reason", "r"], + vec!["unit", "restart", "--reason", "r"], + vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], + vec!["unit", "restart", "nginx.service"], + vec!["unit", "restart", "nginx*", "--reason", "r"], + vec!["package-install", "--reason", "r"], + vec!["package-install", "-y", "x", "--reason", "r"], + vec!["deploy", "--reason", "r"], + vec!["deploy", "a", "b", "--reason", "r"], + vec!["list", "extra"], + vec!["show"], + vec![], + ] { + assert!(parse(&refused).is_err(), "{refused:?} parsed"); + } + } + + #[test] + fn an_order_runs_only_under_the_steward_bound_now() { + let mut h = harness(); + let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The binding changes while both wait: another uid, then another + // principal with the old uid. Each is refused with a record, before + // the brake is asked and before anything runs. + h.ctx.steward_uid = STEWARD + 7; + execute(&h.ctx, &first).unwrap(); + h.ctx.steward_uid = STEWARD; + h.ctx.principal = "void".into(); + execute(&h.ctx, &second).unwrap(); + for id in [&first, &second] { + assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); + assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); + } + assert!(h.runs().is_empty()); + assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); + // The refusal is posted like any other. + let ledger = h.ctx.store.ledger().unwrap(); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert!(due.iter().any(|due| due.kind == PostKind::Result + && due.text.contains("refused: the issuer is not the current steward"))); + // The other direction: an order the bound steward issued after the + // rebinding, and a recovered one under an unchanged binding, run and + // are gated by the bound principal's brake. + let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); + let resumed = harness(); + let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); + assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&resumed.ctx, &waiting).unwrap(); + execute(&h.ctx, &third).unwrap(); + assert_eq!(resumed.outcome(&waiting), Outcome::Ran); + assert_eq!(h.outcome(&third), Outcome::Ran); + assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); + assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); + } + + #[test] + fn root_reads_orders_and_never_issues_one() { + let mut h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + h.ctx.peers = Arc::new(FakePeers(0)); + let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); + assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("root could not read an order"); + }; + assert_eq!(view.order.order_id, id); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("root could not list orders"); + }; + assert_eq!(lines.len(), 1, "root's refused order left no record"); + h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); + assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); + } + + #[test] + fn every_order_posts_once_in_its_turn() { + let h = harness(); + let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] + .into_iter() + .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) + .collect(); + for id in &ids { + execute(&h.ctx, id).unwrap(); + } + assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); + assert_eq!(post_pass(&h.ctx).unwrap(), 0); + let sent = h.home.sent.lock().unwrap().clone(); + let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); + keys.sort(); + let mut expected: Vec = ids + .iter() + .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) + .collect(); + expected.sort(); + assert_eq!(keys, expected); + // An order accepted later is posted by the next pass alone. + let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + execute(&h.ctx, &late).unwrap(); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + } + + #[test] + fn a_deploys_result_is_posted_once_its_command_has_finished() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let ledger = h.ctx.store.ledger().unwrap(); + let results = |state: Option<(&'static str, &str)>| { + let owed = due_posts(&ledger, &|_| match state { + Some((state, detail)) => Ok((state, detail.to_owned())), + None => Err(anyhow!("the command cannot be read")), + }); + owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() + }; + assert!(results(Some(("queued", ""))).is_empty()); + assert!(results(Some(("running", "vili:Sealing"))).is_empty()); + assert!(results(None).is_empty()); + for state in ["succeeded", "failed"] { + let owed = results(Some((state, "vili"))); + assert_eq!(owed.len(), 1, "{state}"); + assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); + assert!(owed[0].text.contains("submitted deployment up-")); + } + } + + #[test] + fn intake_gives_up_on_a_peer_that_sends_no_frame() { + let h = harness(); + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + // Half a length prefix, then nothing: the connection must end by itself. + client.write_all(&[0, 0]).unwrap(); + let (done, ended) = mpsc::channel(); + let worker = thread::spawn(move || { + let result = handle(&h.ctx, &mut server, &h.queue); + let _ = done.send(result.is_ok()); + h + }); + let answered = ended + .recv_timeout(Duration::from_secs(20)) + .expect("intake never gave up on a silent peer"); + assert!(answered, "a silent peer is refused as malformed, not left as an error"); + let h = worker.join().unwrap(); + assert_eq!( + rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn a_record_under_another_schema_is_not_read() { + let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { + "order" => h.ctx.store.put( + key, + schema, + &OrderRecord { + schema_version: schema.into(), + order_id: key.into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }, + ), + "start" => h.ctx.store.put( + key, + schema, + &StartRecord { + schema_version: schema.into(), + order_id: key.into(), + started_at_unix_millis: 1, + actuation: "idunn-order-x".into(), + }, + ), + "outcome" => h.ctx.store.put( + key, + schema, + &OutcomeRecord { + schema_version: schema.into(), + order_id: key.into(), + ended_at_unix_millis: 1, + outcome: Outcome::Lost, + stdout_tail: Vec::new(), + stderr_tail: Vec::new(), + }, + ), + _ => h.ctx.store.put( + key, + schema, + &PostRecord { + schema_version: schema.into(), + post_key: key.into(), + event_id: "$e".into(), + posted_at_unix_millis: 1, + }, + ), + }; + for (kind, current, key) in [ + ("order", ORDER_SCHEMA, "ord-a"), + ("start", START_SCHEMA, "ord-a"), + ("outcome", OUTCOME_SCHEMA, "ord-a"), + ("post", POST_SCHEMA, "ord-a:accepted"), + ] { + let h = harness(); + assert!(put(&h, kind, current, key).unwrap()); + assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); + let h = harness(); + let other = current.replace(".v1", ".v0"); + assert!(put(&h, kind, &other, key).unwrap()); + assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); + } + } + + #[test] + fn posts_are_retried_after_a_widening_pause() { + let pauses: Vec = (0..9).map(post_pause).collect(); + assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); + } + + #[test] + fn serve_options_are_read_from_every_flag() { + let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); + let defaults = parse(&["serve"]).unwrap(); + assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); + assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); + assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); + let given = parse(&[ + "serve", + "--state-store", + "/a/control.cc", + "--store", + "/b", + "--steward-binding", + "/c.toml", + "--socket", + "/d.sock", + ]) + .unwrap(); + assert_eq!( + given, + ServeOptions { + state_store: "/a/control.cc".into(), + store: "/b".into(), + steward_binding: "/c.toml".into(), + socket: "/d.sock".into(), + } + ); + assert!(parse(&["serve", "--store"]).is_err()); + assert!(parse(&["serve", "--other", "/x"]).is_err()); + assert!(parse(&["start"]).is_err()); + } + + #[test] + fn the_steward_user_is_a_real_user_and_never_root() { + assert!(user_uid("root").is_err()); + assert!(user_uid("no-such-user-here").is_err()); + assert!(user_uid("bad\0name").is_err()); + assert_eq!(user_uid("nobody").unwrap(), 65534); + } + + #[test] + fn the_orders_directory_is_made_private() { + let directory = TempDir::new().unwrap(); + let store = OrderStore::new(&directory.path().join("orders")); + store.prepare().unwrap(); + let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + #[test] + fn the_socket_belongs_to_the_steward_alone() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("steward.sock"); + let me = std::fs::metadata(directory.path()).unwrap().uid(); + let _listener = bind_socket(&path, me).unwrap(); + let metadata = std::fs::metadata(&path).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); + // A stale socket is replaced; anything else in the way is not. + let _again = bind_socket(&path, me).unwrap(); + let other = directory.path().join("other"); + std::fs::write(&other, "x").unwrap(); + assert!(bind_socket(&other, me).is_err()); + } + + #[test] + fn synapse_gives_up_on_a_reply_that_never_ends() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + thread::spawn(move || { + // The first connection stays silent; the second drips a byte at a time. + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + thread::spawn(move || { + let mut chunk = [0u8; 4096]; + let _ = socket.read(&mut chunk); + for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { + if index == 0 || socket.write_all(&[*byte]).is_err() { + break; + } + thread::sleep(Duration::from_millis(40)); + } + thread::sleep(Duration::from_secs(5)); + }); + } + }); + let synapse = Synapse { + address, + room: "!r:e".into(), + token_file, + reply_within: Duration::from_millis(600), + }; + let synapse = Arc::new(synapse); + for _ in 0..2 { + let started = std::time::Instant::now(); + let (done, ended) = mpsc::channel(); + let poster = Arc::clone(&synapse); + thread::spawn(move || { + let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); + }); + let error = ended + .recv_timeout(Duration::from_secs(4)) + .expect("the post never gave up") + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error}"); + assert!(!error.contains("tok-123")); + } + } + + /// The whole service, as root, against fake `systemctl`, `systemd-run` + /// and a loopback homeserver, with the steward connecting from another + /// uid. It runs only where the test process is root (the verify container). + #[test] + fn serve_end_to_end_as_root() { + // SAFETY: geteuid reads the process credentials. + if unsafe { libc::geteuid() } != 0 { + eprintln!("serve_end_to_end_as_root: skipped, not root"); + return; + } + let directory = TempDir::new().unwrap(); + let root = directory.path().to_owned(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let bin = root.join("bin"); + std::fs::create_dir(&bin).unwrap(); + let systemctl = format!( + "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ + case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + root.display() + ); + let run = format!( + "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", + root.display() + ); + for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { + let path = bin.join(name); + std::fs::write(&path, text).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let credentials = root.join("cred"); + std::fs::create_dir(&credentials).unwrap(); + std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (posted, posts) = mpsc::channel::(); + thread::spawn(move || { + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + let mut received = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + let text = String::from_utf8_lossy(&received).into_owned(); + if let Some(split) = text.find("\r\n\r\n") { + let length = text + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if received.len() >= split + 4 + length { + break; + } + } + let count = socket.read(&mut chunk).unwrap_or(0); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); + let _ = socket.write_all( + format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), + ); + } + }); + let binding = root.join("steward.toml"); + std::fs::write( + &binding, + format!( + "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ + token_credential = \"idunn-matrix-token\"\n" + ), + ) + .unwrap(); + std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); + let run_directory = root.join("run"); + std::fs::create_dir(&run_directory).unwrap(); + std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let socket = run_directory.join("steward.sock"); + let store = root.join("orders"); + let control = root.join("state").join("control.cc"); + // SAFETY: the only test that reads these variables is this one; PATH + // keeps its old entries. + unsafe { + std::env::set_var( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), + ); + std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); + } + let options = ServeOptions { + state_store: control.clone(), + store: store.clone(), + steward_binding: binding, + socket: socket.clone(), + }; + // The unit's umask must not decide the modes of what the service writes. + // SAFETY: umask only sets the process file-mode mask. + let before = unsafe { libc::umask(0o117) }; + let (ended, serve_end) = mpsc::channel::(); + thread::spawn(move || { + let result = serve(options); + let _ = ended.send(format!("{result:?}")); + }); + for _ in 0..100 { + if socket.exists() || serve_end.try_recv().is_ok() { + break; + } + thread::sleep(Duration::from_millis(100)); + } + assert!(socket.exists(), "serve never bound its socket"); + let metadata = std::fs::metadata(&socket).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); + // Root is not the steward: it may read, it may not order. + let unit = |name: &str| { + Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: name.into(), + }, + reason: "end to end".into(), + }) + }; + let as_root = |request: &Request| { + let mut stream = UnixStream::connect(&socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + write_frame(&mut stream, request).unwrap(); + rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + }; + assert_eq!( + as_root(&unit("nginx.service")), + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert_eq!(as_root(&Request::List), Response::List(Vec::new())); + let orders = OrderStore::new(&store); + let finished = |order_id: &str| { + for _ in 0..200 { + if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { + return record.clone(); + } + thread::sleep(Duration::from_millis(100)); + } + panic!("{order_id} never concluded"); + }; + // The steward is uid 65534 with gid 0: a service that read the gid for + // the uid would take it for root. + let first = steward_asks(&socket, &unit("nginx.service")); + let Response::Accepted { order_id: first } = first else { + panic!("the steward was not accepted: {first:?}"); + }; + assert_eq!(finished(&first).outcome, Outcome::Ran); + std::fs::write(root.join("fail"), "").unwrap(); + let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { + panic!("the second order was not accepted"); + }; + let failed = finished(&second); + assert_eq!( + failed.outcome, + Outcome::Failed { + exit_code: Some(3), + signal: None + } + ); + assert_eq!(failed.stderr_tail, b"boom\n"); + let deploy = Request::Order(OrderRequest { + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "end to end".into(), + }); + let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { + panic!("the deploy was not accepted"); + }; + assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); + let ledger = orders.ledger().unwrap(); + let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); + assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // An invocation id is what a succeeding systemctl prints, if anything. + assert_eq!(HostSystemd.invocation("quiet.service"), None); + assert_eq!(HostSystemd.invocation("broken.service"), None); + assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); + // What the service called, exactly. + let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); + assert!( + run_log.contains(&format!( + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", + order_uuid(&first) + )), + "{run_log}" + ); + let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); + assert!( + systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), + "{systemctl_log}" + ); + // Every record is posted; the deploy's result waits for its command. + let mut seen = Vec::new(); + while seen.len() < 5 { + seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); + } + for (id, kinds) in [ + (&first, ["accepted", "result"].as_slice()), + (&second, &["accepted", "result"]), + (&third, &["accepted"]), + ] { + for kind in kinds { + let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); + assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); + } + } + // Modes: the store is private, and the deploy wrote control.cc under + // Idunn's umask, not the 0117 the process was started with. + let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; + assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); + let control_mode = mode(&control); + // SAFETY: umask only sets the process file-mode mask. + unsafe { libc::umask(before) }; + assert_eq!(control_mode, 0o640); + } + + /// One request as the steward: uid 65534 and gid 0, from a forked child + /// that only makes system calls. + fn steward_asks(socket: &Path, request: &Request) -> Response { + let body = rmp_serde::to_vec_named(request).unwrap(); + let mut frame = (body.len() as u32).to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + // SAFETY: plain data. + let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { + address.sun_path[index] = *byte as libc::c_char; + } + let mut fds = [0i32; 2]; + // SAFETY: `fds` holds two descriptors. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + // SAFETY: the child allocates nothing and ends with _exit. + let pid = unsafe { libc::fork() }; + if pid == 0 { + // SAFETY: only system calls on buffers made before the fork. + unsafe { + libc::close(fds[0]); + libc::alarm(10); + if libc::setgroups(0, std::ptr::null()) != 0 + || libc::setgid(0) != 0 + || libc::setuid(65534) != 0 + { + libc::_exit(2); + } + let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if libc::connect( + fd, + (&address as *const libc::sockaddr_un).cast(), + std::mem::size_of::() as libc::socklen_t, + ) != 0 + { + libc::_exit(3); + } + let mut offset = 0usize; + while offset < frame.len() { + let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); + if count <= 0 { + libc::_exit(4); + } + offset += count as usize; + } + let mut buffer = [0u8; 4096]; + loop { + let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); + if count <= 0 { + break; + } + libc::write(fds[1], buffer.as_ptr().cast(), count as usize); + } + libc::_exit(0); + } + } + // SAFETY: the parent's copy of the write end. + unsafe { libc::close(fds[1]) }; + let mut reply = Vec::new(); + // SAFETY: `fds[0]` is an open descriptor this function now owns. + let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; + pipe.read_to_end(&mut reply).unwrap(); + let mut status = 0; + // SAFETY: `pid` is our child. + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); + rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + } + + fn host_refusal(verb: Verb, unit: &str) -> Option { + refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) + } + + #[test] + #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] + fn real_host_unit_set() { + for name in [ + "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", + "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", + "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", + "wg-quick@wg-gamecult", "cron", "apt-daily", + ] { + let unit = format!("{name}.service"); + for verb in all_verbs() { + assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); + } + } + for name in [ + "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", + "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", + "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", + "systemd-storagetm", + ] { + let unit = format!("{name}.service"); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + host_refusal(verb, &unit), + Some(refused(&unit, Via::OnEnd)), + "{verb:?} {unit}" + ); + } + } + for unit in HOST_NAMES { + for verb in all_verbs() { + assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); + } + } + for unit in [ + "systemd-networkd.service", + "user-runtime-dir@1000.service", + "user-runtime-dir@105.service", + "user-runtime-dir@978.service", + ] { + assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); + assert!( + matches!( + host_refusal(Verb::Stop, unit), + Some(Refused::AlwaysRefused { via: Via::Bound, .. }) + ), + "{unit}" + ); + } + let listing = Command::new("systemctl") + .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) + .output() + .unwrap(); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let units: Vec<&str> = listing + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|unit| validate_unit(unit).is_ok()) + .collect(); + for verb in all_verbs() { + let mut counts = BTreeMap::new(); + for unit in &units { + let key = match refusal_for(&HostSystemd, unit, verb) { + Ok(None) => "ok".to_owned(), + Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), + Ok(Some(other)) => format!("{other:?}"), + Err(_) => "unresolved".to_owned(), + }; + *counts.entry(key).or_insert(0usize) += 1; + } + println!("{verb:?} over {} units: {counts:?}", units.len()); + } + } + + #[test] + #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] + fn one_job_argv_against_a_real_manager() { + let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); + std::fs::create_dir_all(&directory).unwrap(); + let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); + let body = "[Service]\nExecStart=/bin/sleep 600\n"; + std::fs::write(directory.join(a), body).unwrap(); + std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); + let user = |args: &[&str]| { + let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); + (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) + }; + let outcome = std::panic::catch_unwind(|| { + let one_job = |verb: Verb, unit: &str| { + let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); + let mut argv = step.argv.clone(); + argv.insert(1, "--user".into()); + let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); + assert!(status.success(), "{verb:?} {unit}"); + }; + assert!(user(&["daemon-reload"]).0); + // A start of b leaves a running, though b conflicts with it. + assert!(user(&["start", a]).0); + one_job(Verb::Start, b); + assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); + // A restart of b, with both running, leaves a running. + one_job(Verb::Restart, b); + assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); + }); + user(&["stop", a, b]); + std::fs::remove_file(directory.join(a)).unwrap(); + std::fs::remove_file(directory.join(b)).unwrap(); + user(&["daemon-reload"]); + outcome.unwrap(); + } +} diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 70060bf..b9f2637 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -10,3 +10,19 @@ It is checked in because the TypeScript runtime emits a camelCase MessagePack map while Rust `DatabaseEntry` derives tuple payloads. Idunn must prove it can decode the producer's real wire shape rather than round-tripping a fixture it serialized itself. + +# systemd 257 `systemctl show` captures + +`systemd-257/*.show` are the verbatim output of systemd 257.13 for the 27 +properties `show_unit` requested at `112a7a5`, captured read-only on Yggdrasil +on 2026-10-24: `vili-presence-identity-only.show` (a failed idunn-vili unit whose +only credential is the presence identity) and +`odin-presence-and-one-service-credential.show` (the live Odin unit). systemd 257 +prints `LoadCredential=[unprintable]` in both. Tests read these bytes; do not edit. + +# Pre-cut systemd observation records + +`pre-cut-systemd-observation.{compact,named}.hex` are the two MessagePack +encodings of a `SystemdWorkloadObservation` with `load_credential` set to +`[unprintable]`, produced by the code at `112a7a5` (the form live incumbents +persisted). They pin that later Idunns still decode such records. diff --git a/tests/fixtures/pre-cut-systemd-observation.compact.hex b/tests/fixtures/pre-cut-systemd-observation.compact.hex new file mode 100644 index 0000000..e7d09fb --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.compact.hex @@ -0,0 +1 @@ +dc0033d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365aa4964756e6e2074657374a3696e7601a465786563a26e6fa56d69786564c3a6753631303030a6753631303030a0a0a0c3c3a9696e76697369626c65a3616c6cc3a430303037a0ad5b756e7072696e7461626c655dcd10920194cdee48cdee48cdee48cdee4894cdee48cdee48cdee48cdee4891cdee480000000000c391010708b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d3232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323290d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190 \ No newline at end of file diff --git a/tests/fixtures/pre-cut-systemd-observation.named.hex b/tests/fixtures/pre-cut-systemd-observation.named.hex new file mode 100644 index 0000000..8898adf --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.named.hex @@ -0,0 +1 @@ +de0033a4756e6974d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365b0756e69745f6465736372697074696f6eaa4964756e6e2074657374ad696e766f636174696f6e5f6964a3696e76d923657865635f6d61696e5f73746172745f74696d657374616d705f6d6f6e6f746f6e696301ac736572766963655f74797065a465786563ae726573746172745f706f6c696379a26e6fa96b696c6c5f6d6f6465a56d69786564ac64796e616d69635f75736572c3ac73797374656d645f75736572a6753631303030ad73797374656d645f67726f7570a6753631303030b4737570706c656d656e746172795f67726f757073a0b76361706162696c6974795f626f756e64696e675f736574a0b4616d6269656e745f6361706162696c6974696573a0ae707269766174655f6d6f756e7473c3ac707269766174655f70696473c3ac70726f746563745f70726f63a9696e76697369626c65ab70726f635f737562736574a3616c6cb16e6f5f6e65775f70726976696c65676573c3a5756d61736ba430303037b2696e61636365737369626c655f7061746873a0af6c6f61645f63726564656e7469616cad5b756e7072696e7461626c655da86d61696e5f706964cd1092b270726f636573735f73746172745f74696d6501ac70726f636573735f7569647394cdee48cdee48cdee48cdee48ac70726f636573735f6769647394cdee48cdee48cdee48cdee48ae70726f636573735f67726f75707391cdee48b770726f636573735f6361705f696e686572697461626c6500b570726f636573735f6361705f7065726d697474656400b570726f636573735f6361705f65666665637469766500b470726f636573735f6361705f626f756e64696e6700b370726f636573735f6361705f616d6269656e7400b970726f636573735f6e6f5f6e65775f70726976696c65676573c3b670726f636573735f6e616d6573706163655f706964739101b26d6f756e745f6e616d6573706163655f696407b07069645f6e616d6573706163655f696408aa65786563757461626c65b52f6f70742f746573742f62696e2f73657276696365b165786563757461626c655f64657669636501b065786563757461626c655f696e6f646501b165786563757461626c655f736861323536d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131b372756e74696d655f696e7374616e63655f6964d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161b1776f726b696e675f6469726563746f7279a92f6f70742f74657374ae72756e74696d655f62756e646c65a92f72756e2f74657374b3636f6d6d616e645f6c696e655f736861323536d9477368613235362d32323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232b1656e7669726f6e6d656e745f6e616d657390bb656e7669726f6e6d656e745f636f6e74726163745f736861323536d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333ad636f6e74726f6c5f67726f7570d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365b563726564656e7469616c735f6469726563746f7279c0bc706172656e745f6f6e6c795f66696c655f64657363726970746f727390bd61637469766174696f6e5f7369676e65725f6964656e746974795f6964aa61637469766174696f6ebc61637469766174696f6e5f7369676e65725f7075626c69635f6b6579dc00200101010101010101010101010101010101010101010101010101010101010101b3736572766963655f63726564656e7469616c7390 \ No newline at end of file diff --git a/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show new file mode 100644 index 0000000..21e4573 --- /dev/null +++ b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=2571 +OpenFile=/run/idunn/activation-credentials/sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/odin/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=20630000 +ControlGroup=/system.slice/idunn-odin-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.service +UMask=0007 +WorkingDirectory=/srv/odin/releases/sha256-e5c7d0b2b9c0a8ffba368aeb08abdbe37ae0d17484a785e60a66ec4a62b2f18c +CapabilityBoundingSet= +AmbientCapabilities= +User=_duc23369f1b8b167db +Group=odin +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn odin sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8 +LoadState=loaded +ActiveState=active +SubState=running +InvocationID=ada48921043449f39dacc87b14a1bf70 diff --git a/tests/fixtures/systemd-257/vili-presence-identity-only.show b/tests/fixtures/systemd-257/vili-presence-identity-only.show new file mode 100644 index 0000000..fc78fd1 --- /dev/null +++ b/tests/fixtures/systemd-257/vili-presence-identity-only.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=0 +OpenFile=/run/idunn/activation-credentials/sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/vili/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=520083420000 +ControlGroup= +UMask=0007 +WorkingDirectory=/srv/vili-idunn/releases/sha256-aeaee4e1669742572c45a3f26bc86893c44b2a4149520975eb7d8a28aa931906 +CapabilityBoundingSet= +AmbientCapabilities= +User=_du8c06c563aac6d205 +Group=vili-state +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn vili sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041 +LoadState=loaded +ActiveState=failed +SubState=failed +InvocationID=8f7a349f2f8f49fe87ac5d89239260b4