diff --git a/Cargo.lock b/Cargo.lock index dfc3087..44c9457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "libc", "rmp-serde", "serde", + "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index 713ad33..ee797ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,10 @@ path = "src/bin/idunn-provision.rs" name = "idunn-host" path = "src/bin/idunn-host.rs" +[[bin]] +name = "idunn-launch" +path = "src/bin/idunn-launch.rs" + [lib] name = "idunn_daemon" path = "src/lib.rs" @@ -34,6 +38,7 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } +serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/README.md b/README.md index 503409e..8c6935c 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,64 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. +### Steward orders + +A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), +takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. +It has its own store, `orders.cc`, and its own binding, `steward.toml`; the +survival daemon takes no order. There is still no command string: + +```bash +idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them +idunn order unit --reason TEXT +idunn order deploy --reason TEXT # the same commit as `idunn up` +idunn order show # status and output tails +idunn order list +``` + +A unit order submits exactly one job, for the unit it names: every verb runs +`systemctl --job-mode=ignore-requirements -- `, so systemd builds a +transaction of one job. It does not start what the unit requires or restart what +is part of it: order the dependency first, and order `postgresql@16-main.service` +rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is +not: systemd still runs full transactions of its own for `Restart=` after a +failure, for a `StopWhenUnneeded=` dependency left without a user, and for a +`Conflicts=` partner of a unit that fails, so an ordered stop of a unit +that something restarts does not last, and a start of a `Restart=always` +unit can report failure and be active seconds later. A unit +order may act on any unit except Idunn's own (`idunn-*`) and the minds' own +(`mind-self@*`, `mind-persona@*`), and the units that end or take the host +(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, +`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled +into the binary and read from no configuration. An order is refused, recorded +and posted when the ordered unit, or an alias it resolves from, is one of them; +when it has an emergency action (`SuccessAction=`, `FailureAction=`, +`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` +or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another +unit is `BoundBy=` it, because systemd stops that unit with it whatever the job +mode; for a start or restart, when it has an `Upholds=` unit, which systemd +starts in a full transaction whatever the job mode; or, for a stop, restart or +reload, when it has a `ConsistsOf=` member, a wrapper such as +`postgresql.service` whose verbs would exit 0 and touch nothing. A unit +`systemctl show` cannot resolve fails the order. Slices, scopes, +targets and globs are not units an order can name. + +A package order installs the packages it names and nothing else: apt runs with +`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` +match themselves, and a last step fails the order unless every named package +is installed. + +An order runs once, after the steward's uid is checked and the instance's +brake check (`mind-control --instance check`) exits 0. The check +that the order's principal and uid are the bound steward's runs again at every +execution, so an order accepted before `steward.toml` changed is refused and +recorded, not run. The socket belongs to the steward user alone (mode 0600); +root may `show` and `list` orders but never issue one. An order that had +started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to +`#operator` as a retried projection of `orders.cc`: a chat outage delays the +posts and not the orders. A refusal names the field and a code, never the value +it refused. + ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. @@ -150,6 +208,79 @@ omits one. Unknown fields are rejected on both sides. A repository that gets compromised can therefore change what is built, but not what it is allowed to touch on the host. +An upstream daemon that does not speak Idunn's runtime contract is deployed +with `idunn-launch` as its executable artifact and the upstream tree as a +required adjacent artifact; the launcher finds the child inside the release that +holds it. The launcher is the unit's process: it binds the +candidate endpoint and answers Idunn's challenge there, renders the child's +config, starts a stateful child only once it holds the write lease, and signs +presence. Members never pass through it: the binding's `[route.member]` makes +Idunn route a second listener straight to the child, which binds the address +Idunn gives it in `GAMECULT_IDUNN_MEMBER_BIND`. + +```toml +# recipe fragment — the upstream tree is built or fetched as an artifact +[service] +executable_artifact = "idunn-launch" +arguments = [ + { kind = "literal", value = "--child" }, + { kind = "literal", value = "upstream/bin/daemon" }, + { kind = "literal", value = "--template" }, + { kind = "literal", value = "config.tmpl" }, + { kind = "literal", value = "--health-path" }, + { kind = "literal", value = "/health" }, + { kind = "literal", value = "--health-contract" }, + { kind = "literal", value = "example.presence-launcher.v1" }, +] +required_environment = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", +] +``` + +A directory artifact is sealed read-only: its files are 0444 and its +directories 0555, whatever mode the build or git gave them. A file the tree +must run, such as the launcher's `upstream/bin/daemon` child, is declared in +`executable_members` and sealed 0555. The members are part of the sealed release +address, so changing the list makes a new release. Idunn refuses the release if a +member is absolute, climbs with `..`, is missing, is a symlink or sits behind +one, or is not a regular file; `executable = true` on a directory output and +`executable_members` on a file output are refused too. + +A tree may hold relative symlinks whose `..` steps all come first and climb no +higher than the artifact's top (`share/terminfo/4/4025ex -> ../t/tek4025ex` is +kept). An absolute link, a `..` after a name, or a climb out of the artifact +refuses the release when it is sealed. Host targets (Windows) record +`executable` and `executable_members` in the receipt and do not apply them. + +```toml +# recipe fragment — the upstream artifact declares the child it launches +[[artifacts]] +id = "upstream" +source_kind = "runner-output" +runner = "build" +source = "dist/upstream" +destination = "upstream" +executable_members = ["bin/daemon"] +``` + +```toml +# binding fragment — members connect to the member endpoint, which Idunn +# forwards to the private port derived from the candidate: member +# private_port_start + (candidate port - route private_port_start) +[route] +stable_endpoint = "tcp://10.77.0.1:8008" +private_port_start = 18831 +private_port_end = 18839 +# ... + +[route.member] +stable_endpoint = "tcp://10.77.0.1:8448" +private_port_start = 18841 +# max_connections = 128 # nginx limit_conn on the member listener (1-65535) +``` + ## Contracts Idunn's typed contracts are published by diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service new file mode 100644 index 0000000..f2681c2 --- /dev/null +++ b/deploy/idunn-orders.service @@ -0,0 +1,47 @@ +[Unit] +Description=Idunn steward orders: typed package, unit and deploy orders from the steward +# Deliberately no After= or Requires= on any managed target, Synapse or +# idunn-yggdrasil: orders are how an operator repairs those, so none of them +# may be a condition of this unit starting. Posts are a retried projection and +# wait for Synapse on their own. + +[Service] +Type=simple + +# Root, from the same binary as idunn-yggdrasil, with its own store and socket. +# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: +# no order code runs in `idunn serve`. +# +# The unit name matches idunn-*, which is in the always-refused set, so no +# order can stop or restart this service. +User=root +# No Group= and no UMask=: the service gives its socket to the steward user +# (mode 0600) itself and sets Idunn's own umask for the files it writes, so +# nothing here reaches control.cc. The steward user is the only peer the +# service answers: it checks SO_PEERCRED on every connection. +ExecStart=/usr/local/bin/idunn orders serve \ + --store /var/lib/gamecult/idunn-orders \ + --state-store /var/lib/gamecult/idunn/control.cc \ + --steward-binding /etc/gamecult/idunn/steward.toml \ + --socket /run/idunn-orders/steward.sock +Restart=always +RestartSec=5 +TimeoutStopSec=20 + +RuntimeDirectory=idunn-orders +RuntimeDirectoryMode=0755 + +# The Synapse access token of @idunn, copied from a root-only file, and read +# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. +LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token + +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +# Its own order store, and the control store a deploy order commits a command +# to. Transient units started with systemd-run run in the manager's namespace, +# so packages and units are not limited by this list. +ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn + +[Install] +WantedBy=multi-user.target diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index da82bd2..1c45594 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -104,6 +104,11 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web +# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a +# target whose roots are not created yet cannot stop Idunn itself from starting +# (Idunn startup never depends on a managed target). A root created after Idunn +# starts is writable only from Idunn's next restart. +ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases [Install] WantedBy=multi-user.target diff --git a/docs/authority-map.md b/docs/authority-map.md index 1cbf3ca..c8a8546 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -55,7 +55,10 @@ A deployable repository publishes one strict declaration, normally - immutable artifact outputs, optionally pinned to an expected SHA-256; - a constrained launch contract; - one health-contract identity; -- an optional typed state contract with state schema generation and slots; +- an optional typed state contract with state schema generation and slots + (kinds `cultcache-file`, `external-file` and `external-directory`; the + external kinds are process-owned state an external runtime imposes, which + Idunn preserves and fences but never interprets); - provided capabilities, dependency kinds, startup requirements, capacity, and conflicts. @@ -67,6 +70,52 @@ recipe fields. Stateless services declare no state block. Their operator binding therefore has no state root, state-transition policy, or process write-lease record. +## Foreign workloads: idunn-launch + +An upstream daemon that cannot speak Idunn's runtime contract (Synapse, MAS) +runs under `idunn-launch`, the unit's own process. Idunn's two-witness rule is +unchanged: Idunn observes the unit, and the launcher, inside the unit and +supervising the child, signs presence with the provider identity and the +activation key. + +- **Owner:** the launcher owns presence, config rendering and the child's + lifetime inside the unit, and the route answers it signs for that child. + Idunn owns admission, lease, route and continuity, as before. +- **Inputs:** the runtime bundle, the activation credential and provider + identity descriptors, the process write lease, the systemd credentials the + binding names, and the child's health over TCP at the member bind. +- **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active + and failed to Odin; route observations on the candidate endpoint), one child + process, and a rendered config in the unit's private tmp that is never read + back. +- **Route:** Idunn owns both listeners of the route: the Expected route's + stable endpoint (presence) and, when the binding has `[route.member]`, the + member endpoint, rendered as a second server block in the one nginx fragment + with a `limit_conn` on one per-route zone (`max_connections`, default 128), so + members cannot exhaust the connections the presence block shares with them. + The launcher binds the candidate endpoint and answers Idunn's challenge there, + one connection at a time within a one-second deadline; any other first byte + closes the connection. The answer never waits on the child: it reports the + lifecycle's last probe, and a failed, missing or stale probe reads `degraded`. + The child binds the member bind Idunn sets in + `GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes + never touch the launcher. +- **Lease:** with `write_lease_required` the launcher starts the child only + after it holds the lease (bound to this Expected, this activation and one of + its own Warming proofs) and keeps the lease store share-locked while the + child lives. Losing the lease stops the child. +- **Config:** one template from the release is rendered with the closed + placeholder set `{{child.listen}}`, `{{child.listen_host}}`, + `{{child.listen_port}}` (all three the member bind), `{{credential:NAME}}`, + `{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder + refuses to start. +- **Forbidden:** member traffic through the launcher; the child binding the + candidate endpoint; a child of a lease-requiring target running without a held lease; any presence not signed + by both keys. +- **State kind `external-directory`:** a process-owned directory imposed by an + external runtime contract. Same writer, recovery and startup rules as + `external-file`: it is lease-gated, preserved and fenced, never interpreted. + ## Operator binding The root-admitted operator binding supplies the physical policy a repository @@ -173,24 +222,40 @@ health plus Ready correlation. It changes and observes route membership only afterward. Only after the proxy has adopted the candidate may Idunn clear the promotion fence and drain the incumbent. -Ownership of a target is exclusive at each instant. A live -`DeploymentTransaction` owns it from Fencing on (a stop-first rollout, from -Starting); before Fencing a deployment owns only its private candidate and -continuity owns the incumbent slot. Continuity owns the admitted generation's -survival and never stops. There is no stopped state: six charged attempts in -an hour (`restarts_page`) open the ContinuityExhausted incident and gate -nothing, and no deployment path yields to continuity. When continuity next acts -is derived only from its own record in `TargetSupervision` (schema v2, a v1 -record lifts at store open): the attempt streak, the backoff anchor (written -by the dead-and-free pass, the only writer), and the once-per-admission -re-materialization bound. The workload driver alone answers whether a lost unit -was a host reboot (`WorkloadPort::lost_to_host_reboot`). Crash restart, restart -after a sustained run and reboot re-materialization all mint through one CAS; -continuity supervision never aborts a transaction. Operator-facing incidents -and `idunn status` are projections that decide nothing. Cut line: cuts 2 to 4 -of `reincarnation-map.md` still owe continuity restarting alongside a -pre-fencing build, dependency failures not growing the streak, and the -dead-generation route refresh. +A target has two slots: the incumbent (the admitted generation) and a +transaction's candidate. Continuity owns the incumbent slot and keeps the +admitted generation alive; it never stops. A live `DeploymentTransaction` owns +its private candidate before Fencing and takes the target itself at Fencing (a +stop-then-start rollout, at Starting) through one `take_target` CAS, so of two +transactions entering from one view exactly one succeeds. A continuity +restart is itself a transaction and takes the target the same way. + +Phases from Warming through Committing carry deadlines, as do aborts and +drains; an expired deadline never waits silently. A stateless deployment +aborts to its incumbent. A deployment whose candidate was issued a write lease +is left to the operator (`operator-required` incident, the second incident +condition beside `continuity-exhausted`; its subject is the transaction id and +it closes when the record ends or its step goes on). A continuity after +Fencing retries its phase forward on a capped backoff and never stops its live +candidate, since that is the only running copy of the admitted body; the +deadline opens the same incident. The one exception is yielding: a continuity +whose route proof has not succeeded gives way, through its own post-fencing +abort, to a deployment the deployment brake has released and that awaits the +handover; a continuity that has proven its route never yields. The +`ContinuityExhausted` incident (six charged attempts in an hour, +`restarts_page`) pages and gates nothing. When continuity next acts is derived +only from its own record in `TargetSupervision` (schema v2): the attempt +streak, the backoff anchor (written by the dead-and-free pass, the only +writer), and the once-per-admission re-materialization bound. The workload +driver alone answers whether a lost unit was a host reboot +(`WorkloadPort::lost_to_host_reboot`). Operator-facing incidents and +`idunn status` are projections that decide nothing. A boot lift returns a +continuity abort left by an older Idunn, one that had not stopped its running +candidate, to forward retry; if it cannot decide, every continuity abort is +held until a lift succeeds. It is a migration with a deletion line (follow-up +`c2c-delete-boot-lift`). Cut line: cuts 3 and 4 of `reincarnation-map.md` +still owe dependency failures not growing the streak and the dead-generation +route refresh. The deployment brake gates changes to source, release, configuration, schema, unit, or authority binding. The lifecycle brake separately gates restart of an diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md new file mode 100644 index 0000000..95c0eee --- /dev/null +++ b/docs/deploy-provenance-map.md @@ -0,0 +1,394 @@ +# Deployment provenance: map + +Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, +session `self-2026-10-23-eureka-b`). The ends are in +[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, +rulings, cut specs and follow-ups live in Eureka's mind; this page holds what +has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are +at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`, +the head of Hands attempt 1, which revision 2 of the spec builds on. + +## Cut `tree-executable-members` + +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2` +(revision 2, after Soul pass 1 found the real interpreter tree refused). It +serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's +admission waits on it) and will serve gamecult-chat C3 too: the presence +launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact +(README, launcher fragment), and Synapse needs a Python interpreter there. + +### Body facts + +- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable` + bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and + `validate` (`:740-760`) requires it to equal the declaration. +- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its + executable set from `root.join(receipt.destination)` only. `harden_root_tree` + (`:6858-6892`) seals every directory 0555 and every file 0444 unless that + exact path is in the set. A directory artifact with `executable = true` is + therefore silently meaningless today: its files are all 0444. Nothing refuses + it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin, + StreamPixels service and web surveyed): `executable = true` is on files only. +- T3. A tree output is copied by `copy_artifact`/`copy_tree` + (`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest + (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not + mode bits. So runner or git exec bits never reach the release: harden is the + only mode authority. +- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink + only if its target is relative with no `..`. It runs only inside + `harden_root_tree`, at install, so a tree with a `..` or absolute link copies, + digests and seals, then fails at `install_release` (Soul s1: copying and + digesting the real tree twice took 160 s before harden refused it). The rule + is sound but too narrow: it refuses the real interpreter tree (T13). The + frozen-source path has a physical resolver (`resolve_frozen_source_symlink`, + `:6617`) that walks the filesystem with a 40-link budget and a memo. Its + first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs + the tree on disk to answer. Revision 2 does not reuse it (T14). +- T5. The sealed release id hashes `(schema, plan_id, artifacts, + external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes + the whole plan including `recipe_blob`. A changed member list is therefore + already a new plan and a new release. The receipt must still carry the list, + because `harden_installed_release` reads receipts only; its two callers + (`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden + every installed release on each observation. +- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with + `#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every + existing receipt's bytes and release id: the plan's `deadlines` field + (`:574-577`) is the precedent. +- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file + whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB) + is held in Idunn's memory three times per install (seal, install, every + validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB + chunks (S9). The cut collapses both onto one streaming helper. +- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the + artifact destination a single filename. Members are paths inside that + destination, so it does not change. `require_relative_path` (`:1936-1949`) + with `allow_current = false` already refuses absolute, `..`, root and prefix + components. +- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy, + mode check, digest, pinned-digest check and receipt, the same steps as + `collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode + check survives every test. Revision 2 collapses both onto one `seal_artifact` + in `drivers.rs`. Each caller keeps only how it finds the source path. +- T10. The transient unit runs with `ProtectSystem=strict` and the release in + `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no + `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. + Shared libraries need no exec bit. +- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`, + installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is + identical between `8760a4f` and `6e2a31c`, so the live code is the + reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in + `src/`: the presence launcher and member route). Installing this cut ships all + of them. Idunn's own body changes only through gamecult-ops + `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, + bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T13. Link census of the real interpreter tree (Yggdrasil probe of + `cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`, + sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049 + after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all + their `..` components before the first name (every one is + `share/terminfo// -> ..//`, one `..` each). 0 have a `..` + after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it. + Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1). +- T14. Lexically normalising `/` does not check + containment. The kernel resolves a target one component at a time, so a `..` + after a symlinked name climbs from where that link points, not from where its + name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x` + normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x` + beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same + way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`, + `L -> D/D/../../../etc/passwd` passed a lexical check). + The sound lexical rule constrains the target's shape: all its `..` components + come before its first name, and they number no more than the link's depth + below the artifact's top. The climb then starts from the link's own + directory, which the walk reached through real directories only (copy and + digest never descend through a link), so it lands on a real ancestor inside + the artifact. After that the target only descends, and any link it passes + through obeys the same rule. By induction on the kernel's hops (capped at 40, + then ELOOP), every hop stays inside. The check is one pass over the target's + components with no filesystem reads, so its cost is linear in the tree and no + resolver is needed. The shapes it refuses (`s/../x`) occur in neither real + tree (T13). + Prior art, from documentation as known to Imagination (high confidence for Go + and securejoin, medium for the tar details): + - Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account + for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug. + - Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc, + containerd and umoci) resolve each component inside the root when a path is + opened, refusing or clamping an escape. They guard one later access, need + the tree on disk, and check again on every open. Idunn instead certifies a + sealed tree once, so that the kernel's own resolution stays inside on every + later access. + - Zip-slip defences and archive extractors check entry names lexically and + protect against link targets by refusing to write through links + (libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for + names; GNU tar creates such links last). They protect the extraction, not + what a kept link points at afterwards. + - OCI layer extraction keeps link targets verbatim, absolute ones included, + because the runtime resolves them inside the container's root. Idunn runs a + release in the host's mount namespace, so absolute targets stay refused. + - Nix keeps link targets verbatim and tracks dependencies by scanning for + references. The whole store is its root, so it has no per-artifact + containment to compare with. + None of these is a static per-link rule for a tree that runs unconfined. The + T14 shape rule fits that case. +- T15. Host targets: `idunn-host` observes, starts and stops processes on + Windows only (the non-Windows `platform` module in `src/host.rs` bails), and + `host.rs` sets no file modes. On Windows a file runs by its type, not by a + mode bit, so `executable` and `executable_members` have nothing to set there. + The file `executable` flag has always been carried and never applied on host + targets. A recipe does not know which workload kind its binding will choose. +- T16. The release root holds only artifact destinations. Runner workspaces, + where external inputs land, are removed before `SealedRelease::new` + (`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and + `validate_installed_release` (`:2815-2822`) digest every receipt's + destination before calling harden. So a rule in the digest walk runs at seal, + at install and at every validate, over every link harden would see. +- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three + copies (runner workspace, staging, installed). Nothing collects old releases or + staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every + Vili deploy leaves about 7 GiB behind. + +### Contract, corrected against the body + +- Any directory artifact may declare `executable_members` (a set of relative + paths). The follow-up limited it to runner-output; that limit protects no + invariant, and the launcher's worktree or fetched trees need the same feature. +- `executable = true` names a file artifact. On a directory output the release + is refused, and `executable_members` on a file output is refused (T2). +- A member is refused at recipe parse if it is empty, absolute, or holds `.`, + `..`, a root or a prefix. At seal it is refused if it is missing, is a + symlink, passes through a symlink or non-directory, or is not a regular file. + Refusal happens before `SealedRelease::new`, so no sealed release can name a + member it cannot seal. +- Harden seals each member 0555 and fails if any declared executable path was + not visited as a regular file. "Every declared executable is sealed 0555, or + the install fails" holds structurally, not by a lookup that can miss. +- Symlinks: a link in a tree artifact is kept when its target is relative, its + `..` components all come before its first name, and they number no more than + the link's depth below the artifact's top (T14). One function, + `check_release_link`, decides this. Only the digest walk calls it, so a bad + link refuses the release at seal, before `SealedRelease::new`, and again at + install and validate (T16). Harden no longer judges links. The climb is + bounded by the artifact's top, not the release root: no live release has a + `..` link (the old rule refused them all), so nothing installed is newly + refused, and each artifact's digest describes a closed tree. +- Host targets carry `executable` and `executable_members` in the receipt, + shape-checked at seal by the shared `seal_artifact`, and do not apply them + (T15). Refusing members there would make a recipe's validity depend on its + binding. +- Vili's trees must + not come from `python -m venv`: its `bin/python` is an absolute link and its + `pyvenv.cfg` names the build workspace. Install packages into the relocatable + interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). + +### Authority + +Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd +targets and by `materialize` in `host.rs` for host targets) decides +membership from the recipe and writes it into the receipt. Harden enforces the +receipt and decides nothing. Runner exec bits and git `100755` modes are dead +inputs for release modes. Link containment belongs to `check_release_link`, +which only `digest_tree` calls. `validate_release_symlink` is deleted, and +harden's symlink arm leaves links as they are. + +### Acceptance sizing + +The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch +on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It +removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 +terminfo links (T13). If a tree holds a link the T14 rule refuses, the job +prints it and stops, and the link is reported to Self, not deleted. + +## Cut `isolation-credential-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, +at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It +serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, +first deviation): every `idunn up vili` is refused before fencing. + +### Body facts (probed read-only on Yggdrasil, 2026-10-24) + +- I1. `idunn up vili` runs 6 and 7 failed with + `systemd workload isolation properties differ from the Idunn contract`, the + message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, + only one differs on the failed Vili unit: `LoadCredential`. Idunn expects + `""` because Vili has no service credential besides the presence identity + (`service_credential_names` empty); systemd shows `[unprintable]`. Every + other conjunct matches, InaccessiblePaths included (one value across all + five idunn-vili/odin units). +- I2. systemd 257.13 `systemctl show -p LoadCredential` prints + `LoadCredential=[unprintable]` for every unit, whatever the credentials: + `ssh.service` and `systemd-journald.service` (not DynamicUser, zero + LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin + unit (one). The display form carries no information, so the conjunct was + vacuous whenever credentials existed and always false when none did. Odin + and StreamPixels passed only because each carries a regular credential. +- I3. The exact value is readable: D-Bus + `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) + answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 + "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. +- I4. What the workload can actually read is the delivered directory. On the + host, `/run/credentials/` holds a directory only for units with credentials + (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID + root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` + (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` + descriptor, never a credential. +- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn + deployment transaction authority"), with its twin clause in + `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit + touched it. No ruling or finding in any campaign concerns workload isolation + verification; gamecult-chat's foreign-workloads cut touched + `CREDENTIALS_DIRECTORY` reservation, not this check. +- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both + MessagePack encodings, compact (positional) included + (`persisted_systemd_observations_decode_as_the_systemd_variant`, + `:9915-9930`), so deleting the `load_credential` field shifts every later + slot of a live incumbent's record. The field is also in the drift list + (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed + workload: changing what the field holds without leaving that list would make + the next Idunn refuse to stop the live Odin and StreamPixels incumbents. +- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 + properties for a Vili unit (presence identity only) and the Odin unit + (presence identity plus one service credential), captured 2026-10-24 into + this Imagination's scratchpad `fixtures/`. +- I8. The new rule holds for every live systemd incumbent: inside each + MainPID root the delivered entries equal its binding's `secret_files` keys + minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, + StreamPixels web 1; key names compared, no values read). An Idunn restart onto + this cut re-observes them (`:3955-3970`) and passes. + +### Decision + +A property systemd cannot display does not belong in the comparison. The +invariant it reached for, that the workload holds exactly the credentials Idunn +delivered, is observed where it is decided: the entries of +`/run/credentials/` in the MainPID's root equal Idunn's service +credential names, absent directory equal to none. That covers +`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and +needs no new program. The D-Bus read (I3) is exact but sees only one of those +properties and adds a second systemd client and parser; rejected. + +### Authority + +Owner: `observe_unit` decides the credential rule, from the delivered +directory. `LoadCredential`'s display form is no longer read or compared; the +`load_credential` field is a retired wire slot, written empty, out of the drift +list, kept only so persisted records decode. Forbidden writers: the +`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping +the slot waits on the next schema revision of the observation. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). After it, Vili's Hands reruns from step 4 of +`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. + +## Cut `runner-failure-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at +`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area +Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, +transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 +UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and +nobody could tell which recipe step failed. + +### Body facts (source read at `804ed75`; host read-only, 2026-10-25) + +- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by + `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and + `Command::output()`: stdout and stderr are each buffered whole in the + daemon's memory, unbounded. On a non-zero exit the error is + `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The + container is gone the moment it exits, so `docker ps -a` and `docker logs` + hold nothing. +- R2. The empty message: the failing process wrote nothing to stderr, and + stdout, the only stream that carried its report, is discarded at + `:2182-2193`. The same interpolation echoes whatever a step does write to + stderr into `last_error`, `pre_fencing_abort.error` and + `completion.error` (`bounded_detail`, 2048 bytes), and from there into + `idunn up`'s `deployment failed:` line and `idunn status`. +- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops + over `declaration.steps` and calls `run_in_workspace` with argv only; + `step.id` is used only in the environment precondition. External inputs + (`curl` in the runner, `:2247-2258`) fail through the same helper with the + same anonymous message. +- R4. Which step failed on 2026-10-25, inferred from the failed transaction's + staging root, which survives (926 MiB, + `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, + `node_modules/.package-lock.json` was written at 00:51:24 (so + `install-vili-dependencies`, `npm ci`, finished), nothing outside + `node_modules` is newer, and `release/vili` (the next step's output) is + absent; the python workspace holds only fetched inputs. The failing step was + `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test + reporter writes to stdout: exactly the stream R1 discards. Inference from + residue, not from output. +- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) + sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which + persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal + on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only + provider-exclusion lines. +- R6. Stores. `control.cc` (root 0640) holds commands, transactions + (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and + expiry requests; `ControlSnapshot::read` refuses any other type + (`:3420`). Records must re-encode byte-exactly (`decode_record`, + `:3717-3727`), so a new transaction field is a schema bump and a v4 lift + (`:4100-4110`). Terminal transactions move verbatim to `history.cc` + (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. + `incidents.cc` lives in the published projection directory and by its own + doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, + because a type added there stops continuity for every target on a rollback. +- R7. The daemon runs as root (`User=root`). Runner secrets reach the + container as read-only file mounts named by env (`:2041-2047`), so any text + a step derives from them can land in its output. +- R8. Staging residue: 475 transaction roots, 55 GiB under + `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces + because `materialize` returns before `remove_tree_inside`. + +### Decision + +The failure becomes a typed value at the runner, `RunnerStepFailure`: the +subject (recipe step or external input, by id), the runner id, the exit code or +signal, and the last 8 KiB of stdout and of stderr with each stream's total +byte count. Its `Display` carries the subject and status, never output. +Capture streams both pipes through bounded tails, so a verbose build no longer +buffers whole in the daemon. + +The tail lives in a new private store, `runner-evidence.cc`, beside +`control.cc` (mode 0600), one record per transaction +(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the +transaction record: that is a v5 bump whose rollback makes every older Idunn +refuse its control store (R6), the scar `incident.rs` already names. Not in +`incidents.cc`: published, and free text is barred there. Not in the journal: +journal readers are a wider set than root, so the journal line names the +transaction, the subject, the status and the store, and no output. + +No redaction. Matching secret bytes would make the daemon read every secret, +and still misses any encoding, prefix or transform of one; a claimed redaction +that leaks is worse than a stated rule. The rule instead is the reader set: the +store's only readers (root) can already read every secret the runner was +given. + +Keeping the failed container (dropping `--rm`) is worse: its output then lives +in Docker's json-file log, unbounded, untyped, outside Idunn's state and +readable by the docker group; containers accumulate with no owner to reap them; +`docker system prune` erases the evidence; and nothing ties a container to the +transaction in `idunn status`. + +### Authority + +Owner: the runner decides what failed and keeps the bounded tail; the engine's +pre-fence failure path is the one writer of `runner-evidence.cc`. +`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the +failure's `Display`. Forbidden writers: the `stderr` interpolation in +`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the +store needs no external owner; staging roots (R8) stay with +`ghostlight-verse:follow_up:idunn-generation-retention`. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). The next failed Sealing then shows its step and tail on +`sudo idunn status --command `. diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index 6d464e5..bdc7efc 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -313,10 +313,36 @@ written when the pass first finds the admitted generation dead and the target free, and a capped doubling wait from that anchor (5 s base, 300 s cap). A run of 600 s or more by the admitted body resets the streak. A generation lost to a host reboot (the workload driver's evidence, `lost_to_host_reboot`) is -re-materialized once per admission without charge or wait. The target is free -for a deployment's candidate between attempts, and a deployment never yields to -continuity before Fencing; from Fencing on (or from Starting for a stop-first -rollout) the live transaction alone owns the target. +re-materialized once per admission without charge or wait. + +Each target has two slots, the incumbent and a transaction's candidate. The +incumbent is continuity's until a transaction takes the target, at Fencing (at +Starting for a stop-then-start rollout), through one `take_target` CAS; of two +transactions entering from one view exactly one succeeds. Before that, a +deployment owns only its private candidate and the target is free for it +between continuity attempts. + +Phases Warming through Committing, post-fencing aborts and drains all carry +deadlines (the plan's frozen `PhaseDeadlines`; `WARMING_DEADLINE_SECONDS`, and +`PRE_FENCING_ABORT_DEADLINE_SECONDS` for a pre-fencing abort). What an expired +post-fencing deadline does depends on the owner. A deployment whose candidate +was never issued a write lease aborts to the incumbent. One whose candidate was +issued a lease may have written under a new state contract and is left to the +operator. A continuity retries its phase forward on a capped backoff and never +stops its live candidate, because that candidate is the only running copy of +the admitted body. Both operator cases open an `operator-required` incident +(subject: the transaction id) that closes when the record ends or its step goes +on. + +The one yield: a continuity whose route proof has not succeeded gives way to a +deployment the deployment brake has released that awaits the handover of the +target. It begins its own post-fencing abort and the deployment fences what is +admitted. A continuity that has proven its route is serving never yields. + +At boot, a lift returns to forward retry any continuity abort an older Idunn +left over a running candidate. If the lift errs or cannot observe the +candidate, the abort is held, not run. The lift is a migration with a deletion +line (follow-up `c2c-delete-boot-lift`). An admitted generation keeps its Ready and latest Odin receipts as durable proof of the exact activation at their recorded admission times. Dependency diff --git a/docs/host-actuator.md b/docs/host-actuator.md index cdb8cde..c595c5d 100644 --- a/docs/host-actuator.md +++ b/docs/host-actuator.md @@ -97,7 +97,8 @@ Each of these was found by running the pipe against Raven, not by reading. bind time. `rollout.strategy = "stop-then-start"` exists for exactly a route-less host-actuator workload: the admitted incumbent is stopped at Starting, before the candidate starts, and supervision leaves the target - alone from Starting on. A systemd-transient workload may not declare it. + alone from Starting on: the transaction takes the target there (one + `take_target` CAS) rather than at Fencing. A systemd-transient workload may not declare it. - **An unobservable host is silence.** The hub returns a typed `HostUnobservable` for not-attached, detached, reconnected and unanswered. Supervision skips that tick and logs it. Before this, an Idunn restart diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md index d63a6b3..4f99aed 100644 --- a/docs/reincarnation-map.md +++ b/docs/reincarnation-map.md @@ -575,3 +575,26 @@ its stores and `idunn.running` together, because the cb60965 binary cannot read "migrated 5 control record(s) … idunn.target_supervision.v2". streampixels-service kept running through the restart (unit 73ca40ba, PID 968752, NRestarts 0). The catalog returned 200, and 4443/udp was listening. + +## Installed: cut 2 + +Cuts c2-phase-split, c2b-owner-deadlines, c2c-post-fence-forward and +c2d-continuity-yields (branch tip `8760a4f`) were installed on Yggdrasil on +2026-10-20 at 00:34:06 UTC, with the operator's OK. + +- **The hotfix before it.** `ec45906` (`bounded_detail`) was installed on its + own on 2026-10-19 at 17:32 UTC. It unwedged tx-f978ab86. +- **Build.** `/srv/build/idunn-8760a4f`; `cargo test --lib` passed 471. +- **Backup.** `/root/idunn-store-backup-*-pre-reincarnation-c2` holds the + stores, `incidents.cc` and the binaries. A rollback must restore them + together: older binaries refuse two live transactions on one target, and + they refuse the `operator-required` incident condition. +- **First boot.** The boot lift returned exactly tx-039b91e9 (ghostlight) and + tx-4eaed669 (odin) to forward retry, as the Soul install gate predicted. + Odin's continuity, wedged in Routing since the 2026-10-18 reboot, committed + 4 s later. Generation tx-4eaed669 is admitted, its route is healthy, and the + same PID 2571 still runs. Ghostlight's continuity is retrying forward in + AwaitingReady. Its daemon's readiness is a ghostlight-side fault + (`Permission denied` reading its session-wrapping key). +- **Live processes.** Every live MainPID was the same before and after the + install. The streampixels catalog returned 200, and 4443/udp was listening. diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index 4c9ada7..4bc8814 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -21,8 +21,8 @@ probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC | I2 | Continuity exhaustion (since `idunn-reincarnation` cut 1: `restarts_page`, a paging threshold only; nothing stops, and the line numbers below are pre-cut) | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | | I3 | Route degradation | `RouteSupervisionState` `:890-942`; `record_failed_challenge` sets `degraded_since_unix_millis` on the **first** failed challenge, `record_proved_challenge` clears it; reported at `:5975` under `route:` | Durable per incarnation (inside `admitted_generation.v4`); a new generation starts from `default()`. Backoff doubles from the topology max age to a cap. | | I4 | Other operator-facing faults | `:5645-5651` (`generation-down:`, "holds a dead admitted generation down"), `:6016` (`route-record:`), `note_fault` per transaction | Also stderr plus `last_error`. Not named by the target; listed so the paging-set question sees them. | -| I5 | `OperatorRequired` on main | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | -| I6 | `OperatorRequired` on B5 | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | +| I5 | `OperatorRequired` on main (pre-cut-2 reading; since `idunn-reincarnation` cut 2 `resolve` reaches it in production through `PhaseEnd::OperatorRequired` and `PhaseEnd::RetryForward`, both calling `leave_to_operator`, which opens the `operator-required` incident `IncidentCondition::OperatorRequired`, subject the transaction id; `close_spent_operator_incidents` closes it when the record no longer `leaves_to_operator`) | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | +| I6 | `OperatorRequired` on B5 (superseded by cut 2: B5's deadlines merged as `PhaseDeadline`, and a continuity past its deadline now also lands here) | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | | I7 | Store kinds | `:55-61`, `:936-966`; B5 adds `idunn.expiry_request.v1` | `control.cc`: `idunn.deployment_command`, `idunn.deployment_transaction`, `idunn.admitted_generation.v4`, `idunn.target_supervision.v1`. `history.cc` beside it holds terminal transactions (`:3653`), retired one per tick (`:5068-5091`). | | I8 | Single-writer precedent inside `control.cc` | B5 `ExpiryRequest` doc comment | "The CLI only writes this record. The daemon's deadline resolver reads it ... No CLI writes a transaction field." One writer per kind. | | I9 | Serve loop | `:5041-5058`, `run_scheduler_tick` `:5098-5110` | 500 ms poll. A tick error is logged, never propagated. | @@ -166,9 +166,10 @@ cut spec fixes them. Empty cells, stated plainly: -- `operator-required` closure: no path on `hands/b5` ends a transaction left to - the operator (I6). The incident can open and never close until B5 names that - action. +- `operator-required` closure: the incident closes when the transaction ends or + its step goes on (`close_spent_operator_incidents`). A stateful deployment + left to the operator still has no Idunn-owned terminal action; the operator's + `idunn expire` re-enters the same arm. - Delivery-failure visibility: no owner. - Discord `enforce_nonce` window: not verified against Discord's current docs in this pass. diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index 5d21c75..509cb6e 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -30,9 +30,12 @@ journald (`report_once`, `control_plane.rs:4599`): - An admitted route is degraded. It fails its stable-route challenge (`supervise_admitted_route`, ~5973). The degraded mark is durable, but no one is told. -- A post-fencing phase deadline expires into `OperatorRequired`. This lives on - B5, which is unmerged on `hands/b5`. The failure record is durable and the - notice is a log line. +- A post-fencing phase deadline leaves a transaction to the operator: a + stateful deployment whose candidate was issued a lease, or a continuity + retrying forward past its deadline. Since `idunn-reincarnation` cut 2 it is + the second incident condition, `operator-required` (subject: the transaction + id), beside `continuity-exhausted`; it closes when the record ends or its + step goes on. Odin's own store condition (`odin.store_condition.v1`, `contested`) is mapped but not built. Idunn reads nothing about Odin's store health today. @@ -85,8 +88,8 @@ the machine has a detector with no record, and a deliverer that nothing feeds. - Odin's store-recovery cuts, O2 included. They produce `odin.store_condition` and are Odin's campaign, still in prose. This campaign consumes the record once it exists. -- Idunn's B5 phase deadlines. B5 is in prose on `hands/b5`. This campaign - reports its `OperatorRequired` outcome once B5 merges. +- Idunn's phase deadlines. They landed in `idunn-reincarnation` cut 2, which + also owns the `operator-required` incident condition. - VoidBot's `notify_owner` MCP tool and its separate DM code. - Bifrost's Persona delivery crossing. - Eve/TUI lowering of incidents. diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs new file mode 100644 index 0000000..3cf71e6 --- /dev/null +++ b/src/bin/idunn-launch.rs @@ -0,0 +1,96 @@ +//! The unit's process for an upstream daemon: parse the recipe's argv, read +//! what Idunn set, run the launcher, and map the outcome to an exit code. +#[cfg(unix)] +fn main() -> std::process::ExitCode { + use std::sync::atomic::{AtomicBool, Ordering}; + + use anyhow::Context; + use idunn_daemon::deployment::{ + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + }; + use idunn_daemon::launcher::{self, *}; + + static STOP: AtomicBool = AtomicBool::new(false); + extern "C" fn on_signal(_: libc::c_int) { + STOP.store(true, Ordering::SeqCst); + } + + fn launch() -> anyhow::Result { + let release_root = std::env::current_exe()? + .parent() + .context("the launcher is not inside a release")? + .to_owned(); + let contract = LaunchContract::parse(release_root, std::env::args().skip(1))?; + let bundle = std::env::var_os(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT) + .context("the Idunn runtime bundle is not set")?; + let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + .context("the Idunn candidate bind is not set")?; + let member = launcher::parse_member_bind( + &std::env::var(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + .context("the Idunn member bind is not set")?, + &bind, + )?; + let (activation, provider) = launcher::inherited_authority_files()?; + let authority = RuntimeAuthority::open(bundle.as_ref(), activation, provider)?; + authority.require_candidate_bind(&bind)?; + let lease_path = std::env::var_os(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT); + anyhow::ensure!( + lease_path.is_some() == authority.expected().write_lease_required, + "the process write lease binding differs from Expected" + ); + let route = std::net::TcpListener::bind(&bind).context("binding the candidate endpoint")?; + let sink = OdinRudpSink::from_expected(authority.expected())?; + let lease = DriverLeaseReader::new( + &authority.expected().target, + lease_path.as_deref().unwrap_or_default().as_ref(), + ); + let health_path = contract.health_path.clone(); + let ports = Ports { + credentials: std::sync::Arc::new(CredentialsDirectory::from_environment()), + clock: std::sync::Arc::new(SystemClock), + sink: std::sync::Arc::new(sink), + lease: std::sync::Arc::new(lease), + spawner: Box::new(ProcessSpawner), + health: Box::new(move |member| { + std::sync::Arc::new(HttpHealthProbe { + member, + path: health_path.clone(), + }) + }), + }; + // SAFETY: the handler only stores to an atomic. + unsafe { + libc::signal(libc::SIGTERM, on_signal as *const () as libc::sighandler_t); + libc::signal(libc::SIGINT, on_signal as *const () as libc::sighandler_t); + } + Ok(launcher::run( + &contract, + authority, + route, + member, + &std::env::temp_dir(), + ports, + &STOP, + )) + } + + match launch() { + Ok(outcome) => { + if let Outcome::Failed(code) = &outcome { + eprintln!("idunn-launch ended: {code}"); + } + std::process::ExitCode::from(outcome.exit_code() as u8) + } + Err(error) => { + eprintln!("idunn-launch refused: {error}"); + std::process::ExitCode::from(2) + } + } +} + +#[cfg(not(unix))] +fn main() { + eprintln!("idunn-launch runs only on Unix"); + std::process::exit(2); +} diff --git a/src/control_plane.rs b/src/control_plane.rs index d2a7635..cd49073 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -42,11 +42,14 @@ use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, RunnerStepFailure, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; +use crate::runner_evidence::{ + self, Kept, RunnerFailureRecord, runner_evidence_store_path, unreadable_store_path, +}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -2931,6 +2934,10 @@ enum Command { recipe: PathBuf, binding: Option, }, + #[cfg(unix)] + OrdersServe(crate::steward_orders::ServeOptions), + #[cfg(unix)] + Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { @@ -2970,6 +2977,10 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + #[cfg(unix)] + Command::OrdersServe(options) => crate::steward_orders::serve(options), + #[cfg(unix)] + Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -3022,6 +3033,10 @@ fn parse(args: impl Iterator) -> Result { "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), + #[cfg(unix)] + "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), + #[cfg(unix)] + "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -4727,20 +4742,21 @@ fn resolve_selector( Ok(targets) } -fn submit( +/// The one commit primitive for a deployment command: `idunn up` and a steward +/// deploy order both write theirs here. Returns the commit time. +pub(crate) fn commit_deployment_command( store_path: &Path, + command_id: &str, selector: &str, requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { +) -> Result { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), + command_id: command_id.into(), kind: CommandKind::Deploy, selector: selector.into(), requested_by: requested_by.into(), @@ -4758,7 +4774,33 @@ fn submit( )?, "deployment command id collided" ); - println!("{}", command.command_id); + Ok(now) +} + +/// A command's present state as `idunn status --command` derives it, for the +/// steward order poster: `queued` and `running` are not terminal. +#[cfg(unix)] +pub(crate) fn command_state( + state_store: &Path, + command_id: &str, +) -> Result<(&'static str, String)> { + let snapshot = ControlSnapshot::read(state_store)?; + let archived = read_history_transactions(state_store); + let mut transactions = snapshot.transaction_for_command(command_id); + transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); + Ok(derived_command_status(&transactions)) +} + +fn submit( + store_path: &Path, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + let command_id = format!("up-{}", Uuid::new_v4()); + let now = commit_deployment_command(store_path, &command_id, selector, requested_by)?; + println!("{command_id}"); if !wait { return Ok(()); } @@ -4769,10 +4811,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command.command_id), + .any(|stored| stored.value.command_id == command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command.command_id); + let transactions = snapshot.transaction_for_command(&command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4797,7 +4839,7 @@ fn submit( } } -fn status( +pub(crate) fn status( out: &mut impl Write, store_path: &Path, incident_store: &Path, @@ -4877,6 +4919,13 @@ fn status( } if let Some(abort) = &transaction.pre_fencing_abort { writeln!(out, " pre-fencing abort {abort:?}")?; + // The step that failed and its output tail live in their + // own private store. An unreadable store costs its lines. + runner_evidence::show_status( + out, + &runner_evidence_store_path(store_path), + &transaction.transaction_id, + )?; } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -5374,6 +5423,32 @@ impl Engine { IncidentStore::new(&self.options.incident_store) } + /// A failed runner step keeps its output tail in the runner evidence + /// store and says so on the journal, naming no output. The store write is + /// best effort: its failure is named in the line and never stops the + /// abort that follows. + fn keep_runner_failure( + &self, + transaction: &DeploymentTransaction, + error: &anyhow::Error, + ) -> Option { + let failure = error.downcast_ref::()?; + let path = runner_evidence_store_path(&self.options.state_store); + let recorded = now_millis().and_then(|now| { + runner_evidence::record( + &path, + &RunnerFailureRecord::from_failure( + &transaction.transaction_id, + &transaction.command_id, + &transaction.target, + failure, + now, + ), + ) + }); + Some(runner_failure_journal_line(transaction, failure, &path, &recorded)) + } + /// Settle the outcome of one incident operation. A failure never leaves /// this function: it is traced once per episode, under the operation and /// the subject it was decided for (`*` for a pass over the whole store), @@ -6315,6 +6390,9 @@ impl Engine { if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() { + if let Some(line) = self.keep_runner_failure(&latest.value, &error) { + eprintln!("{line}"); + } self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -10553,7 +10631,13 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -fn now_millis() -> Result { +/// The control store `idunn up` and `idunn serve` use unless told otherwise. +#[cfg(unix)] +pub(crate) fn default_state_store() -> PathBuf { + RuntimeOptions::default().state_store +} + +pub(crate) fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -10573,12 +10657,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -fn string_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -fn path_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -10594,7 +10678,7 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -fn require_selector(value: &str) -> Result<()> { +pub(crate) fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } @@ -10641,6 +10725,43 @@ const DETAIL_EMPTY: &str = "(no detail)"; /// only judge, and this output always satisfies it. NUL is dropped, empty /// input gets a fixed placeholder, and an over-long value keeps its head and /// its tail (where a build log's error is) cut on char boundaries. +/// The journal line for a failed runner step: the transaction, the subject and +/// status (the failure's `Display`), and where the evidence is. Never output. +fn runner_failure_journal_line( + transaction: &DeploymentTransaction, + failure: &RunnerStepFailure, + store_path: &Path, + recorded: &Result, +) -> String { + let head = format!( + "Idunn transaction {} (command {}, target {}) {failure}", + transaction.transaction_id, transaction.command_id, transaction.target + ); + match recorded { + Ok(kept) => { + let aside = match kept { + Kept::Stored => String::new(), + Kept::SetAside { replaced } => format!( + "; an undecodable evidence store was set aside as {}{}", + unreadable_store_path(store_path).display(), + match replaced { + Some(last_written) => format!( + ", replacing an earlier set-aside store last written at unix millis {last_written}" + ), + None => String::new(), + } + ), + }; + format!( + "{head}; evidence in {}{aside}; read it with idunn status --command {}", + store_path.display(), + transaction.command_id + ) + } + Err(error) => format!("{head}; evidence was not kept: {error:#}"), + } +} + fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -10681,7 +10802,10 @@ fn usage() -> &'static str { idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\ + idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ + idunn order --reason TEXT\n\ + idunn order show ID | idunn order list\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -10723,6 +10847,7 @@ mod tests { private_port_end, config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), reload_unit: "nginx.service".into(), + member: None, } } @@ -11892,9 +12017,45 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["order", "run", "sh", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], + vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], + vec!["orders", "serve", "--allow-units", "nginx.service"], + vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } + // The declarative order forms parse, and only as order commands. + #[cfg(unix)] + for arguments in [ + vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r"], + vec!["order", "list"], + vec!["orders", "serve"], + ] { + let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); + assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); + } + } + + #[test] + fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + commit_deployment_command(&store, "up-one", "vili", "steward:eureka order ord-1")?; + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.commands.len(), 1); + let command = &snapshot.commands[0].value; + assert_eq!(command.command_id, "up-one"); + assert_eq!(command.selector, "vili"); + assert_eq!(command.requested_by, "steward:eureka order ord-1"); + assert!(commit_deployment_command(&store, "up-one", "other", "x").is_err()); + assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); + Ok(()) } #[test] @@ -24656,4 +24817,435 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } } + + // BEGIN runner-failure-evidence tests + // A failed runner step is named, its output tail is kept in + // runner-evidence.cc, and no output reaches an error, control.cc or the + // journal line. The runner is the real DockerRunnerDriver driving a fake + // docker program, through the engine's Sealing path. + + const RUNNER_CANARY: &str = "CANARY-7c41e0d9"; + + /// A source that freezes a fixed tree, so a deployment reaches the runner. + struct TreeSource { + recipe: String, + root: PathBuf, + } + + impl SourcePort for TreeSource { + fn resolve( + &self, + _: &OperatorBinding, + _: &str, + _: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result { + Ok(FrozenSourceReceipt { + transaction_id: transaction_id.to_owned(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: format!("sha256-{}", "a".repeat(64)), + }) + } + + fn observe_frozen( + &self, + plan: &CompiledDeploymentPlan, + receipt: &FrozenSourceReceipt, + ) -> Result { + Ok(crate::drivers::FrozenSource::fixture( + receipt.clone(), + plan.source.clone(), + plan.recipe_blob.clone(), + self.root.clone(), + )) + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// The unrouted recipe with two steps ahead of its build, and no external + /// input: the second step, `test-daemon`, is the one the fake docker fails. + fn two_step_recipe() -> String { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let (head, rest) = recipe + .split_once("[[external_inputs]]") + .expect("the recipe declares an external input"); + let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts"); + let head = head.replacen( + "[[steps]]\nid = \"build\"", + "[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\ + [[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\ + [[steps]]\nid = \"build\"", + 1, + ); + format!("{head}[[artifacts]]{tail}") + } + + /// A Sealing world whose runner is the real driver over a fake docker: the + /// first call succeeds, the second prints the canary and an escape + /// sequence on both streams and exits 3. + #[cfg(unix)] + fn failing_runner_world() -> Result<(EngineFixture, TempDir)> { + use std::os::unix::fs::PermissionsExt; + let tools = TempDir::new()?; + let docker = tools.path().join("docker"); + std::fs::write( + &docker, + format!( + "#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n" + ), + )?; + std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?; + let recipe = two_step_recipe(); + let mut world = sealing_world(&recipe)?; + let tree = tools.path().join("tree"); + std::fs::create_dir_all(&tree)?; + std::fs::write(tree.join("README"), b"source")?; + world.engine.source = Arc::new(TreeSource { recipe, root: tree }); + world.engine.docker_runner = DockerRunnerDriver { docker_program: docker }; + Ok((world, tools)) + } + + /// Scheduler ticks until the deployment ends (or the budget is spent). + fn tick_until_ended(world: &EngineFixture) -> Result { + for _ in 0..24 { + world.engine.resume_backoff.lock().unwrap().clear(); + let _ = world.engine.run_scheduler_tick(); + if latest(world)?.completion.is_some() { + break; + } + } + latest(world) + } + + fn printed_status(world: &EngineFixture) -> Result { + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?; + Ok(String::from_utf8(printed)?) + } + + fn canary_failure(stdout: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject: crate::drivers::RunnerSubject::Step("test-daemon".into()), + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stdout.to_vec(), + stderr_total_bytes: stdout.len() as u64, + } + } + + #[cfg(unix)] + #[test] + fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error); + }; + // Named: the second step, on its runner, with the exit status. + assert!( + error.contains("runner step test-daemon on runner rust exited with exit status 3"), + "{error}" + ); + // Never echoed: no error text, no control.cc byte, carries the output. + let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; + for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); + } + // The terminal transaction has moved to history.cc: both stores. + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + let bytes = std::fs::read(&store)?; + assert!(!bytes.is_empty(), "{} is empty", store.display()); + assert!( + !bytes.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), + "runner output reached {}", + store.display() + ); + } + // Kept: in the private store, tails of both streams, and on status. + let evidence = runner_evidence_store_path(&world.state_store); + assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("no evidence for the failed transaction")?; + assert_eq!(record.subject_id, "test-daemon"); + assert_eq!(record.exit_code, Some(3)); + assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY)); + let printed = printed_status(&world)?; + assert!(printed.contains("failed step test-daemon"), "{printed}"); + assert!(printed.contains(RUNNER_CANARY), "{printed}"); + assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte"); + Ok(()) + } + + #[test] + fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> { + let world = EngineFixture::new()?; + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let line = world + .engine + .keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing")) + .context("a runner failure produced no journal line")?; + assert!(!line.contains(RUNNER_CANARY), "{line}"); + for needle in [ + transaction.transaction_id.as_str(), + "up-ghostlight", + "ghostlight", + "runner step test-daemon on runner rust exited with exit status 3", + "runner-evidence.cc", + "idunn status --command up-ghostlight", + ] { + assert!(line.contains(needle), "{needle} missing from {line}"); + } + // Not a runner failure: nothing is kept and nothing is said. + assert!( + world + .engine + .keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure")) + .is_none() + ); + // The line a failed write produces says so, with no output. + let unwritable = runner_failure_journal_line( + &transaction, + &failure, + Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"), + &Err(anyhow::anyhow!("disk full")), + ); + assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}"); + assert!(!unwritable.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + // A directory where the store should be: every write to it fails. + std::fs::create_dir(runner_evidence_store_path(&world.state_store))?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b""))) + .context("no journal line")?; + assert!(line.contains("evidence was not kept"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_undecodable_evidence_store_does_not_stop_the_next_failure_keeping_its_evidence() -> Result<()> { + let damage = format!("\u{c1} not a store {RUNNER_CANARY}-in-the-damage"); + let (world, _tools) = failing_runner_world()?; + let evidence = runner_evidence_store_path(&world.state_store); + std::fs::write(&evidence, damage.as_bytes())?; + let ended = tick_until_ended(&world)?; + // The transaction's own outcome is the runner failure, not the store. + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the damaged store changed the outcome: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + assert!(!error.contains("damage"), "{error}"); + // The damage is kept aside; the failure's evidence is in a fresh store. + assert_eq!(std::fs::read(unreadable_store_path(&evidence))?, damage.as_bytes()); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("the failure kept no evidence beside a damaged store")?; + assert_eq!(record.subject_id, "test-daemon"); + assert!(printed_status(&world)?.contains("failed step test-daemon")); + // The journal says it, naming the file and none of its content. + std::fs::write(&evidence, damage.as_bytes())?; + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"x"))) + .context("no journal line")?; + assert!(line.contains("an undecodable evidence store was set aside as"), "{line}"); + assert!(line.contains("runner-evidence.unreadable.cc"), "{line}"); + assert!(!line.contains("damage") && !line.contains("not a store"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?; + let printed = printed_status(&world)?; + assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}"); + assert!(printed.contains("target service phase"), "{printed}"); + assert!(printed.contains("pre-fencing abort"), "{printed}"); + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + Ok(()) + } + + /// Everything status prints from the evidence store goes through one + /// allowlist, so a hostile build log reaches the terminal as visible + /// escapes whatever its class. + #[cfg(unix)] + #[test] + fn status_prints_hostile_runner_output_as_visible_escapes_only() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let hostile: Vec<(&str, &str)> = vec![ + ("\u{1b}[2J", "\\u{1b}[2J"), + ("\u{7}", "\\u{7}"), + ("\r", "\\u{d}"), + ("\u{202e}", "\\u{202e}"), + ("\u{2066}", "\\u{2066}"), + ("\u{85}", "\\u{85}"), + ("\u{e0041}", "\\u{e0041}"), + ("\u{200b}", "\\u{200b}"), + ("\u{feff}", "\\u{feff}"), + ]; + let mut tail: Vec = Vec::new(); + for &(raw, _) in &hostile { + tail.extend_from_slice(b"a"); + tail.extend_from_slice(raw.as_bytes()); + tail.extend_from_slice(b"b\n"); + } + tail.extend_from_slice(b"bad \xff\xc1 bytes\n"); + runner_evidence::record( + &runner_evidence_store_path(&world.state_store), + &RunnerFailureRecord::from_failure( + &ended.transaction_id, + &ended.command_id, + &ended.target, + &canary_failure(&tail), + 9, + ), + )?; + let printed = printed_status(&world)?; + for &(raw, escaped) in &hostile { + assert!(!printed.contains(raw), "{escaped} reached the terminal raw"); + assert!(printed.contains(&format!("a{escaped}b")), "{escaped} missing: {printed}"); + } + assert!(printed.contains("bad \\xff\\xc1 bytes"), "{printed}"); + let section = printed.find(" failed step").context("no evidence section")?; + assert!(printed[section..].is_ascii(), "non-ASCII reached the terminal: {printed}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn status_prints_the_unreadable_store_line_escaped() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + let evidence = runner_evidence_store_path(&world.state_store); + // The decode error of this store quotes the stored subject kind. + let mut bytes = std::fs::read(&evidence)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker in the evidence store")?; + bytes[at..at + 4].copy_from_slice(b"\x1b[2J"); + std::fs::write(&evidence, bytes)?; + let printed = printed_status(&world)?; + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + assert!(!printed.contains('\u{1b}'), "an escape byte reached the terminal"); + assert!(printed.contains("\\u{1b}[2J"), "{printed}"); + Ok(()) + } + + #[test] + fn the_journal_line_names_a_displaced_set_aside_store_by_time_only() -> Result<()> { + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let path = Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"); + let first = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: None }), + ); + assert!(first.contains("was set aside as"), "{first}"); + assert!(!first.contains("replacing"), "{first}"); + let second = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: Some(1234) }), + ); + assert!( + second.contains("replacing an earlier set-aside store last written at unix millis 1234"), + "{second}" + ); + assert!(!second.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + // The store a rolled-back Idunn reads: it refuses any foreign type. + ControlSnapshot::read(&world.state_store)?; + let mut transactions = 0; + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + for envelope in SingleFileMessagePackBackingStore::new(&store).pull_all_read_only_snapshot()? { + assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); + if envelope.r#type == DeploymentTransaction::TYPE { + transactions += 1; + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); + } + } + } + assert_eq!(transactions, 1, "the failed transaction is in neither store"); + Ok(()) + } + + /// The records control.cc holds re-encode byte for byte. The hex was + /// captured at base 804ed75, before this cut touched anything: a runner + /// failure adds no field, no type and no schema to the control store. + #[test] + fn control_records_encode_exactly_as_they_did_at_base() -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?; + transaction.transaction_id = "tx-pinned".into(); + transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into()); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "runner step test-daemon on runner rust exited with exit status 3".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); + let command_hex = hex(&command_envelope(&command, 100)?.payload); + let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload); + assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed"); + assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed"); + Ok(()) + } + + const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; + const PINNED_TRANSACTION_HEX: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; + // END runner-failure-evidence tests } diff --git a/src/deployment.rs b/src/deployment.rs index a67b4bc..b7cbdc8 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -9,12 +9,27 @@ pub const TARGET_DECLARATION_SCHEMA: &str = "gamecult.idunn.target_declaration.v pub const OPERATOR_BINDING_SCHEMA: &str = "gamecult.idunn.operator_binding.v2"; pub const IDUNN_RUNTIME_BUNDLE_ENVIRONMENT: &str = "GAMECULT_IDUNN_RUNTIME_BUNDLE"; pub const IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_CANDIDATE_BIND"; +pub const IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_MEMBER_BIND"; pub const IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT: &str = "GAMECULT_IDUNN_PROCESS_WRITE_LEASE"; pub const RUNTIME_PRESENCE_IDENTITY_BINDING: &str = "GAMECULT_RUNTIME_PRESENCE_IDENTITY"; /// A host actuator hands the workload its activation credential as a file /// named by this variable; on the Idunn host it arrives as a parent-only /// descriptor instead and this name is never set. pub const IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT: &str = "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL"; +/// systemd hands a unit its credential directory under this name; Idunn owns +/// the directory, so no binding may set it. +pub const SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT: &str = "CREDENTIALS_DIRECTORY"; +/// Every launch-environment name Idunn sets or reserves. A binding may name +/// none of them; this is the one list both the binding check and the +/// launch-input check read. +pub(crate) const IDUNN_OWNED_LAUNCH_ENVIRONMENT: [&str; 6] = [ + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT, +]; pub const RUNTIME_PRESENCE_IDENTITY_FD_NAME: &str = "gamecult-runtime-presence-identity"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -98,6 +113,10 @@ pub struct ArtifactOutput { pub expected_sha256: Option, #[serde(default)] pub executable: bool, + /// Regular files inside a directory artifact that are sealed executable + /// (0555); every other file in the tree is sealed 0444. + #[serde(default)] + pub executable_members: BTreeSet, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -184,6 +203,11 @@ pub enum StateKind { /// Process-owned state imposed by an external runtime contract. Idunn /// preserves and fences the file but never interprets its payload. ExternalFile, + /// A process-owned directory imposed by an external runtime contract (for + /// example Synapse's media store). Idunn preserves and fences it and never + /// interprets its contents. Same writer, recovery and startup rules as + /// `ExternalFile`. + ExternalDirectory, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -504,8 +528,7 @@ impl WorkloadBinding { { require_environment_name(name)?; ensure!( - name != IDUNN_RUNTIME_BUNDLE_ENVIRONMENT - && name != IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + !IDUNN_OWNED_LAUNCH_ENVIRONMENT.contains(&name.as_str()), "operator binding attempts to replace Idunn-owned workload environment" ); } @@ -638,9 +661,7 @@ pub struct ProcessWriteLeaseBinding { impl ProcessWriteLeaseBinding { pub fn lock_path(&self) -> PathBuf { - let mut lock_path = self.record_path.as_os_str().to_os_string(); - lock_path.push(".lock"); - PathBuf::from(lock_path) + crate::drivers::authority_lock_path(&self.record_path) } } @@ -662,40 +683,99 @@ pub struct RouteBinding { pub private_port_end: u16, pub config_path: PathBuf, pub reload_unit: String, + /// A second listener for the route's members. When set, members reach the + /// workload straight through a second nginx server block, and the + /// workload's launcher answers presence only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub member: Option, +} + +/// The member listener of a route: where members connect, and the first of the +/// private ports the workload binds for them. The private range is as wide as +/// the route's own. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemberListenerBinding { + pub stable_endpoint: String, + pub private_port_start: u16, + /// Most member connections nginx holds open at once for this route. Each + /// costs nginx two connections (client and upstream) from the pool the + /// route's presence block shares, so this limit is what keeps members from + /// exhausting it. + #[serde(default = "default_member_max_connections")] + pub max_connections: u16, +} + +const DEFAULT_MEMBER_MAX_CONNECTIONS: u16 = 128; + +fn default_member_max_connections() -> u16 { + DEFAULT_MEMBER_MAX_CONNECTIONS } impl RouteBinding { pub(crate) fn stable_socket(&self) -> Result<(IpAddr, u16)> { - let authority = match self.driver { - RouteDriver::NginxStreamTcp => self - .stable_endpoint - .strip_prefix("http://") - .or_else(|| self.stable_endpoint.strip_prefix("tcp://")), - RouteDriver::NginxStreamUdp => self.stable_endpoint.strip_prefix("rudp://"), - } - .context("stable endpoint transport does not match its route driver")?; - ensure!( - !authority - .chars() - .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), - "stable endpoint is not a plain IP address and port" - ); - let (host, port) = authority - .rsplit_once(':') - .context("stable endpoint has no port")?; - let host = host - .parse::() - .context("stable endpoint host is not an IP address")?; - ensure!( - matches!(host, IpAddr::V4(_)), - "route binding v2 stable endpoints use explicit IPv4 addresses" - ); - let port = port - .parse::() - .context("stable endpoint port is not a u16")?; - ensure!(port > 0, "stable endpoint port is zero"); - Ok((host, port)) + parse_stable_socket(self.driver, &self.stable_endpoint, "stable endpoint") } + + /// The member listener's stable socket, when the route has one. + pub(crate) fn member_stable_socket(&self) -> Result> { + self.member + .as_ref() + .map(|member| { + parse_stable_socket(self.driver, &member.stable_endpoint, "member endpoint") + }) + .transpose() + } + + /// The private port the workload binds for members when its presence + /// candidate is `candidate_port`. Derived, never stored: the member range + /// is the private range shifted to the member start. + pub(crate) fn member_candidate_port(&self, candidate_port: u16) -> Result> { + let Some(member) = self.member.as_ref() else { + return Ok(None); + }; + ensure!( + (self.private_port_start..=self.private_port_end).contains(&candidate_port), + "candidate port is outside the route binding" + ); + let offset = candidate_port - self.private_port_start; + let port = member + .private_port_start + .checked_add(offset) + .context("member private port overflows")?; + Ok(Some(port)) + } +} + +fn parse_stable_socket(driver: RouteDriver, endpoint: &str, what: &str) -> Result<(IpAddr, u16)> { + let authority = match driver { + RouteDriver::NginxStreamTcp => endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("tcp://")), + RouteDriver::NginxStreamUdp => endpoint.strip_prefix("rudp://"), + } + .with_context(|| format!("{what} transport does not match its route driver"))?; + ensure!( + !authority + .chars() + .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), + "{what} is not a plain IP address and port" + ); + let (host, port) = authority + .rsplit_once(':') + .with_context(|| format!("{what} has no port"))?; + let host = host + .parse::() + .with_context(|| format!("{what} host is not an IP address"))?; + ensure!( + matches!(host, IpAddr::V4(_)), + "route binding v2 endpoints use explicit IPv4 addresses" + ); + let port = port + .parse::() + .with_context(|| format!("{what} port is not a u16"))?; + ensure!(port > 0, "{what} port is zero"); + Ok((host, port)) } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -836,6 +916,14 @@ impl TargetDeclaration { if let Some(expected_sha256) = &artifact.expected_sha256 { require_sha256(expected_sha256, "expected artifact sha256")?; } + ensure!( + !(artifact.executable && !artifact.executable_members.is_empty()), + "artifact {} declares both executable and executable_members", + artifact.id + ); + for member in &artifact.executable_members { + require_relative_path(member, "executable member", false)?; + } ensure!( destinations.insert(artifact.destination.as_path()), "artifact destination {} is declared twice", @@ -919,6 +1007,14 @@ impl TargetDeclaration { == self.service.route_required, "routed services must require exactly one Idunn-owned candidate bind" ); + ensure!( + !self + .service + .required_environment + .contains(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + || self.service.route_required, + "only routed services may require the Idunn-owned member bind" + ); ensure!( self.service .required_environment @@ -1376,7 +1472,39 @@ impl OperatorBinding { route.private_port_start < route.private_port_end, "candidate rollout requires at least two private ports" ); - route.stable_socket()?; + let stable = route.stable_socket()?; + if let Some(member) = route.member.as_ref() { + ensure!( + route.driver == RouteDriver::NginxStreamTcp, + "only a TCP route may carry a member listener" + ); + require_value(&member.stable_endpoint, "member endpoint")?; + let member_stable = route + .member_stable_socket()? + .context("route has no member listener")?; + ensure!( + member_stable != stable, + "member endpoint equals the route's stable endpoint" + ); + ensure!( + member.private_port_start > 0, + "member private port range starts at zero" + ); + ensure!( + member.max_connections > 0, + "member connection limit is zero" + ); + let width = route.private_port_end - route.private_port_start; + let member_end = member + .private_port_start + .checked_add(width) + .context("member private port range overflows")?; + ensure!( + member_end < route.private_port_start + || member.private_port_start > route.private_port_end, + "member private port range overlaps the route's private range" + ); + } require_absolute_path(&route.config_path, "route config")?; require_unit(&route.reload_unit, "route reload unit")?; } @@ -1554,12 +1682,7 @@ impl OperatorBinding { .keys() .chain(self.workload.secret_environment_names()) .collect(); - for owned in [ - IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, - IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, - IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, - ] { + for owned in IDUNN_OWNED_LAUNCH_ENVIRONMENT { ensure!( !workload_names.contains(&owned.to_string()), "operator binding cannot replace Idunn-owned launch environment {owned}" @@ -1581,6 +1704,9 @@ impl OperatorBinding { } if declaration.service.route_required { available_environment.insert(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT.into()); + if self.route.as_ref().is_some_and(|route| route.member.is_some()) { + available_environment.insert(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into()); + } } if declaration.write_lease_required() { available_environment.insert(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT.into()); @@ -1928,10 +2054,10 @@ fn require_host(value: &str) -> Result<()> { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - const RECIPE: &str = r#" + pub(crate) const RECIPE: &str = r#" schema = "gamecult.idunn.target_declaration.v1" target = "test-service" source_stamp_environment = "TEST_SERVICE_BUILD_COMMIT" @@ -2003,7 +2129,7 @@ compatibility = "v1" startup = "before-promotion" "#; - const BINDING: &str = r#" + pub(crate) const BINDING: &str = r#" schema = "gamecult.idunn.operator_binding.v2" target = "test-service" profiles = ["aetheria", "full-gamecult"] @@ -2097,6 +2223,76 @@ nodes = ["yggdrasil"] assert!(TargetDeclaration::parse(&input).is_err()); } + const MEMBER_CANARY: &str = "zzcanary"; + + fn recipe_with_members(members: &str) -> String { + let input = RECIPE.replace("executable = true", &format!("executable_members = [{members}]")); + assert_ne!(input, RECIPE, "the fixture lost its executable line"); + input + } + + /// A refused member list is refused by name of the field, never by echoing + /// the member. + fn assert_members_refused(members: &[&str]) { + let control = TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).unwrap(); + assert!(control.artifacts[0].executable_members.contains(Path::new("bin/daemon"))); + for member in members { + let error = TargetDeclaration::parse(&recipe_with_members(&format!("\"{member}\""))) + .expect_err(member); + assert!( + !format!("{error:#}").contains(MEMBER_CANARY), + "the refusal echoed a member path" + ); + } + } + + #[test] + fn recipe_executable_member_refuses_parent_traversal() { + assert_members_refused(&[ + "zzcanary/../b", + "../zzcanary", + "bin/zzcanary/..", + "bin/../../zzcanary", + ]); + } + + #[test] + fn recipe_executable_member_refuses_absolute_path() { + assert_members_refused(&["/zzcanary/sh", "/zzcanary"]); + } + + #[test] + fn recipe_executable_member_refuses_empty_and_current_dir() { + assert_members_refused(&["", ".", "./", "zzcanary/.."]); + } + + #[test] + fn recipe_refuses_executable_flag_with_members() { + let input = RECIPE.replace( + "executable = true", + "executable = true\nexecutable_members = [\"bin/daemon\"]", + ); + let error = TargetDeclaration::parse(&input).unwrap_err(); + assert!(format!("{error:#}").contains("executable_members")); + assert!(!format!("{error:#}").contains("bin/daemon")); + // executable = true alone, and members alone, both stay admitted. + assert!(TargetDeclaration::parse(RECIPE).is_ok()); + assert!(TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).is_ok()); + } + + #[test] + fn recipe_admits_members_on_both_source_kinds() { + let runner_output = TargetDeclaration::parse(&recipe_with_members("\"a/b\", \"c\"")).unwrap(); + assert_eq!(runner_output.artifacts[0].source_kind, ArtifactSource::RunnerOutput); + assert_eq!(runner_output.artifacts[0].executable_members.len(), 2); + let worktree = recipe_with_members("\"a/b\", \"c\"") + .replace("source_kind = \"runner-output\"\nrunner = \"rust\"\n", "source_kind = \"worktree-tree\"\n") + .replace("expected_sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"\n", ""); + let worktree = TargetDeclaration::parse(&worktree).unwrap(); + assert_eq!(worktree.artifacts[0].source_kind, ArtifactSource::WorktreeTree); + assert_eq!(worktree.artifacts[0].executable_members.len(), 2); + } + #[test] fn operator_cannot_omit_an_affordance_derived_from_the_recipe() { let recipe = TargetDeclaration::parse(RECIPE).unwrap(); @@ -2230,6 +2426,203 @@ nodes = ["yggdrasil"] } } + /// The recipe and binding of a routed service whose members connect to a + /// listener of their own. + pub(crate) fn member_listener_pair() -> (String, String) { + let recipe = RECIPE.replace( + "required_environment = [", + "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", + ); + let binding = BINDING.replace( + "reload_unit = \"nginx.service\"\n", + "reload_unit = \"nginx.service\"\n\n[route.member]\nstable_endpoint = \"http://127.0.0.1:18840\"\nprivate_port_start = 18841\n", + ); + (recipe, binding) + } + + fn member_binding_with(replacement: (&str, &str)) -> Result { + let (recipe, binding) = member_listener_pair(); + let recipe = TargetDeclaration::parse(&recipe)?; + let binding = OperatorBinding::parse(&binding.replace(replacement.0, replacement.1))?; + binding.admit(&recipe)?; + Ok(binding) + } + + #[test] + fn a_member_listener_admits_with_a_recipe_that_requires_the_member_bind() { + let binding = member_binding_with(("never-matches", "")).unwrap(); + let route = binding.route.unwrap(); + assert_eq!(route.member_stable_socket().unwrap().unwrap().1, 18840); + assert_eq!(route.member_candidate_port(18831).unwrap(), Some(18841)); + assert_eq!(route.member_candidate_port(18839).unwrap(), Some(18849)); + assert!(route.member_candidate_port(18840).is_err()); + } + + #[test] + fn a_binding_without_a_member_listener_has_no_member_ports() { + let binding = OperatorBinding::parse(BINDING).unwrap(); + let route = binding.route.unwrap(); + assert_eq!(route.member_stable_socket().unwrap(), None); + assert_eq!(route.member_candidate_port(18831).unwrap(), None); + } + + #[test] + fn a_member_listener_needs_a_recipe_that_requires_the_member_bind() { + let (recipe, binding) = member_listener_pair(); + // Bound but not required: the recipe never asked for the variable. + let plain_recipe = TargetDeclaration::parse(RECIPE).unwrap(); + assert!( + OperatorBinding::parse(&binding) + .unwrap() + .admit(&plain_recipe) + .is_err() + ); + // Required but not bound: the variable would never be set. + let recipe = TargetDeclaration::parse(&recipe).unwrap(); + assert!( + OperatorBinding::parse(BINDING) + .unwrap() + .admit(&recipe) + .is_err() + ); + } + + #[test] + fn only_a_routed_recipe_may_require_the_member_bind() { + let recipe = RECIPE + .replace("route_required = true", "route_required = false") + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", ", + "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", + ); + assert!(recipe.contains("GAMECULT_IDUNN_MEMBER_BIND")); + assert!(TargetDeclaration::parse(&recipe).is_err()); + } + + #[test] + fn member_listener_bindings_are_validated() { + let start = |value: &str| { + member_binding_with(( + "private_port_start = 18841", + &format!("private_port_start = {value}"), + )) + }; + // Overlapping the route's own private range, in either direction. + assert!(start("18835").is_err()); + assert!(start("18825").is_err()); + // Adjacent but disjoint is fine, above the route's range or below it. + assert!(start("18840").is_ok()); + assert!(start("18822").is_ok()); + assert!(start("18823").is_err()); + // The connection limit defaults, and zero is refused. + let limit = |value: &str| { + member_binding_with(( + "private_port_start = 18841", + &format!("private_port_start = 18841 +max_connections = {value}"), + )) + }; + assert_eq!( + member_binding_with(("never-matches", "")).unwrap().route.unwrap().member.unwrap().max_connections, + 128 + ); + assert_eq!(limit("7").unwrap().route.unwrap().member.unwrap().max_connections, 7); + assert!(limit("0").is_err()); + assert!(limit("65535").is_ok()); + assert!(limit("65536").is_err()); + // Zero, and a width that overflows u16. + assert!(start("0").is_err()); + assert!(start("65530").is_err()); + // The member endpoint may not be the route's stable endpoint. + assert!(member_binding_with(("http://127.0.0.1:18840", "http://127.0.0.1:18830")).is_err()); + // The same transport and address rules as the stable endpoint. + assert!(member_binding_with(("http://127.0.0.1:18840", "rudp://127.0.0.1:18840")).is_err()); + assert!(member_binding_with(("http://127.0.0.1:18840", "http://example.test:18840")).is_err()); + } + + #[test] + fn a_udp_route_cannot_carry_a_member_listener() { + let (recipe, binding) = member_listener_pair(); + let binding = binding + .replace("nginx-stream-tcp", "nginx-stream-udp") + .replace("http://127.0.0.1:18830", "rudp://127.0.0.1:18830") + .replace("http://127.0.0.1:18840", "rudp://127.0.0.1:18840"); + let recipe = TargetDeclaration::parse(&recipe).unwrap(); + assert!( + OperatorBinding::parse(&binding) + .and_then(|binding| binding.admit(&recipe)) + .is_err() + ); + } + + #[test] + fn idunn_owned_names_are_reserved_in_one_list() { + let recipe = TargetDeclaration::parse(RECIPE).unwrap(); + let owned = [ + "GAMECULT_IDUNN_RUNTIME_BUNDLE", + "GAMECULT_IDUNN_CANDIDATE_BIND", + "GAMECULT_IDUNN_MEMBER_BIND", + "GAMECULT_IDUNN_PROCESS_WRITE_LEASE", + "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", + "CREDENTIALS_DIRECTORY", + ]; + let mut listed = IDUNN_OWNED_LAUNCH_ENVIRONMENT.to_vec(); + listed.sort_unstable(); + let mut expected = owned.to_vec(); + expected.sort_unstable(); + assert_eq!(listed, expected); + + let as_environment = |name: &str| { + BINDING.replace( + "[workload.argument_bindings]", + &format!("[workload.environment]\n{name} = \"value\"\n\n[workload.argument_bindings]"), + ) + }; + let as_secret = |name: &str| { + BINDING.replace( + "[workload.secret_files]", + &format!("[workload.secret_files]\n{name} = \"/etc/gamecult/test-service/other.cc\""), + ) + }; + // The fixtures are well formed: an unowned name parses in both + // shapes, so the refusals below are about the name. + assert!(OperatorBinding::parse(&as_environment("UNOWNED_NAME")).is_ok()); + assert!(OperatorBinding::parse(&as_secret("UNOWNED_NAME")).is_ok()); + for name in owned { + // Refused when the binding is parsed... + assert!( + OperatorBinding::parse(&as_environment(name)).is_err(), + "{name} accepted as environment at parse" + ); + assert!( + OperatorBinding::parse(&as_secret(name)).is_err(), + "{name} accepted as a secret file at parse" + ); + // ...and again at admission, for a binding that did not come + // through parse. Each site reads the one list. + for secret in [false, true] { + let mut binding = OperatorBinding::parse(BINDING).unwrap(); + let WorkloadBinding::SystemdTransient(workload) = &mut binding.workload else { + panic!("the fixture is a systemd workload"); + }; + if secret { + workload.secret_files.insert(name.into(), "/etc/gamecult/test-service/other.cc".into()); + } else { + workload.environment.insert(name.into(), "forged".into()); + } + assert!( + binding.workload.validate_launch_environment().is_err(), + "{name} (secret: {secret}) accepted by the binding check" + ); + let error = binding.admit(&recipe).unwrap_err().to_string(); + assert!( + error.contains("Idunn-owned launch environment"), + "{name} (secret: {secret}) accepted by the launch-input check: {error}" + ); + } + } + } + #[test] fn state_slot_path_cannot_escape_the_operator_owned_root() { let input = RECIPE.replace( @@ -2261,6 +2654,17 @@ nodes = ["yggdrasil"] assert!(declaration.write_lease_required()); } + #[test] + fn external_directory_state_is_lease_gated_like_external_file() { + let input = RECIPE.replace("kind = \"cultcache-file\"", "kind = \"external-directory\""); + let declaration = TargetDeclaration::parse(&input).unwrap(); + let slot = &declaration.state.as_ref().unwrap().slots[0]; + assert_eq!(slot.kind, StateKind::ExternalDirectory); + assert_eq!(slot.writer, StateWriter::ProcessBoundSingleWriter); + assert_eq!(slot.startup, StateStartup::CreateOrOpenAfterWriteLease); + assert!(declaration.write_lease_required()); + } + #[test] fn state_slots_cannot_claim_overlapping_paths() { let input = RECIPE.replace( diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 1363c48..eceb513 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -734,6 +734,11 @@ pub struct ArtifactReceipt { pub sha256: String, pub size_bytes: u64, pub executable: bool, + /// Files sealed executable inside a directory artifact. Skipped when empty + /// so a receipt with no members keeps its original encoding, and so its + /// sealed release id. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub executable_members: BTreeSet, } impl ArtifactReceipt { @@ -747,7 +752,9 @@ impl ArtifactReceipt { .find(|artifact| artifact.id == self.artifact_id) .with_context(|| format!("plan declares no artifact {}", self.artifact_id))?; ensure!( - self.destination == declared.destination && self.executable == declared.executable, + self.destination == declared.destination + && self.executable == declared.executable + && self.executable_members == declared.executable_members, "artifact receipt differs from declared output" ); if let Some(expected_sha256) = &declared.expected_sha256 { @@ -1425,6 +1432,7 @@ nodes = ["yggdrasil"] sha256: digest(b'c'), size_bytes: 42, executable: true, + executable_members: BTreeSet::new(), } } @@ -1791,6 +1799,113 @@ nodes = ["yggdrasil"] assert!(changed.validate_against(&plan).is_err()); } + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + } + + #[test] + fn receipt_without_members_encodes_as_before() { + // Both values were computed outside this code, from the five-field + // msgpack array the receipt encoded to at base d74ac8d. + const BASE_RECEIPT: &str = "95a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac3"; + const BASE_RELEASE_ID: &str = + "sha256-b9143d8f546ad71c493c70037813547aaec10fd87994a6f3bdc77ac3a8b3495e"; + let receipt = artifact_receipt(); + let encoded = rmp_serde::to_vec(&receipt).unwrap(); + assert_eq!(hex(&encoded), BASE_RECEIPT); + let bytes = (0..BASE_RECEIPT.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&BASE_RECEIPT[at..at + 2], 16).unwrap()) + .collect::>(); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), receipt); + let release = SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![receipt], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + assert_eq!(release.recomputed_release_id().unwrap(), BASE_RELEASE_ID); + } + + #[test] + fn sealed_release_address_covers_executable_members() { + let release_with = |members: &[&str]| SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![ArtifactReceipt { + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + ..artifact_receipt() + }], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + let none = release_with(&[]).recomputed_release_id().unwrap(); + let one = release_with(&["bin/daemon"]).recomputed_release_id().unwrap(); + let other = release_with(&["bin/other"]).recomputed_release_id().unwrap(); + let two = release_with(&["bin/daemon", "bin/other"]) + .recomputed_release_id() + .unwrap(); + let all = [&none, &one, &other, &two]; + for (index, left) in all.iter().enumerate() { + for right in &all[index + 1..] { + assert_ne!(left, right); + } + } + + // The declaration is the other half: a receipt must carry exactly the + // declared list, in a plan that declares one and in one that does not. + let plain = plan(); + let mut with_members = artifact_receipt(); + with_members.executable_members = BTreeSet::from([PathBuf::from("bin/daemon")]); + assert!( + SealedRelease::new(&plain, vec![with_members], vec![external_input_receipt()], 120) + .is_err() + ); + let recipe = RECIPE.replace("executable = true", "executable_members = [\"bin/daemon\", \"lib/tool\"]"); + let declaring = compile_deployment_plan( + recipe.as_bytes(), + BINDING.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(18001), + 110, + &[ready_odin_provider("odin", "odin-yggdrasil", 1)], + ) + .unwrap(); + let honest = ArtifactReceipt { + executable: false, + executable_members: BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]), + ..artifact_receipt() + }; + let release = SealedRelease::new( + &declaring, + vec![honest.clone()], + vec![external_input_receipt()], + 120, + ) + .unwrap(); + release.validate_against(&declaring).unwrap(); + for drifted in [ + BTreeSet::from([PathBuf::from("bin/daemon")]), + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/other")]), + BTreeSet::new(), + ] { + let mut receipt = honest.clone(); + receipt.executable_members = drifted; + assert!( + SealedRelease::new(&declaring, vec![receipt], vec![external_input_receipt()], 120) + .is_err() + ); + } + let mut changed = release.clone(); + changed.artifacts[0].executable_members.insert(PathBuf::from("lib/extra")); + assert!(changed.validate_against(&declaring).is_err()); + } + fn binding_with_deadlines(table: &str) -> String { BINDING.replace("[placement]", &format!("{table}\n[placement]")) } diff --git a/src/drivers.rs b/src/drivers.rs index fdf925c..3ea1145 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; @@ -22,7 +22,8 @@ use crate::control_plane::SequenceAdmittedWarming; use crate::deployment::{ ArtifactOutput, ArtifactSource, DockerRunnerBinding, GitlinkBinding, IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, LaunchArgument, OperatorBinding, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, + LaunchArgument, OperatorBinding, RUNTIME_PRESENCE_IDENTITY_BINDING, RUNTIME_PRESENCE_IDENTITY_FD_NAME, RouteBinding, RouteDriver, SourceSelectionPolicy, TargetDeclaration, WorkloadNetwork, }; @@ -156,6 +157,24 @@ pub struct FrozenSourceReceipt { pub snapshot_sha256: String, } +#[cfg(test)] +impl FrozenSource { + /// A frozen source over an existing tree, for tests that drive a runner. + pub(crate) fn fixture( + receipt: FrozenSourceReceipt, + facts: SourceSelectionFacts, + recipe_bytes: Vec, + root: PathBuf, + ) -> Self { + Self { + receipt, + facts, + recipe_bytes, + root, + } + } +} + impl FrozenSourceReceipt { pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> { plan.validate()?; @@ -469,6 +488,9 @@ pub struct SystemdWorkloadObservation { pub no_new_privileges: bool, pub umask: String, pub inaccessible_paths: String, + /// Retired wire slot. systemd 257 displays `LoadCredential` as `[unprintable]` for + /// every unit, so it was never evidence: written empty, never compared. Kept so + /// compact (positional) MessagePack records of earlier Idunns still decode. pub load_credential: String, pub main_pid: u32, pub process_start_time: u64, @@ -583,7 +605,9 @@ pub trait WorkloadPort { /// /// Idunn launches candidates as transient units with `Restart=no`, so a /// unit that has failed, or that systemd no longer knows, will not come - /// back on its own. Past the fence every error is otherwise treated as + /// back on its own. Those units are collected once they fail + /// (`INCARNATION_UNIT_LIFECYCLE`), so a failed candidate usually reads as + /// forgotten rather than failed. Past the fence every error is otherwise treated as /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; @@ -2148,6 +2172,115 @@ pub(crate) fn docker_run_args( Ok(args) } +/// Bytes of each output stream that a failed runner step keeps. The daemon +/// never holds more than this per stream while the step runs. +pub(crate) const RUNNER_OUTPUT_TAIL_BYTES: usize = 8192; + +/// What a runner container was running: a recipe step or an external input, +/// by the id the recipe gave it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RunnerSubject { + Step(String), + ExternalInput(String), +} + +impl RunnerSubject { + pub(crate) fn id(&self) -> &str { + match self { + Self::Step(id) | Self::ExternalInput(id) => id, + } + } + + fn noun(&self) -> &'static str { + match self { + Self::Step(_) => "step", + Self::ExternalInput(_) => "external input", + } + } +} + +/// A runner container that did not exit successfully: what failed, how, and +/// the last `RUNNER_OUTPUT_TAIL_BYTES` of each stream with its total size. +/// The tails are evidence for `runner-evidence.cc`. `Display` and `Debug` +/// carry none of them, so no error chain, record or log line renders output. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RunnerStepFailure { + pub(crate) subject: RunnerSubject, + pub(crate) runner: String, + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stdout_total_bytes: u64, + pub(crate) stderr_tail: Vec, + pub(crate) stderr_total_bytes: u64, +} + +impl RunnerStepFailure { + /// The status as an operator reads it. + pub(crate) fn status(&self) -> String { + runner_status(self.exit_code, self.signal) + } +} + +/// How a container ended, as an operator reads it. +pub(crate) fn runner_status(exit_code: Option, signal: Option) -> String { + match (exit_code, signal) { + (Some(code), _) => format!("exit status {code}"), + (None, Some(signal)) => format!("signal {signal}"), + (None, None) => "an unknown status".to_owned(), + } +} + +impl std::fmt::Display for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "runner {} {} on runner {} exited with {}; its output tail is kept in runner-evidence.cc", + self.subject.noun(), + self.subject.id(), + self.runner, + self.status() + ) + } +} + +impl std::fmt::Debug for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerStepFailure") + .field("subject", &self.subject) + .field("runner", &self.runner) + .field("exit_code", &self.exit_code) + .field("signal", &self.signal) + .field("stdout_total_bytes", &self.stdout_total_bytes) + .field("stderr_total_bytes", &self.stderr_total_bytes) + .finish_non_exhaustive() + } +} + +impl std::error::Error for RunnerStepFailure {} + +/// Read `reader` to its end keeping only the last `cap` bytes and counting +/// every byte, so a verbose step never buffers whole in the daemon. +fn drain_tail(mut reader: impl Read, cap: usize) -> std::io::Result<(Vec, u64)> { + let mut tail = Vec::new(); + let mut total = 0u64; + let mut chunk = [0u8; 8192]; + loop { + let read = match reader.read(&mut chunk) { + Ok(0) => return Ok((tail, total)), + Ok(read) => read, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + total += read as u64; + tail.extend_from_slice(&chunk[..read]); + if tail.len() > cap { + tail.drain(..tail.len() - cap); + } + } +} + /// Docker is only a runner substrate. Recipe argv stays an argv vector, the /// operator binding selects the exact image/network/mount affordances, and the /// driver returns complete materialization receipts rather than build truth by @@ -2166,7 +2299,7 @@ impl Default for DockerRunnerDriver { } impl DockerRunnerDriver { - fn docker(&self, args: I) -> Result + fn docker(&self, subject: RunnerSubject, runner_id: &str, args: I) -> Result<()> where I: IntoIterator, S: AsRef, @@ -2175,23 +2308,50 @@ impl DockerRunnerDriver { self.docker_program.is_absolute(), "Docker runner program is not absolute" ); - let output = Command::new(&self.docker_program) + let mut child = Command::new(&self.docker_program) .args(args) .stdin(Stdio::null()) - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .with_context(|| format!("starting Docker runner {}", self.docker_program.display()))?; - if !output.status.success() { - bail!( - "Docker runner exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); + let stdout = child.stdout.take().context("Docker runner has no stdout pipe")?; + let stderr = child.stderr.take().context("Docker runner has no stderr pipe")?; + let stdout = thread::spawn(move || drain_tail(stdout, RUNNER_OUTPUT_TAIL_BYTES)); + let stderr = thread::spawn(move || drain_tail(stderr, RUNNER_OUTPUT_TAIL_BYTES)); + let status = child.wait().context("waiting for the Docker runner")?; + let (stdout_tail, stdout_total_bytes) = stdout + .join() + .map_err(|_| anyhow!("the runner stdout reader panicked"))? + .context("reading the runner stdout")?; + let (stderr_tail, stderr_total_bytes) = stderr + .join() + .map_err(|_| anyhow!("the runner stderr reader panicked"))? + .context("reading the runner stderr")?; + if status.success() { + return Ok(()); } - Ok(output) + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal = None; + Err(RunnerStepFailure { + subject, + runner: runner_id.to_owned(), + exit_code: status.code(), + signal, + stdout_tail, + stdout_total_bytes, + stderr_tail, + stderr_total_bytes, + } + .into()) } fn run_in_workspace( &self, + step_id: &str, + runner_id: &str, runner: &DockerRunnerBinding, workspace: &Path, working_directory: &Path, @@ -2208,8 +2368,7 @@ impl DockerRunnerDriver { build_machine_id_file(workspace)?; let spec = ContainerSpec::for_step(runner, required_environment, source_stamp)?; let args = docker_run_args(&spec, workspace, working_directory, argv)?; - self.docker(args)?; - Ok(()) + self.docker(RunnerSubject::Step(step_id.to_owned()), runner_id, args) } fn materialize_external_input( @@ -2251,7 +2410,11 @@ impl DockerRunnerDriver { input.url.clone(), ]; let args = docker_run_args(&spec, workspace, Path::new("."), &curl_argv)?; - self.docker(args)?; + self.docker( + RunnerSubject::ExternalInput(input.id.clone()), + &input.runner, + args, + )?; let bytes = fs::read(&destination).with_context(|| { format!( "reading materialized external input {}", @@ -2301,27 +2464,7 @@ impl DockerRunnerDriver { } ArtifactSource::WorktreeTree => (source.root.clone(), source.root.join(&artifact.source)), }; - ensure!(source_path.exists(), "declared artifact output is absent"); - let destination = staging_root.join(&artifact.destination); - ensure!( - destination.starts_with(staging_root), - "artifact destination escaped its release staging root" - ); - copy_artifact(&containment_root, &source_path, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact output differs from its recipe-pinned digest" - ); - } - Ok(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - }) + seal_artifact(artifact, &containment_root, &source_path, staging_root) } } @@ -2393,6 +2536,8 @@ impl RunnerPort for DockerRunnerDriver { ); } self.run_in_workspace( + &step.id, + &step.runner, runner, workspace, &step.working_directory, @@ -2431,6 +2576,13 @@ impl RunnerPort for DockerRunnerDriver { } } +/// How every incarnation unit lives and dies. It runs once (`Restart=no`) +/// and systemd forgets it once it is inactive or failed (`--collect`): the +/// outcome is Idunn's to record in its own store, and a failed transient unit +/// left loaded is only clutter that marks the host degraded. +const INCARNATION_UNIT_LIFECYCLE: [&str; 3] = + ["--collect", "--property=Type=exec", "--property=Restart=no"]; + /// systemd owns process execution. This driver lowers one validated launch /// contract into a transient unit and then proves the native process and /// executable that systemd actually started. It does not decide admission, @@ -2478,7 +2630,6 @@ fn changed_systemd_workload_identity_fields( no_new_privileges, umask, inaccessible_paths, - load_credential, main_pid, process_start_time, process_uids, @@ -2878,7 +3029,6 @@ impl SystemdTransientWorkloadDriver { OsString::from("--property=NoNewPrivileges"), OsString::from("--property=UMask"), OsString::from("--property=InaccessiblePaths"), - OsString::from("--property=LoadCredential"), OsString::from("--property=OpenFile"), OsString::from("--property=WorkingDirectory"), OsString::from("--property=ControlGroup"), @@ -2990,38 +3140,7 @@ impl SystemdTransientWorkloadDriver { let umask = required_systemd_property(&values, "UMask")?.to_owned(); let inaccessible_paths = required_systemd_property(&values, "InaccessiblePaths")?.to_owned(); - let load_credential = required_systemd_property(&values, "LoadCredential")?.to_owned(); - ensure!( - service_type == "exec" - && restart_policy == "no" - && kill_mode == "mixed" - // `User` is deliberately not asserted here. Idunn never emits a - // User property, and systemd reports the identity it *allocated* - // for a DynamicUser unit -- the unit name, or a generated `_du…` - // when that is too long -- so the property is never empty for a - // running workload and cannot tell an allocated identity from a - // pinned one. The invariant it was reaching for, that the workload - // does not run as a persistent named account, is carried by - // `dynamic_user` below and by Idunn never passing User at all. - && dynamic_user - && supplementary_groups.is_empty() - && capability_bounding_set.is_empty() - && ambient_capabilities.is_empty() - && private_mounts - && private_pids - && protect_proc == "invisible" - && proc_subset == "all" - && no_new_privileges - && umask == "0007" - && inaccessible_paths == self.credential_root.display().to_string() - && load_credential - == if service_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }, - "systemd workload isolation properties differ from the Idunn contract" - ); + require_systemd_isolation_contract(&values, &self.credential_root)?; let working_directory = PathBuf::from(required_systemd_property(&values, "WorkingDirectory")?); ensure!( @@ -3163,6 +3282,11 @@ impl SystemdTransientWorkloadDriver { ); Some(path) }; + require_delivered_credentials( + &process_root.join("root"), + unit, + service_credential_names, + )?; let activation_descriptor = &parent_only_file_descriptors[0]; ensure!( activation_descriptor.fd_name == IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME @@ -3270,7 +3394,7 @@ impl SystemdTransientWorkloadDriver { no_new_privileges, umask, inaccessible_paths, - load_credential, + load_credential: String::new(), main_pid, process_start_time, process_uids: process_security.uids, @@ -3377,10 +3501,32 @@ impl SystemdTransientWorkloadDriver { .is_none(), "operator binding attempts to replace the Idunn candidate bind" ); + let member_port = binding + .route + .as_ref() + .map(|binding_route| binding_route.member_candidate_port(port)) + .transpose()? + .flatten(); + match member_port { + Some(member_port) => ensure!( + environment + .insert( + IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into(), + format!("{host}:{member_port}"), + ) + .is_none(), + "operator binding attempts to replace the Idunn member bind" + ), + None => ensure!( + !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), + "workload without a member listener carries an Idunn member bind" + ), + } } None => ensure!( - !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT), - "unrouted workload carries an Idunn candidate bind" + !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + && !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), + "unrouted workload carries an Idunn route bind" ), } match &binding.process_write_lease { @@ -3462,12 +3608,7 @@ impl SystemdTransientWorkloadDriver { // workload. The same applies to `Group` when no fixed state group is // bound. In that case the numeric primary GID must match the allocated // UID; a fixed state group still requires both its exact name and GID. - let expected_load_credential = if regular_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }; - let launch_contract: [(bool, &str); 27] = [ + let launch_contract: [(bool, &str); 26] = [ ( observation.unit_description == expected_description, "unit description", @@ -3507,10 +3648,6 @@ impl SystemdTransientWorkloadDriver { observation.inaccessible_paths == self.credential_root.display().to_string(), "inaccessible paths", ), - ( - observation.load_credential == expected_load_credential, - "load credential", - ), ( observation.working_directory == installed, "working directory", @@ -3636,8 +3773,9 @@ impl SystemdTransientWorkloadDriver { OsString::from("--no-ask-password"), OsString::from("--expand-environment=no"), OsString::from(format!("--unit={unit}")), - OsString::from("--property=Type=exec"), - OsString::from("--property=Restart=no"), + ]; + args.extend(INCARNATION_UNIT_LIFECYCLE.map(OsString::from)); + args.extend([ OsString::from("--property=KillMode=mixed"), OsString::from("--property=DynamicUser=yes"), OsString::from("--property=SupplementaryGroups="), @@ -3673,7 +3811,7 @@ impl SystemdTransientWorkloadDriver { OsString::from(format!("--working-directory={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", bundle.display())), - ]; + ]); for open_file in parent_only_open_files { args.push(OsString::from(format!("--property=OpenFile={open_file}"))); } @@ -4110,6 +4248,20 @@ impl CultCacheWriteLeaseDriver { fn current(&self) -> Result> { validate_root_authority_path(&self.record_path)?; + self.read_record() + } + + /// The lease as its holder reads it: the same custody, envelope, key and + /// target validation Idunn applies, without requiring the reader to be + /// root. The holder is a service uid by construction; root is who writes. + pub(crate) fn current_for_holder( + &self, + ) -> Result> { + validate_authority_path_custody(&self.record_path)?; + self.read_record() + } + + fn read_record(&self) -> Result> { if !self.record_path.exists() { return Ok(None); } @@ -4156,36 +4308,61 @@ impl CultCacheWriteLeaseDriver { expected.validate()?; activation.validate()?; lease.validate()?; - let expected_sha256 = expected.canonical_sha256()?; - let activation_sha256 = activation.canonical_sha256()?; ensure!( - self.target == expected.target - && lease.target == expected.target - && lease.expected_projection_sha256 == expected_sha256 - && lease.plan_id == expected.plan_id - && lease.incarnation_id == expected.incarnation_id - && lease.sealed_release_id == expected.sealed_release_id - && lease.activation_witness_sha256 == activation_sha256 - && lease.runtime_id == expected.runtime_id - && lease.runtime_instance_id == activation.runtime_instance_id - && lease.state_schema_generation - == expected - .state_schema_generation - .as_deref() - .context("write-lease Expected has no state generation")? - && lease.state_contract_sha256 - == expected - .state_contract_sha256 - .as_deref() - .context("write-lease Expected has no state contract")? - && lease.warming_presence_sha256 == warming.signed_presence_sha256() - && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + self.target == expected.target, "process write lease does not bind the exact warming candidate" ); - Ok(()) + ensure!( + warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), + "process write lease does not bind the exact warming candidate" + ); + lease_binds_candidate( + expected, + activation, + lease, + std::slice::from_ref(&warming.signed_presence_sha256().to_owned()), + ) } } +/// The one rule for what makes a process write lease this incarnation's: it +/// binds this Expected, this activation and one of the given provider-signed +/// Warming proofs. Idunn grants with it and the lease holder accepts with it. +pub(crate) fn lease_binds_candidate( + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + lease: &IdunnProcessWriteLeaseRecord, + warming_presence_sha256es: &[String], +) -> Result<()> { + let expected_sha256 = expected.canonical_sha256()?; + let activation_sha256 = activation.canonical_sha256()?; + ensure!( + lease.target == expected.target + && lease.expected_projection_sha256 == expected_sha256 + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation_sha256 + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id + && lease.state_schema_generation + == expected + .state_schema_generation + .as_deref() + .context("write-lease Expected has no state generation")? + && lease.state_contract_sha256 + == expected + .state_contract_sha256 + .as_deref() + .context("write-lease Expected has no state contract")? + && warming_presence_sha256es + .iter() + .any(|sha256| sha256 == &lease.warming_presence_sha256), + "process write lease does not bind the exact warming candidate" + ); + Ok(()) +} + impl WriteLeasePort for CultCacheWriteLeaseDriver { fn revoke_exact(&self, incumbent: Option<&IdunnProcessWriteLeaseRecord>) -> Result<()> { if let Some(incumbent) = incumbent { @@ -5202,50 +5379,67 @@ impl NginxRouteDriver { } } - /// The one firewall rule this route owns: inbound to the stable endpoint's - /// address and port, its transport's protocol, tagged with the route id. - fn endpoint_rule(&self) -> Result> { - let (host, port) = self.binding.stable_socket()?; + /// The firewall rules this route owns, one per stable socket (the + /// presence endpoint, then the member endpoint when the binding has one): + /// inbound to the socket's address and port, its transport's protocol. + fn endpoint_rules(&self) -> Result>> { let protocol = match self.binding.driver { RouteDriver::NginxStreamTcp => "tcp", RouteDriver::NginxStreamUdp => "udp", }; - Ok(vec![ - OsString::from("allow"), - OsString::from("in"), - OsString::from("to"), - OsString::from(host.to_string()), - OsString::from("port"), - OsString::from(port.to_string()), - OsString::from("proto"), - OsString::from(protocol), - ]) + let mut sockets = vec![self.binding.stable_socket()?]; + sockets.extend(self.binding.member_stable_socket()?); + Ok(sockets + .into_iter() + .map(|(host, port)| { + vec![ + OsString::from("allow"), + OsString::from("in"), + OsString::from("to"), + OsString::from(host.to_string()), + OsString::from("port"), + OsString::from(port.to_string()), + OsString::from("proto"), + OsString::from(protocol), + ] + }) + .collect()) } fn admit_endpoint(&self) -> Result<()> { - let mut args = self.endpoint_rule()?; - args.push(OsString::from("comment")); - args.push(OsString::from(format!( - "Idunn route {}", - self.binding.route_id - ))); - self.command(&self.ufw_program, args) - .context("admitting the stable endpoint on the host firewall")?; + for mut args in self.endpoint_rules()? { + args.push(OsString::from("comment")); + args.push(OsString::from(format!( + "Idunn route {}", + self.binding.route_id + ))); + self.command(&self.ufw_program, args) + .context("admitting a route endpoint on the host firewall")?; + } Ok(()) } fn withdraw_endpoint(&self) -> Result<()> { - let mut args = vec![OsString::from("delete")]; - args.extend(self.endpoint_rule()?); - match self.command(&self.ufw_program, args) { - Ok(_) => Ok(()), - // ufw reports a rule that is already gone as a failure; for a - // withdrawal that is the state being asked for. - Err(error) if format!("{error:#}").contains("non-existent") => Ok(()), - Err(error) => { - Err(error).context("withdrawing the stable endpoint from the host firewall") + let mut first_error = None; + for rule in self.endpoint_rules()? { + let mut args = vec![OsString::from("delete")]; + args.extend(rule); + match self.command(&self.ufw_program, args) { + Ok(_) => {} + // ufw reports a rule that is already gone as a failure; for a + // withdrawal that is the state being asked for. + Err(error) if format!("{error:#}").contains("non-existent") => {} + Err(error) => { + first_error.get_or_insert( + error.context("withdrawing a route endpoint from the host firewall"), + ); + } } } + match first_error { + Some(error) => Err(error), + None => Ok(()), + } } fn command(&self, program: &Path, args: I) -> Result @@ -5306,8 +5500,28 @@ impl NginxRouteDriver { RouteDriver::NginxStreamTcp => { let (stable_host, stable_port) = endpoint_host_port(&route.stable_endpoint, endpoint_prefix)?; + let member_blocks = match ( + self.binding.member_stable_socket()?, + self.binding.member_candidate_port(candidate_port)?, + ) { + (Some((member_host, member_stable_port)), Some(member_port)) => { + let member_upstream = format!("{upstream}_member"); + let max_connections = self + .binding + .member + .as_ref() + .map_or(0, |member| member.max_connections); + // One constant key: the limit is on the route's members + // together, so a flood of them can never take the + // connections the presence block needs. + format!( + "limit_conn_zone \"{member_upstream}\" zone={member_upstream}:64k;\nupstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n limit_conn {member_upstream} {max_connections};\n proxy_pass {member_upstream};\n}}\n" + ) + } + _ => String::new(), + }; format!( - "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n{member_blocks}" ) } RouteDriver::NginxStreamUdp => { @@ -5861,7 +6075,7 @@ impl Answer { } } -fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { +pub(crate) fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) .with_context(|| format!("connecting stable CultNet TCP route {target}")) .map_err(silent)?; @@ -6070,7 +6284,7 @@ fn connect_route_socket(target: SocketAddr) -> std::io::Result { } } -fn exact_route_snapshot_response( +pub(crate) fn exact_route_snapshot_response( response: CultNetMessage, message_id: &str, target: &str, @@ -6711,21 +6925,7 @@ fn hash_frozen_source_tree(root: &Path, current: &Path, hasher: &mut Sha256) -> ); hasher.update(b"\0"); hasher.update(metadata.len().to_le_bytes()); - // S9: streamed in fixed-size chunks rather than `fs::read`ing the - // whole file, so peak RSS is bounded by the buffer, not by the - // largest file in the tree. - let mut file = fs::File::open(&path) - .with_context(|| format!("opening {} to hash it", path.display()))?; - let mut buffer = [0u8; 1 << 16]; - loop { - let read = file - .read(&mut buffer) - .with_context(|| format!("reading {} to hash it", path.display()))?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - } + hash_file_exact(&path, metadata.len(), hasher)?; } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -6747,20 +6947,28 @@ fn frozen_source_sha256(_root: &Path) -> Result { #[cfg(unix)] fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Result<()> { - let executable_paths = artifacts - .iter() - .filter(|artifact| artifact.executable) - .map(|artifact| root.join(&artifact.destination)) - .collect::>(); - harden_root_tree(root, root, &executable_paths) + let mut executable_paths = BTreeSet::new(); + for artifact in artifacts { + let destination = root.join(&artifact.destination); + for member in &artifact.executable_members { + executable_paths.insert(destination.join(member)); + } + if artifact.executable { + executable_paths.insert(destination); + } + } + harden_root_tree(root, &mut executable_paths)?; + ensure!( + executable_paths.is_empty(), + "declared executable is missing or not a regular file" + ); + Ok(()) } +/// Each regular file the walk seals 0555 is removed from `executable_paths`; +/// whatever remains was declared executable and never sealed. #[cfg(unix)] -fn harden_root_tree( - root: &Path, - current: &Path, - executable_paths: &std::collections::BTreeSet, -) -> Result<()> { +fn harden_root_tree(current: &Path, executable_paths: &mut BTreeSet) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let metadata = fs::symlink_metadata(current)?; @@ -6772,48 +6980,28 @@ fn harden_root_tree( let mut entries = fs::read_dir(current)?.collect::>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { - harden_root_tree(root, &entry.path(), executable_paths)?; + harden_root_tree(&entry.path(), &mut *executable_paths)?; } fs::set_permissions(current, fs::Permissions::from_mode(0o555))?; } else if metadata.is_file() { fs::set_permissions( current, - fs::Permissions::from_mode(if executable_paths.contains(current) { + fs::Permissions::from_mode(if executable_paths.remove(current) { 0o555 } else { 0o444 }), )?; } else if metadata.file_type().is_symlink() { - validate_release_symlink(root, current)?; + // Left untouched: containment was decided by check_release_link in + // digest_tree, which install_release and validate_installed_release + // run over every receipt before this walk (map T16). } else { bail!("sealed release contains a special filesystem entry") } Ok(()) } -#[cfg(unix)] -fn validate_release_symlink(root: &Path, path: &Path) -> Result<()> { - let target = fs::read_link(path)?; - ensure!(target.is_relative(), "sealed release symlink is absolute"); - ensure!( - target.components().all(|component| matches!( - component, - std::path::Component::CurDir | std::path::Component::Normal(_) - )), - "sealed release symlink has a parent traversal" - ); - let destination = path - .parent() - .context("sealed release symlink has no parent")? - .join(target); - ensure!( - destination.starts_with(root), - "sealed release symlink escaped its release" - ); - Ok(()) -} - #[cfg(not(unix))] fn harden_installed_release(_root: &Path, _artifacts: &[ArtifactReceipt]) -> Result<()> { bail!("systemd workload installation requires Unix permissions") @@ -7216,6 +7404,44 @@ fn path_inside_process_root(process_root: &Path, absolute_path: &Path) -> Result Ok(process_root.join(absolute_path.strip_prefix(Path::new("/"))?)) } +/// The workload holds exactly the credentials Idunn delivered: the entries of +/// `/run/credentials/` in the MainPID's root (`process_root` is that root) +/// equal `expected`, and an absent directory is the empty set. Only entry names +/// are read; a credential is never opened here. The message names no credential. +fn require_delivered_credentials( + process_root: &Path, + unit: &str, + expected: &[String], +) -> Result<()> { + let directory = path_inside_process_root( + process_root, + &Path::new("/run/credentials").join(unit), + )?; + let mut delivered = Vec::new(); + match fs::read_dir(&directory) { + Ok(entries) => { + for entry in entries { + let name = entry + .context("reading a delivered credential entry")? + .file_name() + .into_string() + .map_err(|_| anyhow!("delivered credential name is not UTF-8"))?; + delivered.push(name); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).context("reading the delivered credential directory"); + } + } + delivered.sort(); + ensure!( + delivered == expected, + "systemd delivered credentials that differ from the Idunn launch" + ); + Ok(()) +} + fn linux_process_security( status_path: &Path, expected_host_pid: u32, @@ -7431,6 +7657,56 @@ fn parse_systemd_boolean(values: &BTreeMap, name: &str) -> Resul } } +/// The isolation properties of a transient workload unit, judged from `systemctl +/// show`. `LoadCredential` is absent on purpose: systemd 257 displays it as +/// `[unprintable]` whatever the unit holds, so what the workload was given is +/// judged by `require_delivered_credentials` from the delivered directory. +fn require_systemd_isolation_contract( + values: &BTreeMap, + credential_root: &Path, +) -> Result<()> { + let service_type = required_systemd_property(values, "Type")?; + let restart_policy = required_systemd_property(values, "Restart")?; + let kill_mode = required_systemd_property(values, "KillMode")?; + let dynamic_user = parse_systemd_boolean(values, "DynamicUser")?; + let supplementary_groups = systemd_property(values, "SupplementaryGroups")?; + let capability_bounding_set = systemd_property(values, "CapabilityBoundingSet")?; + let ambient_capabilities = systemd_property(values, "AmbientCapabilities")?; + let private_mounts = parse_systemd_boolean(values, "PrivateMounts")?; + let private_pids = parse_systemd_boolean(values, "PrivatePIDs")?; + let protect_proc = required_systemd_property(values, "ProtectProc")?; + let proc_subset = required_systemd_property(values, "ProcSubset")?; + let no_new_privileges = parse_systemd_boolean(values, "NoNewPrivileges")?; + let umask = required_systemd_property(values, "UMask")?; + let inaccessible_paths = required_systemd_property(values, "InaccessiblePaths")?; + ensure!( + service_type == "exec" + && restart_policy == "no" + && kill_mode == "mixed" + // `User` is deliberately not asserted here. Idunn never emits a + // User property, and systemd reports the identity it *allocated* + // for a DynamicUser unit -- the unit name, or a generated `_du…` + // when that is too long -- so the property is never empty for a + // running workload and cannot tell an allocated identity from a + // pinned one. The invariant it was reaching for, that the workload + // does not run as a persistent named account, is carried by + // `dynamic_user` below and by Idunn never passing User at all. + && dynamic_user + && supplementary_groups.is_empty() + && capability_bounding_set.is_empty() + && ambient_capabilities.is_empty() + && private_mounts + && private_pids + && protect_proc == "invisible" + && proc_subset == "all" + && no_new_privileges + && umask == "0007" + && inaccessible_paths == credential_root.display().to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + Ok(()) +} + fn read_process_environment(path: &Path) -> Result>> { let bytes = fs::read(path) .with_context(|| format!("reading process environment {}", path.display()))?; @@ -8003,7 +8279,7 @@ fn copy_symlink(source: &Path, destination: &Path) -> Result<()> { /// checked to contain the fully resolved `source` with the same /// whole-chain-aware resolver `validate_frozen_source_symlink` uses, before /// anything is read from it. -pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { +fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { let canonical_root = root.canonicalize().context("resolving artifact root")?; let canonical_source = source .canonicalize() @@ -8026,11 +8302,124 @@ pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> R } } +/// The one seal-time owner of mode declarations for a copied artifact: a +/// directory output is never `executable` (it declares `executable_members`), +/// a file output never declares members, and every member is a regular file +/// reached through real directories, never through a symlink. Errors name the +/// rule, never the member path. +fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { + let is_dir = fs::symlink_metadata(destination) + .context("inspecting a copied artifact")? + .is_dir(); + ensure!( + !(is_dir && artifact.executable), + "directory artifact {} cannot be executable; declare executable_members", + artifact.id + ); + if !is_dir { + ensure!( + artifact.executable_members.is_empty(), + "file artifact {} cannot declare executable_members", + artifact.id + ); + return Ok(()); + } + for member in &artifact.executable_members { + ensure!(!member.as_os_str().is_empty(), "executable member is empty"); + let mut current = destination.to_path_buf(); + let mut components = member.components().peekable(); + while let Some(component) = components.next() { + let std::path::Component::Normal(name) = component else { + bail!("executable member has a forbidden path component"); + }; + current.push(name); + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if error.kind() == ErrorKind::NotFound => { + bail!("executable member is missing") + } + Err(error) => return Err(error).context("inspecting an executable member"), + }; + if components.peek().is_some() { + ensure!( + metadata.is_dir(), + "executable member passes through a symlink or non-directory" + ); + } else { + ensure!(metadata.is_file(), "executable member is not a regular file"); + } + } + } + Ok(()) +} + +/// The one seal step for a declared artifact, for systemd and host targets +/// alike: copy it into the staging root, check its mode declarations, digest it +/// and check the recipe pin. Modes are applied by harden on systemd targets; +/// host targets (Windows, map T15) carry `executable` and `executable_members` +/// in the receipt and do not apply them. +pub(crate) fn seal_artifact( + artifact: &ArtifactOutput, + containment_root: &Path, + source_path: &Path, + staging_root: &Path, +) -> Result { + ensure!( + source_path.exists(), + "declared artifact output {} is absent", + artifact.id + ); + let destination = staging_root.join(&artifact.destination); + ensure!( + destination.starts_with(staging_root), + "artifact {} destination escaped its staging root", + artifact.id + ); + copy_artifact(containment_root, source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; + let (sha256, size_bytes) = digest_artifact(&destination)?; + if let Some(expected) = &artifact.expected_sha256 { + ensure!( + &sha256 == expected, + "artifact {} differs from its recipe-pinned digest", + artifact.id + ); + } + Ok(ArtifactReceipt { + artifact_id: artifact.id.clone(), + destination: artifact.destination.clone(), + sha256: format!("sha256-{sha256}"), + size_bytes, + executable: artifact.executable, + executable_members: artifact.executable_members.clone(), + }) +} + +/// Streams `path` into `hasher` in 64 KiB chunks, so peak memory is the buffer +/// and not the largest file in the tree, and refuses if the bytes read differ +/// from `expected_len`. The only owner of the length check. +pub(crate) fn hash_file_exact(path: &Path, expected_len: u64, hasher: &mut Sha256) -> Result<()> { + let mut file = fs::File::open(path).context("opening a file to hash it")?; + let mut buffer = [0u8; 1 << 16]; + let mut total = 0_u64; + loop { + let read = file.read(&mut buffer).context("reading a file to hash it")?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + total += read as u64; + } + ensure!(total == expected_len, "file changed while hashing"); + Ok(()) +} + pub(crate) fn digest_artifact(path: &Path) -> Result<(String, u64)> { let metadata = fs::symlink_metadata(path)?; if metadata.is_file() { - let bytes = fs::read(path)?; - return Ok((raw_sha256(&bytes), bytes.len().try_into()?)); + let mut hasher = Sha256::new(); + hash_file_exact(path, metadata.len(), &mut hasher)?; + return Ok((format!("{:x}", hasher.finalize()), metadata.len())); } ensure!(metadata.is_dir(), "artifact is not a file or directory"); let mut hasher = Sha256::new(); @@ -8054,15 +8443,16 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) hasher.update(b"\0"); digest_tree(root, &path, hasher, size)?; } else if metadata.is_file() { - let bytes = fs::read(&path)?; + let length = metadata.len(); hasher.update(b"file\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); - hasher.update((bytes.len() as u64).to_le_bytes()); - hasher.update(&bytes); - *size = size.saturating_add(bytes.len().try_into()?); + hasher.update(length.to_le_bytes()); + hash_file_exact(&path, length, hasher)?; + *size = size.saturating_add(length); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; + check_release_link(path.strip_prefix(root)?, &target)?; hasher.update(b"link\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); @@ -8075,6 +8465,35 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) Ok(()) } +/// The one judge of link containment in a release tree, pure and static: +/// `link` is the link's path relative to its artifact's top and `target` what +/// it reads. A target is relative, its `..` steps all come first, and they +/// climb no higher than the link's own directory depth, so the link can only +/// land inside the artifact. Lexical normalisation of parent and target is not +/// enough: with `s -> ..` it admits `s/../x` and escapes (map T14). +fn check_release_link(link: &Path, target: &Path) -> Result<()> { + use std::path::Component; + + let depth = link.components().count().saturating_sub(1); + let mut climbs = 0_usize; + let mut named = false; + for component in target.components() { + match component { + Component::CurDir => {} + Component::RootDir | Component::Prefix(_) => { + bail!("sealed release symlink is absolute") + } + Component::ParentDir => { + ensure!(!named, "sealed release symlink climbs after a name"); + climbs += 1; + } + Component::Normal(_) => named = true, + } + } + ensure!(climbs <= depth, "sealed release symlink escapes its artifact"); + Ok(()) +} + pub(crate) fn raw_sha256(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } @@ -8327,7 +8746,7 @@ fn ensure_source_directory(path: &Path, identity: Option) -> Re } } -fn authority_lock_path(path: &Path) -> PathBuf { +pub(crate) fn authority_lock_path(path: &Path) -> PathBuf { let mut lock = path.as_os_str().to_os_string(); lock.push(".lock"); PathBuf::from(lock) @@ -8335,12 +8754,17 @@ fn authority_lock_path(path: &Path) -> PathBuf { #[cfg(unix)] fn validate_root_authority_path(path: &Path) -> Result<()> { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - ensure!( unsafe { libc::geteuid() } == 0, "process write-lease actuation requires root Idunn" ); + validate_authority_path_custody(path) +} + +#[cfg(unix)] +fn validate_authority_path_custody(path: &Path) -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let parent = path .parent() .context("process write-lease path has no parent")?; @@ -8383,6 +8807,11 @@ fn validate_root_authority_path(_path: &Path) -> Result<()> { bail!("process write-lease actuation requires a Unix authority path") } +#[cfg(not(unix))] +fn validate_authority_path_custody(_path: &Path) -> Result<()> { + bail!("process write-lease actuation requires a Unix authority path") +} + #[cfg(unix)] fn harden_root_authority_file(path: &Path) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -8649,6 +9078,7 @@ mod tests { private_port_end: port, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }) .request_candidate_runtime_presence(&candidate, "candidate-probe") } @@ -9472,6 +9902,7 @@ Content-Le".to_vec()), |_| {}), private_port_end: 17998, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -9521,6 +9952,16 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// An incarnation unit runs once and is collected when it ends, so a + /// failed candidate does not stay loaded and mark the host degraded. + /// Collection is safe because a forgotten unit already reads as + /// permanently stopped (the test below). + #[test] + fn incarnation_units_run_once_and_are_collected() { + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--collect")); + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--property=Restart=no")); + } + /// Admission audit, claim 3: `is_permanently_stopped` now lets the /// Isolation gate skip `prove_isolation` against the incumbent. Enumerate /// what it answers `true` for, using a fake `systemctl show` that replays @@ -9760,6 +10201,168 @@ processes 9 ); } + #[cfg(unix)] + fn fixture_show_properties(text: &str) -> Result> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let shown = temp.path().join("shown"); + std::fs::write(&shown, text)?; + let systemctl = temp.path().join("systemctl"); + std::fs::write( + &systemctl, + format!("#!/bin/sh\ncat '{}'\nexit 0\n", shown.display()), + )?; + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; + let driver = SystemdTransientWorkloadDriver { + systemctl_program: systemctl, + ..Default::default() + }; + Ok(driver + .show_unit("idunn-test.service")? + .context("fixture unit was not found")? + .properties) + } + + /// systemd 257 prints `LoadCredential=[unprintable]` for every unit, so the + /// isolation contract must admit the real output of a unit with no service + /// credential as well as one with one. The inputs are real captures, not a + /// re-spelling. + #[cfg(unix)] + #[test] + fn systemd_257_isolation_contract_admits_a_presence_only_target_and_one_with_service_credentials( + ) -> Result<()> { + let root = Path::new("/run/idunn/activation-credentials"); + let vili = include_str!("../tests/fixtures/systemd-257/vili-presence-identity-only.show"); + let odin = include_str!( + "../tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show" + ); + for text in [vili, odin] { + assert!(text.contains("LoadCredential=[unprintable]")); + require_systemd_isolation_contract(&fixture_show_properties(text)?, root)?; + } + let default_proc = vili.replace("ProtectProc=invisible", "ProtectProc=default"); + assert_ne!(default_proc, vili); + let other_path = vili.replace( + "/run/idunn/activation-credentials", + "/run/idunn/elsewhere", + ); + assert_ne!(other_path, vili); + for text in [default_proc, other_path] { + let error = + require_systemd_isolation_contract(&fixture_show_properties(&text)?, root) + .expect_err("a drifted isolation property must be refused"); + assert_eq!( + error.to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn delivered_credentials_are_exactly_the_launch_names() -> Result<()> { + let unit = "idunn-test.service"; + let names = |list: &[&str]| list.iter().map(|name| name.to_string()).collect::>(); + // `Some(entries)` creates the unit's directory in a fresh process root. + let check = |delivered: Option<&[&str]>, expected: &[&str]| -> Result<()> { + let temp = tempfile::tempdir()?; + if let Some(entries) = delivered { + let directory = temp.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + for entry in entries { + std::fs::write(directory.join(entry), "canary-credential-contents")?; + } + } + let result = require_delivered_credentials(temp.path(), unit, &names(expected)); + if let Err(error) = &result { + let text = format!("{error:#}"); + for canary in ["STOWAWAY", "ODIN_TOPOLOGY_IDENTITY", "canary-credential"] { + assert!(!text.contains(canary), "error echoed {canary}"); + } + assert!(!text.contains(&*temp.path().to_string_lossy())); + } + result + }; + check(None, &[])?; + check(Some(&[]), &[])?; + check(Some(&["ODIN_TOPOLOGY_IDENTITY"]), &["ODIN_TOPOLOGY_IDENTITY"])?; + check(Some(&["B", "A"]), &["A", "B"])?; + for (delivered, expected) in [ + (None, &["ODIN_TOPOLOGY_IDENTITY"][..]), + (Some(&[][..]), &["ODIN_TOPOLOGY_IDENTITY"][..]), + ( + Some(&["ODIN_TOPOLOGY_IDENTITY", "STOWAWAY"][..]), + &["ODIN_TOPOLOGY_IDENTITY"][..], + ), + (Some(&["STOWAWAY"][..]), &[][..]), + (Some(&["A"][..]), &["B"][..]), + (Some(&["A"][..]), &["A", "B"][..]), + ] { + let error = check(delivered, expected) + .expect_err("delivered credentials differing from the launch must be refused"); + assert_eq!( + error.to_string(), + "systemd delivered credentials that differ from the Idunn launch" + ); + } + // A directory that cannot be listed is a refusal, never the empty set. + let temp = tempfile::tempdir()?; + let parent = temp.path().join("run/credentials"); + std::fs::create_dir_all(&parent)?; + std::fs::write(parent.join(unit), "")?; + let error = require_delivered_credentials(temp.path(), unit, &[]) + .expect_err("an unreadable credential directory must be refused"); + assert!(!format!("{error:#}").contains(&*temp.path().to_string_lossy())); + // The rule reads the process root it is given, never the host's /run. + let temp = tempfile::tempdir()?; + let outside = tempfile::tempdir()?; + let directory = outside.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + std::fs::write(directory.join("A"), "")?; + require_delivered_credentials(temp.path(), unit, &[])?; + assert!(require_delivered_credentials(temp.path(), unit, &names(&["A"])).is_err()); + Ok(()) + } + + /// Live Odin and StreamPixels hold records persisted by the Idunn that + /// compared `LoadCredential`: `[unprintable]` in the retired slot. The bytes + /// are that Idunn's own output (tests/fixtures/pre-cut-*); they must decode + /// under both encodings and read as no drift against a new observation, or + /// `stop` refuses to stop the incumbents. + #[test] + fn an_incumbent_recorded_with_the_display_form_is_not_identity_drift() -> Result<()> { + let decode_hex = |text: &str| -> Vec { + (0..text.trim().len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text.trim()[at..at + 2], 16).unwrap()) + .collect() + }; + let mut recorded = systemd_audit_observation("no", 61_000); + recorded.load_credential = "[unprintable]".into(); + let observed = systemd_audit_observation("no", 61_000); + assert!(observed.load_credential.is_empty()); + for hex in [ + include_str!("../tests/fixtures/pre-cut-systemd-observation.compact.hex"), + include_str!("../tests/fixtures/pre-cut-systemd-observation.named.hex"), + ] { + let decoded: WorkloadObservation = rmp_serde::from_slice(&decode_hex(hex))?; + assert_eq!(decoded, WorkloadObservation::Systemd(recorded.clone())); + let WorkloadObservation::Systemd(prior) = decoded else { + unreachable!() + }; + assert!(changed_systemd_workload_identity_fields(&observed, &prior).is_empty()); + } + let mut drifted = observed.clone(); + drifted.main_pid += 1; + assert_eq!( + changed_systemd_workload_identity_fields(&drifted, &recorded), + vec!["main_pid"] + ); + Ok(()) + } + #[test] fn host_isolation_is_two_live_distinct_processes_on_one_host() { let live = |pid: u32, created: u64| { @@ -12204,6 +12807,7 @@ processes 9 private_port_end: 17979, config_path: temp.path().join("odin.conf"), reload_unit: "nginx.service".into(), + member: None, }, nginx_program: program("nginx", "#!/bin/sh\nexit 0\n")?, systemd_run_program: program("systemd-run", "#!/bin/sh\nexit 64\n")?, @@ -12213,12 +12817,15 @@ processes 9 }; assert_eq!( driver - .endpoint_rule()? + .endpoint_rules()? .iter() - .map(|value| value.to_string_lossy().into_owned()) - .collect::>() - .join(" "), - "allow in to 10.77.0.1 port 17971 proto udp" + .map(|rule| rule + .iter() + .map(|value| value.to_string_lossy().into_owned()) + .collect::>() + .join(" ")) + .collect::>(), + vec!["allow in to 10.77.0.1 port 17971 proto udp"] ); driver.restore(None, &Unmetered)?; @@ -12278,6 +12885,7 @@ processes 9 private_port_end: 4109, config_path: temp.join("service.conf"), reload_unit: "nginx.service".into(), + member: None, }, &RouteActuators { nginx: program("nginx")?, @@ -12312,6 +12920,271 @@ processes 9 }) } + fn with_member_listener(driver: &mut NginxRouteDriver) { + driver.binding.member = Some(crate::deployment::MemberListenerBinding { + stable_endpoint: "http://127.0.0.1:4120".into(), + private_port_start: 4121, + max_connections: 200, + }); + } + + /// The fragment as the base commit rendered it, computed from its format + /// string and not from the code under test. + #[cfg(unix)] + fn base_fragment(sha: &str, candidate_port: u16) -> String { + format!( + "# Idunn Expected {sha}\nupstream idunn_service_route {{\n server 127.0.0.1:{candidate_port};\n}}\nserver {{\n listen 127.0.0.1:4103;\n proxy_pass idunn_service_route;\n}}\n" + ) + } + + /// A route without a member listener renders the bytes it always did; a + /// member listener adds a second upstream and server block to the one + /// fragment, under the one Expected comment. + #[cfg(unix)] + #[test] + fn a_member_listener_renders_a_second_block_only_when_bound() -> Result<()> { + let temp = tempfile::tempdir()?; + let (mut driver, mut candidate, _calls) = stubbed_route_driver(temp.path())?; + let sha = candidate.canonical_sha256()?; + assert_eq!( + String::from_utf8(driver.render(&candidate)?)?, + base_fragment(&sha, 4104) + ); + + with_member_listener(&mut driver); + let member_blocks = |member_port: u16| { + format!( + "limit_conn_zone \"idunn_service_route_member\" zone=idunn_service_route_member:64k;\nupstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n limit_conn idunn_service_route_member 200;\n proxy_pass idunn_service_route_member;\n}}\n" + ) + }; + let rendered = String::from_utf8(driver.render(&candidate)?)?; + assert_eq!(rendered, base_fragment(&sha, 4104) + &member_blocks(4121)); + assert_eq!(rendered.matches("# Idunn Expected").count(), 1); + assert_eq!(rendered.matches("server {").count(), 2); + + candidate.route.as_mut().unwrap().candidate_endpoint = "http://127.0.0.1:4106".into(); + let sha = candidate.canonical_sha256()?; + assert_eq!( + String::from_utf8(driver.render(&candidate)?)?, + base_fragment(&sha, 4106) + &member_blocks(4123) + ); + Ok(()) + } + + /// Both stable sockets are admitted on the firewall, and both withdrawn. + #[cfg(unix)] + #[test] + fn both_route_endpoints_are_admitted_and_withdrawn() -> Result<()> { + let temp = tempfile::tempdir()?; + let (mut driver, _candidate, calls) = stubbed_route_driver(temp.path())?; + with_member_listener(&mut driver); + driver.admit_endpoint()?; + driver.withdraw_endpoint()?; + let log = fs::read_to_string(&calls)?; + let ufw: Vec<&str> = log.lines().filter(|line| line.starts_with("ufw ")).collect(); + assert_eq!( + ufw, + vec![ + "ufw allow in to 127.0.0.1 port 4103 proto tcp comment Idunn route service-route", + "ufw allow in to 127.0.0.1 port 4120 proto tcp comment Idunn route service-route", + "ufw delete allow in to 127.0.0.1 port 4103 proto tcp", + "ufw delete allow in to 127.0.0.1 port 4120 proto tcp", + ] + ); + Ok(()) + } + + /// Real nginx over the rendered two-block fragment, a real presence + /// acceptor behind the presence block and a flood of idle members on the + /// member block. Needs an `nginx` on PATH and the fixed loopback ports + /// 4103, 4104, 4120 and 4121, so it runs on demand in a container: + /// `cargo test --lib real_nginx -- --ignored`. + #[cfg(unix)] + fn challenges_answered_during_a_member_flood(max_connections: u16) -> Result { + use std::net::{TcpListener, TcpStream}; + use std::process::{Child, Command}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + /// Stopped through nginx itself: killing the master would orphan its + /// workers, which keep the test's output pipe open. + struct Nginx(Child, PathBuf, PathBuf); + impl Drop for Nginx { + fn drop(&mut self) { + let _ = Command::new("nginx") + .arg("-p") + .arg(&self.1) + .arg("-c") + .arg(&self.2) + .args(["-s", "stop"]) + .status(); + let _ = self.0.wait(); + } + } + + let temp = tempfile::tempdir()?; + let (mut driver, candidate, _calls) = stubbed_route_driver(temp.path())?; + with_member_listener(&mut driver); + driver.binding.member.as_mut().unwrap().max_connections = max_connections; + let fragment = temp.path().join("route.conf"); + fs::write(&fragment, driver.render(&candidate)?)?; + // Debian builds the stream module dynamic. + let module = Path::new("/usr/lib/nginx/modules/ngx_stream_module.so"); + let load = if module.exists() { + format!("load_module {}; +", module.display()) + } else { + String::new() + }; + let conf = temp.path().join("nginx.conf"); + fs::write( + &conf, + format!( + "{load}worker_processes 1;\npid {dir}/nginx.pid;\nerror_log {dir}/error.log;\nevents {{ worker_connections 512; }}\nstream {{ include {fragment}; }}\n", + dir = temp.path().display(), + fragment = fragment.display() + ), + )?; + let halt = Arc::new(AtomicBool::new(false)); + let presence = crate::launcher::PresenceAcceptor::new( + TcpListener::bind("127.0.0.1:4104")?, + Arc::new(|request| match request { + CultNetMessage::SnapshotRequest { message_id, .. } => { + Ok(CultNetMessage::SnapshotResponseRaw { message_id, documents: vec![] }) + } + _ => bail!("not a snapshot request"), + }), + ); + let presence_halt = halt.clone(); + let presence = thread::spawn(move || presence.serve(presence_halt)); + // The member backend accepts and holds every connection. + let members = TcpListener::bind("127.0.0.1:4121")?; + members.set_nonblocking(true)?; + let member_halt = halt.clone(); + let members = thread::spawn(move || { + let mut held = Vec::new(); + while !member_halt.load(Ordering::SeqCst) { + match members.accept() { + Ok((stream, _)) => held.push(stream), + Err(_) => thread::sleep(Duration::from_millis(5)), + } + } + }); + let _nginx = Nginx( + Command::new("nginx") + .args(["-p"]) + .arg(temp.path()) + .arg("-c") + .arg(&conf) + .args(["-g", "daemon off;"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?, + temp.path().to_path_buf(), + conf.clone(), + ); + let stable: SocketAddr = "127.0.0.1:4103".parse()?; + let ready = Instant::now(); + while TcpStream::connect(stable).is_err() { + ensure!(ready.elapsed() < Duration::from_secs(10), "nginx never listened"); + thread::sleep(Duration::from_millis(50)); + } + // The probe connection above reached the presence acceptor as an idle + // peer; let its deadline lapse so only the flood is measured. + thread::sleep(Duration::from_millis(1200)); + + let mut flood = Vec::new(); + for _ in 0..300 { + if let Ok(mut stream) = TcpStream::connect("127.0.0.1:4120") { + let _ = stream.write_all(b"x"); + flood.push(stream); + } + } + thread::sleep(Duration::from_millis(500)); + + let payload = encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: "flood".into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + )?; + let mut answered = 0; + for _ in 0..20 { + let started = Instant::now(); + if request_tcp_snapshot(stable, &payload).is_ok() + && started.elapsed() < Duration::from_secs(3) + { + answered += 1; + } + } + eprintln!("answered {answered} of 20 challenges with a member limit of {max_connections}"); + drop(flood); + halt.store(true, Ordering::SeqCst); + presence.join().unwrap(); + members.join().unwrap(); + Ok(answered) + } + + #[cfg(unix)] + #[test] + #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] + fn real_nginx_keeps_presence_answerable_under_a_member_flood() -> Result<()> { + // The rendered default limit keeps the route's members out of the + // presence block's share of a 512-connection pool. + assert_eq!(challenges_answered_during_a_member_flood(128)?, 20); + Ok(()) + } + + #[cfg(unix)] + #[test] + #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] + fn real_nginx_flood_starves_presence_without_a_member_limit() -> Result<()> { + // The control: the same flood against a limit too high to bind (nginx allows up to 65535) starves + // presence, so the test above measures the limit and not the load. + assert!(challenges_answered_during_a_member_flood(60_000)? < 20); + Ok(()) + } + + /// Idunn sets the member bind for a bound member listener, derived from the + /// candidate, and for no other workload. + #[test] + fn the_member_bind_is_set_only_for_a_bound_member_listener() -> Result<()> { + use crate::deployment::tests::{BINDING, RECIPE, member_listener_pair}; + + let driver = SystemdTransientWorkloadDriver::default(); + let mut routed = expected(); + routed.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "test-service-public".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:18830".into(), + candidate_endpoint: "http://127.0.0.1:18833".into(), + }); + let bundle = Path::new("/etc/gamecult/test-service/runtime/bundle"); + let unit = "idunn-test-service-abc"; + + let (recipe, binding) = member_listener_pair(); + let binding = OperatorBinding::parse(&binding)?; + binding.admit(&TargetDeclaration::parse(&recipe)?)?; + let environment = driver.launch_environment(&binding, bundle, &routed, unit)?; + assert_eq!( + environment.get(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT).map(String::as_str), + Some("127.0.0.1:18843") + ); + assert_eq!( + environment.get(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT).map(String::as_str), + Some("127.0.0.1:18833") + ); + + let plain = OperatorBinding::parse(BINDING)?; + plain.admit(&TargetDeclaration::parse(RECIPE)?)?; + let environment = driver.launch_environment(&plain, bundle, &routed, unit)?; + assert!(!environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT)); + assert!(environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT)); + Ok(()) + } + /// A refused forward change stops before any host mutation: no program /// runs, no fragment is written, and no private-mount unit starts. #[cfg(unix)] @@ -12631,6 +13504,7 @@ processes 9 private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), + member: None, }, nginx_program: program("nginx")?, systemd_run_program: program("systemd-run")?, @@ -12691,6 +13565,7 @@ processes 9 private_port_end: 27879, config_path: config_path.clone(), reload_unit: "nginx.service".into(), + member: None, }); assert_eq!(driver.current_configuration()?, None); // What systemd leaves behind when it materializes the preflight's bind @@ -12717,6 +13592,7 @@ processes 9 private_port_end: 27879, config_path: "/etc/nginx/idunn-stream-routes/odin-rudp.conf".into(), reload_unit: "nginx.service".into(), + member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -12778,6 +13654,7 @@ processes 9 private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), + member: None, }; let driver = NginxRouteDriver { binding, @@ -13424,6 +14301,40 @@ processes 9 Ok(()) } + #[cfg(unix)] + #[test] + fn write_lease_grant_binds_the_exact_warming_candidate_and_the_holder_read_keeps_custody() + -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let path = temp.path().join("lease.cc"); + let driver = CultCacheWriteLeaseDriver::new("service", &path); + let (expected, activation, warming, _) = authenticated_warming(temp.path())?; + let good = lease(&expected, &activation, &warming); + let mut foreign = good.clone(); + foreign.warming_presence_sha256 = digest('8'); + assert!( + driver + .grant(&expected, &activation, &warming, &foreign) + .is_err() + ); + let other = CultCacheWriteLeaseDriver::new("other", temp.path().join("other.cc")); + assert!(other.grant(&expected, &activation, &warming, &good).is_err()); + assert!(driver.current_for_holder()?.is_none()); + driver.grant(&expected, &activation, &warming, &good)?; + assert_eq!( + driver.current_for_holder()?.map(|(_, held)| held), + Some(good) + ); + fs::set_permissions(&path, fs::Permissions::from_mode(0o666))?; + assert!( + driver.current_for_holder().is_err(), + "a service-writable lease store is not authority" + ); + Ok(()) + } + #[cfg(unix)] #[test] fn installed_release_is_root_owned_and_nonwritable() -> Result<()> { @@ -13443,6 +14354,7 @@ processes 9 sha256: sha256_id(b"sealed"), size_bytes: 6, executable: true, + executable_members: BTreeSet::new(), }], )?; let root_metadata = fs::symlink_metadata(&root)?; @@ -13454,6 +14366,554 @@ processes 9 Ok(()) } + #[cfg(unix)] + const MEMBER_CANARY: &str = "zzcanary"; + + #[cfg(unix)] + fn tree_output(executable: bool, members: &[&str]) -> ArtifactOutput { + ArtifactOutput { + id: "upstream".into(), + source_kind: ArtifactSource::WorktreeTree, + runner: None, + source: PathBuf::from("upstream"), + destination: PathBuf::from("upstream"), + expected_sha256: None, + executable, + executable_members: members.iter().map(PathBuf::from).collect(), + } + } + + #[cfg(unix)] + fn write_file(path: &Path, bytes: &[u8]) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::create_dir_all(path.parent().context("file has no parent")?)?; + fs::write(path, bytes)?; + // Builds and git leave arbitrary exec bits; the seal must not inherit them. + fs::set_permissions(path, fs::Permissions::from_mode(0o777))?; + Ok(()) + } + + #[cfg(unix)] + fn sealed_mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + + fs::symlink_metadata(path).unwrap().permissions().mode() & 0o777 + } + + /// `/src/upstream` is the worktree tree; `/staging` the release. + #[cfg(unix)] + fn sample_tree(root: &Path) -> Result<(PathBuf, PathBuf)> { + let source = root.join("src"); + let staging = root.join("staging"); + fs::create_dir_all(&staging)?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/daemon"), b"daemon")?; + write_file(&tree.join("bin/helper"), b"helper")?; + write_file(&tree.join("lib/tool"), b"tool")?; + write_file(&tree.join("lib/data.txt"), b"data")?; + write_file(&tree.join("NOTICE"), b"notice")?; + Ok((source, staging)) + } + + #[cfg(unix)] + fn collect_tree( + source: &Path, + staging: &Path, + output: ArtifactOutput, + ) -> Result { + let mut declaration = + TargetDeclaration::parse(crate::deployment::tests::RECIPE)?; + let id = output.id.clone(); + declaration.artifacts = vec![output]; + let frozen = FrozenSource { + receipt: FrozenSourceReceipt { + transaction_id: "tx".into(), + plan_id: "plan".into(), + snapshot_sha256: "snapshot".into(), + }, + facts: crate::deployment_plan::tests::source(crate::deployment::tests::RECIPE), + recipe_bytes: Vec::new(), + root: source.to_path_buf(), + }; + DockerRunnerDriver::default().collect_artifact( + &declaration, + &frozen, + &BTreeMap::new(), + staging, + &id, + ) + } + + #[cfg(unix)] + fn refusal_text(result: Result) -> String { + let Err(error) = result else { + panic!("the release was not refused"); + }; + let text = format!("{error:#}"); + assert!( + !text.contains(MEMBER_CANARY), + "the refusal echoed a member path: {text}" + ); + text + } + + #[cfg(unix)] + #[test] + fn tree_member_sealed_0555_and_siblings_0444() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let receipt = collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib/tool"]), + )?; + assert_eq!( + receipt.executable_members, + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]) + ); + harden_installed_release(&staging, &[receipt])?; + let tree = staging.join("upstream"); + assert_eq!(sealed_mode(&tree.join("bin/daemon")), 0o555); + assert_eq!(sealed_mode(&tree.join("lib/tool")), 0o555); + assert_eq!(sealed_mode(&tree.join("bin/helper")), 0o444); + assert_eq!(sealed_mode(&tree.join("lib/data.txt")), 0o444); + assert_eq!(sealed_mode(&tree.join("NOTICE")), 0o444); + for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { + assert_eq!(sealed_mode(directory), 0o555); + } + Ok(()) + } + + #[test] + fn release_link_rule_is_containment() { + for (link, target) in [ + ("a/b/l", "../t/x"), + ("a/l", ".."), + ("a/l", "./../b"), + ("a/l", "b/./c"), + ("l", "python3.12"), + ("a/b/c/l", "../../x"), + ] { + check_release_link(Path::new(link), Path::new(target)) + .unwrap_or_else(|error| panic!("({link}, {target}) refused: {error:#}")); + } + for (link, target, rule) in [ + ("l", "/x", "absolute"), + ("a/l", "s/../x", "climbs after a name"), + ("a/l", "D/D/../../../etc/passwd", "climbs after a name"), + ("a/l", "../../x", "escapes its artifact"), + ("l", "..", "escapes its artifact"), + ("a/b/l", "../../../x", "escapes its artifact"), + ("a/l", "./s/../x", "climbs after a name"), + ] { + let text = format!( + "{:#}", + check_release_link(Path::new(link), Path::new(target)) + .expect_err(&format!("({link}, {target}) was admitted")) + ); + assert!(text.contains(rule), "({link}, {target}): {text}"); + } + } + + #[cfg(unix)] + #[test] + fn missing_member_refuses_the_release() -> Result<()> { + // The bad member sorts after the good one in one case and before it in + // the other, so a check that stops at the first (or last) member fails. + for members in [ + &["bin/daemon", "lib/zzcanary-missing"][..], + &["aaa-zzcanary-missing", "bin/daemon"][..], + &["zzcanary-dir/tool"][..], + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, members))); + assert!(text.contains("executable member is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn symlink_member_refuses_the_release() -> Result<()> { + for link_target in ["daemon", "../zzcanary-nowhere"] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink( + link_target, + source.join("upstream/bin/zzcanary-link"), + )?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "bin/zzcanary-link"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn member_through_symlinked_directory_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("real-bin/zzcanary-tool"), b"tool")?; + std::os::unix::fs::symlink("real-bin", tree.join("linked-bin"))?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["linked-bin/zzcanary-tool"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + // A member below a regular file is refused the same way. + let second_staging = temp.path().join("second-staging"); + fs::create_dir_all(&second_staging)?; + let text = refusal_text(collect_tree( + &source, + &second_staging, + tree_output(false, &["bin/daemon/zzcanary-below-file"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn directory_member_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn executable_flag_on_tree_output_refuses() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(true, &[]))); + assert!(text.contains("cannot be executable"), "{text}"); + // The same flag on a file output stays admitted. + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + let mut output = tree_output(true, &[]); + output.id = "file".into(); + check_artifact_modes(&output, &file)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn members_on_file_output_refuse() -> Result<()> { + let temp = tempfile::tempdir()?; + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + check_artifact_modes(&tree_output(false, &[]), &file)?; + let text = refusal_text(check_artifact_modes( + &tree_output(false, &["zzcanary"]), + &file, + )); + assert!(text.contains("cannot declare executable_members"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn check_artifact_modes_refuses_a_member_that_is_not_a_clean_relative_path() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&temp.path().join("zzcanary-outside"), b"x")?; + for member in ["../../zzcanary-outside", "bin/../bin/daemon", "/zzcanary", ""] { + refusal_text(check_artifact_modes(&tree_output(false, &[member]), &tree)); + } + check_artifact_modes(&tree_output(false, &["bin/daemon"]), &tree)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_member_is_not_reported_as_missing() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + // A component past NAME_MAX fails with an io error that is not NotFound. + let long = "z".repeat(300); + let text = refusal_text(check_artifact_modes( + &tree_output(false, &[long.as_str()]), + &source.join("upstream"), + )); + assert!(!text.contains("missing"), "{text}"); + assert!(text.contains("inspecting an executable member"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn harden_fails_when_a_declared_member_is_absent() -> Result<()> { + let receipt = |members: &[&str]| ArtifactReceipt { + artifact_id: "upstream".into(), + destination: PathBuf::from("upstream"), + sha256: sha256_id(b"x"), + size_bytes: 1, + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + }; + let installed = |root: &Path| -> Result<()> { + write_file(&root.join("upstream/bin/daemon"), b"daemon")?; + write_file(&root.join("upstream/lib/tool"), b"tool")?; + fs::create_dir_all(root.join("upstream/zzcanary-dir"))?; + std::os::unix::fs::symlink("daemon", root.join("upstream/bin/zzcanary-link"))?; + Ok(()) + }; + let temp = tempfile::tempdir()?; + installed(temp.path())?; + harden_installed_release(temp.path(), &[receipt(&["bin/daemon", "lib/tool"])])?; + for members in [ + &["bin/daemon", "lib/zzcanary-gone"][..], + &["zzcanary-gone", "bin/daemon"][..], + &["zzcanary-dir"][..], + &["bin/zzcanary-link"][..], + &["bin/daemon/zzcanary-below-file"][..], + ] { + let temp = tempfile::tempdir()?; + installed(temp.path())?; + let text = refusal_text(harden_installed_release(temp.path(), &[receipt(members)])); + assert!(text.contains("declared executable is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn relative_in_tree_symlink_survives_the_seal() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/python3.12"), b"python")?; + std::os::unix::fs::symlink("python3.12", tree.join("bin/python3"))?; + write_file(&tree.join("lib/terminfo/b/real"), b"terminfo")?; + fs::create_dir_all(tree.join("lib/terminfo/a"))?; + std::os::unix::fs::symlink("../b/real", tree.join("lib/terminfo/a/alias"))?; + let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/python3.12"]))?; + let (before, _) = digest_artifact(&staging.join("upstream"))?; + harden_installed_release(&staging, &[receipt])?; + let link = staging.join("upstream/bin/python3"); + assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); + assert_eq!(sealed_mode(&staging.join("upstream/bin/python3.12")), 0o555); + let alias = staging.join("upstream/lib/terminfo/a/alias"); + assert!(fs::symlink_metadata(&alias)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&alias)?, PathBuf::from("../b/real")); + assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); + + // A link that escapes, is absolute or climbs after a name is refused at + // the seal, before any receipt exists. + for (target, rule) in [ + ("../../zzcanary-up", "escapes its artifact"), + ("/zzcanary/absolute", "absolute"), + ("zzcanary-s/../x", "climbs after a name"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink(target, source.join("upstream/lib/zzcanary-link"))?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); + assert!(text.contains(rule), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn links_to_existing_targets_are_refused_at_the_seal() -> Result<()> { + // Every refused shape, aimed at a target that EXISTS (file and + // directory): a judge guarded by `exists()`, `is_dir()` or a file-type + // test passes the dangling fixtures and fails these. + for kind in ["file", "dir"] { + for (shape, rule) in [ + ("absolute", "absolute"), + ("escape", "escapes its artifact"), + ("after-name", "climbs after a name"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let target = match (shape, kind) { + ("absolute", "file") => source.join("upstream/bin/daemon"), + ("absolute", _) => source.join("upstream/lib"), + ("escape", "file") => { + write_file(&source.join("outside-file"), b"outside")?; + PathBuf::from("../../outside-file") + } + ("escape", _) => { + fs::create_dir_all(source.join("outside-dir"))?; + PathBuf::from("../../outside-dir") + } + (_, "file") => PathBuf::from("../bin/../NOTICE"), + _ => PathBuf::from("../bin/../lib"), + }; + let link = source.join("upstream/lib/zzcanary-link"); + std::os::unix::fs::symlink(&target, &link)?; + assert!(fs::metadata(&link).is_ok(), "{shape} {kind} link must resolve"); + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); + assert!(text.contains(rule), "{shape} {kind}: {text}"); + } + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn streamed_digest_is_byte_compatible() -> Result<()> { + // Not a multiple of the 64 KiB read buffer, so a short last chunk + // matters; the aligned and empty files are the other boundaries. + let big = (0..3 * 65536 + 17).map(|i| (i % 251) as u8).collect::>(); + let aligned = vec![7_u8; 65536]; + let temp = tempfile::tempdir()?; + let tree = temp.path().join("tree"); + write_file(&tree.join("a.txt"), b"alpha")?; + write_file(&tree.join("empty"), b"")?; + write_file(&tree.join("sub/aligned.bin"), &aligned)?; + write_file(&tree.join("sub/big.bin"), &big)?; + std::os::unix::fs::symlink("a.txt", tree.join("zlink"))?; + + // The encoding digest_tree had before it streamed, written out by hand. + fn add_file(hasher: &mut Sha256, name: &str, bytes: &[u8]) { + hasher.update(b"file\0"); + hasher.update(name.as_bytes()); + hasher.update(b"\0"); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + let mut expected = Sha256::new(); + add_file(&mut expected, "a.txt", b"alpha"); + add_file(&mut expected, "empty", b""); + let mut expected_size = 5_u64; + expected.update(b"dir\0sub\0"); + add_file(&mut expected, "sub/aligned.bin", &aligned); + add_file(&mut expected, "sub/big.bin", &big); + expected_size += (aligned.len() + big.len()) as u64; + expected.update(b"link\0zlink\0a.txt"); + expected_size += "a.txt".len() as u64; + assert_eq!( + digest_artifact(&tree)?, + (format!("{:x}", expected.finalize()), expected_size) + ); + + let single = tree.join("sub/big.bin"); + assert_eq!( + digest_artifact(&single)?, + (raw_sha256(&big), big.len() as u64) + ); + let mut hasher = Sha256::new(); + hash_file_exact(&single, big.len() as u64, &mut hasher)?; + assert_eq!(format!("{:x}", hasher.finalize()), raw_sha256(&big)); + for wrong in [big.len() as u64 + 1, big.len() as u64 - 1] { + let text = refusal_text(hash_file_exact(&single, wrong, &mut Sha256::new())); + assert!(text.contains("changed while hashing"), "{text}"); + } + Ok(()) + } + + /// Acceptance on the real trees: `IDUNN_BIG_TREE` holds `python/` and + /// `blender/`. Run on Yggdrasil as its own job (see the cut spec). + #[cfg(unix)] + #[test] + #[ignore = "needs IDUNN_BIG_TREE: a CPython+torch tree and a Blender tree"] + fn big_tree_seals_from_env() -> Result<()> { + use std::time::Instant; + + /// (links, links with a `..` step, regular files, files at 0555), never + /// following a link; every regular file must be 0444 or a member's 0555. + fn census( + root: &Path, + path: &Path, + members: &BTreeSet, + modes: bool, + totals: &mut [usize; 4], + ) -> Result<()> { + for entry in fs::read_dir(path)? { + let entry = entry?; + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.is_dir() { + census(root, &entry.path(), members, modes, totals)?; + } else if metadata.file_type().is_symlink() { + totals[0] += 1; + let target = fs::read_link(entry.path())?; + if target + .components() + .any(|part| matches!(part, std::path::Component::ParentDir)) + { + totals[1] += 1; + } + } else { + totals[2] += 1; + let member = members.contains(entry.path().strip_prefix(root)?); + let mode = sealed_mode(&entry.path()); + if modes { + assert_eq!(mode, if member { 0o555 } else { 0o444 }); + totals[3] += usize::from(mode == 0o555); + } + } + } + Ok(()) + } + fn peak_rss() -> String { + fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| { + status + .lines() + .find(|line| line.starts_with("VmHWM")) + .map(str::to_owned) + }) + .unwrap_or_default() + } + + let tree = PathBuf::from( + std::env::var_os("IDUNN_BIG_TREE").context("IDUNN_BIG_TREE is not set")?, + ); + let members = ["python/bin/python3.12", "blender/blender"]; + let temp = tempfile::tempdir()?; + let staging = temp.path().join("staging"); + fs::create_dir_all(&staging)?; + let destination = staging.join("tree"); + let output = ArtifactOutput { + destination: PathBuf::from("tree"), + ..tree_output(false, &members) + }; + + let started = Instant::now(); + let receipt = seal_artifact(&output, &tree, &tree, &staging)?; + println!("seal {:?} {}", started.elapsed(), peak_rss()); + let started = Instant::now(); + let again = digest_artifact(&destination)?; + assert_eq!(format!("sha256-{}", again.0), receipt.sha256); + assert_eq!(again.1, receipt.size_bytes); + println!("digest {:?} {}", started.elapsed(), peak_rss()); + let mut before = [0; 4]; + census(&destination, &destination, &output.executable_members, false, &mut before)?; + let started = Instant::now(); + harden_installed_release(&staging, &[receipt.clone()])?; + println!("harden {:?} {}", started.elapsed(), peak_rss()); + let mut totals = [0; 4]; + census(&destination, &destination, &output.executable_members, true, &mut totals)?; + assert_eq!(totals[0], before[0], "links changed across harden"); + assert_eq!(totals[3], members.len()); + println!( + "dotdot links {} links before {} after {} files {} exec {}", + totals[1], before[0], totals[0], totals[2], totals[3] + ); + println!("bytes {}", receipt.size_bytes); + Ok(()) + } + #[cfg(unix)] #[test] fn runner_cache_is_created_once_for_one_exact_identity() -> Result<()> { @@ -14950,4 +16410,292 @@ processes 9 ); Ok(()) } + + // BEGIN runner-failure-evidence tests + // The runner decides what failed and what it kept: these drive the real + // DockerRunnerDriver over a fake docker program, never the capture alone. + + const RUNNER_CANARY: &str = "CANARY-5d2e91b7"; + + #[cfg(unix)] + fn fake_docker(dir: &Path, body: &str) -> Result { + use std::os::unix::fs::PermissionsExt; + let program = dir.join("docker"); + fs::write(&program, format!("#!/bin/sh\n{body}\n"))?; + fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?; + Ok(DockerRunnerDriver { + docker_program: program, + }) + } + + /// One recipe step through the real `run_in_workspace` path. + #[cfg(unix)] + fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> { + let workspace = dir.join("workspace"); + fs::create_dir_all(&workspace)?; + driver.run_in_workspace( + step_id, + "rust", + &fixture_docker_runner_binding(), + &workspace, + Path::new("."), + &["cargo".to_owned(), "test".to_owned()], + &std::collections::BTreeSet::new(), + ("IDUNN_STAMP", "rev"), + ) + } + + #[cfg(unix)] + fn failure_of(result: Result<()>) -> Result { + let error = result.expect_err("the step did not fail"); + error + .downcast_ref::() + .cloned() + .with_context(|| format!("the failure is not typed: {error:#}")) + } + + #[cfg(unix)] + #[test] + fn a_failed_step_is_named_with_both_tails() -> Result<()> { + let temp = tempfile::tempdir()?; + // The first step passes, the second fails: the count is the fake's. + let driver = fake_docker( + temp.path(), + &format!( + "n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\ + echo 'stderr report' >&2\nexit 3" + ), + )?; + run_step(&driver, temp.path(), "first")?; + let failure = failure_of(run_step(&driver, temp.path(), "second"))?; + assert_eq!(failure.subject, RunnerSubject::Step("second".into())); + assert_eq!(failure.runner, "rust"); + assert_eq!(failure.exit_code, Some(3)); + assert_eq!(failure.signal, None); + assert!( + failure + .stdout_tail + .ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()), + "stdout was dropped: {:?}", + String::from_utf8_lossy(&failure.stdout_tail) + ); + assert_eq!(failure.stderr_tail, b"stderr report\n"); + assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "kill -9 $$")?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert_eq!((failure.exit_code, failure.signal), (None, Some(9))); + assert!(failure.to_string().contains("exited with signal 9"), "{failure}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_external_input_is_named_as_one() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?; + let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?; + let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?; + let input_id = declaration.external_inputs[0].id.clone(); + let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]); + fs::create_dir_all(&workspaces["rust"])?; + let error = driver + .materialize_external_input(&declaration, &binding, &workspaces, &input_id) + .expect_err("the download did not fail"); + let failure = error + .downcast_ref::() + .with_context(|| format!("the failure is not typed: {error:#}"))?; + assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone())); + assert_eq!(failure.exit_code, Some(22)); + assert!( + failure.to_string().contains(&format!("runner external input {input_id} on runner rust")), + "{failure}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn runner_output_never_reaches_the_error_text() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY)); + let error = anyhow::Error::new(failure.clone()).context("sealing"); + for text in [ + format!("{error:#}"), + format!("{error:?}"), + format!("{failure}"), + format!("{failure:?}"), + ] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}"); + } + Ok(()) + } + + /// A reader that answers from a script, so a read error the OS rarely + /// produces (an interrupted call) is produced on demand. + struct ScriptedReader(Vec>>); + + impl Read for ScriptedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.0.is_empty() { + return Ok(0); + } + let bytes = self.0.remove(0)?; + buffer[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + + struct BrokenReader; + + impl Read for BrokenReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("persistent read failure")) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_loses_no_byte() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + Ok(b"cd".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + ]); + let drained = drain_tail(reader, 3).expect("an interrupted read ended the drain"); + assert_eq!(drained, (b"bcd".to_vec(), 4)); + } + + #[test] + fn a_persistent_read_error_ends_the_drain_instead_of_spinning() { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(drain_tail(BrokenReader, 8).is_err()); + }); + assert_eq!( + receiver.recv_timeout(Duration::from_secs(5)), + Ok(true), + "a persistent read error spun or was swallowed" + ); + } + + #[test] + fn a_non_interrupted_read_error_after_data_is_returned_not_retried() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::BrokenPipe)), + Ok(b"cd".to_vec()), + ]); + let error = drain_tail(reader, 8).expect_err("a broken pipe was retried past"); + assert_eq!(error.kind(), ErrorKind::BrokenPipe); + } + + #[cfg(unix)] + #[test] + fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { + let temp = tempfile::tempdir()?; + let data = temp.path().join("data"); + for size in [ + 0, + 1, + RUNNER_OUTPUT_TAIL_BYTES - 1, + RUNNER_OUTPUT_TAIL_BYTES, + RUNNER_OUTPUT_TAIL_BYTES + 1, + 1024 * 1024, + ] { + // Distinct neighbours, so a shifted or head-keeping tail differs. + let bytes: Vec = (0..size).map(|index| (index % 251) as u8).collect(); + fs::write(&data, &bytes)?; + let driver = fake_docker( + temp.path(), + &format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..]; + assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes"); + assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes"); + assert_eq!(failure.stdout_total_bytes, size as u64); + assert_eq!(failure.stderr_total_bytes, size as u64); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> { + let temp = tempfile::tempdir()?; + run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?; + let missing = DockerRunnerDriver { + docker_program: temp.path().join("no-such-docker"), + }; + let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran"); + assert!(error.downcast_ref::().is_none()); + assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); + Ok(()) + } + + /// The 2026-10-25 vili failure: `node --test` reported on stdout alone, + /// stderr stayed empty (the runner's /tmp is a noexec tmpfs) and the step + /// exited 1. The old capture dropped stdout and printed an empty message. + #[cfg(unix)] + #[test] + fn a_step_that_reports_only_on_stdout_keeps_that_report_and_its_name() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'not ok 1 - daemon {RUNNER_CANARY}'\necho '# fail 1'\nexit 1"), + )?; + let error = run_step(&driver, temp.path(), "test-daemon").expect_err("the step passed"); + // The error names the step and carries no output. + let text = format!("{error:#}"); + assert!(text.contains("runner step test-daemon on runner rust exited with exit status 1"), "{text}"); + assert!(!text.contains(RUNNER_CANARY) && !text.contains("not ok"), "{text}"); + // The evidence keeps the stdout tail; the empty stderr is a recorded 0. + let failure = error + .downcast_ref::() + .context("the failure is not typed")?; + let store = temp.path().join("runner-evidence.cc"); + let record = crate::runner_evidence::RunnerFailureRecord::from_failure( + "tx-vili", + "up-vili", + "vili", + failure, + 1, + ); + crate::runner_evidence::record(&store, &record)?; + let kept = crate::runner_evidence::for_transaction(&store, "tx-vili")?.context("no evidence kept")?; + assert_eq!(kept.subject_id, "test-daemon"); + assert_eq!(kept.exit_code, Some(1)); + assert_eq!( + String::from_utf8_lossy(&kept.stdout_tail), + format!("not ok 1 - daemon {RUNNER_CANARY}\n# fail 1\n") + ); + assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); + assert!(kept.stderr_tail.is_empty()); + assert_eq!(kept.stderr_total_bytes, 0); + let mut shown = Vec::new(); + crate::runner_evidence::show_status(&mut shown, &store, "tx-vili")?; + let rendered = String::from_utf8(shown)?; + assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); + assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); + // Debug names the failure and prints no tail. + let debug = format!("{failure:?}"); + assert!(debug.starts_with("RunnerStepFailure") && !debug.contains("not ok"), "{debug}"); + Ok(()) + } + // END runner-failure-evidence tests } diff --git a/src/host.rs b/src/host.rs index 5221f12..4c6cbec 100644 --- a/src/host.rs +++ b/src/host.rs @@ -30,13 +30,13 @@ use serde::Deserialize; use crate::control_plane::read_trust_anchor; use crate::deployment::{ - ArtifactSource, HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, LaunchArgument, OperatorBinding, TargetDeclaration, }; -use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease}; +use crate::deployment_plan::{CompiledDeploymentPlan, SealedRelease}; use crate::drivers::{ - HostWorkloadObservation, InstalledReleaseObservation, copy_artifact, copy_tree, - digest_artifact, release_artifact, remove_tree_inside, sha256_id, write_runtime_bundle_records, + HostWorkloadObservation, InstalledReleaseObservation, copy_tree, digest_artifact, + release_artifact, remove_tree_inside, seal_artifact, sha256_id, write_runtime_bundle_records, }; use crate::host_actuator::{ HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS, @@ -1048,37 +1048,12 @@ fn materialize( fs::create_dir_all(&staging)?; let mut artifacts = Vec::new(); for artifact in &declaration.artifacts { - let source_path = match artifact.source_kind { - ArtifactSource::RunnerOutput | ArtifactSource::WorktreeTree => { - source.join(&artifact.source) - } - }; - ensure!( - source_path.exists(), - "declared artifact output {} is absent", - artifact.id - ); - let destination = staging.join(&artifact.destination); - ensure!( - destination.starts_with(&staging), - "artifact destination escaped its staging root" - ); - copy_artifact(&source, &source_path, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact {} differs from its recipe-pinned digest", - artifact.id - ); - } - artifacts.push(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - }); + artifacts.push(seal_artifact( + artifact, + &source, + &source.join(&artifact.source), + &staging, + )?); } let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?; Ok((release, staging)) diff --git a/src/host_actuator.rs b/src/host_actuator.rs index f7bf4c7..8fd776c 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1151,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - while let Some(frame) = client.receive_once()? { + if let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/launcher.rs b/src/launcher.rs new file mode 100644 index 0000000..435f8b1 --- /dev/null +++ b/src/launcher.rs @@ -0,0 +1,3238 @@ +//! idunn-launch: the unit's process for an upstream daemon that cannot speak +//! Idunn's runtime contract itself (Synapse, MAS). +//! +//! Idunn observes the unit; the launcher, inside it and supervising the child, +//! signs the presence statement with the provider identity and the activation +//! key. The launcher binds the candidate endpoint Idunn challenges, because the +//! child cannot answer a CultNet frame, and answers presence there and nowhere +//! else: members reach the child on the member bind Idunn routes them to. It +//! starts a stateful child only after it holds the process write lease, renders +//! the child's config from a closed set of placeholders, and ends with the +//! child. +//! +//! Errors, logs and refusals name a field and a code. They never echo an +//! argument, a template line, a path or a credential. + +use std::collections::VecDeque; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::fd::{AsRawFd, RawFd}; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::process::CommandExt; +use std::path::{Component, Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; +use cultnet_rs::{ + CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, + CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, CultNetWireContract, + GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, + GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, + GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, + IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, + IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationRecord, + IdunnRuntimeActivationSigner, ServiceIdentitySigner, decode_cultnet_message_from_slice, + encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, +}; + +use crate::deployment::{RUNTIME_PRESENCE_IDENTITY_FD_NAME, SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT}; +use crate::drivers::{CultCacheWriteLeaseDriver, authority_lock_path, lease_binds_candidate}; + +/// The route challenge is the only thing that begins with this byte: a +/// CultNet frame's 4-byte big-endian length is bounded at 1 MiB, and an HTTP +/// request starts with a method letter. +const CHALLENGE_FIRST_BYTE: u8 = 0x00; +const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; +/// One whole presence exchange on the candidate endpoint, from accept to +/// answer, however the peer paces it. It is shorter than the 3 s Idunn waits +/// for a challenge answer, so one stalled peer cannot cost the next challenge +/// its reply. +const PRESENCE_DEADLINE: Duration = Duration::from_secs(1); +/// How long the presence acceptor waits after an accept error before trying +/// again. No accept error ends the acceptor. +const ACCEPT_BACKOFF: Duration = Duration::from_millis(50); +const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); +/// How often the lifecycle probes the child once it is Active. +const HEALTH_REFRESH: Duration = Duration::from_millis(500); +/// A probe result older than this is not evidence of health: the lifecycle +/// that refreshes it is itself stuck. +const HEALTH_STALE: Duration = Duration::from_secs(5); +const POLL: Duration = Duration::from_millis(100); +/// The largest duration a recipe may declare for the lease wait, the startup +/// wait or the heartbeat: one day. Past it a deadline is not a deadline. +const MAX_DECLARED_SECONDS: u64 = 86_400; +const MAX_RECENT_WARMING_PROOFS: usize = 64; +const TERMINATE_GRACE: Duration = Duration::from_secs(20); +const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; +const ODIN_CAPABILITY: &str = "odin.verse-rendezvous"; + +// ---------------------------------------------------------------- contract + +/// Everything the recipe declares about the child, parsed from the launcher's +/// argv (`--flag value` pairs; `--child-arg` repeats). The release root is the +/// sealed release directory that holds the launcher itself. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LaunchContract { + pub release_root: PathBuf, + pub child: PathBuf, + pub child_args: Vec, + /// The flag that carries the rendered config path to the child, when the + /// child takes one. + pub config_flag: Option, + pub template: PathBuf, + pub health_path: String, + pub health_contract: String, + pub state_root: Option, + pub lease_wait: Duration, + pub startup: Duration, + pub heartbeat: Duration, +} + +impl LaunchContract { + pub fn parse(release_root: PathBuf, args: impl IntoIterator) -> Result { + ensure!(release_root.is_absolute(), "the release root is not absolute"); + let mut child = None; + let mut child_args = Vec::new(); + let mut config_flag = None; + let mut template = None; + let mut health_path = None; + let mut health_contract = None; + let mut state_root = None; + let (mut lease_wait, mut startup, mut heartbeat) = (300_u64, 120_u64, 5_u64); + let mut args = args.into_iter(); + while let Some(flag) = args.next() { + let value = args.next().context("launcher argument has no value")?; + let seconds = |field: &str| -> Result { + value + .parse::() + .ok() + .filter(|seconds| (1..=MAX_DECLARED_SECONDS).contains(seconds)) + .with_context(|| { + format!("launcher argument {field} is outside 1 to {MAX_DECLARED_SECONDS} seconds") + }) + }; + match flag.as_str() { + "--child" => child = Some(release_relative(&value, "--child")?), + "--child-arg" => child_args.push(value), + "--config-flag" => config_flag = Some(value), + "--template" => template = Some(release_relative(&value, "--template")?), + "--health-path" => { + ensure!( + value.starts_with('/'), + "launcher argument --health-path is not absolute" + ); + health_path = Some(value) + } + "--health-contract" => health_contract = Some(value), + "--state-root" => state_root = Some(PathBuf::from(value)), + "--lease-wait-secs" => lease_wait = seconds("--lease-wait-secs")?, + "--startup-secs" => startup = seconds("--startup-secs")?, + "--heartbeat-secs" => heartbeat = seconds("--heartbeat-secs")?, + _ => bail!("unknown launcher argument"), + } + } + Ok(Self { + release_root, + child: child.context("launcher argument --child is required")?, + child_args, + config_flag, + template: template.context("launcher argument --template is required")?, + health_path: health_path.context("launcher argument --health-path is required")?, + health_contract: health_contract + .context("launcher argument --health-contract is required")?, + state_root, + lease_wait: Duration::from_secs(lease_wait), + startup: Duration::from_secs(startup), + heartbeat: Duration::from_secs(heartbeat), + }) + } +} + +fn release_relative(value: &str, field: &str) -> Result { + let path = PathBuf::from(value); + ensure!( + !value.is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))), + "launcher argument {field} is not release-relative" + ); + Ok(path) +} + +// ---------------------------------------------------------------- authority + +/// The bundle, activation credential and provider identity Idunn hands the +/// unit, and the two signatures every presence statement needs. +pub struct RuntimeAuthority { + expected: IdunnExpectedIncarnationRecord, + expected_sha256: String, + activation: IdunnRuntimeActivationRecord, + activation_sha256: String, + activation_signer: IdunnRuntimeActivationSigner, + provider_signer: ServiceIdentitySigner, + sequence: u64, +} + +impl RuntimeAuthority { + pub fn open( + bundle: &Path, + activation_credential: impl Read, + provider_credential: impl Read, + ) -> Result { + ensure!(bundle.is_absolute(), "Idunn runtime bundle is not absolute"); + let (key, payload) = read_single_record( + &bundle.join("expected.cc"), + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + )?; + let expected = IdunnExpectedIncarnationRecord::decode_canonical(&payload)?; + expected.validate()?; + ensure!(key == expected.target, "Expected key is substituted"); + let (key, payload) = read_single_record( + &bundle.join("activation.cc"), + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + )?; + let activation = IdunnRuntimeActivationRecord::decode_canonical(&payload)?; + activation.validate()?; + ensure!(key == expected.target, "activation key is substituted"); + let expected_sha256 = expected.canonical_sha256()?; + ensure!( + activation.expected_projection_sha256 == expected_sha256 + && activation.runtime_id == expected.runtime_id, + "activation does not bind this Expected projection" + ); + let activation_signer = IdunnRuntimeActivationSigner::from_credential_reader( + activation_credential, + )?; + ensure!( + activation_signer.identity_id() == activation.activation_signer_identity_id + && activation_signer.public_key() == activation.activation_signer_public_key, + "activation credential does not belong to this activation" + ); + let provider_signer = + open_service_identity_credential_reader::( + provider_credential, + )?; + ensure!( + provider_signer.entry().identity_id == expected.expected_signer_identity_id, + "provider identity is not the signer selected by Expected" + ); + Ok(Self { + activation_sha256: activation.canonical_sha256()?, + expected, + expected_sha256, + activation, + activation_signer, + provider_signer, + sequence: 0, + }) + } + + pub fn expected(&self) -> &IdunnExpectedIncarnationRecord { + &self.expected + } + + pub fn activation(&self) -> &IdunnRuntimeActivationRecord { + &self.activation + } + + /// The candidate endpoint Idunn will challenge, which the launcher binds. + pub fn require_candidate_bind(&self, bind: &str) -> Result<()> { + let route = self.expected.route.as_ref().context("Expected has no route")?; + ensure!( + route.transport == "tcp" && route.candidate_endpoint == format!("tcp://{bind}"), + "candidate bind differs from the Expected route" + ); + Ok(()) + } + + /// The next presence record, signed by the provider identity and the + /// activation. The sequence is the clock: strictly increasing here and + /// above anything an earlier incarnation of this signer published. + pub fn sign( + &mut self, + state: &str, + detail: &str, + write_lease_sha256: Option, + now_millis: u64, + ) -> Result { + self.sequence = now_millis.max( + self.sequence + .checked_add(1) + .ok_or_else(|| anyhow!("runtime presence sequence exhausted"))?, + ); + let expected = &self.expected; + let mut record = GameCultRuntimePresenceHealthRecord { + schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.expected_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation_sha256.clone(), + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: capability.minimum_capacity, + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: state.into(), + detail: detail.into(), + write_lease_sha256, + signer_identity_id: self.provider_signer.entry().identity_id.clone(), + publisher_sequence: self.sequence, + observed_at_unix_millis: now_millis, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation.activation_signer_identity_id.clone(), + activation_signature: Vec::new(), + }; + let payload = record.canonical_proof_payload()?; + record.signature = self + .provider_signer + .sign::(&payload) + .signature; + record.activation_signature = self.activation_signer.sign_presence_proof(&record)?; + record.validate()?; + Ok(record) + } +} + +fn read_single_record(path: &Path, record_type: &str, schema: &str) -> Result<(String, Vec)> { + let entries = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading a runtime bundle record")?; + let [envelope] = entries.as_slice() else { + bail!("runtime bundle record does not hold exactly one entry"); + }; + ensure!( + envelope.r#type == record_type && envelope.schema_id.as_deref() == Some(schema), + "runtime bundle record has the wrong type or schema" + ); + Ok((envelope.key.clone(), envelope.payload.clone())) +} + +/// The two authority descriptors systemd opens for the unit, in the order and +/// under the names Idunn's launch contract fixes. +pub fn inherited_authority_files() -> Result<(File, File)> { + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let fds = std::env::var("LISTEN_FDS").context("systemd did not bind authority FDs")?; + let names = std::env::var("LISTEN_FDNAMES").context("systemd did not name authority FDs")?; + ensure!( + fds == "2" + && names + == format!( + "{IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME}:{RUNTIME_PRESENCE_IDENTITY_FD_NAME}" + ), + "systemd authority FDs differ from the launch contract" + ); + for fd in [3, 4] { + // SAFETY: F_GETFD observes one integer descriptor and transfers nothing. + ensure!( + unsafe { libc::fcntl(fd, libc::F_GETFD) } != -1, + "an authority descriptor is not open" + ); + } + // SAFETY: the LISTEN_* contract above assigns these two distinct, open + // descriptors to this process; this is their first and only consumer. + let (activation, provider) = unsafe { (File::from_raw_fd(3), File::from_raw_fd(4)) }; + for (file, exact) in [(&activation, Some(32_u64)), (&provider, None)] { + // OpenFile descriptors arrive without close-on-exec; the child must + // never inherit a signer. + // SAFETY: F_SETFD on a descriptor `file` owns. + ensure!( + unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) } != -1, + "an authority descriptor could not be sealed" + ); + let metadata = file.metadata()?; + ensure!( + metadata.is_file() + && metadata.uid() == 0 + && metadata.permissions().mode() & 0o777 == 0o400 + && metadata.nlink() == 1 + && exact.map_or(metadata.len() > 0, |size| metadata.len() == size), + "an authority descriptor is not one root-owned 0400 file" + ); + } + Ok((activation, provider)) +} + +// ---------------------------------------------------------------- ports + +pub trait Clock: Send + Sync { + /// Wall-clock time, for values signed or published to another party. No + /// decision about freshness, deadlines or schedules reads it. + fn now_millis(&self) -> u64; + /// Monotonic time: never steps back, unaffected by wall-clock changes. Every + /// age, deadline and schedule in the launcher reads this one source. + fn monotonic_millis(&self) -> u64; + fn sleep(&self, duration: Duration); +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_millis(&self) -> u64 { + u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0) + } + + fn monotonic_millis(&self) -> u64 { + static START: std::sync::OnceLock = std::sync::OnceLock::new(); + u64::try_from(START.get_or_init(std::time::Instant::now).elapsed().as_millis()) + .unwrap_or(u64::MAX) + } + + fn sleep(&self, duration: Duration) { + thread::sleep(duration); + } +} + +/// Systemd credentials as Idunn binds them: the files `NAME` under the unit's +/// credentials directory. +pub trait CredentialSource: Send + Sync { + fn contents(&self, name: &str) -> Result>; + fn path(&self, name: &str) -> Result; +} + +/// Resolves a credential only as a file directly under the credentials +/// directory systemd gives the unit (`$CREDENTIALS_DIRECTORY`, which Idunn +/// pins to `/run/credentials/`). The process environment is never a +/// credential namespace: a name is a plain file name, never a variable. +pub struct CredentialsDirectory(Option); + +impl CredentialsDirectory { + pub fn new(directory: PathBuf) -> Self { + Self(Some(directory)) + } + + /// The directory systemd exported; no directory means no credential is bound. + pub fn from_environment() -> Self { + Self(std::env::var_os(SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT).map(PathBuf::from)) + } +} + +impl CredentialSource for CredentialsDirectory { + fn contents(&self, name: &str) -> Result> { + fs::read(self.path(name)?).map_err(|_| anyhow!("a credential could not be read")) + } + + fn path(&self, name: &str) -> Result { + ensure!(valid_credential_name(name), "credential name is invalid"); + let path = self + .0 + .as_ref() + .ok_or_else(|| anyhow!("a credential is not bound"))? + .join(name); + ensure!(fs::symlink_metadata(&path).is_ok(), "a credential is not bound"); + Ok(path) + } +} + +/// Where signed presence goes (Odin). +pub trait PresenceSink: Send + Sync { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()>; +} + +/// The lease as the launcher sees it: the current record, and a shared lock on +/// its store that Idunn's revocation cannot take while the child lives. +pub trait LeaseReader: Send + Sync { + fn read(&self) -> Result>; + fn hold_shared(&self) -> Result>; +} + +/// The shared lock on the lease store, held until dropped. Its descriptor is +/// handed to the child (`ChildSpawner::spawn`), so the lock lives as long as +/// the launcher or any process that inherited it. +pub trait LeaseLock: Send { + fn descriptor(&self) -> Option; +} + +pub trait HealthProbe: Send + Sync { + fn healthy(&self) -> bool; +} + +pub trait ChildHandle: Send { + /// The exit status once the child has exited. + fn exited(&mut self) -> Option; + /// SIGTERM, then SIGKILL after the grace period; returns once it is gone. + fn terminate(&mut self, grace: Duration); +} + +pub trait ChildSpawner: Send { + /// `inherit` is a descriptor the child keeps open across exec (the lease + /// lock); every other descriptor of the launcher is close-on-exec. + fn spawn( + &self, + program: &Path, + args: &[String], + inherit: Option, + ) -> Result>; +} + +// ---------------------------------------------------------------- production ports + +pub struct DriverLeaseReader { + driver: CultCacheWriteLeaseDriver, +} + +impl DriverLeaseReader { + pub fn new(target: &str, record_path: &Path) -> Self { + Self { + driver: CultCacheWriteLeaseDriver::new(target, record_path), + } + } +} + +struct SharedLockFile(File); + +impl LeaseLock for SharedLockFile { + fn descriptor(&self) -> Option { + Some(self.0.as_raw_fd()) + } +} + +impl LeaseReader for DriverLeaseReader { + fn read(&self) -> Result> { + self.driver + .current_for_holder() + .map(|current| current.map(|(_, lease)| lease)) + .map_err(|_| anyhow!("write-lease store refused custody, envelope or target")) + } + + /// LOCK_SH on `.lock`, the lock Idunn's revocation respects. The + /// descriptor is a plain file, so flock's lock belongs to its open file + /// description: a child that inherits it keeps the lock after the launcher + /// dies. + fn hold_shared(&self) -> Result> { + let file = File::open(authority_lock_path(&self.driver.record_path)).map_err(|_| anyhow!("write-lease lock could not be held"))?; + // SAFETY: flock on a descriptor this function owns. + ensure!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0, + "write-lease lock could not be held" + ); + Ok(Box::new(SharedLockFile(file))) + } +} + +/// Odin takes presence over RUDP at the endpoint Expected's Odin dependency +/// names. +pub struct OdinRudpSink { + endpoint: SocketAddr, +} + +impl OdinRudpSink { + pub fn from_expected(expected: &IdunnExpectedIncarnationRecord) -> Result { + let dependency = expected + .dependencies + .iter() + .find(|dependency| dependency.capability == ODIN_CAPABILITY) + .context("Expected has no Odin dependency")?; + let endpoint = dependency + .provider_endpoint + .as_deref() + .context("Expected Odin dependency has no endpoint")?; + let endpoint = endpoint.strip_prefix("rudp://").unwrap_or(endpoint); + Ok(Self { + endpoint: endpoint + .parse() + .map_err(|_| anyhow!("Expected Odin endpoint is not a socket address"))?, + }) + } +} + +impl PresenceSink for OdinRudpSink { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { + let message = CultNetMessage::DocumentPutRaw { + message_id: format!( + "runtime-presence:{}:{}:{}", + record.target, record.runtime_instance_id, record.publisher_sequence + ), + document: presence_document( + record, + Some(vec![ + "cultnet.transport.rudp.v0".into(), + "runtime-presence".into(), + ]), + )?, + }; + let bind = if self.endpoint.is_ipv4() { + "0.0.0.0:0" + } else { + "[::]:0" + }; + let socket = UdpSocket::bind(bind)?; + socket.set_read_timeout(Some(Duration::from_millis(100)))?; + let mut transport = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client( + &record.runtime_id, + socket, + self.endpoint, + ODIN_CULTMESH_CATALOG_CONNECTION_ID, + ), + )?; + transport.connect(Vec::new())?; + let deadline = Instant::now() + Duration::from_millis(500); + while !transport.connected() { + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "Odin did not accept the connection"); + } + let receipt = transport.send_reliable( + "schema", + encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?, + )?; + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match transport.reliable_send_status(&receipt) { + CultNetRudpReliableSendStatus::Acknowledged => return Ok(()), + CultNetRudpReliableSendStatus::Invalidated => { + bail!("Odin invalidated the presence acknowledgement") + } + CultNetRudpReliableSendStatus::Pending => {} + } + let _ = transport.receive_once()?; + transport.poll_resends()?; + ensure!(Instant::now() < deadline, "Odin did not acknowledge presence"); + } + } +} + +fn presence_document( + record: &GameCultRuntimePresenceHealthRecord, + tags: Option>, +) -> Result { + let payload = rmp_serde::to_vec(record).context("encoding runtime presence")?; + Ok(CultNetRawDocumentRecord { + schema_id: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: record.target.clone(), + stored_at: chrono::DateTime::from_timestamp_millis( + record.observed_at_unix_millis.try_into()?, + ) + .context("runtime presence timestamp is invalid")? + .to_rfc3339(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: Some(record.runtime_id.clone()), + source_agent_id: Some(record.signer_identity_id.clone()), + source_role: Some("runtime-presence-publisher".into()), + tags, + }) +} + +/// `GET ` over TCP at the child's member bind; healthy on a 200. One +/// deadline covers connect, write and read, so a full backlog or a silent child +/// answers false within `HEALTH_PROBE_TIMEOUT`. +pub struct HttpHealthProbe { + pub member: SocketAddr, + pub path: String, +} + +impl HealthProbe for HttpHealthProbe { + fn healthy(&self) -> bool { + let deadline = Instant::now() + HEALTH_PROBE_TIMEOUT; + let Ok(mut stream) = TcpStream::connect_timeout(&self.member, HEALTH_PROBE_TIMEOUT) else { + return false; + }; + let Some(remaining) = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + else { + return false; + }; + let request = format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + self.path + ); + let mut head = [0_u8; 12]; + stream.set_write_timeout(Some(remaining)).is_ok() + && stream.write_all(request.as_bytes()).is_ok() + && read_by(&mut stream, &mut head, deadline).is_ok() + && head.starts_with(b"HTTP/1.") + && &head[8..12] == b" 200" + } +} + +pub struct ProcessSpawner; + +struct ProcessChild(Child); + +impl ChildSpawner for ProcessSpawner { + fn spawn( + &self, + program: &Path, + args: &[String], + inherit: Option, + ) -> Result> { + let mut command = Command::new(program); + // The child gets a bare environment: no Idunn variable, no candidate + // bind, no credential path leaks into it. + command.args(args).env_clear().stdin(Stdio::null()); + for name in ["PATH", "TMPDIR", "LANG", "LC_ALL"] { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } + if let Some(descriptor) = inherit { + // SAFETY: runs in the forked child before exec; fcntl is + // async-signal-safe and the parent's descriptor stays close-on-exec. + unsafe { + command.pre_exec(move || { + if libc::fcntl(descriptor, libc::F_SETFD, 0) == -1 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + } + let child = command.spawn().map_err(|_| anyhow!("the child could not be spawned"))?; + Ok(Box::new(ProcessChild(child))) + } +} + +impl ChildHandle for ProcessChild { + fn exited(&mut self) -> Option { + match self.0.try_wait() { + Ok(Some(status)) => Some(status.code().unwrap_or(1)), + Ok(None) => None, + Err(_) => Some(1), + } + } + + fn terminate(&mut self, grace: Duration) { + if self.exited().is_some() { + return; + } + // SAFETY: the pid is our own live child. + unsafe { libc::kill(self.0.id() as libc::pid_t, libc::SIGTERM) }; + let deadline = Instant::now() + grace; + while Instant::now() < deadline { + if self.exited().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +// ---------------------------------------------------------------- config + +/// The closed placeholder set (decision D6). Any other `{{...}}` refuses. +pub struct ConfigRenderer<'a> { + pub credentials: &'a dyn CredentialSource, + /// The member bind the child serves on. + pub listen: SocketAddr, + pub state_root: Option<&'a Path>, +} + +impl ConfigRenderer<'_> { + pub fn render(&self, template: &str) -> Result { + let mut rendered = String::with_capacity(template.len()); + let mut rest = template; + while let Some(open) = rest.find("{{") { + rendered.push_str(&rest[..open]); + let after = &rest[open + 2..]; + let close = after + .find("}}") + .context("config template has an unterminated placeholder")?; + rendered.push_str(&self.expand(&after[..close])?); + rest = &after[close + 2..]; + } + rendered.push_str(rest); + Ok(rendered) + } + + fn expand(&self, key: &str) -> Result { + match key { + "child.listen" => return Ok(self.listen.to_string()), + "child.listen_host" => return Ok(self.listen.ip().to_string()), + "child.listen_port" => return Ok(self.listen.port().to_string()), + _ => {} + } + if key == "state_root" { + return Ok(self + .state_root + .context("config template needs a state root the launch did not give")? + .display() + .to_string()); + } + if let Some(name) = key.strip_prefix("credential:") { + ensure!(valid_credential_name(name), "credential placeholder name is invalid"); + let bytes = self.credentials.contents(name)?; + let text = String::from_utf8(bytes) + .map_err(|_| anyhow!("a credential is not UTF-8 text"))?; + return Ok(text.strip_suffix('\n').unwrap_or(&text).to_owned()); + } + if let Some(name) = key.strip_prefix("credential_path:") { + ensure!(valid_credential_name(name), "credential placeholder name is invalid"); + return Ok(self.credentials.path(name)?.display().to_string()); + } + bail!("config template uses a placeholder outside the closed set") + } +} + +fn valid_credential_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') +} + +fn write_private_file(directory: &Path, name: &str, contents: &str) -> Result { + let path = directory.join(name); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .map_err(|_| anyhow!("the rendered config could not be created"))?; + file.write_all(contents.as_bytes())?; + Ok(path) +} + +// ---------------------------------------------------------------- lease + +/// A write lease this launcher holds: the record binds this Expected, +/// activation and one of the launcher's own Warming proofs, and its store is +/// share-locked until the hold drops. +pub struct LeaseHold { + reader: Arc, + lease: IdunnProcessWriteLeaseRecord, + sha256: String, + lock: Box, +} + +impl LeaseHold { + pub fn acquire( + reader: Arc, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + warming_proofs: &[String], + ) -> Result { + let lease = reader.read()?.context("no write lease is granted")?; + lease.validate()?; + lease_binds_candidate(expected, activation, &lease, warming_proofs)?; + let lock = reader.hold_shared()?; + ensure!( + reader.read()?.as_ref() == Some(&lease), + "write lease changed while it was being held" + ); + Ok(Self { + sha256: lease.canonical_sha256()?, + reader, + lease, + lock, + }) + } + + pub fn sha256(&self) -> &str { + &self.sha256 + } + + /// The locked descriptor, for the child to inherit. + pub fn lock_descriptor(&self) -> Option { + self.lock.descriptor() + } + + pub fn still_current(&self) -> bool { + matches!(self.reader.read(), Ok(Some(current)) if current == self.lease) + } +} + +// ---------------------------------------------------------------- route + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Phase { + Warming, + Active(Option), + Failed, +} + +/// What the launcher currently is, shared with the presence acceptor. +struct Shared { + authority: Mutex, + phase: Mutex, + lease: Mutex>, + lease_required: bool, + health: Arc, + /// The last probe result and the monotonic time it started. Only the lifecycle + /// probes; a challenge reads this and never waits on the child. + probed: Mutex>, + clock: Arc, +} + +impl Shared { + /// Probes the child now and keeps the result for presence to answer from. + fn refresh_health(&self) -> bool { + let started = self.clock.monotonic_millis(); + let healthy = self.health.healthy(); + if let Ok(mut probed) = self.probed.lock() { + *probed = Some((healthy, started)); + } + healthy + } + + /// Active only while the last probe found the child healthy and is recent. + /// A child never probed, wedged, or probed too long ago reads degraded. A + /// dead presence acceptor does not degrade presence: it fails the unit. + fn serving_state(&self) -> &'static str { + let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); + let now = self.clock.monotonic_millis(); + // A probe can finish between this thread reading the clock and locking + // the result, so a stamp later than `now` is a result fresher than the + // reading, age zero: monotonic time does not run backwards, so the stamp + // is not wrong. + let healthy = self + .probed + .lock() + .ok() + .and_then(|probed| *probed) + .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); + if healthy { "active" } else { "degraded" } + } + + fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { + self.authority + .lock() + .map_err(|_| anyhow!("authority lock poisoned"))? + .sign(state, detail, lease, self.clock.now_millis()) + } + + fn lease_current(&self) -> bool { + !self.lease_required + || self + .lease + .lock() + .is_ok_and(|hold| hold.as_ref().is_some_and(LeaseHold::still_current)) + } + + /// The answer to a route observation: active only if the child is healthy + /// and the lease is still ours. + fn observe(&self, request: CultNetMessage) -> Result { + let CultNetMessage::SnapshotRequest { message_id, .. } = request else { + bail!("route challenge is not a snapshot request"); + }; + let detail = format!("route-observation:{message_id}"); + let phase = self.phase.lock().map_err(|_| anyhow!("phase lock poisoned"))?.clone(); + let record = match phase { + Phase::Warming => self.sign("warming", &detail, None)?, + Phase::Failed => self.sign("failed", &detail, None)?, + Phase::Active(lease) => { + let state = if self.lease_current() { + self.serving_state() + } else { + "degraded" + }; + self.sign(state, &detail, lease)? + } + }; + Ok(CultNetMessage::SnapshotResponseRaw { + message_id, + documents: vec![presence_document(&record, None)?], + }) + } +} + +type Responder = Arc Result + Send + Sync>; + +/// Owns the candidate endpoint and answers presence there and nowhere else. +/// Members never reach it: Idunn routes them to the child's member bind. +/// +/// The endpoint is public, so it spends bounded resources on a peer: one +/// connection at a time, so a connection holds at most one descriptor; a total +/// deadline for the whole exchange; and any first byte but a challenge's closes +/// the connection unanswered. No accept error ends it: it backs off and accepts +/// again, and only `halt` stops it. +pub struct PresenceAcceptor { + listener: TcpListener, + responder: Responder, +} + +impl PresenceAcceptor { + pub fn new(listener: TcpListener, responder: Responder) -> Self { + Self { listener, responder } + } + + pub fn serve(self, halt: Arc) { + self.serve_accepting(halt, |listener| listener.accept()); + } + + fn serve_accepting( + self, + halt: Arc, + mut accept: impl FnMut(&TcpListener) -> io::Result<(TcpStream, SocketAddr)>, + ) { + // A listener that cannot be made non-blocking could never observe + // `halt`; ending here is a dead acceptor, which fails the unit. + if self.listener.set_nonblocking(true).is_err() { + return; + } + while !halt.load(Ordering::SeqCst) { + match accept(&self.listener) { + Ok((connection, _)) => { + let _ = connection.set_nonblocking(false); + let _ = self.answer(connection); + } + // An idle non-blocking accept is an error too; every one waits + // and tries again. + Err(_) => thread::sleep(ACCEPT_BACKOFF), + } + } + } + + fn answer(&self, mut connection: TcpStream) -> Result<()> { + let deadline = Instant::now() + PRESENCE_DEADLINE; + let mut first = [0_u8; 1]; + read_by(&mut connection, &mut first, deadline)?; + ensure!(first[0] == CHALLENGE_FIRST_BYTE, "not a challenge frame"); + let mut rest = [0_u8; 3]; + read_by(&mut connection, &mut rest, deadline)?; + let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; + ensure!( + (1..=CHALLENGE_MAX_BYTES).contains(&length), + "challenge frame is outside its bound" + ); + let mut payload = vec![0_u8; length]; + read_by(&mut connection, &mut payload, deadline)?; + let request = + decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; + let response = (self.responder)(request)?; + let frame = encode_frame(&encode_cultnet_message_to_vec( + &response, + CultNetWireContract::CultNetSchemaV0, + )?)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .context("the answer was not ready in time")?; + connection.set_write_timeout(Some(remaining))?; + connection.write_all(&frame)?; + connection.flush()?; + Ok(()) + } +} + +/// Fills `buffer` or fails when `deadline` passes, however the peer paces it. +fn read_by(connection: &mut TcpStream, buffer: &mut [u8], deadline: Instant) -> Result<()> { + let mut filled = 0; + while filled < buffer.len() { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .context("the peer did not send in time")?; + connection.set_read_timeout(Some(remaining))?; + match connection.read(&mut buffer[filled..]) { + Ok(0) => bail!("the peer closed early"), + Ok(count) => filled += count, + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(_) => bail!("the peer did not send in time"), + } + } + Ok(()) +} + +// ---------------------------------------------------------------- lifecycle + +pub struct Ports { + pub credentials: Arc, + pub clock: Arc, + pub sink: Arc, + pub lease: Arc, + pub spawner: Box, + /// Built from the member bind the child serves on. + pub health: Box Arc + Send>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + /// SIGTERM: the child was stopped and waited for. + Stopped, + /// The unit must die so continuity sees it; the code names why. + Failed(&'static str), +} + +impl Outcome { + pub fn exit_code(&self) -> i32 { + match self { + Self::Stopped => 0, + Self::Failed(_) => 1, + } + } +} + +/// The member bind Idunn hands the unit: a socket address on a loopback IP, +/// other than the candidate bind. Errors name the variable, never its value. +pub fn parse_member_bind(raw: &str, candidate_bind: &str) -> Result { + let member: SocketAddr = raw + .parse() + .map_err(|_| anyhow!("the Idunn member bind is not a socket address"))?; + ensure!( + member.ip().is_loopback(), + "the Idunn member bind is not on a loopback address" + ); + let candidate: SocketAddr = candidate_bind + .parse() + .map_err(|_| anyhow!("the Idunn candidate bind is not a socket address"))?; + ensure!(member != candidate, "the Idunn member bind equals the candidate bind"); + Ok(member) +} + +/// The monotonic reading `span` after `now`; a span too long to represent is a +/// deadline that never arrives. +fn after(now: u64, span: Duration) -> u64 { + now.saturating_add(u64::try_from(span.as_millis()).unwrap_or(u64::MAX)) +} + +/// The unit's whole life: Warming, lease, child, Active, until the child or the +/// lease ends or SIGTERM arrives. `route` is the already-bound candidate +/// listener, answered for presence only; `member` is the address the child +/// serves members on; `tmp` is the unit's private temporary directory. +pub fn run( + contract: &LaunchContract, + authority: RuntimeAuthority, + route: TcpListener, + member: SocketAddr, + tmp: &Path, + ports: Ports, + stop: &AtomicBool, +) -> Outcome { + let expected = authority.expected().clone(); + let activation = authority.activation().clone(); + if expected.health_contract != contract.health_contract { + return Outcome::Failed("health-contract-mismatch"); + } + let health = (ports.health)(member); + let shared = Arc::new(Shared { + authority: Mutex::new(authority), + phase: Mutex::new(Phase::Warming), + lease: Mutex::new(None), + lease_required: expected.write_lease_required, + health: Arc::clone(&health), + probed: Mutex::new(None), + clock: Arc::clone(&ports.clock), + }); + let halt = Arc::new(AtomicBool::new(false)); + let responder: Responder = { + let shared = Arc::clone(&shared); + Arc::new(move |request| shared.observe(request)) + }; + let acceptor = PresenceAcceptor::new(route, responder); + let serving = { + let halt = Arc::clone(&halt); + thread::Builder::new() + .name("presence-acceptor".into()) + .spawn(move || acceptor.serve(halt)) + }; + let Ok(serving) = serving else { + return Outcome::Failed("presence-acceptor"); + }; + let outcome = lifecycle(contract, &expected, &activation, member, tmp, &ports, &shared, &serving, stop); + halt.store(true, Ordering::SeqCst); + let _ = serving.join(); + outcome +} + +#[allow(clippy::too_many_arguments)] +fn lifecycle( + contract: &LaunchContract, + expected: &IdunnExpectedIncarnationRecord, + activation: &IdunnRuntimeActivationRecord, + member: SocketAddr, + tmp: &Path, + ports: &Ports, + shared: &Shared, + presence: &thread::JoinHandle<()>, + stop: &AtomicBool, +) -> Outcome { + let clock = &*ports.clock; + let publish = |state: &str, detail: &str, lease: Option| -> Option { + let record = shared.sign(state, detail, lease).ok()?; + let sha = record.canonical_sha256().ok()?; + // A refused or lost publish is retried by the heartbeat; Odin may + // refuse Warming for a while after Idunn starts the unit. + let _ = ports.sink.put(&record); + Some(sha) + }; + let set_phase = |phase: Phase| { + if let Ok(mut current) = shared.phase.lock() { + *current = phase; + } + }; + let fail = |child: Option<&mut Box>, code: &'static str| { + if let Some(child) = child { + child.terminate(TERMINATE_GRACE); + } + set_phase(Phase::Failed); + publish("failed", code, None); + Outcome::Failed(code) + }; + + // 1. Warming reaches Odin before any child exists. + let mut proofs: VecDeque = VecDeque::new(); + let remember = |sha: Option, proofs: &mut VecDeque| { + if let Some(sha) = sha { + proofs.push_back(sha); + while proofs.len() > MAX_RECENT_WARMING_PROOFS { + proofs.pop_front(); + } + } + }; + remember(publish("warming", "launcher-started", None), &mut proofs); + let heartbeat = contract.heartbeat; + let mut next_heartbeat = after(clock.monotonic_millis(), heartbeat); + + // 2. A stateful child waits for the lease; a stateless one does not. + let mut lease_sha = None; + let mut lease_fd = None; + if expected.write_lease_required { + let deadline = after(clock.monotonic_millis(), contract.lease_wait); + loop { + if stop.load(Ordering::SeqCst) { + return Outcome::Stopped; + } + if presence.is_finished() { + return fail(None, "presence-acceptor"); + } + let held = LeaseHold::acquire( + Arc::clone(&ports.lease), + expected, + activation, + proofs.make_contiguous(), + ); + if let Ok(hold) = held { + lease_sha = Some(hold.sha256().to_owned()); + lease_fd = hold.lock_descriptor(); + if let Ok(mut slot) = shared.lease.lock() { + *slot = Some(hold); + } + break; + } + let now = clock.monotonic_millis(); + if now >= deadline { + return fail(None, "lease-timeout"); + } + if now >= next_heartbeat { + remember(publish("warming", "write-lease-pending", None), &mut proofs); + next_heartbeat = after(now, heartbeat); + } + clock.sleep(POLL); + } + } + + // 3. Render, spawn, and wait for health. + let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { + return fail(None, "child-start-refused"); + }; + let startup_deadline = after(clock.monotonic_millis(), contract.startup); + loop { + if stop.load(Ordering::SeqCst) { + child.terminate(TERMINATE_GRACE); + return Outcome::Stopped; + } + if child.exited().is_some() { + return fail(None, "child-exited"); + } + if presence.is_finished() { + return fail(Some(&mut child), "presence-acceptor"); + } + if shared.refresh_health() { + break; + } + if clock.monotonic_millis() >= startup_deadline { + return fail(Some(&mut child), "startup-timeout"); + } + clock.sleep(POLL); + } + + // 4. Active, and heartbeats while it holds. + set_phase(Phase::Active(lease_sha.clone())); + publish(shared.serving_state(), "child-healthy", lease_sha.clone()); + let mut next_probe = after(clock.monotonic_millis(), HEALTH_REFRESH); + next_heartbeat = after(clock.monotonic_millis(), heartbeat); + loop { + if stop.load(Ordering::SeqCst) { + child.terminate(TERMINATE_GRACE); + return Outcome::Stopped; + } + if child.exited().is_some() { + return fail(None, "child-exited"); + } + if presence.is_finished() { + return fail(Some(&mut child), "presence-acceptor"); + } + if !shared.lease_current() { + return fail(Some(&mut child), "lease-lost"); + } + let now = clock.monotonic_millis(); + if now >= next_probe { + shared.refresh_health(); + next_probe = after(now, HEALTH_REFRESH); + } + if now >= next_heartbeat { + publish(shared.serving_state(), "heartbeat", lease_sha.clone()); + next_heartbeat = after(now, heartbeat); + } + clock.sleep(POLL); + } +} + +fn start_child( + contract: &LaunchContract, + member: SocketAddr, + tmp: &Path, + ports: &Ports, + lease_fd: Option, +) -> Option> { + let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; + let rendered = ConfigRenderer { + credentials: &*ports.credentials, + listen: member, + state_root: contract.state_root.as_deref(), + } + .render(&template) + .ok()?; + let config = write_private_file(tmp, "child-config", &rendered).ok()?; + let mut args = contract.child_args.clone(); + if let Some(flag) = &contract.config_flag { + args.push(flag.clone()); + args.push(config.display().to_string()); + } + ports + .spawner + .spawn(&contract.release_root.join(&contract.child), &args, lease_fd) + .ok() +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize}; + + use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; + use cultnet_rs::{ + IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, + IdunnServiceIdentity, RuntimePresenceAuthenticationContext, VerifiedRuntimeAuthority, + authenticate_runtime_presence_claim, enroll_service_identity_at, verify_runtime_authority, + }; + + use super::*; + use crate::drivers::{WriteLeasePort, exact_route_snapshot_response, request_tcp_snapshot}; + + /// The member bind Idunn would hand these fixtures' unit. + fn member() -> SocketAddr { + "127.0.0.1:18845".parse().unwrap() + } + + fn digest(byte: char) -> String { + format!("sha256-{}", byte.to_string().repeat(64)) + } + + struct Fixture { + dir: tempfile::TempDir, + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + activation_credential: Vec, + provider_credential: Vec, + verified: VerifiedRuntimeAuthority, + } + + fn write_record(path: &Path, key: &str, kind: &str, schema: &str, payload: Vec) { + let envelope = CultCacheEnvelope { + key: key.into(), + r#type: kind.into(), + payload, + stored_at: chrono::Utc::now().to_rfc3339(), + schema_id: Some(schema.into()), + }; + let outcome = SingleFileMessagePackBackingStore::new(path) + .try_compare_exchange_snapshot(&[], &[envelope]) + .unwrap(); + assert!(matches!( + outcome, + TryCompareExchangeSnapshotOutcome::Exchanged + )); + } + + fn fixture(lease_required: bool) -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let provider_path = root.join("provider.cc"); + let provider = + enroll_service_identity_at::(&provider_path).unwrap(); + let idunn = + enroll_service_identity_at::(&root.join("idunn.cc")).unwrap(); + let expected = IdunnExpectedIncarnationRecord { + schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), + target: "synapse".into(), + plan_id: digest('1'), + incarnation_id: "incarnation-1".into(), + sealed_release_id: digest('2'), + source_repository: "github.com/GameCult/gamecult-chat".into(), + source_revision: "3".repeat(40), + recipe_sha256: digest('4'), + runtime_id: "synapse-yggdrasil".into(), + expected_signer_identity_id: provider.entry().identity_id.clone(), + health_contract: "gamecult.chat.synapse.presence-launcher.v1".into(), + artifact_sha256: digest('5'), + state_schema_generation: lease_required.then(|| "state-v1".into()), + state_contract_sha256: lease_required.then(|| digest('6')), + write_lease_required: lease_required, + route: Some(IdunnExpectedRoute { + route_id: "synapse-tcp".into(), + transport: "tcp".into(), + stable_endpoint: "tcp://127.0.0.1:18834".into(), + candidate_endpoint: "tcp://127.0.0.1:18835".into(), + }), + capabilities: vec![IdunnExpectedCapability { + capability: "gamecult.chat.homeserver".into(), + schema: "matrix.client.v3".into(), + compatibility: "v3".into(), + minimum_capacity: 1, + }], + dependencies: Vec::new(), + }; + let launch = + IdunnRuntimeActivationLaunch::issue(&expected, digest('7'), 100, &idunn).unwrap(); + let activation = launch.activation().clone(); + let mut activation_credential = Vec::new(); + launch.write_credential(&mut activation_credential).unwrap(); + let verified = verify_runtime_authority( + &expected, + &activation, + &idunn.trust_anchor().unwrap(), + &provider.entry().public_key, + ) + .unwrap(); + let bundle = root.join("bundle"); + fs::create_dir(&bundle).unwrap(); + write_record( + &bundle.join("expected.cc"), + &expected.target, + IdunnExpectedIncarnationRecord::TYPE, + IDUNN_EXPECTED_INCARNATION_SCHEMA, + expected.canonical_bytes().unwrap(), + ); + write_record( + &bundle.join("activation.cc"), + &expected.target, + IdunnRuntimeActivationRecord::TYPE, + IDUNN_RUNTIME_ACTIVATION_SCHEMA, + activation.canonical_bytes().unwrap(), + ); + let provider_credential = fs::read(&provider_path).unwrap(); + for directory in ["release/bin", "tmp"] { + fs::create_dir_all(root.join(directory)).unwrap(); + } + Fixture { + dir, + expected, + activation, + activation_credential, + provider_credential, + verified, + } + } + + impl Fixture { + fn authority(&self) -> RuntimeAuthority { + RuntimeAuthority::open( + &self.dir.path().join("bundle"), + &self.activation_credential[..], + &self.provider_credential[..], + ) + .unwrap() + } + + fn lease(&self, warming_sha256: &str) -> IdunnProcessWriteLeaseRecord { + IdunnProcessWriteLeaseRecord { + schema_version: cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), + target: self.expected.target.clone(), + expected_projection_sha256: self.expected.canonical_sha256().unwrap(), + plan_id: self.expected.plan_id.clone(), + incarnation_id: self.expected.incarnation_id.clone(), + sealed_release_id: self.expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256().unwrap(), + state_schema_generation: self.expected.state_schema_generation.clone().unwrap(), + state_contract_sha256: self.expected.state_contract_sha256.clone().unwrap(), + runtime_id: self.expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + warming_presence_sha256: warming_sha256.into(), + lease_epoch: 1, + issued_at_unix_millis: 200, + } + } + + fn contract(&self, template: &str) -> LaunchContract { + fs::write(self.dir.path().join("release/config.tmpl"), template).unwrap(); + LaunchContract { + release_root: self.dir.path().join("release"), + child: PathBuf::from("bin/child"), + child_args: vec!["--serve".into()], + config_flag: Some("--config".into()), + template: PathBuf::from("config.tmpl"), + health_path: "/health".into(), + health_contract: self.expected.health_contract.clone(), + state_root: Some(PathBuf::from("/var/lib/synapse")), + lease_wait: Duration::from_secs(3600), + startup: Duration::from_secs(3600), + heartbeat: Duration::from_secs(5), + } + } + } + + const WALL_EPOCH_MILLIS: i64 = 1_760_000_000_000; + + /// Virtual time runs a hundred times faster than the wall clock. + struct FastClock { + monotonic: AtomicU64, + wall_step: AtomicI64, + } + + impl FastClock { + /// The wall clock starts a Unix-epoch-sized distance from the monotonic + /// one, as the real pair does, so a deadline or schedule that reads the + /// wrong source is off by that distance and no test can pass by it. + fn new(start: u64) -> Self { + Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(WALL_EPOCH_MILLIS) } + } + + /// Steps the wall clock alone, as timesyncd or a VM restore would. + fn step_wall(&self, millis: i64) { + self.wall_step.fetch_add(millis, Ordering::SeqCst); + } + } + + impl Clock for FastClock { + fn now_millis(&self) -> u64 { + let wall = self.monotonic.load(Ordering::SeqCst) as i64 + self.wall_step.load(Ordering::SeqCst); + u64::try_from(wall).unwrap_or(0) + } + + fn monotonic_millis(&self) -> u64 { + self.monotonic.load(Ordering::SeqCst) + } + + fn sleep(&self, duration: Duration) { + self.monotonic + .fetch_add(duration.as_millis() as u64, Ordering::SeqCst); + thread::sleep(Duration::from_millis(1)); + } + } + + #[derive(Default)] + struct Sink(Mutex>); + + impl PresenceSink for Sink { + fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { + self.0.lock().unwrap().push(record.clone()); + Ok(()) + } + } + + impl Sink { + fn states(&self) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .map(|record| record.state.clone()) + .collect() + } + + fn first(&self, state: &str) -> Option { + self.0 + .lock() + .unwrap() + .iter() + .find(|record| record.state == state) + .cloned() + } + } + + #[derive(Default)] + struct FakeLease(Mutex>); + + impl LeaseReader for FakeLease { + fn read(&self) -> Result> { + Ok(self.0.lock().unwrap().clone()) + } + + fn hold_shared(&self) -> Result> { + Ok(Box::new(FakeLock)) + } + } + + /// The descriptor number a fake lock claims; the launcher must hand it on. + const FAKE_LOCK_FD: RawFd = 1234; + + struct FakeLock; + + impl LeaseLock for FakeLock { + fn descriptor(&self) -> Option { + Some(FAKE_LOCK_FD) + } + } + + struct Health(AtomicBool); + + impl HealthProbe for Health { + fn healthy(&self) -> bool { + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct Child { + spawns: AtomicUsize, + exit: Mutex>, + terminated: AtomicBool, + args: Mutex>>, + inherited: Mutex>>, + } + + struct FakeChild(Arc); + + impl ChildHandle for FakeChild { + fn exited(&mut self) -> Option { + *self.0.exit.lock().unwrap() + } + + fn terminate(&mut self, _grace: Duration) { + self.0.terminated.store(true, Ordering::SeqCst); + *self.0.exit.lock().unwrap() = Some(143); + } + } + + struct FakeSpawner(Arc); + + impl ChildSpawner for FakeSpawner { + fn spawn( + &self, + _program: &Path, + args: &[String], + inherit: Option, + ) -> Result> { + self.0.inherited.lock().unwrap().push(inherit); + self.0.spawns.fetch_add(1, Ordering::SeqCst); + self.0.args.lock().unwrap().push(args.to_vec()); + Ok(Box::new(FakeChild(Arc::clone(&self.0)))) + } + } + + struct Credentials(BTreeMap>); + + impl CredentialSource for Credentials { + fn contents(&self, name: &str) -> Result> { + self.0.get(name).cloned().ok_or_else(|| anyhow!("unbound")) + } + + fn path(&self, name: &str) -> Result { + self.0 + .contains_key(name) + .then(|| PathBuf::from(format!("/run/credentials/unit/{name}"))) + .ok_or_else(|| anyhow!("unbound")) + } + } + + struct Run { + sink: Arc, + lease: Arc, + child: Arc, + stop: Arc, + outcome: Option>, + } + + fn start(fx: &Fixture, contract: LaunchContract, healthy: bool) -> Run { + let sink = Arc::new(Sink::default()); + let lease = Arc::new(FakeLease::default()); + let health = Arc::new(Health(AtomicBool::new(healthy))); + let child = Arc::new(Child::default()); + let stop = Arc::new(AtomicBool::new(false)); + let mut secrets = BTreeMap::new(); + secrets.insert("SYNAPSE_KEY".to_owned(), b"s3cret-value\n".to_vec()); + let ports = Ports { + credentials: Arc::new(Credentials(secrets)), + clock: Arc::new(FastClock::new(1_000_000)), + sink: sink.clone(), + lease: lease.clone(), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(move |_| health.clone()), + }; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let flag = stop.clone(); + let outcome = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + Run { + sink, + lease, + child, + stop, + outcome: Some(outcome), + } + } + + impl Run { + fn finish(&mut self) -> Outcome { + self.outcome.take().unwrap().join().unwrap() + } + + fn stop(&mut self) -> Outcome { + self.stop.store(true, Ordering::SeqCst); + self.finish() + } + } + + fn wait_until(what: &str, mut condition: impl FnMut() -> bool) { + for _ in 0..4000 { + if condition() { + return; + } + thread::sleep(Duration::from_millis(5)); + } + panic!("timed out waiting for {what}"); + } + + #[test] + fn stateful_child_never_starts_before_its_lease() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("listen {{child.listen}}"), true); + wait_until("two warming heartbeats", || { + launch.sink.states().iter().filter(|s| *s == "warming").count() >= 2 + }); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); + let warming = launch.sink.first("warming").unwrap(); + assert!(warming.write_lease_sha256.is_none()); + + // The lease binds the launcher's own Warming proof; only then it starts. + let lease = fx.lease(&warming.canonical_sha256().unwrap()); + *launch.lease.0.lock().unwrap() = Some(lease.clone()); + wait_until("child spawn", || { + launch.child.spawns.load(Ordering::SeqCst) == 1 + }); + wait_until("active", || launch.sink.first("active").is_some()); + let active = launch.sink.first("active").unwrap(); + assert_eq!( + active.write_lease_sha256.as_deref(), + Some(lease.canonical_sha256().unwrap().as_str()) + ); + assert_eq!(launch.stop(), Outcome::Stopped); + assert!(launch.child.terminated.load(Ordering::SeqCst)); + } + + #[test] + fn a_lease_for_another_activation_or_expected_is_not_accepted() { + let fx = fixture(true); + let foreign: [(&str, fn(&mut IdunnProcessWriteLeaseRecord)); 3] = [ + ("another runtime instance", |lease| { + lease.runtime_instance_id = "other".into() + }), + ("another Expected", |lease| { + lease.expected_projection_sha256 = digest('9') + }), + ("another warming proof", |lease| { + lease.warming_presence_sha256 = digest('9') + }), + ]; + for (name, edit) in foreign { + let mut contract = fx.contract("x"); + contract.lease_wait = Duration::from_secs(5); + let mut launch = start(&fx, contract, true); + // Installed at once; the launcher's own proofs are never digest('8'). + let mut lease = fx.lease(&digest('8')); + edit(&mut lease); + *launch.lease.0.lock().unwrap() = Some(lease); + let (child, sink) = (launch.child.clone(), launch.sink.clone()); + // Bounded: a lease deadline that never arrives fails here, not at the + // suite's outer timeout. + let outcome = within(Duration::from_secs(20), move || launch.finish()); + assert_eq!(outcome, Outcome::Failed("lease-timeout"), "{name}"); + assert_eq!(child.spawns.load(Ordering::SeqCst), 0, "{name}"); + assert_eq!(sink.states().last().unwrap(), "failed", "{name}"); + } + } + + #[test] + fn lease_loss_stops_the_child() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("warming", || launch.sink.first("warming").is_some()); + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + *launch.lease.0.lock().unwrap() = None; + let outcome = launch.finish(); + assert_eq!(outcome, Outcome::Failed("lease-lost")); + assert_ne!(outcome.exit_code(), 0); + assert!(launch.child.terminated.load(Ordering::SeqCst)); + assert_eq!(launch.sink.states().last().unwrap(), "failed"); + } + + #[test] + fn stateless_child_starts_without_a_lease() { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 1); + assert_eq!(launch.sink.states().first().unwrap(), "warming"); + assert!( + launch + .sink + .first("active") + .unwrap() + .write_lease_sha256 + .is_none() + ); + launch.stop(); + } + + #[test] + fn child_exit_ends_the_launcher() { + for code in [0, 3] { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + *launch.child.exit.lock().unwrap() = Some(code); + let outcome = launch.finish(); + assert_eq!(outcome, Outcome::Failed("child-exited"), "exit {code}"); + assert_eq!(outcome.exit_code(), 1); + assert_eq!(launch.sink.states().last().unwrap(), "failed"); + } + } + + #[test] + fn closed_placeholder_set_renders_and_refuses() { + let fx = fixture(false); + let template = "listen={{child.listen}}\nhost={{child.listen_host}}\nport={{child.listen_port}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; + let mut launch = start(&fx, fx.contract(template), true); + wait_until("spawn", || launch.child.spawns.load(Ordering::SeqCst) == 1); + let args = launch.child.args.lock().unwrap()[0].clone(); + assert_eq!(args[..2], ["--serve", "--config"]); + let config = PathBuf::from(&args[2]); + assert_eq!(config.parent().unwrap(), fx.dir.path().join("tmp")); + assert_eq!( + fs::metadata(&config).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let rendered = fs::read_to_string(&config).unwrap(); + assert_eq!( + rendered, + "listen=127.0.0.1:18845\nhost=127.0.0.1\nport=18845\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n" + ); + assert!( + !rendered.contains("18835"), + "the candidate bind never reaches the child" + ); + launch.stop(); + + // Any other placeholder refuses to start, and the refusal names no input. + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("a={{canary_bind}}"), true); + assert_eq!(launch.finish(), Outcome::Failed("child-start-refused")); + assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); + let credentials = Credentials(BTreeMap::new()); + let renderer = ConfigRenderer { + credentials: &credentials, + listen: member(), + state_root: None, + }; + for template in [ + "{{canary_bind}}", + "{{credential:CANARY-MISSING}}", + "{{credential:canary b}}", + "{{child.listen}} {{canary", + "{{state_root}}", + ] { + let error = format!("{:#}", renderer.render(template).unwrap_err()); + assert!(!error.to_lowercase().contains("canary"), "{error}"); + } + } + + fn shared(fx: &Fixture, health: Arc) -> Arc { + shared_on(fx, health, Arc::new(SystemClock)) + } + + fn shared_on(fx: &Fixture, health: Arc, clock: Arc) -> Arc { + Arc::new(Shared { + authority: Mutex::new(fx.authority()), + phase: Mutex::new(Phase::Active(None)), + lease: Mutex::new(None), + lease_required: false, + health, + probed: Mutex::new(None), + clock, + }) + } + + #[test] + fn route_answer_degrades_with_the_child() { + let fx = fixture(false); + let health = Arc::new(Health(AtomicBool::new(true))); + let shared = shared(&fx, health.clone()); + let state_of = |shared: &Shared| { + let request = CultNetMessage::SnapshotRequest { + message_id: "challenge-1".into(), + schema_ids: None, + record_keys: None, + }; + let CultNetMessage::SnapshotResponseRaw { documents, .. } = + shared.observe(request).unwrap() + else { + panic!("not a raw snapshot response") + }; + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&documents[0].payload).unwrap(); + assert_eq!(record.detail, "route-observation:challenge-1"); + record.state + }; + // Never probed: nothing says the child is healthy. + assert_eq!(state_of(&shared), "degraded"); + shared.refresh_health(); + assert_eq!(state_of(&shared), "active"); + // A challenge answers from the last probe and never runs one. + health.0.store(false, Ordering::SeqCst); + assert_eq!(state_of(&shared), "active"); + shared.refresh_health(); + assert_eq!(state_of(&shared), "degraded"); + } + + #[test] + fn route_challenge_is_answered_with_both_signatures_through_idunns_client() { + let fx = fixture(false); + let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); + shared.refresh_health(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let responder: Responder = Arc::new(move |request| shared.observe(request)); + let halt = Arc::new(AtomicBool::new(false)); + let acceptor = PresenceAcceptor::new(listener, responder); + let serving = { + let halt = halt.clone(); + thread::spawn(move || acceptor.serve(halt)) + }; + let request = CultNetMessage::SnapshotRequest { + message_id: "challenge-7".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + let response = request_tcp_snapshot(address, &payload).ok().unwrap(); + let message = + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .unwrap(); + let answer = + exact_route_snapshot_response(message, "challenge-7", &fx.expected.target).unwrap(); + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&answer.canonical_presence).unwrap(); + let claim = authenticate_runtime_presence_claim( + &answer.canonical_presence, + &fx.verified, + RuntimePresenceAuthenticationContext { + trusted_received_at_unix_millis: record.observed_at_unix_millis, + maximum_age_millis: 10_000, + maximum_future_skew_millis: 10, + }, + ) + .unwrap(); + assert_eq!(claim.record().state, "active"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn contract_parsing_refuses_escapes_and_echoes_nothing() { + let base = |extra: &[&str]| -> Result { + let mut args: Vec = [ + "--child", + "bin/c", + "--template", + "t", + "--health-path", + "/health", + "--health-contract", + "h", + ] + .map(String::from) + .to_vec(); + args.extend(extra.iter().map(|value| value.to_string())); + LaunchContract::parse(PathBuf::from("/r"), args) + }; + assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); + assert!(base(&["--child", "../escape"]).is_err()); + // The child serves members on Idunn's member bind: there is no + // listener to choose. + assert!(base(&["--listener", "loopback"]).is_err()); + assert!(base(&["--template", "/etc/passwd"]).is_err()); + let error = base(&["--canary-flag", "canary-value"]) + .unwrap_err() + .to_string(); + assert!(!error.contains("canary")); + } + + #[test] + fn real_lease_reader_holds_the_shared_lock_idunn_must_respect() { + let fx = fixture(true); + let path = fx.dir.path().join("lease.cc"); + let warming = digest('a'); + let lease = fx.lease(&warming); + write_record( + &path, + &lease.target, + IdunnProcessWriteLeaseRecord::TYPE, + cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes().unwrap(), + ); + let reader: Arc = Arc::new(DriverLeaseReader::new(&lease.target, &path)); + let hold = LeaseHold::acquire(reader, &fx.expected, &fx.activation, &[warming]).unwrap(); + assert!(hold.still_current()); + let idunn = CultCacheWriteLeaseDriver::new(&lease.target, &path); + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer")); + drop(hold); + idunn.revoke_exact(Some(&lease)).unwrap(); + } + + #[test] + fn authority_binds_the_candidate_and_signs_strictly_increasing_sequences() { + let fx = fixture(false); + let mut authority = fx.authority(); + assert!(authority.require_candidate_bind("127.0.0.1:18835").is_ok()); + assert!(authority.require_candidate_bind("127.0.0.1:18836").is_err()); + let first = authority.sign("warming", "a", None, 5_000).unwrap(); + let second = authority.sign("warming", "b", None, 5_000).unwrap(); + assert_eq!(first.publisher_sequence, 5_000); + assert_eq!(second.publisher_sequence, 5_001); + assert_eq!( + first.bound_endpoint.as_deref(), + Some("tcp://127.0.0.1:18835") + ); + } + + #[test] + fn health_probe_is_healthy_only_on_a_200_from_the_child() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let server = thread::spawn(move || { + for status in ["200 OK", "503 Service Unavailable"] { + let (mut stream, _) = listener.accept().unwrap(); + let mut line = String::new(); + io::BufRead::read_line(&mut io::BufReader::new(stream.try_clone().unwrap()), &mut line) + .unwrap(); + assert_eq!(line, "GET /health HTTP/1.1\r\n"); + write!(stream, "HTTP/1.1 {status}\r\nContent-Length: 2\r\n\r\nOK").unwrap(); + } + }); + assert!(probe.healthy()); + assert!(!probe.healthy()); + server.join().unwrap(); + assert!(!probe.healthy(), "no listener is not healthy"); + } + + #[test] + fn contract_parsing_reads_every_declared_field() { + let args: Vec = [ + "--child", "bin/c", "--child-arg", "-x", "--child-arg", "y", "--config-flag", "-c", + "--template", "t", "--health-path", "/h", + "--health-contract", "hc", "--state-root", "/s", "--lease-wait-secs", "11", + "--startup-secs", "22", "--heartbeat-secs", "33", + ] + .map(String::from) + .to_vec(); + let contract = LaunchContract::parse(PathBuf::from("/r"), args.clone()).unwrap(); + assert_eq!(contract.child_args, ["-x", "y"]); + assert_eq!(contract.config_flag.as_deref(), Some("-c")); + assert_eq!(contract.state_root, Some(PathBuf::from("/s"))); + assert_eq!( + (contract.lease_wait, contract.startup, contract.heartbeat), + (Duration::from_secs(11), Duration::from_secs(22), Duration::from_secs(33)) + ); + for zero in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { + let mut bad = args.clone(); + bad.extend([zero.to_owned(), "0".to_owned()]); + assert!(LaunchContract::parse(PathBuf::from("/r"), bad).is_err(), "{zero}"); + } + } + + #[test] + fn credential_names_cannot_escape_or_be_empty() { + struct Any; + impl CredentialSource for Any { + fn contents(&self, _: &str) -> Result> { + Ok(b"v".to_vec()) + } + fn path(&self, name: &str) -> Result { + Ok(PathBuf::from(name)) + } + } + let renderer = ConfigRenderer { credentials: &Any, listen: member(), state_root: None }; + for ok in ["A", "a-b_9", "Z0"] { + assert_eq!(renderer.render(&format!("{{{{credential:{ok}}}}}")).unwrap(), "v"); + } + for bad in ["", "../x", "a/b", "a b", "a.b", "a:b", "é"] { + assert!(renderer.render(&format!("{{{{credential:{bad}}}}}")).is_err(), "{bad}"); + assert!(renderer.render(&format!("{{{{credential_path:{bad}}}}}")).is_err(), "{bad}"); + } + assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "a127.0.0.1:18845b127.0.0.1:18845c"); + } + + #[test] + fn credentials_resolve_only_as_files_under_the_credentials_directory() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("cred"), b"abc\n").unwrap(); + let credentials = CredentialsDirectory::new(dir.path().to_owned()); + assert_eq!(credentials.path("cred").unwrap(), dir.path().join("cred")); + assert_eq!(credentials.contents("cred").unwrap(), b"abc\n"); + assert!(credentials.path("unbound").is_err()); + for bad in ["", "..", "../cred", "a/b", ".", "a.b"] { + assert!(credentials.path(bad).is_err(), "{bad}"); + assert!(credentials.contents(bad).is_err(), "{bad}"); + } + assert!(CredentialsDirectory(None).path("cred").is_err()); + } + + #[test] + fn credential_placeholders_never_read_the_process_environment() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("elsewhere"); + fs::write(&file, b"canary-file\n").unwrap(); + // SAFETY: variables no other test reads. + unsafe { + std::env::set_var("GAMECULT_IDUNN_CANDIDATE_BIND", "127.0.0.1:18835"); + std::env::set_var("IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", &file); + } + let credentials = CredentialsDirectory::new(dir.path().to_owned()); + let renderer = ConfigRenderer { credentials: &credentials, listen: member(), state_root: None }; + for name in [ + "GAMECULT_IDUNN_CANDIDATE_BIND", + "IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", + "../elsewhere", + "a/b", + "..", + ] { + for kind in ["credential", "credential_path"] { + let error = renderer.render(&format!("x={{{{{kind}:{name}}}}}")); + assert!(error.is_err(), "{kind}:{name}"); + assert!(!format!("{:#}", error.unwrap_err()).contains("18835")); + } + } + // The same name resolves when it is a credential file, never otherwise. + fs::write(dir.path().join("GAMECULT_IDUNN_CANDIDATE_BIND"), b"file-value\n").unwrap(); + assert_eq!( + renderer.render("{{credential:GAMECULT_IDUNN_CANDIDATE_BIND}}").unwrap(), + "file-value" + ); + } + + #[test] + fn odin_sink_takes_its_endpoint_only_from_the_odin_dependency() { + let fx = fixture(false); + let dependency = |capability: &str, endpoint: &str| cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: capability.into(), + schema: "s".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: None, + provider_authority: None, + provider_expected_projection_sha256: None, + provider_endpoint: Some(endpoint.into()), + }; + let mut expected = fx.expected.clone(); + assert!(OdinRudpSink::from_expected(&expected).is_err()); + expected.dependencies = vec![dependency("other.thing", "rudp://127.0.0.1:1")]; + assert!(OdinRudpSink::from_expected(&expected).is_err()); + expected.dependencies = vec![ + dependency("other.thing", "rudp://127.0.0.1:1"), + dependency(ODIN_CAPABILITY, "rudp://127.0.0.1:4100"), + ]; + assert_eq!( + OdinRudpSink::from_expected(&expected).unwrap().endpoint, + "127.0.0.1:4100".parse().unwrap() + ); + expected.dependencies = vec![dependency(ODIN_CAPABILITY, "rudp://not-an-address")]; + assert!(OdinRudpSink::from_expected(&expected).is_err()); + } + + + #[test] + fn a_probe_result_is_evidence_only_while_it_is_recent() { + let fx = fixture(false); + let clock = Arc::new(FastClock::new(1_000_000)); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + let stale = HEALTH_STALE.as_millis() as u64; + clock.monotonic.fetch_add(stale, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "active", "a result exactly at the bound is still recent"); + clock.monotonic.fetch_add(1, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "degraded", "a result past the bound is not evidence"); + shared.refresh_health(); + assert_eq!(shared.serving_state(), "active"); + } + + /// A lease reader that grants, at the first read once `after` Warming + /// records exist, a lease bound to the Warming record `back` places before + /// the newest. The launcher reads it from its own thread between publishes, + /// so the record it names is at a known place in the retained window. + struct WindowLease { + sink: Arc, + template: IdunnProcessWriteLeaseRecord, + after: usize, + back: usize, + granted: Mutex>, + } + + impl LeaseReader for WindowLease { + fn read(&self) -> Result> { + let mut granted = self.granted.lock().unwrap(); + if granted.is_none() { + let records = self.sink.0.lock().unwrap(); + let warming: Vec<_> = records.iter().filter(|r| r.state == "warming").collect(); + if warming.len() >= self.after { + let mut lease = self.template.clone(); + lease.warming_presence_sha256 = + warming[warming.len() - self.back].canonical_sha256().unwrap(); + *granted = Some(lease); + } + } + Ok(granted.clone()) + } + + fn hold_shared(&self) -> Result> { + Ok(Box::new(FakeLock)) + } + } + + /// Starts a stateful unit waiting on a `WindowLease` and reports whether it + /// accepted the lease (spawned the child) or retried well past the grant. + fn warming_window_accepts(back: usize) -> bool { + let fx = fixture(true); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let lease = Arc::new(WindowLease { + sink: sink.clone(), + template: fx.lease(""), + after: MAX_RECENT_WARMING_PROOFS + 6, + back, + granted: Mutex::new(None), + }); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock::new(1_000_000)), + sink: sink.clone(), + lease: lease.clone(), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), + }; + let mut contract = fx.contract("x"); + contract.heartbeat = POLL; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("the lease is granted", || lease.granted.lock().unwrap().is_some()); + let warming = || sink.states().iter().filter(|s| *s == "warming").count(); + let granted_at = warming(); + let mut spawned = false; + for _ in 0..4000 { + spawned = child.spawns.load(Ordering::SeqCst) == 1; + if spawned || warming() >= granted_at + 10 { + break; + } + thread::sleep(Duration::from_millis(5)); + } + stop.store(true, Ordering::SeqCst); + // A mutant may wedge the unit; only a unit that took the lease is joined. + if spawned { + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + } + spawned + } + + #[test] + fn the_warming_proof_window_keeps_exactly_the_cap() { + assert!( + warming_window_accepts(MAX_RECENT_WARMING_PROOFS), + "the oldest of the retained Warming proofs is accepted" + ); + assert!( + !warming_window_accepts(MAX_RECENT_WARMING_PROOFS + 1), + "a Warming proof past the cap is no longer accepted" + ); + } + + /// The child is healthy, then the wall clock steps by `step_millis`, then + /// the child wedges. A challenge must keep reading the truth throughout. + fn a_wall_clock_step_never_decides_freshness(step_millis: i64) { + struct Toggle { + healthy: AtomicBool, + probes: AtomicUsize, + } + impl HealthProbe for Toggle { + fn healthy(&self) -> bool { + self.probes.fetch_add(1, Ordering::SeqCst); + self.healthy.load(Ordering::SeqCst) + } + } + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let clock = Arc::new(FastClock::new(100_000_000)); + let probe = Arc::new(Toggle { healthy: AtomicBool::new(true), probes: AtomicUsize::new(0) }); + let given = probe.clone(); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: clock.clone(), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(move |_| given.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); + + clock.step_wall(step_millis); + for _ in 0..10 { + assert_eq!( + challenge_state(address, &fx).as_deref(), + Some("active"), + "a healthy child reads active across a wall-clock step of {step_millis} ms" + ); + thread::sleep(Duration::from_millis(20)); + } + + probe.healthy.store(false, Ordering::SeqCst); + let probed = probe.probes.load(Ordering::SeqCst); + let published = sink.0.lock().unwrap().len(); + let started = Instant::now(); + let mut state = None; + while started.elapsed() < Duration::from_secs(5) { + state = challenge_state(address, &fx); + if state.as_deref() == Some("degraded") { + break; + } + thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + state.as_deref(), + Some("degraded"), + "a wedged child reads degraded within the freshness window after a wall-clock step of {step_millis} ms" + ); + assert!(probe.probes.load(Ordering::SeqCst) > probed, "probing continued"); + wait_until("a heartbeat after the step", || sink.0.lock().unwrap().len() > published); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + } + + #[test] + fn a_backward_wall_clock_step_does_not_keep_a_wedged_child_active() { + a_wall_clock_step_never_decides_freshness(-600_000); + } + + #[test] + fn a_forward_wall_clock_step_does_not_make_a_healthy_child_degraded() { + a_wall_clock_step_never_decides_freshness(600_000); + } + + #[test] + fn the_cached_result_ages_on_the_monotonic_clock_alone() { + let fx = fixture(false); + let clock = Arc::new(FastClock::new(100_000_000)); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + for step in [-600_000, 1_200_000] { + clock.step_wall(step); + assert_eq!(shared.serving_state(), "active", "wall step {step}"); + } + clock.monotonic.fetch_add(HEALTH_STALE.as_millis() as u64 + 1, Ordering::SeqCst); + assert_eq!(shared.serving_state(), "degraded", "monotonic age decides, whatever the wall says"); + } + + /// A monotonic clock whose read, on a thread that marks itself, returns the + /// current value and then waits to be released: the presence thread + /// descheduled right after it reads the time. + struct HeldClock { + monotonic: AtomicU64, + held: Mutex>>, + release: Mutex>>, + } + + thread_local! { + static HOLD_THIS_READ: std::cell::Cell = const { std::cell::Cell::new(false) }; + } + + impl Clock for HeldClock { + fn now_millis(&self) -> u64 { + 0 + } + + fn monotonic_millis(&self) -> u64 { + let value = self.monotonic.load(Ordering::SeqCst); + if HOLD_THIS_READ.with(std::cell::Cell::get) { + self.held.lock().unwrap().take().unwrap().send(()).unwrap(); + self.release.lock().unwrap().take().unwrap().recv().unwrap(); + } + value + } + + fn sleep(&self, _: Duration) {} + } + + #[test] + fn a_presence_read_descheduled_across_a_probe_still_sees_a_healthy_child() { + let fx = fixture(false); + let (held_tx, held_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let clock = Arc::new(HeldClock { + monotonic: AtomicU64::new(1_000), + held: Mutex::new(Some(held_tx)), + release: Mutex::new(Some(release_rx)), + }); + let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); + shared.refresh_health(); + let reader = { + let shared = shared.clone(); + thread::spawn(move || { + HOLD_THIS_READ.with(|hold| hold.set(true)); + shared.serving_state() + }) + }; + // The reader is descheduled in its clock read. The lifecycle probes a + // millisecond later and stores a result stamped after the reader's time. + held_rx.recv().unwrap(); + clock.monotonic.store(1_001, Ordering::SeqCst); + assert!(shared.refresh_health()); + release_tx.send(()).unwrap(); + assert_eq!(reader.join().unwrap(), "active"); + } + + #[test] + fn the_declared_durations_are_bounded_at_parse() { + let base: Vec = [ + "--child", "bin/c", "--template", "t", "--health-path", "/h", "--health-contract", "hc", + ] + .map(String::from) + .to_vec(); + for flag in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { + for (value, accepted) in [ + ("86400".to_owned(), true), + ("86401".to_owned(), false), + (u64::MAX.to_string(), false), + ] { + let mut args = base.clone(); + args.extend([flag.to_owned(), value.clone()]); + let parsed = LaunchContract::parse(PathBuf::from("/r"), args); + assert_eq!(parsed.is_ok(), accepted, "{flag} {value}"); + if let Err(error) = parsed { + assert!(!format!("{error:#}").contains(&value), "the refusal echoes its input"); + } + } + } + } + + #[test] + fn a_span_too_long_for_the_clock_is_a_deadline_that_never_arrives() { + assert_eq!(after(10, Duration::from_millis(5)), 15); + assert_eq!(after(u64::MAX - 1, Duration::from_secs(1)), u64::MAX); + assert_eq!(after(10, Duration::MAX), u64::MAX); + } + + /// Runs `work` and returns its result, failing the test when it has not + /// finished within `limit`, so a mutant that makes it wait forever fails + /// instead of hanging the suite. + fn within(limit: Duration, work: impl FnOnce() -> T + Send + 'static) -> T { + join_within(limit, thread::spawn(work)) + } + + fn join_within(limit: Duration, handle: thread::JoinHandle) -> T { + let started = Instant::now(); + while !handle.is_finished() { + assert!(started.elapsed() < limit, "still running after {limit:?}"); + thread::sleep(Duration::from_millis(10)); + } + handle.join().unwrap() + } + + /// The state the presence endpoint signs in answer to a challenge, or None + /// when it is not answered promptly. + fn challenge_state(address: SocketAddr, fx: &Fixture) -> Option { + let request = CultNetMessage::SnapshotRequest { + message_id: "wedge".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + let started = Instant::now(); + let answer = request_tcp_snapshot(address, &payload).ok()?; + if started.elapsed() > Duration::from_millis(750) { + return None; + } + let CultNetMessage::SnapshotResponseRaw { documents, .. } = + decode_cultnet_message_from_slice(&answer, CultNetWireContract::CultNetSchemaV0).ok()? + else { + return None; + }; + let record: GameCultRuntimePresenceHealthRecord = + rmp_serde::from_slice(&documents.first()?.payload).ok()?; + Some(record.state) + } + + #[derive(Clone, Copy)] + enum Wedge { + /// Accepts and never answers. + Silent, + /// Never accepts; the kernel backlog fills. + FullBacklog, + /// Accepts and answers 200 after 1.1 s, past the probe's deadline. + Slow, + } + + type WedgeSwitch = Arc>>; + + /// A child that answers 200 until wedged. Returns the member address, the + /// switch, and the thread to join after `halt`. + fn serving_child(halt: Arc) -> (SocketAddr, WedgeSwitch, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let wedge: WedgeSwitch = Arc::default(); + let mode = wedge.clone(); + let serving = thread::spawn(move || { + while !halt.load(Ordering::SeqCst) { + let current = *mode.lock().unwrap(); + if matches!(current, Some(Wedge::FullBacklog)) { + thread::sleep(Duration::from_millis(5)); + continue; + } + let Ok((mut stream, _)) = listener.accept() else { + thread::sleep(Duration::from_millis(5)); + continue; + }; + stream.set_nonblocking(false).unwrap(); + match current { + None => { + let mut request = [0_u8; 256]; + let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); + let _ = stream.read(&mut request); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + } + Some(Wedge::Slow) => { + thread::spawn(move || { + thread::sleep(Duration::from_millis(1100)); + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); + }); + } + _ => { + thread::spawn(move || { + thread::sleep(Duration::from_secs(8)); + drop(stream); + }); + } + } + } + }); + (address, wedge, serving) + } + + /// Whatever wedges the child, every challenge is still answered promptly, + /// and the answer turns to a signed degraded within a few probes. + fn a_wedged_child_is_answered_degraded(wedge: Wedge) { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let halt = Arc::new(AtomicBool::new(false)); + let (member, mode, child) = serving_child(halt.clone()); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(SystemClock), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(|member| Arc::new(HttpHealthProbe { member, path: "/health".into() })), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member, &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); + + *mode.lock().unwrap() = Some(wedge); + // A backlog fills only once clients have connected without being + // accepted. + let mut queued = Vec::new(); + if matches!(wedge, Wedge::FullBacklog) { + for _ in 0..4096 { + match TcpStream::connect_timeout(&member, Duration::from_millis(20)) { + Ok(stream) => queued.push(stream), + Err(_) => break, + } + } + } + let started = Instant::now(); + let mut degraded = false; + while started.elapsed() < Duration::from_secs(6) { + let state = challenge_state(address, &fx); + assert!(state.is_some(), "a challenge went unanswered after {:?}", started.elapsed()); + if state.as_deref() == Some("degraded") { + degraded = true; + break; + } + thread::sleep(Duration::from_millis(100)); + } + assert!(degraded, "the wedged child was never reported degraded"); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + halt.store(true, Ordering::SeqCst); + join_within(Duration::from_secs(10), child); + drop(queued); + } + + #[test] + fn a_silent_child_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::Silent); + } + + #[test] + fn a_child_with_a_full_backlog_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::FullBacklog); + } + + #[test] + fn a_child_answering_after_the_probe_deadline_is_answered_degraded_never_silent() { + a_wedged_child_is_answered_degraded(Wedge::Slow); + } + + /// The virtual time (milliseconds) of each record that has the given + /// detail. + fn times_of(sink: &Sink, detail: &str) -> Vec { + sink.0 + .lock() + .unwrap() + .iter() + .filter(|record| record.detail == detail) + .map(|record| record.observed_at_unix_millis) + .collect() + } + + #[test] + fn a_child_that_never_becomes_healthy_fails_the_unit_at_the_startup_timeout() { + let fx = fixture(false); + let mut contract = fx.contract("x"); + contract.startup = Duration::from_secs(5); + let mut launch = start(&fx, contract, false); + let sink = launch.sink.clone(); + let child = launch.child.clone(); + let outcome = within(Duration::from_secs(20), move || launch.finish()); + assert_eq!(outcome, Outcome::Failed("startup-timeout")); + assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); + let started = times_of(&sink, "launcher-started")[0]; + let failed = times_of(&sink, "startup-timeout")[0]; + // Virtual time: the deadline, found within one poll of it. + let waited = failed - started; + assert!( + (5_000..=5_000 + POLL.as_millis() as u64).contains(&waited), + "failed after {waited} ms of virtual time" + ); + } + + #[test] + fn heartbeats_follow_the_contracts_cadence_while_waiting_for_the_lease_and_while_active() { + let fx = fixture(true); + let mut contract = fx.contract("x"); + contract.heartbeat = Duration::from_secs(1); + let mut launch = start(&fx, contract, true); + let cadence = 1_000..=1_000 + POLL.as_millis() as u64; + wait_until("lease-wait heartbeats", || { + times_of(&launch.sink, "write-lease-pending").len() >= 4 + }); + let warming = launch.sink.first("warming").unwrap(); + let mut beats = vec![warming.observed_at_unix_millis]; + beats.extend(times_of(&launch.sink, "write-lease-pending").into_iter().take(4)); + for pair in beats.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "lease-wait beats {beats:?}"); + } + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active heartbeats", || times_of(&launch.sink, "heartbeat").len() >= 4); + let mut beats = vec![times_of(&launch.sink, "child-healthy")[0]]; + beats.extend(times_of(&launch.sink, "heartbeat").into_iter().take(4)); + for pair in beats.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "active beats {beats:?}"); + } + assert_eq!(launch.stop().exit_code(), 0); + } + + + /// Healthy, and notes the virtual time of every probe. + struct RecordingProbe { + clock: Arc, + at: Mutex>, + } + + impl HealthProbe for RecordingProbe { + fn healthy(&self) -> bool { + self.at.lock().unwrap().push(self.clock.monotonic_millis()); + true + } + } + + #[test] + fn an_active_child_is_probed_at_the_refresh_cadence() { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let clock = Arc::new(FastClock::new(1_000_000)); + let probe = Arc::new(RecordingProbe { clock: clock.clone(), at: Mutex::new(Vec::new()) }); + let seen = probe.clone(); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock, + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), + health: Box::new(move |_| probe.clone()), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("five probes", || seen.at.lock().unwrap().len() >= 5); + stop.store(true, Ordering::SeqCst); + assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); + let at = seen.at.lock().unwrap().clone(); + // The first probe ends startup; every later one follows the cadence. + let cadence = HEALTH_REFRESH.as_millis() as u64..=HEALTH_REFRESH.as_millis() as u64 + POLL.as_millis() as u64; + for pair in at.windows(2) { + assert!(cadence.contains(&(pair[1] - pair[0])), "probes at {at:?}"); + } + } + + #[test] + fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { + let mut stubborn = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()], None) + .unwrap(); + thread::sleep(Duration::from_millis(200)); + let started = Instant::now(); + stubborn.terminate(Duration::from_millis(400)); + assert!(started.elapsed() >= Duration::from_millis(400), "SIGTERM gets its grace period"); + assert!(stubborn.exited().is_some(), "then SIGKILL ends it"); + } + + #[test] + fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { + let mut gone = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()], None) + .unwrap(); + wait_until("child exit", || gone.exited().is_some()); + assert_eq!(gone.exited(), Some(3)); + let mut running = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()], None) + .unwrap(); + assert_eq!(running.exited(), None); + running.terminate(Duration::from_secs(5)); + assert!(running.exited().is_some()); + } + + // ------------------------------------------------------------ presence acceptor + + type Accepted = io::Result<(TcpStream, SocketAddr)>; + + fn serve_presence_accepting( + fx: &Fixture, + responder: Responder, + accept: impl FnMut(&TcpListener) -> Accepted + Send + 'static, + ) -> (SocketAddr, Arc, thread::JoinHandle<()>) { + let _ = fx; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let acceptor = PresenceAcceptor::new(listener, responder); + let halt = Arc::new(AtomicBool::new(false)); + let flag = halt.clone(); + (address, halt, thread::spawn(move || acceptor.serve_accepting(flag, accept))) + } + + fn observing(fx: &Fixture) -> Responder { + let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); + shared.refresh_health(); + Arc::new(move |request| shared.observe(request)) + } + + fn serve_presence(fx: &Fixture) -> (SocketAddr, Arc, thread::JoinHandle<()>) { + serve_presence_accepting(fx, observing(fx), |listener| listener.accept()) + } + + /// True when the peer closed (EOF or reset) within `within`, having sent + /// nothing. + fn closed_within(stream: &mut TcpStream, within: Duration) -> bool { + stream.set_read_timeout(Some(within)).unwrap(); + let mut byte = [0_u8; 64]; + match stream.read(&mut byte) { + Ok(0) => true, + Ok(_) => false, + Err(error) => !matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut), + } + } + + fn challenge_answered(address: SocketAddr, fx: &Fixture) -> bool { + let request = CultNetMessage::SnapshotRequest { + message_id: "bounded".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some(vec![fx.expected.target.clone()]), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + request_tcp_snapshot(address, &payload).is_ok() + } + + #[test] + fn only_challenge_frames_are_answered_and_anything_else_is_closed_unanswered() { + let fx = fixture(false); + let reached = Arc::new(AtomicUsize::new(0)); + let responder: Responder = { + let (inner, reached) = (observing(&fx), reached.clone()); + Arc::new(move |request| { + reached.fetch_add(1, Ordering::SeqCst); + inner(request) + }) + }; + let (address, halt, serving) = + serve_presence_accepting(&fx, responder, |listener| listener.accept()); + // Member traffic is not presence: an HTTP request gets no byte back. + let mut member = TcpStream::connect(address).unwrap(); + member + .write_all(b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 2\r\n\r\n\x00\x01") + .unwrap(); + assert!(closed_within(&mut member, Duration::from_secs(3)), "member bytes were answered"); + // A leading 0x00 with a frame length outside its bound never reaches + // the responder. + let mut malformed = TcpStream::connect(address).unwrap(); + malformed.write_all(&[0x00, 0xff, 0xff, 0xff]).unwrap(); + assert!(closed_within(&mut malformed, Duration::from_secs(3)), "a malformed frame was answered"); + assert_eq!(reached.load(Ordering::SeqCst), 0); + // A real challenge is answered, and is the only thing that reaches it. + assert!(challenge_answered(address, &fx)); + assert_eq!(reached.load(Ordering::SeqCst), 1); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn a_challenge_up_to_the_frame_bound_is_answered_and_one_past_it_is_not() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); + // A request far larger than a few KiB, well inside the 1 MiB bound. + let request = CultNetMessage::SnapshotRequest { + message_id: "large".into(), + schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), + record_keys: Some((0..300).map(|index| format!("{index:0>40}")).collect()), + }; + let payload = + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert!(payload.len() > 10_000 && payload.len() < CHALLENGE_MAX_BYTES); + assert!(request_tcp_snapshot(address, &payload).is_ok()); + // One byte past the bound is closed unanswered. + let past = (CHALLENGE_MAX_BYTES as u32 + 1).to_be_bytes(); + assert_eq!(past[0], CHALLENGE_FIRST_BYTE); + let mut oversized = TcpStream::connect(address).unwrap(); + oversized.write_all(&past).unwrap(); + assert!(closed_within(&mut oversized, Duration::from_secs(3)), "an oversized frame was read"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn an_idle_peer_holds_the_acceptor_for_the_presence_deadline_and_no_longer() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); + let mut idle = TcpStream::connect(address).unwrap(); + let started = Instant::now(); + // Idunn waits 3 s for a challenge answer; one stalled peer ahead of it + // must not cost it that. + assert!(challenge_answered(address, &fx)); + let waited = started.elapsed(); + assert!(waited >= Duration::from_millis(800), "the idle peer was cut early: {waited:?}"); + assert!(waited < Duration::from_millis(2200), "the idle peer held the acceptor: {waited:?}"); + assert!(closed_within(&mut idle, Duration::from_secs(1)), "the idle peer stayed connected"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_the_presence_deadline() { + let fx = fixture(false); + let (address, halt, serving) = serve_presence(&fx); + let mut slow = TcpStream::connect(address).unwrap(); + slow.write_all(&[0x00, 0x00, 0x01, 0x00]).unwrap(); + let started = Instant::now(); + let mut closed = false; + while started.elapsed() < Duration::from_secs(4) { + if closed_within(&mut slow, Duration::from_millis(150)) || slow.write_all(&[0x01]).is_err() { + closed = true; + break; + } + } + assert!(closed, "the trickle was never cut off"); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(800), "cut before its deadline: {elapsed:?}"); + assert!(elapsed < Duration::from_millis(2200), "cut after its deadline: {elapsed:?}"); + assert!(challenge_answered(address, &fx)); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + #[test] + fn accept_errors_never_end_the_presence_acceptor() { + let fx = fixture(false); + // Descriptor exhaustion: the first accepts fail, then descriptors free. + let failures = Arc::new(AtomicUsize::new(0)); + let (address, halt, serving) = { + let failures = failures.clone(); + serve_presence_accepting(&fx, observing(&fx), move |listener| { + if failures.fetch_add(1, Ordering::SeqCst) < 3 { + Err(io::Error::from_raw_os_error(libc::EMFILE)) + } else { + listener.accept() + } + }) + }; + assert!(challenge_answered(address, &fx), "the acceptor did not recover"); + assert!(failures.load(Ordering::SeqCst) >= 4); + assert!(!serving.is_finished()); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + /// A listener whose accept always fails: a datagram socket's descriptor. + fn failing_listener() -> TcpListener { + use std::os::fd::{FromRawFd, IntoRawFd}; + let datagram = UdpSocket::bind("127.0.0.1:0").unwrap(); + // SAFETY: the descriptor is owned by the new listener alone. + unsafe { TcpListener::from_raw_fd(datagram.into_raw_fd()) } + } + + #[test] + fn an_acceptor_whose_every_accept_fails_runs_until_halted() { + let fx = fixture(false); + let acceptor = PresenceAcceptor::new(failing_listener(), observing(&fx)); + let halt = Arc::new(AtomicBool::new(false)); + let flag = halt.clone(); + let serving = thread::spawn(move || acceptor.serve(flag)); + thread::sleep(Duration::from_millis(400)); + assert!(!serving.is_finished(), "the acceptor ended on an accept error"); + halt.store(true, Ordering::SeqCst); + serving.join().unwrap(); + } + + /// A clock that kills the presence thread when it reads the time: the + /// stand-in for a presence acceptor that dies while the child is fine. + /// Answering a challenge reads the cached result's age, which reads the + /// monotonic clock. + struct PresenceTripwire(FastClock); + + impl Clock for PresenceTripwire { + fn now_millis(&self) -> u64 { + self.0.now_millis() + } + + fn monotonic_millis(&self) -> u64 { + assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); + self.0.monotonic_millis() + } + + fn sleep(&self, duration: Duration) { + self.0.sleep(duration); + } + } + + #[test] + fn a_dead_presence_acceptor_fails_the_unit() { + let fx = fixture(false); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(PresenceTripwire(FastClock::new(1_000_000))), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), + }; + let contract = fx.contract("x"); + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = route.local_addr().unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + // The first challenge reaches the tripwire on the presence thread. + assert!(!challenge_answered(address, &fx)); + assert_eq!(join_within(Duration::from_secs(20), handle), Outcome::Failed("presence-acceptor")); + assert_eq!(sink.first("failed").unwrap().detail, "presence-acceptor"); + assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); + } + + #[test] + fn a_live_acceptor_keeps_presence_active() { + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("two active heartbeats", || { + launch.sink.states().iter().filter(|state| *state == "active").count() >= 2 + }); + assert!(launch.sink.first("degraded").is_none()); + launch.stop(); + } + + /// A store with a lease for `fx`, and Idunn's own driver over it. + fn lease_store( + fx: &Fixture, + name: &str, + ) -> (DriverLeaseReader, CultCacheWriteLeaseDriver, IdunnProcessWriteLeaseRecord) { + let dir = fx.dir.path().join(name); + fs::create_dir(&dir).unwrap(); + let path = dir.join("lease.cc"); + let lease = fx.lease(&digest('a')); + write_record( + &path, + &lease.target, + IdunnProcessWriteLeaseRecord::TYPE, + cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + lease.canonical_bytes().unwrap(), + ); + fs::write(dir.join("lease.cc.lock"), b"").unwrap(); + ( + DriverLeaseReader::new(&lease.target, &path), + CultCacheWriteLeaseDriver::new(&lease.target, &path), + lease, + ) + } + + #[test] + fn a_child_that_inherits_the_lease_lock_holds_it_after_the_launcher_is_gone() { + let fx = fixture(true); + let (reader, idunn, lease) = lease_store(&fx, "inherited"); + let guard = reader.hold_shared().unwrap(); + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], guard.descriptor()) + .unwrap(); + // The launcher dies: its own descriptor goes with it. + drop(guard); + // Held for a full second: a fork of some other test could pin the lock + // for an instant, only the child under test keeps it that long. + for _ in 0..10 { + let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); + assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); + thread::sleep(Duration::from_millis(100)); + } + child.terminate(Duration::from_secs(5)); + wait_until("child gone", || child.exited().is_some()); + idunn.revoke_exact(Some(&lease)).unwrap(); + } + + #[test] + fn a_child_spawned_without_the_descriptor_leaves_the_lock_with_the_launcher() { + let fx = fixture(true); + let (reader, idunn, lease) = lease_store(&fx, "uninherited"); + let guard = reader.hold_shared().unwrap(); + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], None) + .unwrap(); + drop(guard); + // Another test's fork may hold a copy for the instant before its exec; + // the child under test would hold it for thirty seconds. + wait_until("lock released", || idunn.revoke_exact(Some(&lease)).is_ok()); + child.terminate(Duration::from_secs(5)); + } + + #[test] + fn the_launcher_hands_its_lease_descriptor_to_the_child_it_spawns() { + let fx = fixture(true); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("warming", || launch.sink.first("warming").is_some()); + let warming = launch.sink.first("warming").unwrap(); + *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(*launch.child.inherited.lock().unwrap(), [Some(FAKE_LOCK_FD)]); + launch.stop(); + + let fx = fixture(false); + let mut launch = start(&fx, fx.contract("x"), true); + wait_until("active", || launch.sink.first("active").is_some()); + assert_eq!(*launch.child.inherited.lock().unwrap(), [None]); + launch.stop(); + } + + /// Unhealthy for its first `probes` calls, healthy from then on. + struct LateHealth { + calls: AtomicUsize, + probes: usize, + } + + impl HealthProbe for LateHealth { + fn healthy(&self) -> bool { + self.calls.fetch_add(1, Ordering::SeqCst) >= self.probes + } + } + + #[test] + fn a_child_that_becomes_healthy_late_is_not_killed_at_its_first_probe() { + let fx = fixture(false); + let mut contract = fx.contract("x"); + contract.startup = Duration::from_secs(60); + let sink = Arc::new(Sink::default()); + let child = Arc::new(Child::default()); + let health = Arc::new(LateHealth { calls: AtomicUsize::new(0), probes: 5 }); + let ports = Ports { + credentials: Arc::new(Credentials(BTreeMap::new())), + clock: Arc::new(FastClock::new(1_000_000)), + sink: sink.clone(), + lease: Arc::new(FakeLease::default()), + spawner: Box::new(FakeSpawner(child.clone())), + health: Box::new(move |_| health.clone()), + }; + let authority = fx.authority(); + let route = TcpListener::bind("127.0.0.1:0").unwrap(); + let tmp = fx.dir.path().join("tmp"); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = + thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); + wait_until("active", || sink.first("active").is_some()); + assert!(!child.terminated.load(Ordering::SeqCst), "killed while it was still starting"); + assert_eq!(child.spawns.load(Ordering::SeqCst), 1); + assert!(sink.first("failed").is_none()); + stop.store(true, Ordering::SeqCst); + assert_eq!(handle.join().unwrap(), Outcome::Stopped); + } + + #[test] + fn a_silent_child_fails_the_probe_within_its_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + // Accepts and never answers. + let held = thread::spawn(move || { + let _connection = listener.accept().unwrap(); + thread::sleep(Duration::from_secs(4)); + }); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let started = Instant::now(); + assert!(!within(Duration::from_secs(5), move || probe.healthy())); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(900), "gave up before its timeout: {elapsed:?}"); + assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its timeout: {elapsed:?}"); + drop(held); + } + + #[test] + fn the_probe_deadline_covers_the_whole_response() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let member = listener.local_addr().unwrap(); + // Every byte arrives inside any per-read timeout, but the head takes + // three seconds in all. + let dribbler = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + for byte in b"HTTP/1.1 200 OK" { + if stream.write_all(&[*byte]).is_err() { + return; + } + thread::sleep(Duration::from_millis(250)); + } + }); + let probe = HttpHealthProbe { member, path: "/health".into() }; + let started = Instant::now(); + assert!(!within(Duration::from_secs(5), move || probe.healthy())); + let elapsed = started.elapsed(); + assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its deadline: {elapsed:?}"); + join_within(Duration::from_secs(10), dribbler); + } + + #[test] + fn a_process_spawner_refuses_an_invalid_lease_descriptor_and_runs_no_child() { + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("ran"); + let script = format!("touch '{}'", marker.display()); + for invalid in [-1, 9999] { + let refused = ProcessSpawner.spawn( + Path::new("/bin/sh"), + &["-c".to_owned(), script.clone()], + Some(invalid), + ); + assert!(refused.is_err(), "descriptor {invalid} was accepted"); + } + thread::sleep(Duration::from_millis(300)); + assert!(!marker.exists(), "a child ran past an invalid descriptor"); + // Control: the same spawn without a descriptor runs the child. + let mut child = ProcessSpawner + .spawn(Path::new("/bin/sh"), &["-c".to_owned(), script], None) + .unwrap(); + wait_until("the control child to exit", || child.exited().is_some()); + assert!(marker.exists()); + } + + #[test] + fn the_member_bind_is_a_loopback_address_other_than_the_candidate() { + let candidate = "127.0.0.1:18835"; + assert_eq!( + parse_member_bind("127.0.0.1:18845", candidate).unwrap(), + "127.0.0.1:18845".parse().unwrap() + ); + for bad in [ + "127.0.0.1:18835", + "10.0.0.5:18845", + "0.0.0.0:18845", + "canary-not-an-address", + "", + ] { + let error = parse_member_bind(bad, candidate).unwrap_err().to_string(); + assert!(!error.contains("canary") && !error.contains("10.0.0.5"), "{error}"); + } + assert!(parse_member_bind("127.0.0.1:18845", "not-an-address").is_err()); + } +} diff --git a/src/lib.rs b/src/lib.rs index b96d835..b5499a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,5 +4,10 @@ pub mod deployment_plan; pub mod drivers; pub mod host; pub mod host_actuator; +#[cfg(unix)] +pub mod launcher; mod incident; +mod runner_evidence; pub mod provisioning; +#[cfg(unix)] +mod steward_orders; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs new file mode 100644 index 0000000..30dd1d2 --- /dev/null +++ b/src/runner_evidence.rs @@ -0,0 +1,805 @@ +//! The output tail of a failed runner step. +//! +//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the +//! last bytes of a failed runner container's stdout and stderr, one record per +//! transaction. The engine's pre-fence failure path is its only writer, and +//! `idunn status --command` its only reader. It is never `control.cc`, which +//! refuses any type it does not know, and never `incidents.cc`, which is +//! published and carries no free text. +//! +//! The tail is raw runner output. It is kept unredacted and readable by root +//! alone (mode 0600): root can already read every secret the runner was given. +//! It never enters an error, a transaction record or the journal. The status +//! reader is a terminal, so everything it prints from this store goes through +//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + TryCompareExchangeSnapshotOutcome, +}; +use serde::{Deserialize, Serialize}; + +use crate::control_plane::{decode_record, require_id, typed_envelope}; +use crate::drivers::{ + LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject, + exchange_behind_private_lock, runner_status, +}; + +const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1"; + +/// Records kept; the writer drops the oldest past this, so the store needs no +/// other owner. +pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32; + +/// `runner-evidence.cc`, beside the control store. +pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf { + state_store.with_file_name("runner-evidence.cc") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum SubjectKind { + Step, + ExternalInput, +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")] +pub(crate) struct RunnerFailureRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) transaction_id: String, + #[cultcache(key = 2)] + pub(crate) command_id: String, + #[cultcache(key = 3)] + pub(crate) target: String, + #[cultcache(key = 4)] + pub(crate) subject_kind: SubjectKind, + #[cultcache(key = 5)] + pub(crate) subject_id: String, + #[cultcache(key = 6)] + pub(crate) runner: String, + #[cultcache(key = 7)] + pub(crate) exit_code: Option, + #[cultcache(key = 8)] + pub(crate) signal: Option, + #[cultcache(key = 9)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 10)] + pub(crate) stdout_total_bytes: u64, + #[cultcache(key = 11)] + pub(crate) stderr_tail: Vec, + #[cultcache(key = 12)] + pub(crate) stderr_total_bytes: u64, + #[cultcache(key = 13)] + pub(crate) recorded_at_unix_millis: u64, +} + +/// Tails stay out of `Debug`, so a failed assertion or a logged record never +/// prints runner output. +impl std::fmt::Debug for RunnerFailureRecord { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerFailureRecord") + .field("transaction_id", &self.transaction_id) + .field("subject_id", &self.subject_id) + .finish_non_exhaustive() + } +} + +impl RunnerFailureRecord { + pub(crate) fn from_failure( + transaction_id: &str, + command_id: &str, + target: &str, + failure: &RunnerStepFailure, + now: u64, + ) -> Self { + let (subject_kind, subject_id) = match &failure.subject { + RunnerSubject::Step(id) => (SubjectKind::Step, id), + RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id), + }; + Self { + schema_version: RUNNER_FAILURE_SCHEMA.to_owned(), + transaction_id: transaction_id.to_owned(), + command_id: command_id.to_owned(), + target: target.to_owned(), + subject_kind, + subject_id: subject_id.clone(), + runner: failure.runner.clone(), + exit_code: failure.exit_code, + signal: failure.signal, + stdout_tail: failure.stdout_tail.clone(), + stdout_total_bytes: failure.stdout_total_bytes, + stderr_tail: failure.stderr_tail.clone(), + stderr_total_bytes: failure.stderr_total_bytes, + recorded_at_unix_millis: now, + } + } + + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == RUNNER_FAILURE_SCHEMA, + "runner failure schema is unsupported" + ); + require_id(&self.transaction_id, "runner failure transaction")?; + require_id(&self.command_id, "runner failure command")?; + require_id(&self.target, "runner failure target")?; + require_id(&self.subject_id, "runner failure subject")?; + require_id(&self.runner, "runner failure runner")?; + for (tail, total, stream) in [ + (&self.stdout_tail, self.stdout_total_bytes, "stdout"), + (&self.stderr_tail, self.stderr_total_bytes, "stderr"), + ] { + ensure!( + tail.len() <= RUNNER_OUTPUT_TAIL_BYTES, + "runner failure {stream} tail exceeds its bound" + ); + ensure!( + tail.len() as u64 <= total, + "runner failure {stream} tail exceeds its stream" + ); + } + Ok(()) + } + + fn envelope(&self) -> Result { + self.validate()?; + typed_envelope( + &self.transaction_id, + Self::TYPE, + RUNNER_FAILURE_SCHEMA, + self, + self.recorded_at_unix_millis, + ) + } +} + +/// One read of the store: the envelopes exactly as stored, and the records +/// decoded from them in the same order. +fn snapshot(path: &Path) -> Result<(Vec, Vec)> { + let envelopes = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading the runner evidence store")?; + let mut records = Vec::new(); + for envelope in &envelopes { + ensure!( + envelope.r#type == RunnerFailureRecord::TYPE + && envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA), + "the runner evidence store holds a document that is not a runner failure" + ); + let record: RunnerFailureRecord = decode_record(envelope)?; + record.validate()?; + ensure!( + envelope.key == record.transaction_id, + "runner evidence store key differs from the record's transaction" + ); + records.push(record); + } + Ok((envelopes, records)) +} + +/// How a write left the store. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kept { + Stored, + /// The store held bytes that do not decode; it was moved to + /// `unreadable_store_path` and the record opened a fresh one. `replaced` + /// is the last-written time (unix milliseconds) of an earlier set-aside + /// store this one displaced, which is gone: one slot, and it says so. + SetAside { replaced: Option }, +} + +/// Where an undecodable store is kept. One slot: the next one replaces it, so +/// the damage a bad store can cost is one extra file. +pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf { + path.with_file_name("runner-evidence.unreadable.cc") +} + +/// Moves the store over the previous set-aside one, root-only, and reports +/// what it displaced. Nothing of the store's content is named: a decode error +/// can quote stored values, and none is carried here. +fn set_aside_unreadable(path: &Path) -> Result> { + let aside = unreadable_store_path(path); + let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| { + earlier + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |since| since.as_millis() as u64) + }); + std::fs::rename(path, &aside) + .context("the runner evidence store is undecodable and could not be set aside")?; + keep_private(&aside)?; + Ok(replaced) +} + +/// How the writer reads the store's bytes; tests inject an I/O error here. +type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result>; + +fn unreadable_fault(error: &std::io::Error) -> anyhow::Error { + anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind()) +} + +/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the +/// oldest records go in the same exchange. The file ends at mode 0600. +/// +/// A store is set aside only when bytes that were read do not decode. An I/O +/// error leaves the store in place, keeps no evidence for that failure, and +/// the error names its kind for the journal. +pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { + record_reading(path, record, &|path| std::fs::read(path)) +} + +fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result { + let first = match read(path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(unreadable_fault(&error)), + }; + let (envelopes, records, how) = match snapshot(path) { + Ok((envelopes, records)) => (envelopes, records, Kept::Stored), + Err(_) => { + let Some(first) = first else { + bail!("the runner evidence store could not be decoded"); + }; + // Undecodable only if the same bytes are still there to read: a + // second read that fails, or finds other bytes, is not a verdict. + match read(path) { + Ok(second) if second == first => {} + Ok(_) => bail!("the runner evidence store changed while it was being read"), + Err(error) => return Err(unreadable_fault(&error)), + } + let replaced = set_aside_unreadable(path)?; + (Vec::new(), Vec::new(), Kept::SetAside { replaced }) + } + }; + let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes + .iter() + .zip(&records) + .filter(|(_, stored)| stored.transaction_id != record.transaction_id) + .map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone())) + .collect(); + kept.push((record.recorded_at_unix_millis, record.envelope()?)); + // Stable: records of one instant keep their stored order, so the record + // being written survives a tie. + kept.sort_by_key(|(recorded_at, _)| *recorded_at); + let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS); + let replacements: Vec<_> = kept + .into_iter() + .skip(excess) + .map(|(_, envelope)| envelope) + .collect(); + match exchange_behind_private_lock(path, &envelopes, &replacements)? { + TryCompareExchangeSnapshotOutcome::Exchanged => {} + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("the runner evidence store changed while it was being written") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + return Err(LockContended { + store: "runner evidence store", + } + .into()); + } + } + keep_private(path)?; + Ok(how) +} + +#[cfg(unix)] +fn keep_private(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("making the runner evidence store private") +} + +#[cfg(not(unix))] +fn keep_private(_path: &Path) -> Result<()> { + Ok(()) +} + +/// The one reader. An absent store holds nothing. +pub(crate) fn for_transaction( + path: &Path, + transaction_id: &str, +) -> Result> { + Ok(snapshot(path)? + .1 + .into_iter() + .find(|stored| stored.transaction_id == transaction_id)) +} + +/// The record as `idunn status --command` shows it, as raw lines. They are +/// not text yet: `show_status` is the only caller, and `emit` the only +/// writer. +fn render(record: &RunnerFailureRecord) -> Vec> { + let noun = match record.subject_kind { + SubjectKind::Step => "step", + SubjectKind::ExternalInput => "external input", + }; + let mut lines: Vec> = vec![ + format!(" failed {noun} {}", record.subject_id).into_bytes(), + format!(" runner {}", record.runner).into_bytes(), + format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(), + ]; + for (stream, tail, total) in [ + ("stdout", &record.stdout_tail, record.stdout_total_bytes), + ("stderr", &record.stderr_tail, record.stderr_total_bytes), + ] { + lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes()); + let text = tail.strip_suffix(b"\n").unwrap_or(tail); + if !text.is_empty() { + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); + } + } + lines +} + +/// What `idunn status --command` prints for a transaction's runner evidence: +/// its record, or one line naming why the store is unreadable. Nothing is +/// written when the transaction has no record. +pub(crate) fn show_status( + out: &mut impl std::io::Write, + store: &Path, + transaction_id: &str, +) -> std::io::Result<()> { + let lines = match for_transaction(store, transaction_id) { + Ok(Some(record)) => render(&record), + Ok(None) => return Ok(()), + Err(error) => { + vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()] + } + }; + emit(out, &lines) +} + +/// The one sink: every byte the status read prints from the evidence store, +/// or from an error about it, is written here and nowhere else. An allowlist: +/// printable ASCII and tab pass; every other code point is a visible +/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline +/// inside a line is escaped too: this function writes the only ones. +pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { + for line in lines { + out.write_all(escape(line).as_bytes())?; + out.write_all(b"\n")?; + } + Ok(()) +} + +pub(crate) fn escape(bytes: &[u8]) -> String { + let mut shown = String::with_capacity(bytes.len()); + for chunk in bytes.utf8_chunks() { + for character in chunk.valid().chars() { + match character { + ' '..='~' | '\t' => shown.push(character), + _ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))), + } + } + for byte in chunk.invalid() { + shown.push_str(&format!("\\x{byte:02x}")); + } + } + shown +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject, + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stderr.to_vec(), + stderr_total_bytes: stderr.len() as u64, + } + } + + /// What status writes for a record, through the one sink. + fn shown(record: &RunnerFailureRecord) -> String { + let mut written = Vec::new(); + emit(&mut written, &render(record)).expect("a Vec takes any write"); + String::from_utf8(written).expect("the sink writes ASCII only") + } + + fn shown_lines(record: &RunnerFailureRecord) -> Vec { + shown(record).lines().map(str::to_owned).collect() + } + + fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { + RunnerFailureRecord::from_failure( + transaction, + "up-service", + "service", + &failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"), + now, + ) + } + + #[test] + fn the_evidence_store_keeps_the_newest_32() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // The literal, not the constant: the spec's bound is 32. + for index in 0..=32 { + record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; + } + let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); + assert_eq!(kept.len(), 32); + assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); + assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); + assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); + // Rewriting a resident transaction replaces it and drops nothing. + let mut again = record_at("tx-05", 5000); + again.stdout_tail = b"rewritten\n".to_vec(); + again.stdout_total_bytes = 10; + record(&path, &again)?; + let kept = snapshot(&path)?.1; + assert_eq!(kept.len(), 32); + assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); + let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; + assert_eq!(stored.stdout_tail, b"rewritten\n"); + assert_eq!(for_transaction(&path, "tx-00")?, None); + Ok(()) + } + + #[test] + fn a_record_round_trips_byte_for_byte() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let written = RunnerFailureRecord::from_failure( + "tx-a", + "up-service", + "service", + &failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""), + 77, + ); + record(&path, &written)?; + assert_eq!(for_transaction(&path, "tx-a")?, Some(written)); + assert_eq!(for_transaction(&path, "tx-b")?, None); + Ok(()) + } + + #[test] + fn the_evidence_store_is_private() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // Whatever the umask, the mode is the store's own rule, on every write. + for index in 0..2 { + record(&path, &record_at(&format!("tx-{index}"), 10 + index))?; + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn a_tail_over_its_bound_or_its_stream_is_refused() { + let mut oversize = record_at("tx-a", 1); + oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1]; + oversize.stdout_total_bytes = u64::MAX; + assert!(oversize.validate().is_err()); + let mut impossible = record_at("tx-a", 1); + impossible.stderr_total_bytes = 1; + assert!(impossible.validate().is_err()); + let mut exact = record_at("tx-a", 1); + exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES]; + exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64; + assert!(exact.validate().is_ok()); + } + + #[test] + fn status_renders_the_tail_with_control_bytes_escaped() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec(); + stored.stdout_total_bytes = 5000; + stored.stderr_tail = b"bad\r\x07".to_vec(); + stored.stderr_total_bytes = 6; + let lines = shown_lines(&stored); + let text = lines.join("\n"); + assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); + assert!(!text.contains('\u{7}')); + assert_eq!( + lines, + vec![ + " failed step build", + " runner rust", + " status exit status 3", + " stdout (last 30 of 5000 bytes)", + " | first", + " | \\u{1b}[2Jsecond\\u{0}end\tTab", + " | last", + " stderr (last 5 of 6 bytes)", + " | bad\\u{d}\\u{7}", + ] + ); + } + + #[test] + fn debug_names_the_record_and_prints_no_tail() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"SECRET-OUTPUT".to_vec(); + stored.stdout_total_bytes = 13; + let debug = format!("{stored:?}"); + assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}"); + } + + #[test] + fn an_empty_tail_renders_only_its_header() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = Vec::new(); + stored.stdout_total_bytes = 0; + let lines = shown_lines(&stored); + assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); + assert!(!lines.iter().any(|line| line == " | ")); + } + + #[test] + fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 not a store")?; + assert!(for_transaction(&path, "tx-a").is_err()); + Ok(()) + } + + fn files_beside(path: &Path) -> Result> { + let mut names: Vec = std::fs::read_dir(path.parent().context("no parent")?)? + .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned())) + .collect::>()?; + names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock")); + names.sort(); + Ok(names) + } + + /// A store a newer Idunn wrote: well-formed, but a schema this one does not know. + fn write_future_store(path: &Path) -> Result<()> { + let mut future = record_at("tx-future", 1); + future.schema_version = "idunn.runner_failure.v2".into(); + let envelope = typed_envelope( + &future.transaction_id, + RunnerFailureRecord::TYPE, + "idunn.runner_failure.v2", + &future, + 1, + )?; + let outcome = exchange_behind_private_lock(path, &[], &[envelope])?; + ensure!( + matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged), + "the future store was not written" + ); + Ok(()) + } + + #[test] + fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damage = b"\xc1 not a store \x00 damage"; + std::fs::write(&path, damage)?; + let kept = record(&path, &record_at("tx-a", 5))?; + assert_eq!(kept, Kept::SetAside { replaced: None }); + // Kept, not deleted: the damage is where the journal says, root-only. + let aside = unreadable_store_path(&path); + assert_eq!(std::fs::read(&aside)?, damage); + assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600); + // The failure's evidence is in a fresh, private store. + assert_eq!( + for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id), + Some("tx-a".to_owned()) + ); + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + // A healthy store is not set aside again. + assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored); + assert_eq!(std::fs::read(&aside)?, damage); + Ok(()) + } + + #[test] + fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + write_future_store(&path)?; + let before = std::fs::read(&path)?; + assert!(for_transaction(&path, "tx-future").is_err()); + assert_eq!( + record(&path, &record_at("tx-a", 5))?, + Kept::SetAside { replaced: None } + ); + assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); + assert!(for_transaction(&path, "tx-a")?.is_some()); + assert_eq!(for_transaction(&path, "tx-future")?, None); + Ok(()) + } + + #[test] + fn at_most_one_unreadable_store_is_kept() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"]; + for (index, damage) in damages.into_iter().enumerate() { + std::fs::write(&path, damage)?; + record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?; + assert_eq!( + std::fs::read(unreadable_store_path(&path))?, + damage, + "the newest damage is the one kept" + ); + } + assert_eq!( + files_beside(&path)?, + vec![ + "runner-evidence.cc".to_owned(), + "runner-evidence.unreadable.cc".to_owned() + ] + ); + Ok(()) + } + + #[test] + fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // A directory where the store should be: unreadable, not undecodable. + std::fs::create_dir(&path)?; + assert!(record(&path, &record_at("tx-a", 1)).is_err()); + assert!(path.is_dir()); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn every_code_point_but_printable_ascii_is_a_visible_escape() { + // The allowlist, exhaustively: no class list decides anything. + for code in 0..=0x10ffffu32 { + let Some(character) = char::from_u32(code) else { + continue; + }; + let text = character.to_string(); + let shown = escape(text.as_bytes()); + if matches!(character, ' '..='~' | '\t') { + assert_eq!(shown, text, "U+{code:04X} should pass"); + } else { + assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}"); + } + } + // Invalid UTF-8 is escaped byte by byte; valid text round it passes. + assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82"); + assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}"); + } + + #[test] + fn a_newline_inside_a_line_cannot_forge_a_line() { + let mut stored = record_at("tx-a", 1); + stored.subject_id = "build\n status exit status 0".into(); + let lines = shown_lines(&stored); + assert_eq!(lines[0], " failed step build\\u{a} status exit status 0"); + assert_eq!( + lines.iter().filter(|line| line.starts_with(" status ")).count(), + 1 + ); + } + + /// A store whose first record is well-formed except that its subject kind + /// holds `marker`: the decode error quotes it. + fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> { + record(path, &record_at("tx-a", 1))?; + let mut bytes = std::fs::read(path)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker")?; + bytes[at..at + 4].copy_from_slice(marker); + std::fs::write(path, bytes)?; + Ok(()) + } + + #[test] + fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"\x1b[2J")?; + let mut written = Vec::new(); + show_status(&mut written, &path, "tx-a")?; + assert!(!written.contains(&0x1b), "an escape byte reached the terminal"); + assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n'))); + let text = String::from_utf8(written)?; + assert!(text.starts_with(" runner evidence unreadable: "), "{text}"); + assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}"); + Ok(()) + } + + #[test] + fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"CNRY")?; + let damage = std::fs::read(&path)?; + // Sanity: this damage decodes to an error that quotes the canary. + assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY")); + // A non-empty directory where the aside goes: rename fails even as root. + let aside = unreadable_store_path(&path); + std::fs::create_dir(&aside)?; + std::fs::write(aside.join("occupied"), b"x")?; + let error = record(&path, &record_at("tx-b", 2)).unwrap_err(); + for text in [format!("{error:#}"), format!("{error:?}")] { + assert!(text.contains("could not be set aside"), "{text}"); + assert!(!text.contains("CNRY"), "stored content reached the error: {text}"); + } + // The failure cost nothing: the store is as it was, with no evidence + // written over it. + assert_eq!(std::fs::read(&path)?, damage); + assert!(aside.join("occupied").exists()); + Ok(()) + } + + #[test] + fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + record(&path, &record_at("tx-a", 1))?; + let before = std::fs::read(&path)?; + let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied); + // The first read fails on a healthy store. + let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, before, "the store changed"); + assert!(!unreadable_store_path(&path).exists()); + assert!(for_transaction(&path, "tx-b")?.is_none()); + // The second read, the one that confirms the bytes do not decode, fails. + std::fs::write(&path, b"\xc1 not a store")?; + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| { + calls.set(calls.get() + 1); + if calls.get() == 2 { + return Err(denied()); + } + std::fs::read(path) + }; + let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + // Bytes that changed between the reads are no verdict either. + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| -> std::io::Result> { + calls.set(calls.get() + 1); + let mut bytes = std::fs::read(path)?; + bytes.push(calls.get() as u8); + Ok(bytes) + }; + assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err()); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 first")?; + assert_eq!( + record(&path, &record_at("tx-a", 1))?, + Kept::SetAside { replaced: None } + ); + std::fs::write(&path, b"\xc1 second")?; + let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else { + bail!("a displaced set-aside store went unreported"); + }; + assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second"); + Ok(()) + } +} diff --git a/src/steward_orders.rs b/src/steward_orders.rs new file mode 100644 index 0000000..d4be1ee --- /dev/null +++ b/src/steward_orders.rs @@ -0,0 +1,4079 @@ +//! Typed orders from Idunn's steward. +//! +//! `idunn orders serve` is a second root unit from the same binary as +//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints +//! order ids, decides acceptance and refusal, actuates, and records outcomes. +//! The survival daemon takes no part: `control.cc` never learns an order type. +//! +//! An order is a closed enum, never a command string: install packages, act on +//! a systemd unit, or deploy a target through the same commit primitive +//! `idunn up` uses. Every unit is allowed except the always-refused set, which +//! is compiled in here and read from nowhere: no binding field, record or flag +//! holds it. The `#operator` post is a retried projection of the store. +//! +//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the +//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and +//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE +//! `mind-control check`; START record before actuation, at most once; RUN; +//! RESULT outcome; POST accepted and result. +//! +//! No refusal or error here carries a value the issuer supplied: it names the +//! field and a code. A refused unit that a record names was resolved by +//! systemd and passed the unit grammar. + +use std::fmt; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::control_plane::{ + command_state, commit_deployment_command, decode_record, default_state_store, now_millis, + path_value, require_id, require_selector, string_value, typed_envelope, +}; +use crate::runner_evidence::{emit, escape}; + +const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; +const START_SCHEMA: &str = "idunn.steward_order_start.v1"; +const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; +const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; + +const REQUEST_FRAME_LIMIT: usize = 16 * 1024; +const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; +const TAIL_BYTES: usize = 8 * 1024; +const REASON_LIMIT: usize = 300; +const PACKAGE_LIMIT: usize = 16; +const UNIT_LIMIT: usize = 200; +const LIST_LIMIT: usize = 50; +const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; + +/// The always-refused set: Idunn's own units and the minds' own. Matched on +/// every name a unit answers to, ASCII case-insensitively (systemd units are +/// case-sensitive, so the fold only widens the refusal). +fn is_refused_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + ["idunn-", "mind-self@", "mind-persona@"] + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +/// Units whose binaries suspend, hibernate or open recovery: no property +/// reveals it, so they are refused by name, on every name they answer to. +const HOST_NAMES: [&str; 6] = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", +]; + +fn is_host_name(name: &str) -> bool { + HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) +} + +// ---------------------------------------------------------------- grammar -- + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Code { + Empty, + TooLong, + TooMany, + BadCharacter, + BadShape, + Malformed, + Unauthorized, + Unknown, + Unavailable, +} + +/// A refusal names the field and a code. It never carries the offending value. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Refusal { + pub(crate) field: String, + pub(crate) code: Code, +} + +impl Refusal { + fn new(field: &'static str, code: Code) -> Self { + Self { + field: field.to_owned(), + code, + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{} refused: {:?}", self.field, self.code) + } +} + +impl std::error::Error for Refusal {} + +fn validate_package(spec: &str) -> Result<(), Refusal> { + const FIELD: &str = "package"; + if spec.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if spec.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let (name, version) = match spec.split_once('=') { + Some((name, version)) => (name, Some(version)), + None => (spec, None), + }; + let bytes = name.as_bytes(); + if bytes.len() < 2 { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); + let rest = bytes[1..].iter().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') + }); + if !head || !rest { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + // apt reads `name-` as a removal and `name.deb` as a local file. + if name.ends_with('-') || name.ends_with(".deb") { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if let Some(version) = version { + if version.is_empty() { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if !version.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') + }) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + } + Ok(()) +} + +fn validate_unit(unit: &str) -> Result<(), Refusal> { + const FIELD: &str = "unit"; + if unit.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if unit.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let allowed = unit.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') + }); + if !allowed || unit.starts_with('-') { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + match UNIT_SUFFIXES + .iter() + .find_map(|suffix| unit.strip_suffix(suffix)) + { + Some(stem) if !stem.is_empty() => Ok(()), + _ => Err(Refusal::new(FIELD, Code::BadShape)), + } +} + +fn validate_reason(reason: &str) -> Result<(), Refusal> { + const FIELD: &str = "reason"; + if reason.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if reason.len() > REASON_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Verb { + Start, + Stop, + Restart, + Reload, +} + +impl Verb { + fn as_str(self) -> &'static str { + match self { + Verb::Start => "start", + Verb::Stop => "stop", + Verb::Restart => "restart", + Verb::Reload => "reload", + } + } + + fn parse(word: &str) -> Option { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + .into_iter() + .find(|verb| verb.as_str() == word) + } +} + +/// The closed set of things an order can ask for. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Operation { + PackageInstall { packages: Vec }, + UnitAction { verb: Verb, unit: String }, + Deploy { selector: String }, +} + +/// One fixed argv for `systemd-run`, built from the typed operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Step { + pub(crate) argv: Vec, + pub(crate) environment: Vec<(String, String)>, +} + +impl Operation { + fn validate(&self) -> Result<(), Refusal> { + match self { + Operation::PackageInstall { packages } => { + if packages.is_empty() { + return Err(Refusal::new("package", Code::Empty)); + } + if packages.len() > PACKAGE_LIMIT { + return Err(Refusal::new("package", Code::TooMany)); + } + packages.iter().try_for_each(|package| validate_package(package)) + } + Operation::UnitAction { unit, .. } => validate_unit(unit), + Operation::Deploy { selector } => require_selector(selector) + .map_err(|_| Refusal::new("selector", Code::BadShape)), + } + } + + fn steps(&self) -> Vec { + let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); + let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; + match self { + Operation::PackageInstall { packages } => { + // Never a removal, never a pattern: each name selects exactly + // itself, and apt aborts rather than displace an installed package. + let mut install: Vec = + words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); + let mut present: Vec = words(&["dpkg", "-s", "--"]); + for package in packages { + let (name, version) = package.split_once('=').unwrap_or((package, "")); + install.push(match version { + "" => format!("?exact-name({name})"), + version => format!("?exact-name({name})={version}"), + }); + present.push(name.to_owned()); + } + // An exact name that matches nothing installs nothing and exits 0, + // so the last step fails unless every named package is installed. + vec![ + Step { + argv: words(&["apt-get", "update", "-q"]), + environment: apt_environment.clone(), + }, + Step { + argv: install, + environment: apt_environment, + }, + Step { + argv: present, + environment: Vec::new(), + }, + ] + } + Operation::UnitAction { verb, unit } => vec![Step { + argv: vec![ + "systemctl".into(), + verb.as_str().into(), + "--job-mode=ignore-requirements".into(), + "--".into(), + unit.clone(), + ], + environment: Vec::new(), + }], + Operation::Deploy { .. } => Vec::new(), + } + } + + fn describe(&self) -> String { + match self { + Operation::PackageInstall { packages } => { + format!("package-install {}", packages.join(" ")) + } + Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), + Operation::Deploy { selector } => format!("deploy {selector}"), + } + } +} + +// ---------------------------------------------------------------- records -- + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] +pub(crate) struct OrderRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) principal: String, + #[cultcache(key = 3)] + pub(crate) issuer_uid: u32, + #[cultcache(key = 4)] + pub(crate) issuer_unit: String, + #[cultcache(key = 5)] + pub(crate) issuer_invocation: String, + #[cultcache(key = 6)] + pub(crate) issued_at_unix_millis: u64, + #[cultcache(key = 7)] + pub(crate) operation: Operation, + #[cultcache(key = 8)] + pub(crate) reason: String, +} + +impl fmt::Debug for OrderRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OrderRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_start", + schema = "idunn.steward_order_start.v1" +)] +pub(crate) struct StartRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) started_at_unix_millis: u64, + /// The transient unit name, or the deployment command id. + #[cultcache(key = 3)] + pub(crate) actuation: String, +} + +impl fmt::Debug for StartRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StartRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Via { + Name, + Alias, + Host, + OnEnd, + Bound, + Upholds, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Refused { + AlwaysRefused { unit: String, via: Via }, + /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload + /// of the wrapper exits 0 and leaves the member alone. + Wrapper { member: String }, + Braked, + /// The order's principal or issuing uid is not the steward bound now. + NotSteward, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Outcome { + Ran, + Failed { + exit_code: Option, + signal: Option, + }, + Refused(Refused), + Lost, + Submitted { command_id: String }, +} + +impl Outcome { + fn describe(&self) -> String { + match self { + Outcome::Ran => "ran (exit 0)".into(), + Outcome::Failed { + exit_code: Some(code), + .. + } => format!("failed (exit {code})"), + Outcome::Failed { + signal: Some(signal), + .. + } => format!("failed (signal {signal})"), + Outcome::Failed { .. } => "failed (not run to an exit)".into(), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( + "refused: always-refused unit {} reached by {}", + escape(unit.as_bytes()), + match via { + Via::Name => "name", + Via::Alias => "alias", + Via::Host => "host", + Via::OnEnd => "on-end", + Via::Bound => "bound", + Via::Upholds => "upholds", + } + ), + Outcome::Refused(Refused::Wrapper { member }) => format!( + "refused: wrapper unit, it only wraps {}", + escape(member.as_bytes()) + ), + Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Refused(Refused::NotSteward) => { + "refused: the issuer is not the current steward".into() + } + Outcome::Lost => "lost: the orders service restarted mid-run".into(), + Outcome::Submitted { command_id } => { + format!("submitted deployment {}", escape(command_id.as_bytes())) + } + } + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_outcome", + schema = "idunn.steward_order_outcome.v1" +)] +pub(crate) struct OutcomeRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) ended_at_unix_millis: u64, + #[cultcache(key = 3)] + pub(crate) outcome: Outcome, + #[cultcache(key = 4)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 5)] + pub(crate) stderr_tail: Vec, +} + +/// Tails stay out of `Debug`: they are raw program output. +impl fmt::Debug for OutcomeRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutcomeRecord") + .field("order_id", &self.order_id) + .field("outcome", &self.outcome) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum PostKind { + Accepted, + Result, +} + +impl PostKind { + fn as_str(self) -> &'static str { + match self { + PostKind::Accepted => "accepted", + PostKind::Result => "result", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_post", + schema = "idunn.steward_order_post.v1" +)] +pub(crate) struct PostRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `:`. + #[cultcache(key = 1)] + pub(crate) post_key: String, + #[cultcache(key = 2)] + pub(crate) event_id: String, + #[cultcache(key = 3)] + pub(crate) posted_at_unix_millis: u64, +} + +// ------------------------------------------------------------------ store -- + +/// `orders.cc`: root-only, written by `idunn orders serve` alone. +pub(crate) struct OrderStore { + directory: PathBuf, +} + +#[derive(Default)] +struct Ledger { + orders: Vec, + starts: Vec, + outcomes: Vec, + posts: Vec, +} + +impl Ledger { + fn start_of(&self, order_id: &str) -> Option<&StartRecord> { + self.starts.iter().find(|start| start.order_id == order_id) + } + + fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { + self.outcomes + .iter() + .find(|outcome| outcome.order_id == order_id) + } + + fn posted(&self, order_id: &str, kind: PostKind) -> bool { + let key = post_key(order_id, kind); + self.posts.iter().any(|post| post.post_key == key) + } +} + +fn post_key(order_id: &str, kind: PostKind) -> String { + format!("{order_id}:{}", kind.as_str()) +} + +impl OrderStore { + pub(crate) fn new(directory: &Path) -> Self { + Self { + directory: directory.to_owned(), + } + } + + fn path(&self) -> PathBuf { + self.directory.join("orders.cc") + } + + fn prepare(&self) -> Result<()> { + std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; + std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) + .context("making the orders directory private") + } + + /// Insert a record whose (type, key) is absent. `false` means it was there. + fn put(&self, key: &str, schema: &str, value: &T) -> Result { + self.prepare()?; + let path = self.path(); + let now = now_millis()?; + let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: T::TYPE.into(), + key: key.into(), + current: None, + }], + &[typed_envelope(key, T::TYPE, schema, value, now)?], + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .context("making the orders store private")?; + Ok(inserted) + } + + fn ledger(&self) -> Result { + let path = self.path(); + let mut ledger = Ledger::default(); + if !path.exists() { + return Ok(ledger); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .context("reading the orders store")?; + for envelope in &envelopes { + let schema = envelope.schema_id.as_deref(); + let key = envelope.key.as_str(); + match envelope.r#type.as_str() { + OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { + let record: OrderRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an order is stored under another key"); + ledger.orders.push(record); + } + StartRecord::TYPE if schema == Some(START_SCHEMA) => { + let record: StartRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "a start is stored under another key"); + ledger.starts.push(record); + } + OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { + let record: OutcomeRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an outcome is stored under another key"); + ledger.outcomes.push(record); + } + PostRecord::TYPE if schema == Some(POST_SCHEMA) => { + let record: PostRecord = decode_record(envelope)?; + ensure!(record.post_key == key, "a post is stored under another key"); + ledger.posts.push(record); + } + _ => bail!("the orders store holds a document that is not an order record"), + } + } + ledger + .orders + .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); + Ok(ledger) + } +} + +// ------------------------------------------------------------------ ports -- + +/// The properties naming what systemd does when a unit ends or its job times +/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). +const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; + +/// The properties naming units systemd starts when a unit ends. +const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; + +/// What `systemctl show` says about one unit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct UnitFacts { + pub(crate) id: String, + pub(crate) names: Vec, + pub(crate) not_found: bool, + /// The properties that make the unit act when it ends: each emergency + /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. + pub(crate) on_end: Vec, + /// The units bound to this one at runtime (`BoundBy=`): systemd stops + /// them when this unit stops, whatever the job mode. + pub(crate) bound_by: Vec, + /// The units this one upholds (`Upholds=`): systemd starts them in a full + /// transaction whenever this unit is active, whatever the job mode. + pub(crate) upholds: Vec, + /// The units this one groups (`ConsistsOf=`): the verbs reach them only + /// through propagation, which the job mode removes. + pub(crate) consists_of: Vec, +} + +impl UnitFacts { + fn identities(&self) -> impl Iterator { + std::iter::once(&self.id).chain(self.names.iter()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RunReport { + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stderr_tail: Vec, +} + +pub(crate) trait Systemd: Send + Sync { + fn show(&self, unit: &str) -> Result; + fn invocation(&self, unit: &str) -> Option; + /// Run one step as the transient unit `unit`, to its end. + fn run(&self, unit: &str, step: &Step) -> Result; +} + +pub(crate) struct Peer { + pub(crate) uid: u32, + pub(crate) issuer_unit: String, +} + +pub(crate) trait PeerCredentials: Send + Sync { + fn peer(&self, stream: &UnixStream) -> Result; +} + +pub(crate) trait BrakeCheck: Send + Sync { + /// `true` only when the instance's brake check exits 0. + fn clear(&self, principal: &str) -> bool; +} + +pub(crate) trait Homeserver: Send + Sync { + /// Idempotent by `txn_id`; returns the event id. + fn post(&self, txn_id: &str, text: &str) -> Result; +} + +pub(crate) struct Ctx { + pub(crate) store: OrderStore, + pub(crate) state_store: PathBuf, + pub(crate) steward_uid: u32, + pub(crate) principal: String, + pub(crate) systemd: Arc, + pub(crate) peers: Arc, + pub(crate) brake: Arc, + pub(crate) homeserver: Arc, +} + +// ------------------------------------------------------------------ scope -- + +fn refused(unit: &str, via: Via) -> Refused { + Refused::AlwaysRefused { + unit: unit.to_owned(), + via, + } +} + +/// Would `verb` on `unit` be refused. systemd builds the transaction and the +/// order tells it to build one job (`--job-mode=ignore-requirements`), so this +/// reads only what that job mode cannot remove from the one named unit: the +/// name it answers to, what fires when it ends, who is bound to it, what it upholds and +/// what it only wraps. +fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { + if is_refused_name(unit) { + return Ok(Some(refused(unit, Via::Name))); + } + let facts = systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + for name in facts.identities() { + if is_refused_name(name) { + return Ok(Some(refused(name, Via::Alias))); + } + if is_host_name(name) { + return Ok(Some(refused(name, Via::Host))); + } + } + if verb != Verb::Reload && !facts.on_end.is_empty() { + return Ok(Some(refused(unit, Via::OnEnd))); + } + if matches!(verb, Verb::Stop | Verb::Restart) + && let Some(bound) = facts.bound_by.first() + { + return Ok(Some(refused(bound, Via::Bound))); + } + if matches!(verb, Verb::Start | Verb::Restart) && !facts.upholds.is_empty() { + return Ok(Some(refused(unit, Via::Upholds))); + } + if verb != Verb::Start + && let Some(member) = facts.consists_of.first() + { + return Ok(Some(Refused::Wrapper { + member: member.clone(), + })); + } + Ok(None) +} + +// -------------------------------------------------------------- execution -- + +fn tail(bytes: &[u8]) -> Vec { + bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() +} + +fn conclude( + ctx: &Ctx, + order_id: &str, + outcome: Outcome, + stdout_tail: &[u8], + stderr_tail: &[u8], +) -> Result<()> { + ctx.store.put( + order_id, + OUTCOME_SCHEMA, + &OutcomeRecord { + schema_version: OUTCOME_SCHEMA.into(), + order_id: order_id.into(), + ended_at_unix_millis: now_millis()?, + outcome, + stdout_tail: tail(stdout_tail), + stderr_tail: tail(stderr_tail), + }, + )?; + Ok(()) +} + +fn order_uuid(order_id: &str) -> &str { + order_id.strip_prefix("ord-").unwrap_or(order_id) +} + +pub(crate) fn requester(principal: &str, order_id: &str) -> String { + format!("steward:{principal} order {order_id}") +} + +/// Run one accepted order. A start record is written before anything is +/// actuated and an order that has one is never run again. +pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { + let ledger = ctx.store.ledger()?; + let order = ledger + .orders + .iter() + .find(|order| order.order_id == order_id) + .context("the order is not in the store")?; + if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { + return Ok(()); + } + // One gate for a fresh order and a recovered one: who may order is what the + // binding says now, not what it said when the order was accepted. + if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { + return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); + } + if order.operation.validate().is_err() { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the stored order no longer satisfies the grammar", + ); + } + if let Operation::UnitAction { verb, unit } = &order.operation { + match refusal_for(ctx.systemd.as_ref(), unit, *verb) { + Ok(None) => {} + Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), + Err(_) => { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the unit could not be resolved, so it was not acted on", + ); + } + } + } + if !ctx.brake.clear(&ctx.principal) { + return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); + } + let uuid = order_uuid(order_id); + let command_id = format!("up-{uuid}"); + let steps = order.operation.steps(); + let actuation = match &order.operation { + Operation::Deploy { .. } => command_id.clone(), + _ => format!("idunn-order-{uuid}"), + }; + let started = StartRecord { + schema_version: START_SCHEMA.into(), + order_id: order_id.into(), + started_at_unix_millis: now_millis()?, + actuation, + }; + if !ctx.store.put(order_id, START_SCHEMA, &started)? { + return Ok(()); + } + if let Operation::Deploy { selector } = &order.operation { + let outcome = match commit_deployment_command( + &ctx.state_store, + &command_id, + selector, + &requester(&order.principal, order_id), + ) { + Ok(_) => Outcome::Submitted { command_id }, + Err(_) => Outcome::Failed { + exit_code: None, + signal: None, + }, + }; + return conclude(ctx, order_id, outcome, b"", b""); + } + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let mut outcome = Outcome::Ran; + for (index, step) in steps.iter().enumerate() { + let unit = if steps.len() == 1 { + format!("idunn-order-{uuid}") + } else { + format!("idunn-order-{uuid}-{}", index + 1) + }; + match ctx.systemd.run(&unit, step) { + Ok(report) => { + stdout = report.stdout_tail; + stderr = report.stderr_tail; + if report.exit_code != Some(0) { + outcome = Outcome::Failed { + exit_code: report.exit_code, + signal: report.signal, + }; + break; + } + } + Err(_) => { + outcome = Outcome::Failed { + exit_code: None, + signal: None, + }; + stderr = b"the transient unit could not be started".to_vec(); + break; + } + } + } + conclude(ctx, order_id, outcome, &stdout, &stderr) +} + +/// On service start: an order with a start record and no outcome ran, or began +/// to, in a life that ended. It is Lost and never run again. Returns the +/// orders that never started, oldest first. +pub(crate) fn recover(ctx: &Ctx) -> Result> { + let ledger = ctx.store.ledger()?; + for start in &ledger.starts { + if ledger.outcome_of(&start.order_id).is_none() { + conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; + } + } + Ok(ledger + .orders + .iter() + .filter(|order| { + ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() + }) + .map(|order| order.order_id.clone()) + .collect()) +} + +// ---------------------------------------------------------------- posting -- + +struct Due { + order_id: String, + kind: PostKind, + text: String, +} + +fn quoted(reason: &str) -> String { + format!("\"{}\"", escape(reason.as_bytes())) +} + +fn accepted_text(order: &OrderRecord) -> String { + format!( + "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", + order.order_id, + escape(order.operation.describe().as_bytes()), + escape(order.issuer_unit.as_bytes()), + escape(order.issuer_invocation.as_bytes()), + quoted(&order.reason) + ) +} + +/// A deploy's command has finished once its state is neither queued nor running. +fn deploy_finished(state: &str) -> bool { + !matches!(state, "queued" | "running") +} + +/// What is owed to the room. `deploy_state` is a deployment command's derived +/// state and detail, read from the control store by the caller. +fn due_posts( + ledger: &Ledger, + deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, +) -> Vec { + let mut due = Vec::new(); + for order in &ledger.orders { + let id = &order.order_id; + if !ledger.posted(id, PostKind::Accepted) { + due.push(Due { + order_id: id.clone(), + kind: PostKind::Accepted, + text: accepted_text(order), + }); + } + if ledger.posted(id, PostKind::Result) { + continue; + } + let Some(record) = ledger.outcome_of(id) else { + continue; + }; + let mut line = record.outcome.describe(); + if let Outcome::Submitted { command_id } = &record.outcome { + match deploy_state(command_id) { + Ok((state, detail)) if deploy_finished(state) => { + line = format!("{line}: {state} {}", escape(detail.as_bytes())); + } + _ => continue, + } + } + due.push(Due { + order_id: id.clone(), + kind: PostKind::Result, + text: format!( + "Steward order {id}: {line}\n{}", + escape(order.operation.describe().as_bytes()) + ), + }); + } + due +} + +/// Post what is due, in order, stopping at the first failure. Returns how many +/// went out. +pub(crate) fn post_pass(ctx: &Ctx) -> Result { + let ledger = ctx.store.ledger()?; + let mut sent = 0; + let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); + for due in due_posts(&ledger, &deploy_state) { + let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); + let event_id = ctx.homeserver.post(&transaction, &due.text)?; + ctx.store.put( + &post_key(&due.order_id, due.kind), + POST_SCHEMA, + &PostRecord { + schema_version: POST_SCHEMA.into(), + post_key: post_key(&due.order_id, due.kind), + event_id, + posted_at_unix_millis: now_millis()?, + }, + )?; + sent += 1; + } + Ok(sent) +} + +// ----------------------------------------------------------------- frames -- + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct OrderRequest { + pub(crate) operation: Operation, + pub(crate) reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ShowRequest { + pub(crate) id: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Request { + Order(OrderRequest), + Show(ShowRequest), + List, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderView { + pub(crate) order: OrderRecord, + pub(crate) start: Option, + pub(crate) outcome: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderLine { + pub(crate) order_id: String, + pub(crate) status: String, + pub(crate) summary: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Response { + Accepted { order_id: String }, + Refused(Refusal), + Show(Box), + List(Vec), +} + +fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { + let bytes = rmp_serde::to_vec_named(value)?; + stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; + stream.write_all(&bytes)?; + Ok(stream.flush()?) +} + +fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { + let mut length = [0u8; 4]; + stream.read_exact(&mut length)?; + let length = u32::from_be_bytes(length) as usize; + ensure!(length <= limit, "a frame exceeds its limit"); + let mut bytes = vec![0u8; length]; + stream.read_exact(&mut bytes)?; + Ok(bytes) +} + +fn status_word(view: &OrderView) -> &'static str { + match (&view.outcome, &view.start) { + (Some(record), _) => match record.outcome { + Outcome::Ran => "ran", + Outcome::Failed { .. } => "failed", + Outcome::Refused(_) => "refused", + Outcome::Lost => "lost", + Outcome::Submitted { .. } => "submitted", + }, + (None, Some(_)) => "running", + (None, None) => "accepted", + } +} + +fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { + OrderView { + order: order.clone(), + start: ledger.start_of(&order.order_id).cloned(), + outcome: ledger.outcome_of(&order.order_id).cloned(), + } +} + +fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { + write_frame(stream, &response) +} + +fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { + answer(stream, Response::Refused(Refusal::new(field, code))) +} + +/// Serve one connection. The peer is authenticated before its frame is read. +pub(crate) fn handle( + ctx: &Ctx, + stream: &mut UnixStream, + queue: &mpsc::Sender, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + let peer = ctx.peers.peer(stream)?; + // The steward may order and read; root, the operator, may only read. + let steward = peer.uid == ctx.steward_uid; + if !steward && peer.uid != 0 { + eprintln!("steward socket: refused a peer that is not the steward"); + return refuse(stream, "peer", Code::Unauthorized); + } + let request = read_frame(stream, REQUEST_FRAME_LIMIT) + .ok() + .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); + let Some(request) = request else { + eprintln!("steward socket: refused a malformed frame"); + return refuse(stream, "frame", Code::Malformed); + }; + match request { + Request::Order(_) if !steward => { + eprintln!("steward socket: refused an order from a peer that is not the steward"); + refuse(stream, "peer", Code::Unauthorized) + } + Request::Order(request) => { + let checked = + validate_reason(&request.reason).and_then(|()| request.operation.validate()); + if let Err(refusal) = checked { + eprintln!("steward socket: refused an order at the grammar"); + return answer(stream, Response::Refused(refusal)); + } + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: format!("ord-{}", Uuid::new_v4()), + principal: ctx.principal.clone(), + issuer_uid: peer.uid, + issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), + issuer_unit: peer.issuer_unit, + issued_at_unix_millis: now_millis()?, + operation: request.operation, + reason: request.reason, + }; + match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { + Ok(true) => {} + _ => return refuse(stream, "store", Code::Unavailable), + } + queue + .send(order.order_id.clone()) + .map_err(|_| anyhow!("the executor has stopped"))?; + answer( + stream, + Response::Accepted { + order_id: order.order_id, + }, + ) + } + Request::Show(request) => { + if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) + { + return refuse(stream, "id", Code::BadShape); + } + let ledger = ctx.store.ledger()?; + match ledger.orders.iter().find(|order| order.order_id == request.id) { + Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), + None => refuse(stream, "id", Code::Unknown), + } + } + Request::List => { + let ledger = ctx.store.ledger()?; + let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); + let lines = ledger + .orders + .iter() + .skip(skip) + .map(|order| { + let view = view_of(&ledger, order); + OrderLine { + order_id: order.order_id.clone(), + status: status_word(&view).into(), + summary: order.operation.describe(), + } + }) + .collect(); + answer(stream, Response::List(lines)) + } + } +} + +// ------------------------------------------------------------- real ports -- + +/// `systemctl` and `systemd-run`, found by name. +pub(crate) struct HostSystemd; + +/// The names in one `systemctl show` list value. systemd quotes a name that +/// holds a backslash and doubles the backslash inside the quotes. +fn parse_names(value: &str) -> Vec { + let mut names = Vec::new(); + let mut current = String::new(); + let (mut quoted, mut open, mut escaped) = (false, false, false); + for character in value.chars() { + match character { + _ if escaped => { + current.push(character); + escaped = false; + } + '\\' if quoted => escaped = true, + '"' => { + quoted = !quoted; + open = true; + } + _ if character.is_whitespace() && !quoted => { + if open { + names.push(std::mem::take(&mut current)); + open = false; + } + } + _ => { + current.push(character); + open = true; + } + } + } + if open { + names.push(current); + } + names +} + +/// The properties one `systemctl show` is asked for: identity, what fires at +/// the unit's end, who is bound to it, and what it upholds or groups. +fn show_properties() -> String { + ["Id", "Names", "LoadState"] + .into_iter() + .chain(ACTIONS) + .chain(ON_END) + .chain(["BoundBy", "Upholds", "ConsistsOf"]) + .collect::>() + .join(",") +} + +fn parse_show(output: &str) -> Result { + let mut facts = UnitFacts::default(); + for line in output.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + match key { + "Id" => facts.id = value.to_owned(), + "Names" => facts.names = parse_names(value), + "LoadState" => facts.not_found = value == "not-found", + "BoundBy" => facts.bound_by = parse_names(value), + "Upholds" => facts.upholds = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), + _ if ACTIONS.contains(&key) => { + if !matches!(value, "" | "none") { + facts.on_end.push(key.to_owned()); + } + } + _ if ON_END.contains(&key) => { + facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); + } + _ => {} + } + } + ensure!(!facts.id.is_empty(), "systemctl named no unit"); + Ok(facts) +} + +impl Systemd for HostSystemd { + fn show(&self, unit: &str) -> Result { + let output = Command::new("systemctl") + .args(["show", "--no-pager"]) + .arg(format!("--property={}", show_properties())) + .args(["--", unit]) + .stdin(Stdio::null()) + .output() + .context("running systemctl show")?; + ensure!(output.status.success(), "systemctl show failed"); + parse_show(&String::from_utf8_lossy(&output.stdout)) + } + + fn invocation(&self, unit: &str) -> Option { + let output = Command::new("systemctl") + .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) + .arg(unit) + .stdin(Stdio::null()) + .output() + .ok()?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + (output.status.success() && !value.is_empty()).then_some(value) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + let mut command = Command::new("systemd-run"); + command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); + command.args(["-p", "RuntimeMaxSec=15min"]); + for (name, value) in &step.environment { + command.arg(format!("--setenv={name}={value}")); + } + let output = command + .arg("--") + .args(&step.argv) + .stdin(Stdio::null()) + .output() + .context("running systemd-run")?; + Ok(RunReport { + exit_code: output.status.code(), + signal: output.status.signal(), + stdout_tail: tail(&output.stdout), + stderr_tail: tail(&output.stderr), + }) + } +} + +/// A system service or scope names the issuer: the first unit under +/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices +/// above it are the manager's, and what a delegated subtree adds below it is +/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. +fn unit_from_cgroup(text: &str) -> String { + let line = text + .lines() + .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) + .unwrap_or_default(); + let path = line.rsplit_once(':').map_or("", |(_, path)| path); + let unit = path.strip_prefix("/system.slice/").and_then(|under| { + for segment in under.split('/') { + if segment.ends_with(".slice") { + continue; + } + let unit = segment.ends_with(".service") || segment.ends_with(".scope"); + return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); + } + None + }); + unit.map_or_else(|| "unknown".to_owned(), str::to_owned) +} + +/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. +pub(crate) struct KernelPeers; + +impl PeerCredentials for KernelPeers { + fn peer(&self, stream: &UnixStream) -> Result { + use std::os::fd::AsRawFd; + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = std::mem::size_of::() as libc::socklen_t; + // SAFETY: the descriptor is a live socket and the pointer and length + // describe `credentials`, which outlives the call. + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&mut credentials as *mut libc::ucred).cast(), + &mut length, + ) + }; + ensure!( + status == 0 && length as usize == std::mem::size_of::(), + "peer credentials are unavailable" + ); + let cgroup = + std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); + Ok(Peer { + uid: credentials.uid, + issuer_unit: unit_from_cgroup(&cgroup), + }) + } +} + +/// `mind-control --instance check`; a missing binary is a held brake. +pub(crate) struct MindControl { + pub(crate) program: PathBuf, +} + +impl BrakeCheck for MindControl { + fn clear(&self, principal: &str) -> bool { + Command::new(&self.program) + .args(["--instance", principal, "check"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} + +/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is +/// never chunked. The token is read from its credential file for each post. +pub(crate) struct Synapse { + pub(crate) address: std::net::SocketAddr, + pub(crate) room: String, + pub(crate) token_file: PathBuf, + /// The whole wait for a reply, however slowly it drips. + pub(crate) reply_within: Duration, +} + +fn percent_encode(text: &str) -> String { + let mut encoded = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +impl Homeserver for Synapse { + fn post(&self, txn_id: &str, text: &str) -> Result { + let token = std::fs::read_to_string(&self.token_file) + .context("the homeserver token is unreadable")?; + let token = token.trim(); + ensure!( + !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), + "the homeserver token is not a bearer token" + ); + let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; + let head = format!( + "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ + Host: {}\r\nAuthorization: Bearer {token}\r\n\ + Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", + percent_encode(&self.room), + self.address, + body.len() + ); + let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) + .context("the homeserver is unreachable")?; + socket.set_write_timeout(Some(self.reply_within))?; + socket.write_all(head.as_bytes())?; + socket.write_all(&body)?; + let deadline = std::time::Instant::now() + self.reply_within; + let mut reply = Vec::new(); + let mut chunk = [0u8; 4096]; + while reply.len() < RESPONSE_FRAME_LIMIT { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|left| !left.is_zero()) + .context("the homeserver reply was too slow")?; + socket.set_read_timeout(Some(remaining))?; + let count = socket + .read(&mut chunk) + .context("the homeserver reply was cut short")?; + if count == 0 { + break; + } + reply.extend_from_slice(&chunk[..count]); + } + let split = reply + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("the homeserver reply has no body")?; + let status = String::from_utf8_lossy(&reply[..split]); + ensure!( + status.split_whitespace().nth(1) == Some("200"), + "the homeserver refused the post" + ); + #[derive(Deserialize)] + struct Sent { + event_id: String, + } + let sent: Sent = serde_json::from_slice(&reply[split + 4..]) + .map_err(|_| anyhow!("the homeserver reply has no event id"))?; + Ok(sent.event_id) + } +} + +// ---------------------------------------------------------------- binding -- + +/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. +/// It lists no unit of any kind, and any field outside this set is refused. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct StewardBinding { + pub(crate) principal: String, + pub(crate) user: String, + pub(crate) mind_control: PathBuf, + pub(crate) room: String, + pub(crate) homeserver: String, + pub(crate) token_credential: String, +} + +impl StewardBinding { + pub(crate) fn parse(text: &str) -> Result { + let binding: Self = toml::from_str(text) + .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; + binding.validate()?; + Ok(binding) + } + + fn validate(&self) -> Result<()> { + require_id(&self.principal, "steward principal")?; + ensure!( + !self.principal.starts_with('-') && !self.principal.contains('/'), + "steward principal is invalid" + ); + let user_ok = !self.user.is_empty() + && self.user.len() <= 32 + && self.user.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) + && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); + ensure!(user_ok && self.user != "root", "steward user is invalid or root"); + ensure!( + self.mind_control.is_absolute(), + "mind_control must be an absolute path" + ); + let room_ok = self.room.starts_with('!') + && self.room.len() <= 255 + && self.room.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') + }); + ensure!(room_ok, "room must be a Matrix room id"); + self.address()?; + let credential_ok = !self.token_credential.is_empty() + && !self.token_credential.starts_with('.') + && self.token_credential.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') + }); + ensure!(credential_ok, "token_credential is not a credential name"); + Ok(()) + } + + fn address(&self) -> Result { + let address: std::net::SocketAddr = self + .homeserver + .strip_prefix("http://") + .and_then(|rest| rest.parse().ok()) + .context("homeserver must be http://:")?; + ensure!(address.ip().is_loopback(), "homeserver must be loopback"); + Ok(address) + } +} + +/// Root-admitted: owned by root and writable by no one else. +fn admitted(owner: u32, mode: u32) -> bool { + owner == 0 && mode & 0o022 == 0 +} + +fn load_binding(path: &Path) -> Result { + let metadata = std::fs::metadata(path).context("reading the steward binding")?; + ensure!( + metadata.is_file() && admitted(metadata.uid(), metadata.mode()), + "the steward binding is not a root-owned file closed to writes" + ); + let text = std::fs::read_to_string(path).context("reading the steward binding")?; + StewardBinding::parse(&text) +} + +fn user_uid(name: &str) -> Result { + let name = std::ffi::CString::new(name)?; + // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in + // `buffer`, which outlives every read of them below (none are read). + let (status, found, uid) = unsafe { + let mut passwd: libc::passwd = std::mem::zeroed(); + let mut buffer = vec![0 as libc::c_char; 4096]; + let mut result: *mut libc::passwd = std::ptr::null_mut(); + let status = libc::getpwnam_r( + name.as_ptr(), + &mut passwd, + buffer.as_mut_ptr(), + buffer.len(), + &mut result, + ); + (status, !result.is_null(), passwd.pw_uid) + }; + ensure!(status == 0 && found, "the steward user is unknown"); + ensure!(uid != 0, "the steward user resolves to root"); + Ok(uid) +} + +// ------------------------------------------------------------------ serve -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ServeOptions { + pub(crate) state_store: PathBuf, + pub(crate) store: PathBuf, + pub(crate) steward_binding: PathBuf, + pub(crate) socket: PathBuf, +} + +const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; +const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; +const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; + +pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { + ensure!( + args.next().as_deref() == Some("serve"), + "idunn orders takes the subcommand serve" + ); + let mut options = ServeOptions { + state_store: default_state_store(), + store: DEFAULT_STORE.into(), + steward_binding: DEFAULT_BINDING.into(), + socket: DEFAULT_SOCKET.into(), + }; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, + "--store" => options.store = path_value(&mut args, "--store")?, + "--steward-binding" => { + options.steward_binding = path_value(&mut args, "--steward-binding")? + } + "--socket" => options.socket = path_value(&mut args, "--socket")?, + _ => bail!("an orders serve option is not part of the grammar"), + } + } + Ok(options) +} + +/// Seconds between post passes: two when the last pass went through, and +/// doubling for each failed pass in a row up to a minute. +fn post_pause(failures: u32) -> u64 { + if failures == 0 { + 2 + } else { + (2u64 << failures.min(5)).min(60) + } +} + +/// Run a thread whose end ends the service: systemd restarts it, and the +/// start recovery settles what it left. +fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { + struct ExitOnDrop(&'static str); + impl Drop for ExitOnDrop { + fn drop(&mut self) { + eprintln!("steward orders: the {} thread ended", self.0); + std::process::exit(1); + } + } + thread::Builder::new() + .name(name.into()) + .spawn(move || { + let _guard = ExitOnDrop(name); + work(); + }) + .expect("a thread can be spawned"); +} + +/// The socket belongs to the steward and no one else may open it; the peer +/// credential check on every connection stays the lock. +fn bind_socket(path: &Path, steward_uid: u32) -> Result { + if let Ok(metadata) = std::fs::symlink_metadata(path) { + use std::os::unix::fs::FileTypeExt; + ensure!( + metadata.file_type().is_socket(), + "the socket path holds something that is not a socket" + ); + std::fs::remove_file(path).context("removing a stale socket")?; + } + let listener = UnixListener::bind(path).context("binding the steward socket")?; + std::os::unix::fs::chown(path, Some(steward_uid), None) + .context("giving the steward socket to the steward")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("setting the steward socket mode")?; + Ok(listener) +} + +/// Files this service creates in Idunn's own stores (a deploy order writes +/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. +const IDUNN_UMASK: libc::mode_t = 0o027; + +pub(crate) fn serve(options: ServeOptions) -> Result<()> { + // SAFETY: umask only sets the process file-mode mask; it cannot fail. + unsafe { libc::umask(IDUNN_UMASK) }; + let binding = load_binding(&options.steward_binding)?; + let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") + .map(PathBuf::from) + .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; + let ctx = Arc::new(Ctx { + store: OrderStore::new(&options.store), + state_store: options.state_store, + steward_uid: user_uid(&binding.user)?, + principal: binding.principal.clone(), + systemd: Arc::new(HostSystemd), + peers: Arc::new(KernelPeers), + brake: Arc::new(MindControl { + program: binding.mind_control.clone(), + }), + homeserver: Arc::new(Synapse { + address: binding.address()?, + room: binding.room.clone(), + token_file: credentials.join(&binding.token_credential), + reply_within: Duration::from_secs(10), + }), + }); + ctx.store.prepare()?; + let pending = recover(&ctx)?; + let listener = bind_socket(&options.socket, ctx.steward_uid)?; + let (queue, orders) = mpsc::channel::(); + for order_id in pending { + queue.send(order_id)?; + } + let executor = Arc::clone(&ctx); + vital("executor", move || { + for order_id in orders { + if let Err(error) = execute(&executor, &order_id) { + eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); + } + } + }); + let poster = Arc::clone(&ctx); + vital("poster", move || { + let mut failures = 0u32; + loop { + match post_pass(&poster) { + Ok(_) => failures = 0, + Err(error) => { + failures = failures.saturating_add(1); + eprintln!("steward orders: posting is delayed: {error:#}"); + } + } + thread::sleep(Duration::from_secs(post_pause(failures))); + } + }); + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + if let Err(error) = handle(&ctx, &mut stream, &queue) { + eprintln!("steward socket: a connection ended in error: {error:#}"); + } + } + Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), + } + } + bail!("the steward socket stopped accepting") +} + +// ----------------------------------------------------------------- client -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OrderAction { + Issue { operation: Operation, reason: String }, + Show { id: String }, + List, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct OrderCommand { + pub(crate) socket: PathBuf, + pub(crate) action: OrderAction, +} + +pub(crate) fn parse_order(mut args: impl Iterator) -> Result { + let kind = args.next().context("idunn order requires an operation")?; + let mut socket = PathBuf::from(DEFAULT_SOCKET); + let mut reason = None; + let mut positional = Vec::new(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--socket" => socket = path_value(&mut args, "--socket")?, + "--reason" => reason = Some(string_value(&mut args, "--reason")?), + word if word.starts_with('-') => bail!("an order option is not part of the grammar"), + _ => positional.push(argument), + } + } + let action = match kind.as_str() { + "list" => { + ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); + OrderAction::List + } + "show" => { + ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); + OrderAction::Show { + id: positional.remove(0), + } + } + "package-install" | "unit" | "deploy" => { + let operation = match kind.as_str() { + "package-install" => Operation::PackageInstall { + packages: positional, + }, + "unit" => { + ensure!(positional.len() == 2, "unit takes a verb and a unit"); + let verb = Verb::parse(&positional[0]) + .context("the unit verb is not start, stop, restart or reload")?; + Operation::UnitAction { + verb, + unit: positional.remove(1), + } + } + _ => { + ensure!(positional.len() == 1, "deploy takes one selector"); + Operation::Deploy { + selector: positional.remove(0), + } + } + }; + let reason = reason.context("an order requires --reason")?; + validate_reason(&reason)?; + operation.validate()?; + OrderAction::Issue { operation, reason } + } + _ => bail!("the order operation is not part of the grammar"), + }; + Ok(OrderCommand { socket, action }) +} + +fn call(socket: &Path, request: &Request) -> Result { + let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + write_frame(&mut stream, request)?; + let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; + rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) +} + +fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); + let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); +} + +/// Raw lines for the terminal; `emit` is their only way out. +fn render_view(view: &OrderView) -> Vec> { + let order = &view.order; + let mut lines: Vec> = vec![ + format!("order {}", order.order_id).into_bytes(), + format!("status {}", status_word(view)).into_bytes(), + format!("operation {}", order.operation.describe()).into_bytes(), + format!( + "issuer {} (invocation {}, uid {})", + order.issuer_unit, order.issuer_invocation, order.issuer_uid + ) + .into_bytes(), + format!("reason {}", quoted(&order.reason)).into_bytes(), + ]; + if let Some(start) = &view.start { + lines.push(format!("started {}", start.actuation).into_bytes()); + } + if let Some(record) = &view.outcome { + lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); + render_tail(&mut lines, "stdout", &record.stdout_tail); + render_tail(&mut lines, "stderr", &record.stderr_tail); + } + lines +} + +pub(crate) fn run_order(command: OrderCommand) -> Result<()> { + let request = match command.action { + OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), + OrderAction::Show { id } => Request::Show(ShowRequest { id }), + OrderAction::List => Request::List, + }; + let mut out = std::io::stdout().lock(); + match call(&command.socket, &request)? { + Response::Accepted { order_id } => writeln!(out, "{order_id}")?, + Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), + Response::Show(view) => emit(&mut out, &render_view(&view))?, + Response::List(lines) => { + let lines: Vec> = lines + .iter() + .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) + .collect(); + emit(&mut out, &lines)? + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tempfile::TempDir; + + use super::*; + + const STEWARD: u32 = 1000; + const CANARY: &str = "CANARY-7f3a91"; + + /// What `systemctl show` prints for a loaded unit that acts on nothing, + /// plus the given properties (a given key replaces its default). + fn show_text(id: &str, properties: &[(&str, &str)]) -> String { + let mut text = format!("Id={id}\n"); + for (key, default) in [("Names", id), ("LoadState", "loaded")] { + if !properties.iter().any(|(given, _)| *given == key) { + text += &format!("{key}={default}\n"); + } + } + for (key, value) in properties { + text += &format!("{key}={value}\n"); + } + text + } + + struct FakeSystemd { + units: Mutex>, + runs: Mutex>, + shows: Mutex>, + report: Mutex, + } + + impl Systemd for FakeSystemd { + fn show(&self, unit: &str) -> Result { + self.shows.lock().unwrap().push(unit.into()); + let known = self.units.lock().unwrap().get(unit).cloned(); + parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) + } + + fn invocation(&self, _unit: &str) -> Option { + Some("invocation-1".into()) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + self.runs.lock().unwrap().push((unit.into(), step.clone())); + Ok(self.report.lock().unwrap().clone()) + } + } + + struct FakePeers(u32); + + impl PeerCredentials for FakePeers { + fn peer(&self, _stream: &UnixStream) -> Result { + Ok(Peer { + uid: self.0, + issuer_unit: "mind-eureka-self.service".into(), + }) + } + } + + /// Whether the brake is clear, and the principals it was asked about. + struct FakeBrake(AtomicBool, Mutex>); + + impl BrakeCheck for FakeBrake { + fn clear(&self, principal: &str) -> bool { + self.1.lock().unwrap().push(principal.into()); + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct FakeHome { + failing: AtomicBool, + sent: Mutex>, + } + + impl Homeserver for FakeHome { + fn post(&self, txn_id: &str, text: &str) -> Result { + ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); + let mut sent = self.sent.lock().unwrap(); + sent.push((txn_id.into(), text.into())); + Ok(format!("$event{}", sent.len())) + } + } + + struct Harness { + directory: TempDir, + ctx: Ctx, + systemd: Arc, + brake: Arc, + home: Arc, + queue: mpsc::Sender, + _orders: mpsc::Receiver, + } + + fn harness_for(peer_uid: u32) -> Harness { + let directory = TempDir::new().unwrap(); + let systemd = Arc::new(FakeSystemd { + units: Mutex::new(BTreeMap::new()), + runs: Mutex::new(Vec::new()), + shows: Mutex::new(Vec::new()), + report: Mutex::new(RunReport { + exit_code: Some(0), + ..RunReport::default() + }), + }); + let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); + let home = Arc::new(FakeHome::default()); + let (queue, orders) = mpsc::channel(); + let ctx = Ctx { + store: OrderStore::new(&directory.path().join("orders")), + state_store: directory.path().join("state").join("control.cc"), + steward_uid: STEWARD, + principal: "eureka".into(), + systemd: systemd.clone(), + peers: Arc::new(FakePeers(peer_uid)), + brake: brake.clone(), + homeserver: home.clone(), + }; + Harness { + directory, + ctx, + systemd, + brake, + home, + queue, + _orders: orders, + } + } + + fn harness() -> Harness { + harness_for(STEWARD) + } + + impl Harness { + fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + write(&mut client); + handle(&self.ctx, &mut server, &self.queue).unwrap(); + let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); + rmp_serde::from_slice(&bytes).unwrap() + } + + fn ask(&self, request: &Request) -> Response { + self.exchange(|client| write_frame(client, request).unwrap()) + } + + fn issue(&self, operation: Operation) -> Response { + self.ask(&Request::Order(OrderRequest { + operation, + reason: "probe".into(), + })) + } + + fn issue_unit(&self, verb: Verb, unit: &str) -> Response { + self.issue(Operation::UnitAction { + verb, + unit: unit.into(), + }) + } + + fn accept(&self, response: Response) -> String { + match response { + Response::Accepted { order_id } => order_id, + other => panic!("expected an accepted order, got {other:?}"), + } + } + + fn outcome(&self, order_id: &str) -> Outcome { + self.ctx + .store + .ledger() + .unwrap() + .outcome_of(order_id) + .expect("the order has an outcome") + .outcome + .clone() + } + + fn runs(&self) -> Vec<(String, Step)> { + self.systemd.runs.lock().unwrap().clone() + } + + /// Tell the fake `systemctl show` what it prints for `name`. + fn describe_unit(&self, name: &str, text: String) { + self.systemd.units.lock().unwrap().insert(name.into(), text); + } + + fn describe(&self, name: &str, properties: &[(&str, &str)]) { + self.describe_unit(name, show_text(name, properties)); + } + } + + fn all_verbs() -> [Verb; 4] { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + } + + #[test] + fn foreign_uid_is_refused_at_the_door() { + let h = harness_for(STEWARD + 1); + let response = h.issue_unit(Verb::Restart, "nginx.service"); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + let response = h.ask(&Request::List); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); + assert!(h.runs().is_empty()); + } + + #[derive(Serialize)] + struct ForgedBody { + operation: Operation, + reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + } + + #[derive(Serialize)] + struct ForgedFrame { + order: ForgedBody, + } + + #[test] + fn order_id_and_requester_are_minted_not_declared() { + let h = harness(); + let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { + let frame = ForgedFrame { + order: ForgedBody { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into(), + }, + reason: "probe".into(), + order_id: order_id.map(str::to_owned), + requested_by: requested_by.map(str::to_owned), + command: command.map(str::to_owned), + }, + }; + h.exchange(|client| write_frame(client, &frame).unwrap()) + }; + // The same frame without the extra field is a good order. + let honest = h.accept(forge(None, None, None)); + for forged in [ + forge(Some("ord-declared"), None, None), + forge(None, Some("steward:someone-else order x"), None), + forge(None, None, Some("reboot-now")), + ] { + assert_eq!( + forged, + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + } + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); + let order = &ledger.orders[0]; + assert_eq!(order.order_id, honest); + let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); + assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); + assert_eq!(order.issuer_unit, "mind-eureka-self.service"); + assert_eq!(order.issuer_invocation, "invocation-1"); + assert_eq!(order.principal, "eureka"); + assert_eq!(order.issuer_uid, STEWARD); + assert_eq!( + requester(&order.principal, &order.order_id), + format!("steward:eureka order {}", order.order_id) + ); + } + + #[test] + fn package_names_outside_grammar_are_refused() { + let h = harness(); + let bad = [ + "-y", + "--allow-unauthenticated", + "-", + "a", + "", + "Foo", + "a/b", + "/tmp/x", + "./x", + "x.deb", + "foo.deb", + "foo bar", + "foo;reboot", + "foo$(id)", + "foo`id`", + "foo|bar", + "foo&bar", + "foo\nbar", + "foo*", + "foo?", + "pkg:amd64", + "pkg/stable", + "libc6-", + "foo=", + "foo=1 2", + "foo=1;2", + "foo=1=2", + "fo\u{ff0f}o", + "f\u{43e}o", + ]; + for name in bad { + let response = h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into(), name.into()], + }); + assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); + } + let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); + assert_eq!( + h.issue(Operation::PackageInstall { packages: many }), + Response::Refused(Refusal::new("package", Code::TooMany)) + ); + assert_eq!( + h.issue(Operation::PackageInstall { packages: vec![] }), + Response::Refused(Refusal::new("package", Code::Empty)) + ); + assert!(!h.ctx.store.path().exists(), "no refusal left a record"); + + let good = [ + "git-lfs", + "matrix-synapse-py3", + "libnss-systemd", + "bubblewrap", + "socat", + "g++", + "libstdc++6", + "python3.11", + "nginx=1.24.0-2ubuntu7", + "foo=1:2.3-4+b1~rc1", + ]; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: good.iter().map(|name| (*name).to_owned()).collect(), + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + let runs = h.runs(); + assert_eq!(runs.len(), 3); + assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); + // Each name selects exactly itself and apt may remove nothing. + let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; + install.extend(["--no-remove".to_owned(), "--".into()]); + install.extend(good.iter().map(|spec| match spec.split_once('=') { + Some((name, version)) => format!("?exact-name({name})={version}"), + None => format!("?exact-name({spec})"), + })); + assert_eq!(runs[1].1.argv, install); + assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); + assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); + // The last step fails unless every named package is installed. + let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; + present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); + assert_eq!(runs[2].1.argv, present); + for (unit, step) in &runs { + assert!(unit.starts_with("idunn-order-")); + let apt = step.argv[0] == "apt-get"; + assert_eq!( + step.environment, + if apt { + vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + } else { + Vec::new() + } + ); + } + assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); + + // A package whose name apt would read as a regex or a conflict is + // exactly itself and never a removal. + for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { + let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); + assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); + } + // A step that failed ends the order: the absent package is a failure. + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(1), + ..RunReport::default() + }; + let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); + execute(&h.ctx, &absent).unwrap(); + assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); + assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); + } + + #[test] + fn any_unit_outside_the_refused_set_runs() { + let h = harness(); + let units = [ + "wg-quick@wg0.service", + "nginx.service", + "matrix-synapse.service", + "ssh.socket", + "huginn@eureka.service", + "claude-connector.service", + "persona-matrix-mouth.service", + "mind-heartbeat.service", + "apt-daily.timer", + // Near misses: the prefix is the rule, not a substring or a stem. + "midunn-x.service", + "not-idunn-x.service", + "idunn.service", + "mind-self.service", + "mind-selfx@a.service", + "mind-persona.service", + "mind-personal@a.service", + ]; + for verb in all_verbs() { + for unit in units { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); + let (transient, step) = h.runs().pop().unwrap(); + assert!(transient.starts_with("idunn-order-")); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] + ); + } + } + assert_eq!(h.runs().len(), units.len() * 4); + } + + #[test] + fn refused_set_is_refused_by_name_and_alias() { + let h = harness(); + h.describe_unit( + "backup-name.service", + show_text( + "idunn-yggdrasil.service", + &[("Names", "idunn-yggdrasil.service backup-name.service")], + ), + ); + h.describe( + "quiet-alias.service", + &[("Names", "quiet-alias.service mind-persona@void.service")], + ); + let direct = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-x.service", + "idunn-order-3f2c.service", + "idunn-yggdrasil.timer", + "idunn-x.socket", + "idunn-.service", + "idunn-yggdrasil.service.service", + "idunn-x@y.service", + "mind-self@eureka.service", + "mind-persona@void.service", + "mind-self@.service", + "mind-persona@a@b.service", + // Case folds: a lookalike is still refused, never run. + "IDUNN-yggdrasil.service", + "Idunn-Orders.service", + "MIND-SELF@eureka.service", + "Mind-Persona@Void.service", + ]; + for verb in all_verbs() { + for unit in direct { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + match h.outcome(&id) { + Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { + assert_eq!(hit, unit, "{verb:?} {unit}"); + assert_eq!(via, Via::Name); + } + other => panic!("{verb:?} {unit} was {other:?}"), + } + } + } + for verb in all_verbs() { + for (unit, hit) in [ + ("backup-name.service", "idunn-yggdrasil.service"), + ("quiet-alias.service", "mind-persona@void.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Alias + }), + "{verb:?} {unit}" + ); + } + } + assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); + // Every refusal is recorded and has a result post due. + let ledger = h.ctx.store.ledger().unwrap(); + let refusals = ledger + .outcomes + .iter() + .filter(|record| matches!(record.outcome, Outcome::Refused(_))) + .count(); + assert_eq!(refusals, direct.len() * 4 + 8); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + let results = due + .iter() + .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) + .count(); + assert_eq!(results, refusals); + // A refused order never started. + assert!(ledger.starts.is_empty()); + } + + fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + h.outcome(&id) + } + + fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { + match outcome { + Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { + Some(unit.as_str()) + } + _ => None, + } + } + + #[test] + fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { + let h = harness(); + // systemctl show quotes a name with a backslash and doubles the backslash. + h.describe( + "huginn.service", + &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], + ); + let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); + assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); + h.describe( + "quiet.service", + &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], + ); + let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); + assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); + assert!(h.runs().is_empty()); + } + + #[test] + fn unit_orders_run_as_one_job() { + let h = harness(); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); + let (_, step) = h.runs().pop().unwrap(); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] + ); + } + } + + #[test] + fn the_rule_reads_only_the_named_unit() { + let h = harness(); + // Whatever the unit lists, no other unit is looked at. + h.describe( + "plain.service", + &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], + ); + h.describe("other.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); + } + + #[test] + fn a_unit_that_acts_when_it_ends_is_refused() { + let h = harness(); + let properties = [ + ("SuccessAction", "reboot"), + ("FailureAction", "poweroff-force"), + ("StartLimitAction", "soft-reboot-immediate"), + ("JobTimeoutAction", "exit"), + ("FailureAction", "a-spelling-nobody-knows"), + ("OnFailure", "rescue.service"), + ("OnSuccess", "cleanup.service"), + ]; + for (property, value) in properties { + h.describe("ender.service", &[(property, value)]); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "ender.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "ender.service".into(), + via: Via::OnEnd + }), + "{verb:?} {property}={value}" + ); + } + assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); + } + // `none` and an empty list say the unit does nothing at its end. + h.describe( + "calm.service", + &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.runs().len(), 11); + } + + #[test] + fn host_names_are_refused() { + let h = harness(); + let names = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", + ]; + h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); + for verb in all_verbs() { + for name in names { + assert_eq!( + outcome_of_unit(&h, verb, name), + Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), + "{verb:?} {name}" + ); + } + assert_eq!( + outcome_of_unit(&h, verb, "sleepy.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "systemd-hibernate.service".into(), + via: Via::Host + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); + } + + #[test] + fn a_unit_others_are_bound_to_only_starts() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service second.service")]); + for verb in [Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "base.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "first.service".into(), + via: Via::Bound + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Start, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_unit_that_upholds_others_is_refused_where_it_would_start() { + let h = harness(); + h.describe("holder.service", &[("Upholds", "held.service")]); + for verb in [Verb::Start, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "holder.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "holder.service".into(), + via: Via::Upholds + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { + let h = harness(); + h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); + for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { + assert_eq!( + outcome_of_unit(&h, verb, "wrapper.service"), + Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); + } + + #[test] + fn empty_dependency_lists_name_no_one() { + let h = harness(); + // systemctl prints every property it is asked for, empty or not. + h.describe( + "alone.service", + &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service")]); + h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); + assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); + h.systemd.shows.lock().unwrap().clear(); + assert_eq!( + hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), + Some("idunn-ghost.service") + ); + assert!(h.systemd.shows.lock().unwrap().is_empty()); + } + + #[test] + fn each_refusal_is_posted_under_its_own_reason() { + for (via, word) in [ + (Via::Name, "name"), + (Via::Alias, "alias"), + (Via::Host, "host"), + (Via::OnEnd, "on-end"), + (Via::Bound, "bound"), + (Via::Upholds, "upholds"), + ] { + assert_eq!( + Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) + .describe(), + format!("refused: always-refused unit u.service reached by {word}") + ); + } + assert_eq!( + Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), + "refused: wrapper unit, it only wraps m.service" + ); + } + + #[test] + fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { + let h = harness(); + h.describe("ghost.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "ghost.service")); + execute(&h.ctx, &id).unwrap(); + assert!( + matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + } + + #[test] + fn globs_slices_scopes_and_targets_are_not_units() { + let h = harness(); + let bad = [ + "idunn*", + "idunn-*.service", + "idunn-?ggdrasil.service", + "mind-?elf@eureka.service", + "mind-self@*.service", + "mind-[s]elf@eureka.service", + "system.slice", + "poweroff.target", + "multi-user.target", + "user.slice", + "session-1.scope", + "init.scope", + "idunn-yggdrasil", + "mind-self@eureka", + "nginx", + ".service", + "", + "-idunn-x.service", + "-x.service", + "/etc/systemd/system/idunn-yggdrasil.service", + "../idunn-yggdrasil.service", + "idunn-yggdrasil.service\n", + "idunn-yggdrasil.service ", + " idunn-yggdrasil.service", + "idunn-yggdrasil.service\0", + "idunn-yggdrasil.service;reboot", + "idunn\\x2dyggdrasil.service", + "idunn%2dyggdrasil.service", + // Unicode lookalikes of a refused name. + "\u{131}dunn-yggdrasil.service", + "idunn\u{2010}yggdrasil.service", + "idunn\u{ff0d}yggdrasil.service", + "idunn-yggdrasil.service\u{200b}", + "\u{200b}idunn-yggdrasil.service", + "idunn-yggdrasil\u{2024}service", + "mind-self\u{ff20}eureka.service", + ]; + for unit in bad { + for verb in all_verbs() { + assert!( + matches!(h.issue_unit(verb, unit), Response::Refused(_)), + "{unit:?} was not refused by the grammar" + ); + } + } + let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); + assert_eq!( + h.issue_unit(Verb::Start, &long), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); + } + + #[test] + fn binding_with_a_unit_list_is_refused() { + let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ + mind_control = \"/usr/local/bin/mind-control\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ + token_credential = \"idunn-matrix-token\"\n"; + let binding = StewardBinding::parse(valid).unwrap(); + assert_eq!(binding.principal, "eureka"); + assert_eq!(binding.address().unwrap().port(), 8008); + for extra in [ + "units = [\"nginx.service\"]", + "allow = [\"nginx.service\"]", + "deny = [\"idunn-*\"]", + "denylist = [\"idunn-*\"]", + "refused = [\"idunn-*\"]", + "unit = \"nginx.service\"", + "[units]\nnginx = true", + ] { + assert!( + StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), + "{extra} loaded" + ); + } + let swap = |from: &str, to: &str| valid.replace(from, to); + for broken in [ + swap("user = \"mind-eureka-self\"", "user = \"root\""), + swap("user = \"mind-eureka-self\"", "user = \"\""), + swap("/usr/local/bin/mind-control", "mind-control"), + swap("!abc:example.org", "#alias:example.org"), + swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), + swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), + swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), + swap("idunn-matrix-token", "../token"), + swap("principal = \"eureka\"\n", ""), + ] { + assert!(StewardBinding::parse(&broken).is_err()); + } + assert!(admitted(0, 0o100600)); + assert!(admitted(0, 0o100644)); + assert!(!admitted(0, 0o100664)); + assert!(!admitted(0, 0o100666)); + assert!(!admitted(0, 0o100602)); + assert!(!admitted(1000, 0o100600)); + } + + #[test] + fn held_brake_refuses_at_start() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); + assert!(h.runs().is_empty()); + // The deploy path is gated the same way and writes no command. + let deploy = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &deploy).unwrap(); + assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); + assert!(!h.ctx.state_store.exists()); + // A brake that cannot be asked is a held brake. + let directory = TempDir::new().unwrap(); + assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); + } + + #[test] + fn brake_check_is_the_exit_status_of_mind_control_check() { + let directory = TempDir::new().unwrap(); + let program = |name: &str, body: &str| { + let path = directory.path().join(name); + std::fs::write(&path, body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + MindControl { program: path } + }; + let recorder = directory.path().join("argv"); + let clear = program( + "clear", + &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), + ); + assert!(clear.clear("eureka")); + assert_eq!( + std::fs::read_to_string(&recorder).unwrap().trim(), + "--instance eureka check" + ); + assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); + } + + #[test] + fn started_order_is_never_rerun() { + let h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The first order began and its service died: a start with no outcome. + h.ctx + .store + .put( + &id, + START_SCHEMA, + &StartRecord { + schema_version: START_SCHEMA.into(), + order_id: id.clone(), + started_at_unix_millis: 1, + actuation: format!("idunn-order-{}", order_uuid(&id)), + }, + ) + .unwrap(); + let pending = recover(&h.ctx).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Lost); + assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); + // Recovery is idempotent, and a finished order is not run again. + assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&h.ctx, &waiting).unwrap(); + execute(&h.ctx, &waiting).unwrap(); + assert_eq!(h.runs().len(), 1); + assert!(recover(&h.ctx).unwrap().is_empty()); + } + + #[test] + fn deploy_order_uses_the_up_primitive() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let command_id = format!("up-{}", order_uuid(&id)); + assert_eq!( + h.outcome(&id), + Outcome::Submitted { + command_id: command_id.clone() + } + ); + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); + assert!(h.runs().is_empty(), "a deploy runs no transient unit"); + let mut shown = Vec::new(); + crate::control_plane::status( + &mut shown, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + Some(&command_id), + ) + .unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); + assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); + // Re-executing writes no second command. + execute(&h.ctx, &id).unwrap(); + let mut again = Vec::new(); + crate::control_plane::status( + &mut again, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + None, + ) + .unwrap(); + assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); + // The result post waits for the transactions: queued is not finished. + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert_eq!(due.len(), 1); + assert_eq!(due[0].kind, PostKind::Accepted); + assert!(!deploy_finished("queued") && !deploy_finished("running")); + assert!(deploy_finished("succeeded") && deploy_finished("failed")); + assert!(deploy_finished("complete")); + } + + #[test] + fn chat_outage_delays_posts_not_orders() { + let h = harness(); + h.home.failing.store(true, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + assert_eq!(h.runs().len(), 1); + assert!(post_pass(&h.ctx).is_err()); + assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); + h.home.failing.store(false, Ordering::SeqCst); + assert_eq!(post_pass(&h.ctx).unwrap(), 2); + assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); + let sent = h.home.sent.lock().unwrap().clone(); + assert_eq!(sent.len(), 2); + assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); + assert_eq!(sent[1].0, format!("idunn-{id}-result")); + assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); + assert!(sent[0].1.contains("mind-eureka-self.service")); + assert!(sent[0].1.contains("reason: \"probe\"")); + assert!(sent[1].1.contains("ran (exit 0)")); + } + + #[test] + fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { + let h = harness(); + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(100), + signal: None, + stdout_tail: vec![b'o'; 3 * TAIL_BYTES], + stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), + }; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into()], + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Failed { + exit_code: Some(100), + signal: None + } + ); + assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); + let ledger = h.ctx.store.ledger().unwrap(); + let record = ledger.outcome_of(&id).unwrap(); + assert_eq!(record.stdout_tail.len(), TAIL_BYTES); + assert!(!format!("{record:?}").contains("secret-looking")); + // Neither the post nor the journal text carries a tail. + let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) + .into_iter() + .filter(|due| due.kind == PostKind::Result) + .collect(); + assert_eq!(results.len(), 1); + assert!(results[0].text.contains("failed (exit 100)")); + assert!(!results[0].text.contains("secret-looking")); + // The steward reads them through the escape sink and nothing raw. + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("expected a show") + }; + let mut shown = Vec::new(); + emit(&mut shown, &render_view(&view)).unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert!(shown.contains("secret-looking")); + assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); + assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); + assert!(shown.contains("status failed")); + // A show of an unknown or malformed id names the field only. + assert_eq!( + h.ask(&Request::Show(ShowRequest { + id: format!("ord-{}", Uuid::new_v4()) + })), + Response::Refused(Refusal::new("id", Code::Unknown)) + ); + assert_eq!( + h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), + Response::Refused(Refusal::new("id", Code::BadShape)) + ); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("expected a list") + }; + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].status, "failed"); + } + + #[test] + fn a_refusal_or_error_never_echoes_an_input_value() { + let h = harness(); + let mut seen: Vec> = Vec::new(); + let mut note = |response: Response| { + let bytes = rmp_serde::to_vec_named(&response).unwrap(); + if let Response::Refused(refusal) = &response { + seen.push(refusal.to_string().into_bytes()); + } + seen.push(bytes); + }; + for unit in [ + format!("{CANARY}*.service"), + format!("{CANARY}/../idunn-orders.service"), + format!("idunn-{CANARY};x.service"), + CANARY.to_string(), + ] { + note(h.issue_unit(Verb::Restart, &unit)); + } + note(h.issue(Operation::PackageInstall { + packages: vec![format!("{CANARY}/x")], + })); + note(h.issue(Operation::PackageInstall { + packages: vec![format!("pkg={CANARY} 1")], + })); + note(h.issue(Operation::Deploy { + selector: format!("{CANARY} bad"), + })); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: format!("{CANARY}\n"), + }))); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: CANARY.repeat(40), + }))); + note(h.ask(&Request::Show(ShowRequest { + id: format!("ord-{CANARY}"), + }))); + // A frame whose own keys and values are the canary, and plain garbage. + let named: std::collections::BTreeMap> = + [( + CANARY.to_owned(), + [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), + )] + .into_iter() + .collect(); + note(h.exchange(|client| write_frame(client, &named).unwrap())); + note(h.exchange(|client| { + let garbage = format!("{CANARY}{CANARY}"); + client + .write_all(&(garbage.len() as u32).to_be_bytes()) + .unwrap(); + client.write_all(garbage.as_bytes()).unwrap(); + })); + note(h.exchange(|client| { + client.write_all(&u32::MAX.to_be_bytes()).unwrap(); + })); + // The client grammar fails the same way. + let mut messages = Vec::new(); + let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); + for line in [ + words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), + words(&["unit", CANARY, "nginx.service", "--reason", "r"]), + words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), + words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), + words(&[CANARY]), + words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), + words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), + words(&["show"]), + ] { + messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); + } + for line in [ + words(&[CANARY]), + words(&["serve", &format!("--{CANARY}")]), + words(&["serve", "--store"]), + ] { + messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); + } + for bytes in &seen { + assert!( + !String::from_utf8_lossy(bytes).contains(CANARY), + "a response carried an input value" + ); + } + for message in &messages { + assert!(!message.contains(CANARY), "an error carried an input value: {message}"); + } + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn cgroup_names_the_issuer_unit() { + assert_eq!( + unit_from_cgroup("0::/system.slice/claude-connector.service\n"), + "claude-connector.service" + ); + // What a delegated subtree adds below the unit is the peer's to name. + assert_eq!( + unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), + "huginn@eureka.service" + ); + assert_eq!( + unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), + "getty@tty1.service" + ); + assert_eq!( + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), + "a.scope" + ); + // A user manager's units, and paths no manager made, name no one. + for path in [ + "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", + "0::/user.slice/user-1000.slice/session-3.scope\n", + "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", + "0::/system.slice/\n", + "0::/\n", + "", + ] { + assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); + } + } + + #[test] + fn systemctl_show_output_is_read_into_facts() { + let facts = parse_show( + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ + SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ + OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ + Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", + ) + .unwrap(); + assert_eq!(facts.id, "a.service"); + assert_eq!(facts.names, ["a.service", "b.service"]); + assert!(!facts.not_found); + assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); + // A quoted name holds one backslash, not two and not a quote mark. + assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); + assert_eq!(facts.upholds, ["u.service"]); + assert_eq!(facts.consists_of, ["m.service", "n.service"]); + assert!(parse_show("Names=a.service\n").is_err()); + // The request asks for exactly the properties the parse reads. + assert_eq!( + show_properties(), + "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ + OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" + ); + // A backslash outside quotes is a backslash; inside, it escapes one character. + assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); + assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); + assert!(parse_names(" ").is_empty()); + let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); + assert!(gone.not_found); + } + + #[test] + fn the_store_refuses_a_second_record_and_a_foreign_document() { + let h = harness(); + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: "ord-x".into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }; + assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert_eq!( + std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); + SingleFileMessagePackBackingStore::new(h.ctx.store.path()) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: "idunn.other".into(), + key: "k".into(), + current: None, + }], + &[foreign], + ) + .unwrap(); + assert!(h.ctx.store.ledger().is_err()); + } + + #[test] + fn synapse_post_is_one_idempotent_put_on_loopback() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (sent, requests) = mpsc::channel(); + let server = thread::spawn(move || { + for reply in [ + "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", + "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", + ] { + let (mut socket, _) = listener.accept().unwrap(); + let mut received = Vec::new(); + let mut chunk = [0u8; 1024]; + while !received.windows(4).any(|window| window == b"\r\n\r\n") + || !String::from_utf8_lossy(&received).contains("\"body\"") + { + let count = socket.read(&mut chunk).unwrap(); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); + socket.write_all(reply.as_bytes()).unwrap(); + } + }); + let synapse = Synapse { + address, + room: "!room:example.org".into(), + token_file, + reply_within: Duration::from_secs(10), + }; + assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); + let request = requests.recv().unwrap(); + assert!(request.starts_with( + "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" + )); + assert!(request.contains("Authorization: Bearer tok-123\r\n")); + assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); + let error = synapse.post("txn-2", "x").unwrap_err(); + assert!(!format!("{error:#}").contains("tok-123")); + server.join().unwrap(); + assert!( + Synapse { + address, + room: "!r:e".into(), + token_file: directory.path().join("absent"), + reply_within: Duration::from_secs(10), + } + .post("t", "x") + .is_err() + ); + } + + #[test] + fn grammar_limits_are_exact_at_their_edges() { + let h = harness(); + let order = |operation: Operation, reason: &str| { + h.ask(&Request::Order(OrderRequest { + operation, + reason: reason.into(), + })) + }; + let packages = |names: Vec| Operation::PackageInstall { packages: names }; + let unit = |name: String| Operation::UnitAction { + verb: Verb::Start, + unit: name, + }; + // Package: two characters is the shortest name, 200 bytes the longest. + assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); + assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); + assert_eq!( + order(packages(vec!["a".repeat(201)]), "r"), + Response::Refused(Refusal::new("package", Code::TooLong)) + ); + assert_eq!( + order(packages(vec!["a".into()]), "r"), + Response::Refused(Refusal::new("package", Code::BadShape)) + ); + let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); + assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); + // Unit: 200 bytes with its suffix is the longest. + let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); + assert_eq!(longest.len(), UNIT_LIMIT); + assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); + let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); + assert_eq!( + order(unit(too_long), "r"), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + // Reason: 300 bytes is the longest, and it must be there. + assert!(matches!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), + Response::Accepted { .. } + )); + assert_eq!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), + Response::Refused(Refusal::new("reason", Code::TooLong)) + ); + assert_eq!( + order(unit("nginx.service".into()), ""), + Response::Refused(Refusal::new("reason", Code::Empty)) + ); + assert_eq!( + order(unit("nginx.service".into()), "caf\u{e9}"), + Response::Refused(Refusal::new("reason", Code::BadCharacter)) + ); + } + + #[test] + fn a_concluded_order_is_never_run_later() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + // Someone else clears the brake; the refused order stays refused. + h.brake.0.store(true, Ordering::SeqCst); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty()); + let ledger = h.ctx.store.ledger().unwrap(); + assert!(ledger.start_of(&id).is_none()); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + } + + #[test] + fn the_orders_cli_has_no_command_string() { + let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); + let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); + assert_eq!( + parsed.action, + OrderAction::Issue { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into() + }, + reason: "why".into() + } + ); + assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); + assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); + assert!(parse(&["show", "ord-x"]).is_ok()); + assert!(parse(&["list"]).is_ok()); + for refused in [ + vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], + vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], + vec!["run", "sh", "--reason", "r"], + vec!["exec", "--reason", "r"], + vec!["unit", "enable", "nginx.service", "--reason", "r"], + vec!["unit", "restart", "--reason", "r"], + vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], + vec!["unit", "restart", "nginx.service"], + vec!["unit", "restart", "nginx*", "--reason", "r"], + vec!["package-install", "--reason", "r"], + vec!["package-install", "-y", "x", "--reason", "r"], + vec!["deploy", "--reason", "r"], + vec!["deploy", "a", "b", "--reason", "r"], + vec!["list", "extra"], + vec!["show"], + vec![], + ] { + assert!(parse(&refused).is_err(), "{refused:?} parsed"); + } + } + + #[test] + fn an_order_runs_only_under_the_steward_bound_now() { + let mut h = harness(); + let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The binding changes while both wait: another uid, then another + // principal with the old uid. Each is refused with a record, before + // the brake is asked and before anything runs. + h.ctx.steward_uid = STEWARD + 7; + execute(&h.ctx, &first).unwrap(); + h.ctx.steward_uid = STEWARD; + h.ctx.principal = "void".into(); + execute(&h.ctx, &second).unwrap(); + for id in [&first, &second] { + assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); + assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); + } + assert!(h.runs().is_empty()); + assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); + // The refusal is posted like any other. + let ledger = h.ctx.store.ledger().unwrap(); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert!(due.iter().any(|due| due.kind == PostKind::Result + && due.text.contains("refused: the issuer is not the current steward"))); + // The other direction: an order the bound steward issued after the + // rebinding, and a recovered one under an unchanged binding, run and + // are gated by the bound principal's brake. + let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); + let resumed = harness(); + let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); + assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&resumed.ctx, &waiting).unwrap(); + execute(&h.ctx, &third).unwrap(); + assert_eq!(resumed.outcome(&waiting), Outcome::Ran); + assert_eq!(h.outcome(&third), Outcome::Ran); + assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); + assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); + } + + #[test] + fn root_reads_orders_and_never_issues_one() { + let mut h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + h.ctx.peers = Arc::new(FakePeers(0)); + let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); + assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("root could not read an order"); + }; + assert_eq!(view.order.order_id, id); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("root could not list orders"); + }; + assert_eq!(lines.len(), 1, "root's refused order left no record"); + h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); + assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); + } + + #[test] + fn every_order_posts_once_in_its_turn() { + let h = harness(); + let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] + .into_iter() + .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) + .collect(); + for id in &ids { + execute(&h.ctx, id).unwrap(); + } + assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); + assert_eq!(post_pass(&h.ctx).unwrap(), 0); + let sent = h.home.sent.lock().unwrap().clone(); + let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); + keys.sort(); + let mut expected: Vec = ids + .iter() + .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) + .collect(); + expected.sort(); + assert_eq!(keys, expected); + // An order accepted later is posted by the next pass alone. + let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + execute(&h.ctx, &late).unwrap(); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + } + + #[test] + fn a_deploys_result_is_posted_once_its_command_has_finished() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let ledger = h.ctx.store.ledger().unwrap(); + let results = |state: Option<(&'static str, &str)>| { + let owed = due_posts(&ledger, &|_| match state { + Some((state, detail)) => Ok((state, detail.to_owned())), + None => Err(anyhow!("the command cannot be read")), + }); + owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() + }; + assert!(results(Some(("queued", ""))).is_empty()); + assert!(results(Some(("running", "vili:Sealing"))).is_empty()); + assert!(results(None).is_empty()); + for state in ["succeeded", "failed"] { + let owed = results(Some((state, "vili"))); + assert_eq!(owed.len(), 1, "{state}"); + assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); + assert!(owed[0].text.contains("submitted deployment up-")); + } + } + + #[test] + fn intake_gives_up_on_a_peer_that_sends_no_frame() { + let h = harness(); + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + // Half a length prefix, then nothing: the connection must end by itself. + client.write_all(&[0, 0]).unwrap(); + let (done, ended) = mpsc::channel(); + let worker = thread::spawn(move || { + let result = handle(&h.ctx, &mut server, &h.queue); + let _ = done.send(result.is_ok()); + h + }); + let answered = ended + .recv_timeout(Duration::from_secs(20)) + .expect("intake never gave up on a silent peer"); + assert!(answered, "a silent peer is refused as malformed, not left as an error"); + let h = worker.join().unwrap(); + assert_eq!( + rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn a_record_under_another_schema_is_not_read() { + let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { + "order" => h.ctx.store.put( + key, + schema, + &OrderRecord { + schema_version: schema.into(), + order_id: key.into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }, + ), + "start" => h.ctx.store.put( + key, + schema, + &StartRecord { + schema_version: schema.into(), + order_id: key.into(), + started_at_unix_millis: 1, + actuation: "idunn-order-x".into(), + }, + ), + "outcome" => h.ctx.store.put( + key, + schema, + &OutcomeRecord { + schema_version: schema.into(), + order_id: key.into(), + ended_at_unix_millis: 1, + outcome: Outcome::Lost, + stdout_tail: Vec::new(), + stderr_tail: Vec::new(), + }, + ), + _ => h.ctx.store.put( + key, + schema, + &PostRecord { + schema_version: schema.into(), + post_key: key.into(), + event_id: "$e".into(), + posted_at_unix_millis: 1, + }, + ), + }; + for (kind, current, key) in [ + ("order", ORDER_SCHEMA, "ord-a"), + ("start", START_SCHEMA, "ord-a"), + ("outcome", OUTCOME_SCHEMA, "ord-a"), + ("post", POST_SCHEMA, "ord-a:accepted"), + ] { + let h = harness(); + assert!(put(&h, kind, current, key).unwrap()); + assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); + let h = harness(); + let other = current.replace(".v1", ".v0"); + assert!(put(&h, kind, &other, key).unwrap()); + assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); + } + } + + #[test] + fn posts_are_retried_after_a_widening_pause() { + let pauses: Vec = (0..9).map(post_pause).collect(); + assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); + } + + #[test] + fn serve_options_are_read_from_every_flag() { + let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); + let defaults = parse(&["serve"]).unwrap(); + assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); + assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); + assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); + let given = parse(&[ + "serve", + "--state-store", + "/a/control.cc", + "--store", + "/b", + "--steward-binding", + "/c.toml", + "--socket", + "/d.sock", + ]) + .unwrap(); + assert_eq!( + given, + ServeOptions { + state_store: "/a/control.cc".into(), + store: "/b".into(), + steward_binding: "/c.toml".into(), + socket: "/d.sock".into(), + } + ); + assert!(parse(&["serve", "--store"]).is_err()); + assert!(parse(&["serve", "--other", "/x"]).is_err()); + assert!(parse(&["start"]).is_err()); + } + + #[test] + fn the_steward_user_is_a_real_user_and_never_root() { + assert!(user_uid("root").is_err()); + assert!(user_uid("no-such-user-here").is_err()); + assert!(user_uid("bad\0name").is_err()); + assert_eq!(user_uid("nobody").unwrap(), 65534); + } + + #[test] + fn the_orders_directory_is_made_private() { + let directory = TempDir::new().unwrap(); + let store = OrderStore::new(&directory.path().join("orders")); + store.prepare().unwrap(); + let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + #[test] + fn the_socket_belongs_to_the_steward_alone() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("steward.sock"); + let me = std::fs::metadata(directory.path()).unwrap().uid(); + let _listener = bind_socket(&path, me).unwrap(); + let metadata = std::fs::metadata(&path).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); + // A stale socket is replaced; anything else in the way is not. + let _again = bind_socket(&path, me).unwrap(); + let other = directory.path().join("other"); + std::fs::write(&other, "x").unwrap(); + assert!(bind_socket(&other, me).is_err()); + } + + #[test] + fn synapse_gives_up_on_a_reply_that_never_ends() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + thread::spawn(move || { + // The first connection stays silent; the second drips a byte at a time. + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + thread::spawn(move || { + let mut chunk = [0u8; 4096]; + let _ = socket.read(&mut chunk); + for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { + if index == 0 || socket.write_all(&[*byte]).is_err() { + break; + } + thread::sleep(Duration::from_millis(40)); + } + thread::sleep(Duration::from_secs(5)); + }); + } + }); + let synapse = Synapse { + address, + room: "!r:e".into(), + token_file, + reply_within: Duration::from_millis(600), + }; + let synapse = Arc::new(synapse); + for _ in 0..2 { + let started = std::time::Instant::now(); + let (done, ended) = mpsc::channel(); + let poster = Arc::clone(&synapse); + thread::spawn(move || { + let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); + }); + let error = ended + .recv_timeout(Duration::from_secs(4)) + .expect("the post never gave up") + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error}"); + assert!(!error.contains("tok-123")); + } + } + + /// The whole service, as root, against fake `systemctl`, `systemd-run` + /// and a loopback homeserver, with the steward connecting from another + /// uid. It runs only where the test process is root (the verify container). + #[test] + fn serve_end_to_end_as_root() { + // SAFETY: geteuid reads the process credentials. + if unsafe { libc::geteuid() } != 0 { + eprintln!("serve_end_to_end_as_root: skipped, not root"); + return; + } + let directory = TempDir::new().unwrap(); + let root = directory.path().to_owned(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let bin = root.join("bin"); + std::fs::create_dir(&bin).unwrap(); + let systemctl = format!( + "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ + case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + root.display() + ); + let run = format!( + "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", + root.display() + ); + for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { + let path = bin.join(name); + std::fs::write(&path, text).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let credentials = root.join("cred"); + std::fs::create_dir(&credentials).unwrap(); + std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (posted, posts) = mpsc::channel::(); + thread::spawn(move || { + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + let mut received = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + let text = String::from_utf8_lossy(&received).into_owned(); + if let Some(split) = text.find("\r\n\r\n") { + let length = text + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if received.len() >= split + 4 + length { + break; + } + } + let count = socket.read(&mut chunk).unwrap_or(0); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); + let _ = socket.write_all( + format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), + ); + } + }); + let binding = root.join("steward.toml"); + std::fs::write( + &binding, + format!( + "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ + token_credential = \"idunn-matrix-token\"\n" + ), + ) + .unwrap(); + std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); + let run_directory = root.join("run"); + std::fs::create_dir(&run_directory).unwrap(); + std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let socket = run_directory.join("steward.sock"); + let store = root.join("orders"); + let control = root.join("state").join("control.cc"); + // SAFETY: the only test that reads these variables is this one; PATH + // keeps its old entries. + unsafe { + std::env::set_var( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), + ); + std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); + } + let options = ServeOptions { + state_store: control.clone(), + store: store.clone(), + steward_binding: binding, + socket: socket.clone(), + }; + // The unit's umask must not decide the modes of what the service writes. + // SAFETY: umask only sets the process file-mode mask. + let before = unsafe { libc::umask(0o117) }; + let (ended, serve_end) = mpsc::channel::(); + thread::spawn(move || { + let result = serve(options); + let _ = ended.send(format!("{result:?}")); + }); + for _ in 0..100 { + if socket.exists() || serve_end.try_recv().is_ok() { + break; + } + thread::sleep(Duration::from_millis(100)); + } + assert!(socket.exists(), "serve never bound its socket"); + let metadata = std::fs::metadata(&socket).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); + // Root is not the steward: it may read, it may not order. + let unit = |name: &str| { + Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: name.into(), + }, + reason: "end to end".into(), + }) + }; + let as_root = |request: &Request| { + let mut stream = UnixStream::connect(&socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + write_frame(&mut stream, request).unwrap(); + rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + }; + assert_eq!( + as_root(&unit("nginx.service")), + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert_eq!(as_root(&Request::List), Response::List(Vec::new())); + let orders = OrderStore::new(&store); + let finished = |order_id: &str| { + for _ in 0..200 { + if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { + return record.clone(); + } + thread::sleep(Duration::from_millis(100)); + } + panic!("{order_id} never concluded"); + }; + // The steward is uid 65534 with gid 0: a service that read the gid for + // the uid would take it for root. + let first = steward_asks(&socket, &unit("nginx.service")); + let Response::Accepted { order_id: first } = first else { + panic!("the steward was not accepted: {first:?}"); + }; + assert_eq!(finished(&first).outcome, Outcome::Ran); + std::fs::write(root.join("fail"), "").unwrap(); + let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { + panic!("the second order was not accepted"); + }; + let failed = finished(&second); + assert_eq!( + failed.outcome, + Outcome::Failed { + exit_code: Some(3), + signal: None + } + ); + assert_eq!(failed.stderr_tail, b"boom\n"); + let deploy = Request::Order(OrderRequest { + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "end to end".into(), + }); + let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { + panic!("the deploy was not accepted"); + }; + assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); + let ledger = orders.ledger().unwrap(); + let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); + assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // An invocation id is what a succeeding systemctl prints, if anything. + assert_eq!(HostSystemd.invocation("quiet.service"), None); + assert_eq!(HostSystemd.invocation("broken.service"), None); + assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); + // What the service called, exactly. + let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); + assert!( + run_log.contains(&format!( + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", + order_uuid(&first) + )), + "{run_log}" + ); + let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); + assert!( + systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), + "{systemctl_log}" + ); + // Every record is posted; the deploy's result waits for its command. + let mut seen = Vec::new(); + while seen.len() < 5 { + seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); + } + for (id, kinds) in [ + (&first, ["accepted", "result"].as_slice()), + (&second, &["accepted", "result"]), + (&third, &["accepted"]), + ] { + for kind in kinds { + let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); + assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); + } + } + // Modes: the store is private, and the deploy wrote control.cc under + // Idunn's umask, not the 0117 the process was started with. + let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; + assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); + let control_mode = mode(&control); + // SAFETY: umask only sets the process file-mode mask. + unsafe { libc::umask(before) }; + assert_eq!(control_mode, 0o640); + } + + /// One request as the steward: uid 65534 and gid 0, from a forked child + /// that only makes system calls. + fn steward_asks(socket: &Path, request: &Request) -> Response { + let body = rmp_serde::to_vec_named(request).unwrap(); + let mut frame = (body.len() as u32).to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + // SAFETY: plain data. + let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { + address.sun_path[index] = *byte as libc::c_char; + } + let mut fds = [0i32; 2]; + // SAFETY: `fds` holds two descriptors. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + // SAFETY: the child allocates nothing and ends with _exit. + let pid = unsafe { libc::fork() }; + if pid == 0 { + // SAFETY: only system calls on buffers made before the fork. + unsafe { + libc::close(fds[0]); + libc::alarm(10); + if libc::setgroups(0, std::ptr::null()) != 0 + || libc::setgid(0) != 0 + || libc::setuid(65534) != 0 + { + libc::_exit(2); + } + let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if libc::connect( + fd, + (&address as *const libc::sockaddr_un).cast(), + std::mem::size_of::() as libc::socklen_t, + ) != 0 + { + libc::_exit(3); + } + let mut offset = 0usize; + while offset < frame.len() { + let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); + if count <= 0 { + libc::_exit(4); + } + offset += count as usize; + } + let mut buffer = [0u8; 4096]; + loop { + let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); + if count <= 0 { + break; + } + libc::write(fds[1], buffer.as_ptr().cast(), count as usize); + } + libc::_exit(0); + } + } + // SAFETY: the parent's copy of the write end. + unsafe { libc::close(fds[1]) }; + let mut reply = Vec::new(); + // SAFETY: `fds[0]` is an open descriptor this function now owns. + let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; + pipe.read_to_end(&mut reply).unwrap(); + let mut status = 0; + // SAFETY: `pid` is our child. + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); + rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + } + + fn host_refusal(verb: Verb, unit: &str) -> Option { + refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) + } + + #[test] + #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] + fn real_host_unit_set() { + for name in [ + "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", + "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", + "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", + "wg-quick@wg-gamecult", "cron", "apt-daily", + ] { + let unit = format!("{name}.service"); + for verb in all_verbs() { + assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); + } + } + for name in [ + "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", + "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", + "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", + "systemd-storagetm", + ] { + let unit = format!("{name}.service"); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + host_refusal(verb, &unit), + Some(refused(&unit, Via::OnEnd)), + "{verb:?} {unit}" + ); + } + } + for unit in HOST_NAMES { + for verb in all_verbs() { + assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); + } + } + for unit in [ + "systemd-networkd.service", + "user-runtime-dir@1000.service", + "user-runtime-dir@105.service", + "user-runtime-dir@978.service", + ] { + assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); + assert!( + matches!( + host_refusal(Verb::Stop, unit), + Some(Refused::AlwaysRefused { via: Via::Bound, .. }) + ), + "{unit}" + ); + } + let listing = Command::new("systemctl") + .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) + .output() + .unwrap(); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let units: Vec<&str> = listing + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|unit| validate_unit(unit).is_ok()) + .collect(); + for verb in all_verbs() { + let mut counts = BTreeMap::new(); + for unit in &units { + let key = match refusal_for(&HostSystemd, unit, verb) { + Ok(None) => "ok".to_owned(), + Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), + Ok(Some(other)) => format!("{other:?}"), + Err(_) => "unresolved".to_owned(), + }; + *counts.entry(key).or_insert(0usize) += 1; + } + println!("{verb:?} over {} units: {counts:?}", units.len()); + } + } + + #[test] + #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] + fn one_job_argv_against_a_real_manager() { + let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); + std::fs::create_dir_all(&directory).unwrap(); + let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); + let body = "[Service]\nExecStart=/bin/sleep 600\n"; + std::fs::write(directory.join(a), body).unwrap(); + std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); + let user = |args: &[&str]| { + let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); + (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) + }; + let outcome = std::panic::catch_unwind(|| { + let one_job = |verb: Verb, unit: &str| { + let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); + let mut argv = step.argv.clone(); + argv.insert(1, "--user".into()); + let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); + assert!(status.success(), "{verb:?} {unit}"); + }; + assert!(user(&["daemon-reload"]).0); + // A start of b leaves a running, though b conflicts with it. + assert!(user(&["start", a]).0); + one_job(Verb::Start, b); + assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); + // A restart of b, with both running, leaves a running. + one_job(Verb::Restart, b); + assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); + }); + user(&["stop", a, b]); + std::fs::remove_file(directory.join(a)).unwrap(); + std::fs::remove_file(directory.join(b)).unwrap(); + user(&["daemon-reload"]); + outcome.unwrap(); + } +} diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 70060bf..b9f2637 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -10,3 +10,19 @@ It is checked in because the TypeScript runtime emits a camelCase MessagePack map while Rust `DatabaseEntry` derives tuple payloads. Idunn must prove it can decode the producer's real wire shape rather than round-tripping a fixture it serialized itself. + +# systemd 257 `systemctl show` captures + +`systemd-257/*.show` are the verbatim output of systemd 257.13 for the 27 +properties `show_unit` requested at `112a7a5`, captured read-only on Yggdrasil +on 2026-10-24: `vili-presence-identity-only.show` (a failed idunn-vili unit whose +only credential is the presence identity) and +`odin-presence-and-one-service-credential.show` (the live Odin unit). systemd 257 +prints `LoadCredential=[unprintable]` in both. Tests read these bytes; do not edit. + +# Pre-cut systemd observation records + +`pre-cut-systemd-observation.{compact,named}.hex` are the two MessagePack +encodings of a `SystemdWorkloadObservation` with `load_credential` set to +`[unprintable]`, produced by the code at `112a7a5` (the form live incumbents +persisted). They pin that later Idunns still decode such records. diff --git a/tests/fixtures/pre-cut-systemd-observation.compact.hex b/tests/fixtures/pre-cut-systemd-observation.compact.hex new file mode 100644 index 0000000..e7d09fb --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.compact.hex @@ -0,0 +1 @@ +dc0033d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365aa4964756e6e2074657374a3696e7601a465786563a26e6fa56d69786564c3a6753631303030a6753631303030a0a0a0c3c3a9696e76697369626c65a3616c6cc3a430303037a0ad5b756e7072696e7461626c655dcd10920194cdee48cdee48cdee48cdee4894cdee48cdee48cdee48cdee4891cdee480000000000c391010708b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d3232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323290d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190 \ No newline at end of file diff --git a/tests/fixtures/pre-cut-systemd-observation.named.hex b/tests/fixtures/pre-cut-systemd-observation.named.hex new file mode 100644 index 0000000..8898adf --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.named.hex @@ -0,0 +1 @@ +de0033a4756e6974d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365b0756e69745f6465736372697074696f6eaa4964756e6e2074657374ad696e766f636174696f6e5f6964a3696e76d923657865635f6d61696e5f73746172745f74696d657374616d705f6d6f6e6f746f6e696301ac736572766963655f74797065a465786563ae726573746172745f706f6c696379a26e6fa96b696c6c5f6d6f6465a56d69786564ac64796e616d69635f75736572c3ac73797374656d645f75736572a6753631303030ad73797374656d645f67726f7570a6753631303030b4737570706c656d656e746172795f67726f757073a0b76361706162696c6974795f626f756e64696e675f736574a0b4616d6269656e745f6361706162696c6974696573a0ae707269766174655f6d6f756e7473c3ac707269766174655f70696473c3ac70726f746563745f70726f63a9696e76697369626c65ab70726f635f737562736574a3616c6cb16e6f5f6e65775f70726976696c65676573c3a5756d61736ba430303037b2696e61636365737369626c655f7061746873a0af6c6f61645f63726564656e7469616cad5b756e7072696e7461626c655da86d61696e5f706964cd1092b270726f636573735f73746172745f74696d6501ac70726f636573735f7569647394cdee48cdee48cdee48cdee48ac70726f636573735f6769647394cdee48cdee48cdee48cdee48ae70726f636573735f67726f75707391cdee48b770726f636573735f6361705f696e686572697461626c6500b570726f636573735f6361705f7065726d697474656400b570726f636573735f6361705f65666665637469766500b470726f636573735f6361705f626f756e64696e6700b370726f636573735f6361705f616d6269656e7400b970726f636573735f6e6f5f6e65775f70726976696c65676573c3b670726f636573735f6e616d6573706163655f706964739101b26d6f756e745f6e616d6573706163655f696407b07069645f6e616d6573706163655f696408aa65786563757461626c65b52f6f70742f746573742f62696e2f73657276696365b165786563757461626c655f64657669636501b065786563757461626c655f696e6f646501b165786563757461626c655f736861323536d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131b372756e74696d655f696e7374616e63655f6964d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161b1776f726b696e675f6469726563746f7279a92f6f70742f74657374ae72756e74696d655f62756e646c65a92f72756e2f74657374b3636f6d6d616e645f6c696e655f736861323536d9477368613235362d32323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232b1656e7669726f6e6d656e745f6e616d657390bb656e7669726f6e6d656e745f636f6e74726163745f736861323536d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333ad636f6e74726f6c5f67726f7570d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365b563726564656e7469616c735f6469726563746f7279c0bc706172656e745f6f6e6c795f66696c655f64657363726970746f727390bd61637469766174696f6e5f7369676e65725f6964656e746974795f6964aa61637469766174696f6ebc61637469766174696f6e5f7369676e65725f7075626c69635f6b6579dc00200101010101010101010101010101010101010101010101010101010101010101b3736572766963655f63726564656e7469616c7390 \ No newline at end of file diff --git a/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show new file mode 100644 index 0000000..21e4573 --- /dev/null +++ b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=2571 +OpenFile=/run/idunn/activation-credentials/sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/odin/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=20630000 +ControlGroup=/system.slice/idunn-odin-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.service +UMask=0007 +WorkingDirectory=/srv/odin/releases/sha256-e5c7d0b2b9c0a8ffba368aeb08abdbe37ae0d17484a785e60a66ec4a62b2f18c +CapabilityBoundingSet= +AmbientCapabilities= +User=_duc23369f1b8b167db +Group=odin +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn odin sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8 +LoadState=loaded +ActiveState=active +SubState=running +InvocationID=ada48921043449f39dacc87b14a1bf70 diff --git a/tests/fixtures/systemd-257/vili-presence-identity-only.show b/tests/fixtures/systemd-257/vili-presence-identity-only.show new file mode 100644 index 0000000..fc78fd1 --- /dev/null +++ b/tests/fixtures/systemd-257/vili-presence-identity-only.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=0 +OpenFile=/run/idunn/activation-credentials/sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/vili/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=520083420000 +ControlGroup= +UMask=0007 +WorkingDirectory=/srv/vili-idunn/releases/sha256-aeaee4e1669742572c45a3f26bc86893c44b2a4149520975eb7d8a28aa931906 +CapabilityBoundingSet= +AmbientCapabilities= +User=_du8c06c563aac6d205 +Group=vili-state +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn vili sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041 +LoadState=loaded +ActiveState=failed +SubState=failed +InvocationID=8f7a349f2f8f49fe87ac5d89239260b4