diff --git a/Cargo.lock b/Cargo.lock index 44c9457..dfc3087 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,7 +482,6 @@ dependencies = [ "libc", "rmp-serde", "serde", - "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index ee797ee..713ad33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,10 +18,6 @@ path = "src/bin/idunn-provision.rs" name = "idunn-host" path = "src/bin/idunn-host.rs" -[[bin]] -name = "idunn-launch" -path = "src/bin/idunn-launch.rs" - [lib] name = "idunn_daemon" path = "src/lib.rs" @@ -38,7 +34,6 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } -serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/README.md b/README.md index 8c6935c..503409e 100644 --- a/README.md +++ b/README.md @@ -58,64 +58,6 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. -### Steward orders - -A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), -takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. -It has its own store, `orders.cc`, and its own binding, `steward.toml`; the -survival daemon takes no order. There is still no command string: - -```bash -idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them -idunn order unit --reason TEXT -idunn order deploy --reason TEXT # the same commit as `idunn up` -idunn order show # status and output tails -idunn order list -``` - -A unit order submits exactly one job, for the unit it names: every verb runs -`systemctl --job-mode=ignore-requirements -- `, so systemd builds a -transaction of one job. It does not start what the unit requires or restart what -is part of it: order the dependency first, and order `postgresql@16-main.service` -rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is -not: systemd still runs full transactions of its own for `Restart=` after a -failure, for a `StopWhenUnneeded=` dependency left without a user, and for a -`Conflicts=` partner of a unit that fails, so an ordered stop of a unit -that something restarts does not last, and a start of a `Restart=always` -unit can report failure and be active seconds later. A unit -order may act on any unit except Idunn's own (`idunn-*`) and the minds' own -(`mind-self@*`, `mind-persona@*`), and the units that end or take the host -(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, -`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled -into the binary and read from no configuration. An order is refused, recorded -and posted when the ordered unit, or an alias it resolves from, is one of them; -when it has an emergency action (`SuccessAction=`, `FailureAction=`, -`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` -or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another -unit is `BoundBy=` it, because systemd stops that unit with it whatever the job -mode; for a start or restart, when it has an `Upholds=` unit, which systemd -starts in a full transaction whatever the job mode; or, for a stop, restart or -reload, when it has a `ConsistsOf=` member, a wrapper such as -`postgresql.service` whose verbs would exit 0 and touch nothing. A unit -`systemctl show` cannot resolve fails the order. Slices, scopes, -targets and globs are not units an order can name. - -A package order installs the packages it names and nothing else: apt runs with -`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` -match themselves, and a last step fails the order unless every named package -is installed. - -An order runs once, after the steward's uid is checked and the instance's -brake check (`mind-control --instance check`) exits 0. The check -that the order's principal and uid are the bound steward's runs again at every -execution, so an order accepted before `steward.toml` changed is refused and -recorded, not run. The socket belongs to the steward user alone (mode 0600); -root may `show` and `list` orders but never issue one. An order that had -started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to -`#operator` as a retried projection of `orders.cc`: a chat outage delays the -posts and not the orders. A refusal names the field and a code, never the value -it refused. - ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. @@ -208,79 +150,6 @@ omits one. Unknown fields are rejected on both sides. A repository that gets compromised can therefore change what is built, but not what it is allowed to touch on the host. -An upstream daemon that does not speak Idunn's runtime contract is deployed -with `idunn-launch` as its executable artifact and the upstream tree as a -required adjacent artifact; the launcher finds the child inside the release that -holds it. The launcher is the unit's process: it binds the -candidate endpoint and answers Idunn's challenge there, renders the child's -config, starts a stateful child only once it holds the write lease, and signs -presence. Members never pass through it: the binding's `[route.member]` makes -Idunn route a second listener straight to the child, which binds the address -Idunn gives it in `GAMECULT_IDUNN_MEMBER_BIND`. - -```toml -# recipe fragment — the upstream tree is built or fetched as an artifact -[service] -executable_artifact = "idunn-launch" -arguments = [ - { kind = "literal", value = "--child" }, - { kind = "literal", value = "upstream/bin/daemon" }, - { kind = "literal", value = "--template" }, - { kind = "literal", value = "config.tmpl" }, - { kind = "literal", value = "--health-path" }, - { kind = "literal", value = "/health" }, - { kind = "literal", value = "--health-contract" }, - { kind = "literal", value = "example.presence-launcher.v1" }, -] -required_environment = [ - "GAMECULT_IDUNN_RUNTIME_BUNDLE", - "GAMECULT_IDUNN_CANDIDATE_BIND", - "GAMECULT_IDUNN_MEMBER_BIND", -] -``` - -A directory artifact is sealed read-only: its files are 0444 and its -directories 0555, whatever mode the build or git gave them. A file the tree -must run, such as the launcher's `upstream/bin/daemon` child, is declared in -`executable_members` and sealed 0555. The members are part of the sealed release -address, so changing the list makes a new release. Idunn refuses the release if a -member is absolute, climbs with `..`, is missing, is a symlink or sits behind -one, or is not a regular file; `executable = true` on a directory output and -`executable_members` on a file output are refused too. - -A tree may hold relative symlinks whose `..` steps all come first and climb no -higher than the artifact's top (`share/terminfo/4/4025ex -> ../t/tek4025ex` is -kept). An absolute link, a `..` after a name, or a climb out of the artifact -refuses the release when it is sealed. Host targets (Windows) record -`executable` and `executable_members` in the receipt and do not apply them. - -```toml -# recipe fragment — the upstream artifact declares the child it launches -[[artifacts]] -id = "upstream" -source_kind = "runner-output" -runner = "build" -source = "dist/upstream" -destination = "upstream" -executable_members = ["bin/daemon"] -``` - -```toml -# binding fragment — members connect to the member endpoint, which Idunn -# forwards to the private port derived from the candidate: member -# private_port_start + (candidate port - route private_port_start) -[route] -stable_endpoint = "tcp://10.77.0.1:8008" -private_port_start = 18831 -private_port_end = 18839 -# ... - -[route.member] -stable_endpoint = "tcp://10.77.0.1:8448" -private_port_start = 18841 -# max_connections = 128 # nginx limit_conn on the member listener (1-65535) -``` - ## Contracts Idunn's typed contracts are published by diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service deleted file mode 100644 index f2681c2..0000000 --- a/deploy/idunn-orders.service +++ /dev/null @@ -1,47 +0,0 @@ -[Unit] -Description=Idunn steward orders: typed package, unit and deploy orders from the steward -# Deliberately no After= or Requires= on any managed target, Synapse or -# idunn-yggdrasil: orders are how an operator repairs those, so none of them -# may be a condition of this unit starting. Posts are a retried projection and -# wait for Synapse on their own. - -[Service] -Type=simple - -# Root, from the same binary as idunn-yggdrasil, with its own store and socket. -# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: -# no order code runs in `idunn serve`. -# -# The unit name matches idunn-*, which is in the always-refused set, so no -# order can stop or restart this service. -User=root -# No Group= and no UMask=: the service gives its socket to the steward user -# (mode 0600) itself and sets Idunn's own umask for the files it writes, so -# nothing here reaches control.cc. The steward user is the only peer the -# service answers: it checks SO_PEERCRED on every connection. -ExecStart=/usr/local/bin/idunn orders serve \ - --store /var/lib/gamecult/idunn-orders \ - --state-store /var/lib/gamecult/idunn/control.cc \ - --steward-binding /etc/gamecult/idunn/steward.toml \ - --socket /run/idunn-orders/steward.sock -Restart=always -RestartSec=5 -TimeoutStopSec=20 - -RuntimeDirectory=idunn-orders -RuntimeDirectoryMode=0755 - -# The Synapse access token of @idunn, copied from a root-only file, and read -# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. -LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token - -PrivateTmp=yes -ProtectHome=yes -ProtectSystem=strict -# Its own order store, and the control store a deploy order commits a command -# to. Transient units started with systemd-run run in the manager's namespace, -# so packages and units are not limited by this list. -ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn - -[Install] -WantedBy=multi-user.target diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index 1c45594..da82bd2 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -104,11 +104,6 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web -# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a -# target whose roots are not created yet cannot stop Idunn itself from starting -# (Idunn startup never depends on a managed target). A root created after Idunn -# starts is writable only from Idunn's next restart. -ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases [Install] WantedBy=multi-user.target diff --git a/docs/authority-map.md b/docs/authority-map.md index c8a8546..1cbf3ca 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -55,10 +55,7 @@ A deployable repository publishes one strict declaration, normally - immutable artifact outputs, optionally pinned to an expected SHA-256; - a constrained launch contract; - one health-contract identity; -- an optional typed state contract with state schema generation and slots - (kinds `cultcache-file`, `external-file` and `external-directory`; the - external kinds are process-owned state an external runtime imposes, which - Idunn preserves and fences but never interprets); +- an optional typed state contract with state schema generation and slots; - provided capabilities, dependency kinds, startup requirements, capacity, and conflicts. @@ -70,52 +67,6 @@ recipe fields. Stateless services declare no state block. Their operator binding therefore has no state root, state-transition policy, or process write-lease record. -## Foreign workloads: idunn-launch - -An upstream daemon that cannot speak Idunn's runtime contract (Synapse, MAS) -runs under `idunn-launch`, the unit's own process. Idunn's two-witness rule is -unchanged: Idunn observes the unit, and the launcher, inside the unit and -supervising the child, signs presence with the provider identity and the -activation key. - -- **Owner:** the launcher owns presence, config rendering and the child's - lifetime inside the unit, and the route answers it signs for that child. - Idunn owns admission, lease, route and continuity, as before. -- **Inputs:** the runtime bundle, the activation credential and provider - identity descriptors, the process write lease, the systemd credentials the - binding names, and the child's health over TCP at the member bind. -- **Outputs:** signed `gamecult.runtime_presence_health.v2` (Warming, Active - and failed to Odin; route observations on the candidate endpoint), one child - process, and a rendered config in the unit's private tmp that is never read - back. -- **Route:** Idunn owns both listeners of the route: the Expected route's - stable endpoint (presence) and, when the binding has `[route.member]`, the - member endpoint, rendered as a second server block in the one nginx fragment - with a `limit_conn` on one per-route zone (`max_connections`, default 128), so - members cannot exhaust the connections the presence block shares with them. - The launcher binds the candidate endpoint and answers Idunn's challenge there, - one connection at a time within a one-second deadline; any other first byte - closes the connection. The answer never waits on the child: it reports the - lifecycle's last probe, and a failed, missing or stale probe reads `degraded`. - The child binds the member bind Idunn sets in - `GAMECULT_IDUNN_MEMBER_BIND`, derived from the candidate port. Member bytes - never touch the launcher. -- **Lease:** with `write_lease_required` the launcher starts the child only - after it holds the lease (bound to this Expected, this activation and one of - its own Warming proofs) and keeps the lease store share-locked while the - child lives. Losing the lease stops the child. -- **Config:** one template from the release is rendered with the closed - placeholder set `{{child.listen}}`, `{{child.listen_host}}`, - `{{child.listen_port}}` (all three the member bind), `{{credential:NAME}}`, - `{{credential_path:NAME}}` and `{{state_root}}`. Any other placeholder - refuses to start. -- **Forbidden:** member traffic through the launcher; the child binding the - candidate endpoint; a child of a lease-requiring target running without a held lease; any presence not signed - by both keys. -- **State kind `external-directory`:** a process-owned directory imposed by an - external runtime contract. Same writer, recovery and startup rules as - `external-file`: it is lease-gated, preserved and fenced, never interpreted. - ## Operator binding The root-admitted operator binding supplies the physical policy a repository @@ -222,40 +173,24 @@ health plus Ready correlation. It changes and observes route membership only afterward. Only after the proxy has adopted the candidate may Idunn clear the promotion fence and drain the incumbent. -A target has two slots: the incumbent (the admitted generation) and a -transaction's candidate. Continuity owns the incumbent slot and keeps the -admitted generation alive; it never stops. A live `DeploymentTransaction` owns -its private candidate before Fencing and takes the target itself at Fencing (a -stop-then-start rollout, at Starting) through one `take_target` CAS, so of two -transactions entering from one view exactly one succeeds. A continuity -restart is itself a transaction and takes the target the same way. - -Phases from Warming through Committing carry deadlines, as do aborts and -drains; an expired deadline never waits silently. A stateless deployment -aborts to its incumbent. A deployment whose candidate was issued a write lease -is left to the operator (`operator-required` incident, the second incident -condition beside `continuity-exhausted`; its subject is the transaction id and -it closes when the record ends or its step goes on). A continuity after -Fencing retries its phase forward on a capped backoff and never stops its live -candidate, since that is the only running copy of the admitted body; the -deadline opens the same incident. The one exception is yielding: a continuity -whose route proof has not succeeded gives way, through its own post-fencing -abort, to a deployment the deployment brake has released and that awaits the -handover; a continuity that has proven its route never yields. The -`ContinuityExhausted` incident (six charged attempts in an hour, -`restarts_page`) pages and gates nothing. When continuity next acts is derived -only from its own record in `TargetSupervision` (schema v2): the attempt -streak, the backoff anchor (written by the dead-and-free pass, the only -writer), and the once-per-admission re-materialization bound. The workload -driver alone answers whether a lost unit was a host reboot -(`WorkloadPort::lost_to_host_reboot`). Operator-facing incidents and -`idunn status` are projections that decide nothing. A boot lift returns a -continuity abort left by an older Idunn, one that had not stopped its running -candidate, to forward retry; if it cannot decide, every continuity abort is -held until a lift succeeds. It is a migration with a deletion line (follow-up -`c2c-delete-boot-lift`). Cut line: cuts 3 and 4 of `reincarnation-map.md` -still owe dependency failures not growing the streak and the dead-generation -route refresh. +Ownership of a target is exclusive at each instant. A live +`DeploymentTransaction` owns it from Fencing on (a stop-first rollout, from +Starting); before Fencing a deployment owns only its private candidate and +continuity owns the incumbent slot. Continuity owns the admitted generation's +survival and never stops. There is no stopped state: six charged attempts in +an hour (`restarts_page`) open the ContinuityExhausted incident and gate +nothing, and no deployment path yields to continuity. When continuity next acts +is derived only from its own record in `TargetSupervision` (schema v2, a v1 +record lifts at store open): the attempt streak, the backoff anchor (written +by the dead-and-free pass, the only writer), and the once-per-admission +re-materialization bound. The workload driver alone answers whether a lost unit +was a host reboot (`WorkloadPort::lost_to_host_reboot`). Crash restart, restart +after a sustained run and reboot re-materialization all mint through one CAS; +continuity supervision never aborts a transaction. Operator-facing incidents +and `idunn status` are projections that decide nothing. Cut line: cuts 2 to 4 +of `reincarnation-map.md` still owe continuity restarting alongside a +pre-fencing build, dependency failures not growing the streak, and the +dead-generation route refresh. The deployment brake gates changes to source, release, configuration, schema, unit, or authority binding. The lifecycle brake separately gates restart of an diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md deleted file mode 100644 index 95c0eee..0000000 --- a/docs/deploy-provenance-map.md +++ /dev/null @@ -1,394 +0,0 @@ -# Deployment provenance: map - -Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, -session `self-2026-10-23-eureka-b`). The ends are in -[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, -rulings, cut specs and follow-ups live in Eureka's mind; this page holds what -has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are -at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`, -the head of Hands attempt 1, which revision 2 of the spec builds on. - -## Cut `tree-executable-members` - -Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2` -(revision 2, after Soul pass 1 found the real interpreter tree refused). It -serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's -admission waits on it) and will serve gamecult-chat C3 too: the presence -launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact -(README, launcher fragment), and Synapse needs a Python interpreter there. - -### Body facts - -- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable` - bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and - `validate` (`:740-760`) requires it to equal the declaration. -- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its - executable set from `root.join(receipt.destination)` only. `harden_root_tree` - (`:6858-6892`) seals every directory 0555 and every file 0444 unless that - exact path is in the set. A directory artifact with `executable = true` is - therefore silently meaningless today: its files are all 0444. Nothing refuses - it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin, - StreamPixels service and web surveyed): `executable = true` is on files only. -- T3. A tree output is copied by `copy_artifact`/`copy_tree` - (`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest - (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not - mode bits. So runner or git exec bits never reach the release: harden is the - only mode authority. -- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink - only if its target is relative with no `..`. It runs only inside - `harden_root_tree`, at install, so a tree with a `..` or absolute link copies, - digests and seals, then fails at `install_release` (Soul s1: copying and - digesting the real tree twice took 160 s before harden refused it). The rule - is sound but too narrow: it refuses the real interpreter tree (T13). The - frozen-source path has a physical resolver (`resolve_frozen_source_symlink`, - `:6617`) that walks the filesystem with a 40-link budget and a memo. Its - first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs - the tree on disk to answer. Revision 2 does not reuse it (T14). -- T5. The sealed release id hashes `(schema, plan_id, artifacts, - external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes - the whole plan including `recipe_blob`. A changed member list is therefore - already a new plan and a new release. The receipt must still carry the list, - because `harden_installed_release` reads receipts only; its two callers - (`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden - every installed release on each observation. -- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with - `#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every - existing receipt's bytes and release id: the plan's `deadlines` field - (`:574-577`) is the precedent. -- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file - whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB) - is held in Idunn's memory three times per install (seal, install, every - validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB - chunks (S9). The cut collapses both onto one streaming helper. -- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the - artifact destination a single filename. Members are paths inside that - destination, so it does not change. `require_relative_path` (`:1936-1949`) - with `allow_current = false` already refuses absolute, `..`, root and prefix - components. -- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy, - mode check, digest, pinned-digest check and receipt, the same steps as - `collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode - check survives every test. Revision 2 collapses both onto one `seal_artifact` - in `drivers.rs`. Each caller keeps only how it finds the source path. -- T10. The transient unit runs with `ProtectSystem=strict` and the release in - `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no - `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. - Shared libraries need no exec bit. -- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`, - installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is - identical between `8760a4f` and `6e2a31c`, so the live code is the - reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in - `src/`: the presence launcher and member route). Installing this cut ships all - of them. Idunn's own body changes only through gamecult-ops - `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, - bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). -- T13. Link census of the real interpreter tree (Yggdrasil probe of - `cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`, - sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049 - after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all - their `..` components before the first name (every one is - `share/terminfo// -> ..//`, one `..` each). 0 have a `..` - after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it. - Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1). -- T14. Lexically normalising `/` does not check - containment. The kernel resolves a target one component at a time, so a `..` - after a symlinked name climbs from where that link points, not from where its - name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x` - normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x` - beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same - way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`, - `L -> D/D/../../../etc/passwd` passed a lexical check). - The sound lexical rule constrains the target's shape: all its `..` components - come before its first name, and they number no more than the link's depth - below the artifact's top. The climb then starts from the link's own - directory, which the walk reached through real directories only (copy and - digest never descend through a link), so it lands on a real ancestor inside - the artifact. After that the target only descends, and any link it passes - through obeys the same rule. By induction on the kernel's hops (capped at 40, - then ELOOP), every hop stays inside. The check is one pass over the target's - components with no filesystem reads, so its cost is linear in the tree and no - resolver is needed. The shapes it refuses (`s/../x`) occur in neither real - tree (T13). - Prior art, from documentation as known to Imagination (high confidence for Go - and securejoin, medium for the tar details): - - Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account - for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug. - - Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc, - containerd and umoci) resolve each component inside the root when a path is - opened, refusing or clamping an escape. They guard one later access, need - the tree on disk, and check again on every open. Idunn instead certifies a - sealed tree once, so that the kernel's own resolution stays inside on every - later access. - - Zip-slip defences and archive extractors check entry names lexically and - protect against link targets by refusing to write through links - (libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for - names; GNU tar creates such links last). They protect the extraction, not - what a kept link points at afterwards. - - OCI layer extraction keeps link targets verbatim, absolute ones included, - because the runtime resolves them inside the container's root. Idunn runs a - release in the host's mount namespace, so absolute targets stay refused. - - Nix keeps link targets verbatim and tracks dependencies by scanning for - references. The whole store is its root, so it has no per-artifact - containment to compare with. - None of these is a static per-link rule for a tree that runs unconfined. The - T14 shape rule fits that case. -- T15. Host targets: `idunn-host` observes, starts and stops processes on - Windows only (the non-Windows `platform` module in `src/host.rs` bails), and - `host.rs` sets no file modes. On Windows a file runs by its type, not by a - mode bit, so `executable` and `executable_members` have nothing to set there. - The file `executable` flag has always been carried and never applied on host - targets. A recipe does not know which workload kind its binding will choose. -- T16. The release root holds only artifact destinations. Runner workspaces, - where external inputs land, are removed before `SealedRelease::new` - (`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and - `validate_installed_release` (`:2815-2822`) digest every receipt's - destination before calling harden. So a rule in the digest walk runs at seal, - at install and at every validate, over every link harden would see. -- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three - copies (runner workspace, staging, installed). Nothing collects old releases or - staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every - Vili deploy leaves about 7 GiB behind. - -### Contract, corrected against the body - -- Any directory artifact may declare `executable_members` (a set of relative - paths). The follow-up limited it to runner-output; that limit protects no - invariant, and the launcher's worktree or fetched trees need the same feature. -- `executable = true` names a file artifact. On a directory output the release - is refused, and `executable_members` on a file output is refused (T2). -- A member is refused at recipe parse if it is empty, absolute, or holds `.`, - `..`, a root or a prefix. At seal it is refused if it is missing, is a - symlink, passes through a symlink or non-directory, or is not a regular file. - Refusal happens before `SealedRelease::new`, so no sealed release can name a - member it cannot seal. -- Harden seals each member 0555 and fails if any declared executable path was - not visited as a regular file. "Every declared executable is sealed 0555, or - the install fails" holds structurally, not by a lookup that can miss. -- Symlinks: a link in a tree artifact is kept when its target is relative, its - `..` components all come before its first name, and they number no more than - the link's depth below the artifact's top (T14). One function, - `check_release_link`, decides this. Only the digest walk calls it, so a bad - link refuses the release at seal, before `SealedRelease::new`, and again at - install and validate (T16). Harden no longer judges links. The climb is - bounded by the artifact's top, not the release root: no live release has a - `..` link (the old rule refused them all), so nothing installed is newly - refused, and each artifact's digest describes a closed tree. -- Host targets carry `executable` and `executable_members` in the receipt, - shape-checked at seal by the shared `seal_artifact`, and do not apply them - (T15). Refusing members there would make a recipe's validity depend on its - binding. -- Vili's trees must - not come from `python -m venv`: its `bin/python` is an absolute link and its - `pyvenv.cfg` names the build workspace. Install packages into the relocatable - interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). - -### Authority - -Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd -targets and by `materialize` in `host.rs` for host targets) decides -membership from the recipe and writes it into the receipt. Harden enforces the -receipt and decides nothing. Runner exec bits and git `100755` modes are dead -inputs for release modes. Link containment belongs to `check_release_link`, -which only `digest_tree` calls. `validate_release_symlink` is deleted, and -harden's symlink arm leaves links as they are. - -### Acceptance sizing - -The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch -on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It -removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 -terminfo links (T13). If a tree holds a link the T14 rule refuses, the job -prints it and stops, and the link is reported to Self, not deleted. - -## Cut `isolation-credential-evidence` - -Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, -at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It -serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, -first deviation): every `idunn up vili` is refused before fencing. - -### Body facts (probed read-only on Yggdrasil, 2026-10-24) - -- I1. `idunn up vili` runs 6 and 7 failed with - `systemd workload isolation properties differ from the Idunn contract`, the - message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, - only one differs on the failed Vili unit: `LoadCredential`. Idunn expects - `""` because Vili has no service credential besides the presence identity - (`service_credential_names` empty); systemd shows `[unprintable]`. Every - other conjunct matches, InaccessiblePaths included (one value across all - five idunn-vili/odin units). -- I2. systemd 257.13 `systemctl show -p LoadCredential` prints - `LoadCredential=[unprintable]` for every unit, whatever the credentials: - `ssh.service` and `systemd-journald.service` (not DynamicUser, zero - LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin - unit (one). The display form carries no information, so the conjunct was - vacuous whenever credentials existed and always false when none did. Odin - and StreamPixels passed only because each carries a regular credential. -- I3. The exact value is readable: D-Bus - `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) - answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 - "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. -- I4. What the workload can actually read is the delivered directory. On the - host, `/run/credentials/` holds a directory only for units with credentials - (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID - root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` - (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` - descriptor, never a credential. -- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn - deployment transaction authority"), with its twin clause in - `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit - touched it. No ruling or finding in any campaign concerns workload isolation - verification; gamecult-chat's foreign-workloads cut touched - `CREDENTIALS_DIRECTORY` reservation, not this check. -- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both - MessagePack encodings, compact (positional) included - (`persisted_systemd_observations_decode_as_the_systemd_variant`, - `:9915-9930`), so deleting the `load_credential` field shifts every later - slot of a live incumbent's record. The field is also in the drift list - (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed - workload: changing what the field holds without leaving that list would make - the next Idunn refuse to stop the live Odin and StreamPixels incumbents. -- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 - properties for a Vili unit (presence identity only) and the Odin unit - (presence identity plus one service credential), captured 2026-10-24 into - this Imagination's scratchpad `fixtures/`. -- I8. The new rule holds for every live systemd incumbent: inside each - MainPID root the delivered entries equal its binding's `secret_files` keys - minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, - StreamPixels web 1; key names compared, no values read). An Idunn restart onto - this cut re-observes them (`:3955-3970`) and passes. - -### Decision - -A property systemd cannot display does not belong in the comparison. The -invariant it reached for, that the workload holds exactly the credentials Idunn -delivered, is observed where it is decided: the entries of -`/run/credentials/` in the MainPID's root equal Idunn's service -credential names, absent directory equal to none. That covers -`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and -needs no new program. The D-Bus read (I3) is exact but sees only one of those -properties and adds a second systemd client and parser; rejected. - -### Authority - -Owner: `observe_unit` decides the credential rule, from the delivered -directory. `LoadCredential`'s display form is no longer read or compared; the -`load_credential` field is a retired wire slot, written empty, out of the drift -list, kept only so persisted records decode. Forbidden writers: the -`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping -the slot waits on the next schema revision of the observation. - -### Deploy - -Idunn's body changes only through gamecult-ops -`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and -bindings unchanged). After it, Vili's Hands reruns from step 4 of -`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. - -## Cut `runner-failure-evidence` - -Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at -`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area -Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, -transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 -UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and -nobody could tell which recipe step failed. - -### Body facts (source read at `804ed75`; host read-only, 2026-10-25) - -- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by - `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and - `Command::output()`: stdout and stderr are each buffered whole in the - daemon's memory, unbounded. On a non-zero exit the error is - `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The - container is gone the moment it exits, so `docker ps -a` and `docker logs` - hold nothing. -- R2. The empty message: the failing process wrote nothing to stderr, and - stdout, the only stream that carried its report, is discarded at - `:2182-2193`. The same interpolation echoes whatever a step does write to - stderr into `last_error`, `pre_fencing_abort.error` and - `completion.error` (`bounded_detail`, 2048 bytes), and from there into - `idunn up`'s `deployment failed:` line and `idunn status`. -- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops - over `declaration.steps` and calls `run_in_workspace` with argv only; - `step.id` is used only in the environment precondition. External inputs - (`curl` in the runner, `:2247-2258`) fail through the same helper with the - same anonymous message. -- R4. Which step failed on 2026-10-25, inferred from the failed transaction's - staging root, which survives (926 MiB, - `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, - `node_modules/.package-lock.json` was written at 00:51:24 (so - `install-vili-dependencies`, `npm ci`, finished), nothing outside - `node_modules` is newer, and `release/vili` (the next step's output) is - absent; the python workspace holds only fetched inputs. The failing step was - `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test - reporter writes to stdout: exactly the stream R1 discards. Inference from - residue, not from output. -- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) - sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which - persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal - on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only - provider-exclusion lines. -- R6. Stores. `control.cc` (root 0640) holds commands, transactions - (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and - expiry requests; `ControlSnapshot::read` refuses any other type - (`:3420`). Records must re-encode byte-exactly (`decode_record`, - `:3717-3727`), so a new transaction field is a schema bump and a v4 lift - (`:4100-4110`). Terminal transactions move verbatim to `history.cc` - (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. - `incidents.cc` lives in the published projection directory and by its own - doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, - because a type added there stops continuity for every target on a rollback. -- R7. The daemon runs as root (`User=root`). Runner secrets reach the - container as read-only file mounts named by env (`:2041-2047`), so any text - a step derives from them can land in its output. -- R8. Staging residue: 475 transaction roots, 55 GiB under - `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces - because `materialize` returns before `remove_tree_inside`. - -### Decision - -The failure becomes a typed value at the runner, `RunnerStepFailure`: the -subject (recipe step or external input, by id), the runner id, the exit code or -signal, and the last 8 KiB of stdout and of stderr with each stream's total -byte count. Its `Display` carries the subject and status, never output. -Capture streams both pipes through bounded tails, so a verbose build no longer -buffers whole in the daemon. - -The tail lives in a new private store, `runner-evidence.cc`, beside -`control.cc` (mode 0600), one record per transaction -(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the -transaction record: that is a v5 bump whose rollback makes every older Idunn -refuse its control store (R6), the scar `incident.rs` already names. Not in -`incidents.cc`: published, and free text is barred there. Not in the journal: -journal readers are a wider set than root, so the journal line names the -transaction, the subject, the status and the store, and no output. - -No redaction. Matching secret bytes would make the daemon read every secret, -and still misses any encoding, prefix or transform of one; a claimed redaction -that leaks is worse than a stated rule. The rule instead is the reader set: the -store's only readers (root) can already read every secret the runner was -given. - -Keeping the failed container (dropping `--rm`) is worse: its output then lives -in Docker's json-file log, unbounded, untyped, outside Idunn's state and -readable by the docker group; containers accumulate with no owner to reap them; -`docker system prune` erases the evidence; and nothing ties a container to the -transaction in `idunn status`. - -### Authority - -Owner: the runner decides what failed and keeps the bounded tail; the engine's -pre-fence failure path is the one writer of `runner-evidence.cc`. -`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the -failure's `Display`. Forbidden writers: the `stderr` interpolation in -`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the -store needs no external owner; staging roots (R8) stay with -`ghostlight-verse:follow_up:idunn-generation-retention`. - -### Deploy - -Idunn's body changes only through gamecult-ops -`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and -bindings unchanged). The next failed Sealing then shows its step and tail on -`sudo idunn status --command `. diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index bdc7efc..6d464e5 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -313,36 +313,10 @@ written when the pass first finds the admitted generation dead and the target free, and a capped doubling wait from that anchor (5 s base, 300 s cap). A run of 600 s or more by the admitted body resets the streak. A generation lost to a host reboot (the workload driver's evidence, `lost_to_host_reboot`) is -re-materialized once per admission without charge or wait. - -Each target has two slots, the incumbent and a transaction's candidate. The -incumbent is continuity's until a transaction takes the target, at Fencing (at -Starting for a stop-then-start rollout), through one `take_target` CAS; of two -transactions entering from one view exactly one succeeds. Before that, a -deployment owns only its private candidate and the target is free for it -between continuity attempts. - -Phases Warming through Committing, post-fencing aborts and drains all carry -deadlines (the plan's frozen `PhaseDeadlines`; `WARMING_DEADLINE_SECONDS`, and -`PRE_FENCING_ABORT_DEADLINE_SECONDS` for a pre-fencing abort). What an expired -post-fencing deadline does depends on the owner. A deployment whose candidate -was never issued a write lease aborts to the incumbent. One whose candidate was -issued a lease may have written under a new state contract and is left to the -operator. A continuity retries its phase forward on a capped backoff and never -stops its live candidate, because that candidate is the only running copy of -the admitted body. Both operator cases open an `operator-required` incident -(subject: the transaction id) that closes when the record ends or its step goes -on. - -The one yield: a continuity whose route proof has not succeeded gives way to a -deployment the deployment brake has released that awaits the handover of the -target. It begins its own post-fencing abort and the deployment fences what is -admitted. A continuity that has proven its route is serving never yields. - -At boot, a lift returns to forward retry any continuity abort an older Idunn -left over a running candidate. If the lift errs or cannot observe the -candidate, the abort is held, not run. The lift is a migration with a deletion -line (follow-up `c2c-delete-boot-lift`). +re-materialized once per admission without charge or wait. The target is free +for a deployment's candidate between attempts, and a deployment never yields to +continuity before Fencing; from Fencing on (or from Starting for a stop-first +rollout) the live transaction alone owns the target. An admitted generation keeps its Ready and latest Odin receipts as durable proof of the exact activation at their recorded admission times. Dependency diff --git a/docs/host-actuator.md b/docs/host-actuator.md index c595c5d..cdb8cde 100644 --- a/docs/host-actuator.md +++ b/docs/host-actuator.md @@ -97,8 +97,7 @@ Each of these was found by running the pipe against Raven, not by reading. bind time. `rollout.strategy = "stop-then-start"` exists for exactly a route-less host-actuator workload: the admitted incumbent is stopped at Starting, before the candidate starts, and supervision leaves the target - alone from Starting on: the transaction takes the target there (one - `take_target` CAS) rather than at Fencing. A systemd-transient workload may not declare it. + alone from Starting on. A systemd-transient workload may not declare it. - **An unobservable host is silence.** The hub returns a typed `HostUnobservable` for not-attached, detached, reconnected and unanswered. Supervision skips that tick and logs it. Before this, an Idunn restart diff --git a/docs/reincarnation-map.md b/docs/reincarnation-map.md index 4f99aed..d63a6b3 100644 --- a/docs/reincarnation-map.md +++ b/docs/reincarnation-map.md @@ -575,26 +575,3 @@ its stores and `idunn.running` together, because the cb60965 binary cannot read "migrated 5 control record(s) … idunn.target_supervision.v2". streampixels-service kept running through the restart (unit 73ca40ba, PID 968752, NRestarts 0). The catalog returned 200, and 4443/udp was listening. - -## Installed: cut 2 - -Cuts c2-phase-split, c2b-owner-deadlines, c2c-post-fence-forward and -c2d-continuity-yields (branch tip `8760a4f`) were installed on Yggdrasil on -2026-10-20 at 00:34:06 UTC, with the operator's OK. - -- **The hotfix before it.** `ec45906` (`bounded_detail`) was installed on its - own on 2026-10-19 at 17:32 UTC. It unwedged tx-f978ab86. -- **Build.** `/srv/build/idunn-8760a4f`; `cargo test --lib` passed 471. -- **Backup.** `/root/idunn-store-backup-*-pre-reincarnation-c2` holds the - stores, `incidents.cc` and the binaries. A rollback must restore them - together: older binaries refuse two live transactions on one target, and - they refuse the `operator-required` incident condition. -- **First boot.** The boot lift returned exactly tx-039b91e9 (ghostlight) and - tx-4eaed669 (odin) to forward retry, as the Soul install gate predicted. - Odin's continuity, wedged in Routing since the 2026-10-18 reboot, committed - 4 s later. Generation tx-4eaed669 is admitted, its route is healthy, and the - same PID 2571 still runs. Ghostlight's continuity is retrying forward in - AwaitingReady. Its daemon's readiness is a ghostlight-side fault - (`Permission denied` reading its session-wrapping key). -- **Live processes.** Every live MainPID was the same before and after the - install. The streampixels catalog returned 200, and 4443/udp was listening. diff --git a/docs/watchdog-reporting-map.md b/docs/watchdog-reporting-map.md index 4bc8814..4c9ada7 100644 --- a/docs/watchdog-reporting-map.md +++ b/docs/watchdog-reporting-map.md @@ -21,8 +21,8 @@ probes ran on Yggdrasil over `ssh ygg`, read-only, 2026-09-30 13:51-13:55 UTC | I2 | Continuity exhaustion (since `idunn-reincarnation` cut 1: `restarts_page`, a paging threshold only; nothing stops, and the line numbers below are pre-cut) | `:68-71` (6 restarts / 3,600,000 ms, 5 s doubling), `:1066-1072` (`restarts_used`, `restarts_exhausted`), decided `:5709-5722` under key `continuity:` (`:5678`) | Exhaustion is **derived** from `TargetSupervision.continuity_restarts` (newest 6 times, sliding window). No write marks it. The fault clears (`:5723`) as soon as one entry leaves the window, so a permanently dead target leaves and re-enters exhaustion several times each hour. | | I3 | Route degradation | `RouteSupervisionState` `:890-942`; `record_failed_challenge` sets `degraded_since_unix_millis` on the **first** failed challenge, `record_proved_challenge` clears it; reported at `:5975` under `route:` | Durable per incarnation (inside `admitted_generation.v4`); a new generation starts from `default()`. Backoff doubles from the topology max age to a cap. | | I4 | Other operator-facing faults | `:5645-5651` (`generation-down:`, "holds a dead admitted generation down"), `:6016` (`route-record:`), `note_fault` per transaction | Also stderr plus `last_error`. Not named by the target; listed so the paging-set question sees them. | -| I5 | `OperatorRequired` on main (pre-cut-2 reading; since `idunn-reincarnation` cut 2 `resolve` reaches it in production through `PhaseEnd::OperatorRequired` and `PhaseEnd::RetryForward`, both calling `leave_to_operator`, which opens the `operator-required` incident `IncidentCondition::OperatorRequired`, subject the transaction id; `close_spent_operator_incidents` closes it when the record no longer `leaves_to_operator`) | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | -| I6 | `OperatorRequired` on B5 (superseded by cut 2: B5's deadlines merged as `PhaseDeadline`, and a continuity past its deadline now also lands here) | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | +| I5 | `OperatorRequired` on main | `TerminalRecovery::OperatorRequired` `:878-885`, validated `:1954`; constructed only in tests (`:12525`, `:12535`) | No production path produces it on main. | +| I6 | `OperatorRequired` on B5 | `hands/b5` `resolve_phase_deadline` (diff hunk at `control_plane.rs` ~8560-8625 on the branch), `PhaseEnd::OperatorRequired` | Deadline passed on a stateful deploy with an issued lease: `report_once("deadline:")`, returns `Ok(None)`, the transaction stays live. No terminal transition and no operator action that ends it were found on the branch (`idunn expire` writes `idunn.expiry_request.v1`, which re-enters the same arm). | | I7 | Store kinds | `:55-61`, `:936-966`; B5 adds `idunn.expiry_request.v1` | `control.cc`: `idunn.deployment_command`, `idunn.deployment_transaction`, `idunn.admitted_generation.v4`, `idunn.target_supervision.v1`. `history.cc` beside it holds terminal transactions (`:3653`), retired one per tick (`:5068-5091`). | | I8 | Single-writer precedent inside `control.cc` | B5 `ExpiryRequest` doc comment | "The CLI only writes this record. The daemon's deadline resolver reads it ... No CLI writes a transaction field." One writer per kind. | | I9 | Serve loop | `:5041-5058`, `run_scheduler_tick` `:5098-5110` | 500 ms poll. A tick error is logged, never propagated. | @@ -166,10 +166,9 @@ cut spec fixes them. Empty cells, stated plainly: -- `operator-required` closure: the incident closes when the transaction ends or - its step goes on (`close_spent_operator_incidents`). A stateful deployment - left to the operator still has no Idunn-owned terminal action; the operator's - `idunn expire` re-enters the same arm. +- `operator-required` closure: no path on `hands/b5` ends a transaction left to + the operator (I6). The incident can open and never close until B5 names that + action. - Delivery-failure visibility: no owner. - Discord `enforce_nonce` window: not verified against Discord's current docs in this pass. diff --git a/docs/watchdog-reporting-target.md b/docs/watchdog-reporting-target.md index 509cb6e..5d21c75 100644 --- a/docs/watchdog-reporting-target.md +++ b/docs/watchdog-reporting-target.md @@ -30,12 +30,9 @@ journald (`report_once`, `control_plane.rs:4599`): - An admitted route is degraded. It fails its stable-route challenge (`supervise_admitted_route`, ~5973). The degraded mark is durable, but no one is told. -- A post-fencing phase deadline leaves a transaction to the operator: a - stateful deployment whose candidate was issued a lease, or a continuity - retrying forward past its deadline. Since `idunn-reincarnation` cut 2 it is - the second incident condition, `operator-required` (subject: the transaction - id), beside `continuity-exhausted`; it closes when the record ends or its - step goes on. +- A post-fencing phase deadline expires into `OperatorRequired`. This lives on + B5, which is unmerged on `hands/b5`. The failure record is durable and the + notice is a log line. Odin's own store condition (`odin.store_condition.v1`, `contested`) is mapped but not built. Idunn reads nothing about Odin's store health today. @@ -88,8 +85,8 @@ the machine has a detector with no record, and a deliverer that nothing feeds. - Odin's store-recovery cuts, O2 included. They produce `odin.store_condition` and are Odin's campaign, still in prose. This campaign consumes the record once it exists. -- Idunn's phase deadlines. They landed in `idunn-reincarnation` cut 2, which - also owns the `operator-required` incident condition. +- Idunn's B5 phase deadlines. B5 is in prose on `hands/b5`. This campaign + reports its `OperatorRequired` outcome once B5 merges. - VoidBot's `notify_owner` MCP tool and its separate DM code. - Bifrost's Persona delivery crossing. - Eve/TUI lowering of incidents. diff --git a/src/bin/idunn-launch.rs b/src/bin/idunn-launch.rs deleted file mode 100644 index 3cf71e6..0000000 --- a/src/bin/idunn-launch.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! The unit's process for an upstream daemon: parse the recipe's argv, read -//! what Idunn set, run the launcher, and map the outcome to an exit code. -#[cfg(unix)] -fn main() -> std::process::ExitCode { - use std::sync::atomic::{AtomicBool, Ordering}; - - use anyhow::Context; - use idunn_daemon::deployment::{ - IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, - }; - use idunn_daemon::launcher::{self, *}; - - static STOP: AtomicBool = AtomicBool::new(false); - extern "C" fn on_signal(_: libc::c_int) { - STOP.store(true, Ordering::SeqCst); - } - - fn launch() -> anyhow::Result { - let release_root = std::env::current_exe()? - .parent() - .context("the launcher is not inside a release")? - .to_owned(); - let contract = LaunchContract::parse(release_root, std::env::args().skip(1))?; - let bundle = std::env::var_os(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT) - .context("the Idunn runtime bundle is not set")?; - let bind = std::env::var(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) - .context("the Idunn candidate bind is not set")?; - let member = launcher::parse_member_bind( - &std::env::var(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) - .context("the Idunn member bind is not set")?, - &bind, - )?; - let (activation, provider) = launcher::inherited_authority_files()?; - let authority = RuntimeAuthority::open(bundle.as_ref(), activation, provider)?; - authority.require_candidate_bind(&bind)?; - let lease_path = std::env::var_os(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT); - anyhow::ensure!( - lease_path.is_some() == authority.expected().write_lease_required, - "the process write lease binding differs from Expected" - ); - let route = std::net::TcpListener::bind(&bind).context("binding the candidate endpoint")?; - let sink = OdinRudpSink::from_expected(authority.expected())?; - let lease = DriverLeaseReader::new( - &authority.expected().target, - lease_path.as_deref().unwrap_or_default().as_ref(), - ); - let health_path = contract.health_path.clone(); - let ports = Ports { - credentials: std::sync::Arc::new(CredentialsDirectory::from_environment()), - clock: std::sync::Arc::new(SystemClock), - sink: std::sync::Arc::new(sink), - lease: std::sync::Arc::new(lease), - spawner: Box::new(ProcessSpawner), - health: Box::new(move |member| { - std::sync::Arc::new(HttpHealthProbe { - member, - path: health_path.clone(), - }) - }), - }; - // SAFETY: the handler only stores to an atomic. - unsafe { - libc::signal(libc::SIGTERM, on_signal as *const () as libc::sighandler_t); - libc::signal(libc::SIGINT, on_signal as *const () as libc::sighandler_t); - } - Ok(launcher::run( - &contract, - authority, - route, - member, - &std::env::temp_dir(), - ports, - &STOP, - )) - } - - match launch() { - Ok(outcome) => { - if let Outcome::Failed(code) = &outcome { - eprintln!("idunn-launch ended: {code}"); - } - std::process::ExitCode::from(outcome.exit_code() as u8) - } - Err(error) => { - eprintln!("idunn-launch refused: {error}"); - std::process::ExitCode::from(2) - } - } -} - -#[cfg(not(unix))] -fn main() { - eprintln!("idunn-launch runs only on Unix"); - std::process::exit(2); -} diff --git a/src/control_plane.rs b/src/control_plane.rs index cd49073..d2a7635 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -42,14 +42,11 @@ use crate::drivers::{ ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, RunnerStepFailure, SourcePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, WriteLeasePort, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; -use crate::runner_evidence::{ - self, Kept, RunnerFailureRecord, runner_evidence_store_path, unreadable_store_path, -}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, @@ -2934,10 +2931,6 @@ enum Command { recipe: PathBuf, binding: Option, }, - #[cfg(unix)] - OrdersServe(crate::steward_orders::ServeOptions), - #[cfg(unix)] - Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { @@ -2977,10 +2970,6 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), - #[cfg(unix)] - Command::OrdersServe(options) => crate::steward_orders::serve(options), - #[cfg(unix)] - Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -3033,10 +3022,6 @@ fn parse(args: impl Iterator) -> Result { "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), - #[cfg(unix)] - "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), - #[cfg(unix)] - "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -4742,21 +4727,20 @@ fn resolve_selector( Ok(targets) } -/// The one commit primitive for a deployment command: `idunn up` and a steward -/// deploy order both write theirs here. Returns the commit time. -pub(crate) fn commit_deployment_command( +fn submit( store_path: &Path, - command_id: &str, selector: &str, requested_by: &str, -) -> Result { + wait: bool, + timeout_seconds: u64, +) -> Result<()> { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: command_id.into(), + command_id: format!("up-{}", Uuid::new_v4()), kind: CommandKind::Deploy, selector: selector.into(), requested_by: requested_by.into(), @@ -4774,33 +4758,7 @@ pub(crate) fn commit_deployment_command( )?, "deployment command id collided" ); - Ok(now) -} - -/// A command's present state as `idunn status --command` derives it, for the -/// steward order poster: `queued` and `running` are not terminal. -#[cfg(unix)] -pub(crate) fn command_state( - state_store: &Path, - command_id: &str, -) -> Result<(&'static str, String)> { - let snapshot = ControlSnapshot::read(state_store)?; - let archived = read_history_transactions(state_store); - let mut transactions = snapshot.transaction_for_command(command_id); - transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); - Ok(derived_command_status(&transactions)) -} - -fn submit( - store_path: &Path, - selector: &str, - requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { - let command_id = format!("up-{}", Uuid::new_v4()); - let now = commit_deployment_command(store_path, &command_id, selector, requested_by)?; - println!("{command_id}"); + println!("{}", command.command_id); if !wait { return Ok(()); } @@ -4811,10 +4769,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command_id), + .any(|stored| stored.value.command_id == command.command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command_id); + let transactions = snapshot.transaction_for_command(&command.command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4839,7 +4797,7 @@ fn submit( } } -pub(crate) fn status( +fn status( out: &mut impl Write, store_path: &Path, incident_store: &Path, @@ -4919,13 +4877,6 @@ pub(crate) fn status( } if let Some(abort) = &transaction.pre_fencing_abort { writeln!(out, " pre-fencing abort {abort:?}")?; - // The step that failed and its output tail live in their - // own private store. An unreadable store costs its lines. - runner_evidence::show_status( - out, - &runner_evidence_store_path(store_path), - &transaction.transaction_id, - )?; } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -5423,32 +5374,6 @@ impl Engine { IncidentStore::new(&self.options.incident_store) } - /// A failed runner step keeps its output tail in the runner evidence - /// store and says so on the journal, naming no output. The store write is - /// best effort: its failure is named in the line and never stops the - /// abort that follows. - fn keep_runner_failure( - &self, - transaction: &DeploymentTransaction, - error: &anyhow::Error, - ) -> Option { - let failure = error.downcast_ref::()?; - let path = runner_evidence_store_path(&self.options.state_store); - let recorded = now_millis().and_then(|now| { - runner_evidence::record( - &path, - &RunnerFailureRecord::from_failure( - &transaction.transaction_id, - &transaction.command_id, - &transaction.target, - failure, - now, - ), - ) - }); - Some(runner_failure_journal_line(transaction, failure, &path, &recorded)) - } - /// Settle the outcome of one incident operation. A failure never leaves /// this function: it is traced once per episode, under the operation and /// the subject it was decided for (`*` for a pass over the whole store), @@ -6390,9 +6315,6 @@ impl Engine { if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() { - if let Some(line) = self.keep_runner_failure(&latest.value, &error) { - eprintln!("{line}"); - } self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -10631,13 +10553,7 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -/// The control store `idunn up` and `idunn serve` use unless told otherwise. -#[cfg(unix)] -pub(crate) fn default_state_store() -> PathBuf { - RuntimeOptions::default().state_store -} - -pub(crate) fn now_millis() -> Result { +fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -10657,12 +10573,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { +fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { +fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -10678,7 +10594,7 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -pub(crate) fn require_selector(value: &str) -> Result<()> { +fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } @@ -10725,43 +10641,6 @@ const DETAIL_EMPTY: &str = "(no detail)"; /// only judge, and this output always satisfies it. NUL is dropped, empty /// input gets a fixed placeholder, and an over-long value keeps its head and /// its tail (where a build log's error is) cut on char boundaries. -/// The journal line for a failed runner step: the transaction, the subject and -/// status (the failure's `Display`), and where the evidence is. Never output. -fn runner_failure_journal_line( - transaction: &DeploymentTransaction, - failure: &RunnerStepFailure, - store_path: &Path, - recorded: &Result, -) -> String { - let head = format!( - "Idunn transaction {} (command {}, target {}) {failure}", - transaction.transaction_id, transaction.command_id, transaction.target - ); - match recorded { - Ok(kept) => { - let aside = match kept { - Kept::Stored => String::new(), - Kept::SetAside { replaced } => format!( - "; an undecodable evidence store was set aside as {}{}", - unreadable_store_path(store_path).display(), - match replaced { - Some(last_written) => format!( - ", replacing an earlier set-aside store last written at unix millis {last_written}" - ), - None => String::new(), - } - ), - }; - format!( - "{head}; evidence in {}{aside}; read it with idunn status --command {}", - store_path.display(), - transaction.command_id - ) - } - Err(error) => format!("{head}; evidence was not kept: {error:#}"), - } -} - fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -10802,10 +10681,7 @@ fn usage() -> &'static str { idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\ - idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ - idunn order --reason TEXT\n\ - idunn order show ID | idunn order list\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -10847,7 +10723,6 @@ mod tests { private_port_end, config_path: PathBuf::from(format!("/etc/nginx/idunn-stream-routes/{route_id}.conf")), reload_unit: "nginx.service".into(), - member: None, } } @@ -12017,45 +11892,9 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], - vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], - vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], - vec!["order", "run", "sh", "--reason", "r"], - vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], - vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], - vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], - vec!["orders", "serve", "--allow-units", "nginx.service"], - vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } - // The declarative order forms parse, and only as order commands. - #[cfg(unix)] - for arguments in [ - vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], - vec!["order", "package-install", "git-lfs", "--reason", "r"], - vec!["order", "deploy", "vili", "--reason", "r"], - vec!["order", "list"], - vec!["orders", "serve"], - ] { - let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); - assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); - } - } - - #[test] - fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { - let temp = TempDir::new()?; - let store = temp.path().join("control.cc"); - commit_deployment_command(&store, "up-one", "vili", "steward:eureka order ord-1")?; - let snapshot = ControlSnapshot::read(&store)?; - assert_eq!(snapshot.commands.len(), 1); - let command = &snapshot.commands[0].value; - assert_eq!(command.command_id, "up-one"); - assert_eq!(command.selector, "vili"); - assert_eq!(command.requested_by, "steward:eureka order ord-1"); - assert!(commit_deployment_command(&store, "up-one", "other", "x").is_err()); - assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); - Ok(()) } #[test] @@ -24817,435 +24656,4 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } } - - // BEGIN runner-failure-evidence tests - // A failed runner step is named, its output tail is kept in - // runner-evidence.cc, and no output reaches an error, control.cc or the - // journal line. The runner is the real DockerRunnerDriver driving a fake - // docker program, through the engine's Sealing path. - - const RUNNER_CANARY: &str = "CANARY-7c41e0d9"; - - /// A source that freezes a fixed tree, so a deployment reaches the runner. - struct TreeSource { - recipe: String, - root: PathBuf, - } - - impl SourcePort for TreeSource { - fn resolve( - &self, - _: &OperatorBinding, - _: &str, - _: u64, - ) -> Result { - Ok(crate::drivers::ResolvedSource { - facts: crate::deployment_plan::tests::source(&self.recipe), - recipe_bytes: self.recipe.clone().into_bytes(), - }) - } - - fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result { - Ok(FrozenSourceReceipt { - transaction_id: transaction_id.to_owned(), - plan_id: plan.plan_id.clone(), - snapshot_sha256: format!("sha256-{}", "a".repeat(64)), - }) - } - - fn observe_frozen( - &self, - plan: &CompiledDeploymentPlan, - receipt: &FrozenSourceReceipt, - ) -> Result { - Ok(crate::drivers::FrozenSource::fixture( - receipt.clone(), - plan.source.clone(), - plan.recipe_blob.clone(), - self.root.clone(), - )) - } - - fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { - Ok(()) - } - } - - /// The unrouted recipe with two steps ahead of its build, and no external - /// input: the second step, `test-daemon`, is the one the fake docker fails. - fn two_step_recipe() -> String { - let recipe = unrouted_recipe().replace( - "capability = \"service.runtime\"", - &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), - ); - let (head, rest) = recipe - .split_once("[[external_inputs]]") - .expect("the recipe declares an external input"); - let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts"); - let head = head.replacen( - "[[steps]]\nid = \"build\"", - "[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\ - [[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\ - [[steps]]\nid = \"build\"", - 1, - ); - format!("{head}[[artifacts]]{tail}") - } - - /// A Sealing world whose runner is the real driver over a fake docker: the - /// first call succeeds, the second prints the canary and an escape - /// sequence on both streams and exits 3. - #[cfg(unix)] - fn failing_runner_world() -> Result<(EngineFixture, TempDir)> { - use std::os::unix::fs::PermissionsExt; - let tools = TempDir::new()?; - let docker = tools.path().join("docker"); - std::fs::write( - &docker, - format!( - "#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\ - [ \"$n\" -lt 2 ] && exit 0\n\ - printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n" - ), - )?; - std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?; - let recipe = two_step_recipe(); - let mut world = sealing_world(&recipe)?; - let tree = tools.path().join("tree"); - std::fs::create_dir_all(&tree)?; - std::fs::write(tree.join("README"), b"source")?; - world.engine.source = Arc::new(TreeSource { recipe, root: tree }); - world.engine.docker_runner = DockerRunnerDriver { docker_program: docker }; - Ok((world, tools)) - } - - /// Scheduler ticks until the deployment ends (or the budget is spent). - fn tick_until_ended(world: &EngineFixture) -> Result { - for _ in 0..24 { - world.engine.resume_backoff.lock().unwrap().clear(); - let _ = world.engine.run_scheduler_tick(); - if latest(world)?.completion.is_some() { - break; - } - } - latest(world) - } - - fn printed_status(world: &EngineFixture) -> Result { - let mut printed = Vec::new(); - status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?; - Ok(String::from_utf8(printed)?) - } - - fn canary_failure(stdout: &[u8]) -> RunnerStepFailure { - RunnerStepFailure { - subject: crate::drivers::RunnerSubject::Step("test-daemon".into()), - runner: "rust".into(), - exit_code: Some(3), - signal: None, - stdout_tail: stdout.to_vec(), - stdout_total_bytes: stdout.len() as u64, - stderr_tail: stdout.to_vec(), - stderr_total_bytes: stdout.len() as u64, - } - } - - #[cfg(unix)] - #[test] - fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> { - use std::os::unix::fs::PermissionsExt; - let (world, _tools) = failing_runner_world()?; - let ended = tick_until_ended(&world)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error); - }; - // Named: the second step, on its runner, with the exit status. - assert!( - error.contains("runner step test-daemon on runner rust exited with exit status 3"), - "{error}" - ); - // Never echoed: no error text, no control.cc byte, carries the output. - let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; - for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { - assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); - } - // The terminal transaction has moved to history.cc: both stores. - for store in [world.state_store.clone(), history_store_path(&world.state_store)] { - let bytes = std::fs::read(&store)?; - assert!(!bytes.is_empty(), "{} is empty", store.display()); - assert!( - !bytes.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), - "runner output reached {}", - store.display() - ); - } - // Kept: in the private store, tails of both streams, and on status. - let evidence = runner_evidence_store_path(&world.state_store); - assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); - let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? - .context("no evidence for the failed transaction")?; - assert_eq!(record.subject_id, "test-daemon"); - assert_eq!(record.exit_code, Some(3)); - assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY)); - assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY)); - let printed = printed_status(&world)?; - assert!(printed.contains("failed step test-daemon"), "{printed}"); - assert!(printed.contains(RUNNER_CANARY), "{printed}"); - assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte"); - Ok(()) - } - - #[test] - fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> { - let world = EngineFixture::new()?; - let (transaction, _) = terminal_transaction_with_command("ghostlight")?; - let failure = canary_failure(RUNNER_CANARY.as_bytes()); - let line = world - .engine - .keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing")) - .context("a runner failure produced no journal line")?; - assert!(!line.contains(RUNNER_CANARY), "{line}"); - for needle in [ - transaction.transaction_id.as_str(), - "up-ghostlight", - "ghostlight", - "runner step test-daemon on runner rust exited with exit status 3", - "runner-evidence.cc", - "idunn status --command up-ghostlight", - ] { - assert!(line.contains(needle), "{needle} missing from {line}"); - } - // Not a runner failure: nothing is kept and nothing is said. - assert!( - world - .engine - .keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure")) - .is_none() - ); - // The line a failed write produces says so, with no output. - let unwritable = runner_failure_journal_line( - &transaction, - &failure, - Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"), - &Err(anyhow::anyhow!("disk full")), - ); - assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}"); - assert!(!unwritable.contains(RUNNER_CANARY)); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - // A directory where the store should be: every write to it fails. - std::fs::create_dir(runner_evidence_store_path(&world.state_store))?; - let ended = tick_until_ended(&world)?; - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); - let line = world - .engine - .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b""))) - .context("no journal line")?; - assert!(line.contains("evidence was not kept"), "{line}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_undecodable_evidence_store_does_not_stop_the_next_failure_keeping_its_evidence() -> Result<()> { - let damage = format!("\u{c1} not a store {RUNNER_CANARY}-in-the-damage"); - let (world, _tools) = failing_runner_world()?; - let evidence = runner_evidence_store_path(&world.state_store); - std::fs::write(&evidence, damage.as_bytes())?; - let ended = tick_until_ended(&world)?; - // The transaction's own outcome is the runner failure, not the store. - let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { - bail!("the damaged store changed the outcome: {:?} {:?}", ended.phase, ended.last_error); - }; - assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); - assert!(!error.contains("damage"), "{error}"); - // The damage is kept aside; the failure's evidence is in a fresh store. - assert_eq!(std::fs::read(unreadable_store_path(&evidence))?, damage.as_bytes()); - let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? - .context("the failure kept no evidence beside a damaged store")?; - assert_eq!(record.subject_id, "test-daemon"); - assert!(printed_status(&world)?.contains("failed step test-daemon")); - // The journal says it, naming the file and none of its content. - std::fs::write(&evidence, damage.as_bytes())?; - let line = world - .engine - .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"x"))) - .context("no journal line")?; - assert!(line.contains("an undecodable evidence store was set aside as"), "{line}"); - assert!(line.contains("runner-evidence.unreadable.cc"), "{line}"); - assert!(!line.contains("damage") && !line.contains("not a store"), "{line}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - let ended = tick_until_ended(&world)?; - std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?; - let printed = printed_status(&world)?; - assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}"); - assert!(printed.contains("target service phase"), "{printed}"); - assert!(printed.contains("pre-fencing abort"), "{printed}"); - assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); - Ok(()) - } - - /// Everything status prints from the evidence store goes through one - /// allowlist, so a hostile build log reaches the terminal as visible - /// escapes whatever its class. - #[cfg(unix)] - #[test] - fn status_prints_hostile_runner_output_as_visible_escapes_only() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - let ended = tick_until_ended(&world)?; - let hostile: Vec<(&str, &str)> = vec![ - ("\u{1b}[2J", "\\u{1b}[2J"), - ("\u{7}", "\\u{7}"), - ("\r", "\\u{d}"), - ("\u{202e}", "\\u{202e}"), - ("\u{2066}", "\\u{2066}"), - ("\u{85}", "\\u{85}"), - ("\u{e0041}", "\\u{e0041}"), - ("\u{200b}", "\\u{200b}"), - ("\u{feff}", "\\u{feff}"), - ]; - let mut tail: Vec = Vec::new(); - for &(raw, _) in &hostile { - tail.extend_from_slice(b"a"); - tail.extend_from_slice(raw.as_bytes()); - tail.extend_from_slice(b"b\n"); - } - tail.extend_from_slice(b"bad \xff\xc1 bytes\n"); - runner_evidence::record( - &runner_evidence_store_path(&world.state_store), - &RunnerFailureRecord::from_failure( - &ended.transaction_id, - &ended.command_id, - &ended.target, - &canary_failure(&tail), - 9, - ), - )?; - let printed = printed_status(&world)?; - for &(raw, escaped) in &hostile { - assert!(!printed.contains(raw), "{escaped} reached the terminal raw"); - assert!(printed.contains(&format!("a{escaped}b")), "{escaped} missing: {printed}"); - } - assert!(printed.contains("bad \\xff\\xc1 bytes"), "{printed}"); - let section = printed.find(" failed step").context("no evidence section")?; - assert!(printed[section..].is_ascii(), "non-ASCII reached the terminal: {printed}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn status_prints_the_unreadable_store_line_escaped() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - tick_until_ended(&world)?; - let evidence = runner_evidence_store_path(&world.state_store); - // The decode error of this store quotes the stored subject kind. - let mut bytes = std::fs::read(&evidence)?; - let at = bytes - .windows(4) - .position(|window| window == b"step") - .context("no step marker in the evidence store")?; - bytes[at..at + 4].copy_from_slice(b"\x1b[2J"); - std::fs::write(&evidence, bytes)?; - let printed = printed_status(&world)?; - assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); - assert!(!printed.contains('\u{1b}'), "an escape byte reached the terminal"); - assert!(printed.contains("\\u{1b}[2J"), "{printed}"); - Ok(()) - } - - #[test] - fn the_journal_line_names_a_displaced_set_aside_store_by_time_only() -> Result<()> { - let (transaction, _) = terminal_transaction_with_command("ghostlight")?; - let failure = canary_failure(RUNNER_CANARY.as_bytes()); - let path = Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"); - let first = runner_failure_journal_line( - &transaction, - &failure, - path, - &Ok(Kept::SetAside { replaced: None }), - ); - assert!(first.contains("was set aside as"), "{first}"); - assert!(!first.contains("replacing"), "{first}"); - let second = runner_failure_journal_line( - &transaction, - &failure, - path, - &Ok(Kept::SetAside { replaced: Some(1234) }), - ); - assert!( - second.contains("replacing an earlier set-aside store last written at unix millis 1234"), - "{second}" - ); - assert!(!second.contains(RUNNER_CANARY)); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { - let (world, _tools) = failing_runner_world()?; - tick_until_ended(&world)?; - // The store a rolled-back Idunn reads: it refuses any foreign type. - ControlSnapshot::read(&world.state_store)?; - let mut transactions = 0; - for store in [world.state_store.clone(), history_store_path(&world.state_store)] { - for envelope in SingleFileMessagePackBackingStore::new(&store).pull_all_read_only_snapshot()? { - assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); - if envelope.r#type == DeploymentTransaction::TYPE { - transactions += 1; - assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v4")); - } - } - } - assert_eq!(transactions, 1, "the failed transaction is in neither store"); - Ok(()) - } - - /// The records control.cc holds re-encode byte for byte. The hex was - /// captured at base 804ed75, before this cut touched anything: a runner - /// failure adds no field, no type and no schema to the control store. - #[test] - fn control_records_encode_exactly_as_they_did_at_base() -> Result<()> { - let command = DeploymentCommand { - schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-service".into(), - kind: CommandKind::Deploy, - selector: "service".into(), - requested_by: "test".into(), - requested_at_unix_millis: 100, - }; - let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?; - transaction.transaction_id = "tx-pinned".into(); - transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into()); - transaction.pre_fencing_abort = Some(PreFencingAbort { - error: "runner step test-daemon on runner rust exited with exit status 3".into(), - candidate_cleanup: CleanupEvidence::Skipped, - topology_reconciliation: CleanupEvidence::Skipped, - source_cleanup: CleanupEvidence::Complete, - }); - let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); - let command_hex = hex(&command_envelope(&command, 100)?.payload); - let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload); - assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed"); - assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed"); - Ok(()) - } - - const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; - const PINNED_TRANSACTION_HEX: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; - // END runner-failure-evidence tests } diff --git a/src/deployment.rs b/src/deployment.rs index b7cbdc8..a67b4bc 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -9,27 +9,12 @@ pub const TARGET_DECLARATION_SCHEMA: &str = "gamecult.idunn.target_declaration.v pub const OPERATOR_BINDING_SCHEMA: &str = "gamecult.idunn.operator_binding.v2"; pub const IDUNN_RUNTIME_BUNDLE_ENVIRONMENT: &str = "GAMECULT_IDUNN_RUNTIME_BUNDLE"; pub const IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_CANDIDATE_BIND"; -pub const IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT: &str = "GAMECULT_IDUNN_MEMBER_BIND"; pub const IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT: &str = "GAMECULT_IDUNN_PROCESS_WRITE_LEASE"; pub const RUNTIME_PRESENCE_IDENTITY_BINDING: &str = "GAMECULT_RUNTIME_PRESENCE_IDENTITY"; /// A host actuator hands the workload its activation credential as a file /// named by this variable; on the Idunn host it arrives as a parent-only /// descriptor instead and this name is never set. pub const IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT: &str = "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL"; -/// systemd hands a unit its credential directory under this name; Idunn owns -/// the directory, so no binding may set it. -pub const SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT: &str = "CREDENTIALS_DIRECTORY"; -/// Every launch-environment name Idunn sets or reserves. A binding may name -/// none of them; this is the one list both the binding check and the -/// launch-input check read. -pub(crate) const IDUNN_OWNED_LAUNCH_ENVIRONMENT: [&str; 6] = [ - IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, - IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, - IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, - IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, - SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT, -]; pub const RUNTIME_PRESENCE_IDENTITY_FD_NAME: &str = "gamecult-runtime-presence-identity"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -113,10 +98,6 @@ pub struct ArtifactOutput { pub expected_sha256: Option, #[serde(default)] pub executable: bool, - /// Regular files inside a directory artifact that are sealed executable - /// (0555); every other file in the tree is sealed 0444. - #[serde(default)] - pub executable_members: BTreeSet, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -203,11 +184,6 @@ pub enum StateKind { /// Process-owned state imposed by an external runtime contract. Idunn /// preserves and fences the file but never interprets its payload. ExternalFile, - /// A process-owned directory imposed by an external runtime contract (for - /// example Synapse's media store). Idunn preserves and fences it and never - /// interprets its contents. Same writer, recovery and startup rules as - /// `ExternalFile`. - ExternalDirectory, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -528,7 +504,8 @@ impl WorkloadBinding { { require_environment_name(name)?; ensure!( - !IDUNN_OWNED_LAUNCH_ENVIRONMENT.contains(&name.as_str()), + name != IDUNN_RUNTIME_BUNDLE_ENVIRONMENT + && name != IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, "operator binding attempts to replace Idunn-owned workload environment" ); } @@ -661,7 +638,9 @@ pub struct ProcessWriteLeaseBinding { impl ProcessWriteLeaseBinding { pub fn lock_path(&self) -> PathBuf { - crate::drivers::authority_lock_path(&self.record_path) + let mut lock_path = self.record_path.as_os_str().to_os_string(); + lock_path.push(".lock"); + PathBuf::from(lock_path) } } @@ -683,101 +662,42 @@ pub struct RouteBinding { pub private_port_end: u16, pub config_path: PathBuf, pub reload_unit: String, - /// A second listener for the route's members. When set, members reach the - /// workload straight through a second nginx server block, and the - /// workload's launcher answers presence only. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub member: Option, -} - -/// The member listener of a route: where members connect, and the first of the -/// private ports the workload binds for them. The private range is as wide as -/// the route's own. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MemberListenerBinding { - pub stable_endpoint: String, - pub private_port_start: u16, - /// Most member connections nginx holds open at once for this route. Each - /// costs nginx two connections (client and upstream) from the pool the - /// route's presence block shares, so this limit is what keeps members from - /// exhausting it. - #[serde(default = "default_member_max_connections")] - pub max_connections: u16, -} - -const DEFAULT_MEMBER_MAX_CONNECTIONS: u16 = 128; - -fn default_member_max_connections() -> u16 { - DEFAULT_MEMBER_MAX_CONNECTIONS } impl RouteBinding { pub(crate) fn stable_socket(&self) -> Result<(IpAddr, u16)> { - parse_stable_socket(self.driver, &self.stable_endpoint, "stable endpoint") - } - - /// The member listener's stable socket, when the route has one. - pub(crate) fn member_stable_socket(&self) -> Result> { - self.member - .as_ref() - .map(|member| { - parse_stable_socket(self.driver, &member.stable_endpoint, "member endpoint") - }) - .transpose() - } - - /// The private port the workload binds for members when its presence - /// candidate is `candidate_port`. Derived, never stored: the member range - /// is the private range shifted to the member start. - pub(crate) fn member_candidate_port(&self, candidate_port: u16) -> Result> { - let Some(member) = self.member.as_ref() else { - return Ok(None); - }; + let authority = match self.driver { + RouteDriver::NginxStreamTcp => self + .stable_endpoint + .strip_prefix("http://") + .or_else(|| self.stable_endpoint.strip_prefix("tcp://")), + RouteDriver::NginxStreamUdp => self.stable_endpoint.strip_prefix("rudp://"), + } + .context("stable endpoint transport does not match its route driver")?; ensure!( - (self.private_port_start..=self.private_port_end).contains(&candidate_port), - "candidate port is outside the route binding" + !authority + .chars() + .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), + "stable endpoint is not a plain IP address and port" ); - let offset = candidate_port - self.private_port_start; - let port = member - .private_port_start - .checked_add(offset) - .context("member private port overflows")?; - Ok(Some(port)) + let (host, port) = authority + .rsplit_once(':') + .context("stable endpoint has no port")?; + let host = host + .parse::() + .context("stable endpoint host is not an IP address")?; + ensure!( + matches!(host, IpAddr::V4(_)), + "route binding v2 stable endpoints use explicit IPv4 addresses" + ); + let port = port + .parse::() + .context("stable endpoint port is not a u16")?; + ensure!(port > 0, "stable endpoint port is zero"); + Ok((host, port)) } } -fn parse_stable_socket(driver: RouteDriver, endpoint: &str, what: &str) -> Result<(IpAddr, u16)> { - let authority = match driver { - RouteDriver::NginxStreamTcp => endpoint - .strip_prefix("http://") - .or_else(|| endpoint.strip_prefix("tcp://")), - RouteDriver::NginxStreamUdp => endpoint.strip_prefix("rudp://"), - } - .with_context(|| format!("{what} transport does not match its route driver"))?; - ensure!( - !authority - .chars() - .any(|character| matches!(character, '/' | '?' | '#' | '@' | '[' | ']')), - "{what} is not a plain IP address and port" - ); - let (host, port) = authority - .rsplit_once(':') - .with_context(|| format!("{what} has no port"))?; - let host = host - .parse::() - .with_context(|| format!("{what} host is not an IP address"))?; - ensure!( - matches!(host, IpAddr::V4(_)), - "route binding v2 endpoints use explicit IPv4 addresses" - ); - let port = port - .parse::() - .with_context(|| format!("{what} port is not a u16"))?; - ensure!(port > 0, "{what} port is zero"); - Ok((host, port)) -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum RouteDriver { @@ -916,14 +836,6 @@ impl TargetDeclaration { if let Some(expected_sha256) = &artifact.expected_sha256 { require_sha256(expected_sha256, "expected artifact sha256")?; } - ensure!( - !(artifact.executable && !artifact.executable_members.is_empty()), - "artifact {} declares both executable and executable_members", - artifact.id - ); - for member in &artifact.executable_members { - require_relative_path(member, "executable member", false)?; - } ensure!( destinations.insert(artifact.destination.as_path()), "artifact destination {} is declared twice", @@ -1007,14 +919,6 @@ impl TargetDeclaration { == self.service.route_required, "routed services must require exactly one Idunn-owned candidate bind" ); - ensure!( - !self - .service - .required_environment - .contains(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) - || self.service.route_required, - "only routed services may require the Idunn-owned member bind" - ); ensure!( self.service .required_environment @@ -1472,39 +1376,7 @@ impl OperatorBinding { route.private_port_start < route.private_port_end, "candidate rollout requires at least two private ports" ); - let stable = route.stable_socket()?; - if let Some(member) = route.member.as_ref() { - ensure!( - route.driver == RouteDriver::NginxStreamTcp, - "only a TCP route may carry a member listener" - ); - require_value(&member.stable_endpoint, "member endpoint")?; - let member_stable = route - .member_stable_socket()? - .context("route has no member listener")?; - ensure!( - member_stable != stable, - "member endpoint equals the route's stable endpoint" - ); - ensure!( - member.private_port_start > 0, - "member private port range starts at zero" - ); - ensure!( - member.max_connections > 0, - "member connection limit is zero" - ); - let width = route.private_port_end - route.private_port_start; - let member_end = member - .private_port_start - .checked_add(width) - .context("member private port range overflows")?; - ensure!( - member_end < route.private_port_start - || member.private_port_start > route.private_port_end, - "member private port range overlaps the route's private range" - ); - } + route.stable_socket()?; require_absolute_path(&route.config_path, "route config")?; require_unit(&route.reload_unit, "route reload unit")?; } @@ -1682,7 +1554,12 @@ impl OperatorBinding { .keys() .chain(self.workload.secret_environment_names()) .collect(); - for owned in IDUNN_OWNED_LAUNCH_ENVIRONMENT { + for owned in [ + IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, + IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, + IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + ] { ensure!( !workload_names.contains(&owned.to_string()), "operator binding cannot replace Idunn-owned launch environment {owned}" @@ -1704,9 +1581,6 @@ impl OperatorBinding { } if declaration.service.route_required { available_environment.insert(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT.into()); - if self.route.as_ref().is_some_and(|route| route.member.is_some()) { - available_environment.insert(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into()); - } } if declaration.write_lease_required() { available_environment.insert(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT.into()); @@ -2054,10 +1928,10 @@ fn require_host(value: &str) -> Result<()> { } #[cfg(test)] -pub(crate) mod tests { +mod tests { use super::*; - pub(crate) const RECIPE: &str = r#" + const RECIPE: &str = r#" schema = "gamecult.idunn.target_declaration.v1" target = "test-service" source_stamp_environment = "TEST_SERVICE_BUILD_COMMIT" @@ -2129,7 +2003,7 @@ compatibility = "v1" startup = "before-promotion" "#; - pub(crate) const BINDING: &str = r#" + const BINDING: &str = r#" schema = "gamecult.idunn.operator_binding.v2" target = "test-service" profiles = ["aetheria", "full-gamecult"] @@ -2223,76 +2097,6 @@ nodes = ["yggdrasil"] assert!(TargetDeclaration::parse(&input).is_err()); } - const MEMBER_CANARY: &str = "zzcanary"; - - fn recipe_with_members(members: &str) -> String { - let input = RECIPE.replace("executable = true", &format!("executable_members = [{members}]")); - assert_ne!(input, RECIPE, "the fixture lost its executable line"); - input - } - - /// A refused member list is refused by name of the field, never by echoing - /// the member. - fn assert_members_refused(members: &[&str]) { - let control = TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).unwrap(); - assert!(control.artifacts[0].executable_members.contains(Path::new("bin/daemon"))); - for member in members { - let error = TargetDeclaration::parse(&recipe_with_members(&format!("\"{member}\""))) - .expect_err(member); - assert!( - !format!("{error:#}").contains(MEMBER_CANARY), - "the refusal echoed a member path" - ); - } - } - - #[test] - fn recipe_executable_member_refuses_parent_traversal() { - assert_members_refused(&[ - "zzcanary/../b", - "../zzcanary", - "bin/zzcanary/..", - "bin/../../zzcanary", - ]); - } - - #[test] - fn recipe_executable_member_refuses_absolute_path() { - assert_members_refused(&["/zzcanary/sh", "/zzcanary"]); - } - - #[test] - fn recipe_executable_member_refuses_empty_and_current_dir() { - assert_members_refused(&["", ".", "./", "zzcanary/.."]); - } - - #[test] - fn recipe_refuses_executable_flag_with_members() { - let input = RECIPE.replace( - "executable = true", - "executable = true\nexecutable_members = [\"bin/daemon\"]", - ); - let error = TargetDeclaration::parse(&input).unwrap_err(); - assert!(format!("{error:#}").contains("executable_members")); - assert!(!format!("{error:#}").contains("bin/daemon")); - // executable = true alone, and members alone, both stay admitted. - assert!(TargetDeclaration::parse(RECIPE).is_ok()); - assert!(TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).is_ok()); - } - - #[test] - fn recipe_admits_members_on_both_source_kinds() { - let runner_output = TargetDeclaration::parse(&recipe_with_members("\"a/b\", \"c\"")).unwrap(); - assert_eq!(runner_output.artifacts[0].source_kind, ArtifactSource::RunnerOutput); - assert_eq!(runner_output.artifacts[0].executable_members.len(), 2); - let worktree = recipe_with_members("\"a/b\", \"c\"") - .replace("source_kind = \"runner-output\"\nrunner = \"rust\"\n", "source_kind = \"worktree-tree\"\n") - .replace("expected_sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"\n", ""); - let worktree = TargetDeclaration::parse(&worktree).unwrap(); - assert_eq!(worktree.artifacts[0].source_kind, ArtifactSource::WorktreeTree); - assert_eq!(worktree.artifacts[0].executable_members.len(), 2); - } - #[test] fn operator_cannot_omit_an_affordance_derived_from_the_recipe() { let recipe = TargetDeclaration::parse(RECIPE).unwrap(); @@ -2426,203 +2230,6 @@ nodes = ["yggdrasil"] } } - /// The recipe and binding of a routed service whose members connect to a - /// listener of their own. - pub(crate) fn member_listener_pair() -> (String, String) { - let recipe = RECIPE.replace( - "required_environment = [", - "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", - ); - let binding = BINDING.replace( - "reload_unit = \"nginx.service\"\n", - "reload_unit = \"nginx.service\"\n\n[route.member]\nstable_endpoint = \"http://127.0.0.1:18840\"\nprivate_port_start = 18841\n", - ); - (recipe, binding) - } - - fn member_binding_with(replacement: (&str, &str)) -> Result { - let (recipe, binding) = member_listener_pair(); - let recipe = TargetDeclaration::parse(&recipe)?; - let binding = OperatorBinding::parse(&binding.replace(replacement.0, replacement.1))?; - binding.admit(&recipe)?; - Ok(binding) - } - - #[test] - fn a_member_listener_admits_with_a_recipe_that_requires_the_member_bind() { - let binding = member_binding_with(("never-matches", "")).unwrap(); - let route = binding.route.unwrap(); - assert_eq!(route.member_stable_socket().unwrap().unwrap().1, 18840); - assert_eq!(route.member_candidate_port(18831).unwrap(), Some(18841)); - assert_eq!(route.member_candidate_port(18839).unwrap(), Some(18849)); - assert!(route.member_candidate_port(18840).is_err()); - } - - #[test] - fn a_binding_without_a_member_listener_has_no_member_ports() { - let binding = OperatorBinding::parse(BINDING).unwrap(); - let route = binding.route.unwrap(); - assert_eq!(route.member_stable_socket().unwrap(), None); - assert_eq!(route.member_candidate_port(18831).unwrap(), None); - } - - #[test] - fn a_member_listener_needs_a_recipe_that_requires_the_member_bind() { - let (recipe, binding) = member_listener_pair(); - // Bound but not required: the recipe never asked for the variable. - let plain_recipe = TargetDeclaration::parse(RECIPE).unwrap(); - assert!( - OperatorBinding::parse(&binding) - .unwrap() - .admit(&plain_recipe) - .is_err() - ); - // Required but not bound: the variable would never be set. - let recipe = TargetDeclaration::parse(&recipe).unwrap(); - assert!( - OperatorBinding::parse(BINDING) - .unwrap() - .admit(&recipe) - .is_err() - ); - } - - #[test] - fn only_a_routed_recipe_may_require_the_member_bind() { - let recipe = RECIPE - .replace("route_required = true", "route_required = false") - .replace( - "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", ", - "required_environment = [\"GAMECULT_IDUNN_MEMBER_BIND\", ", - ); - assert!(recipe.contains("GAMECULT_IDUNN_MEMBER_BIND")); - assert!(TargetDeclaration::parse(&recipe).is_err()); - } - - #[test] - fn member_listener_bindings_are_validated() { - let start = |value: &str| { - member_binding_with(( - "private_port_start = 18841", - &format!("private_port_start = {value}"), - )) - }; - // Overlapping the route's own private range, in either direction. - assert!(start("18835").is_err()); - assert!(start("18825").is_err()); - // Adjacent but disjoint is fine, above the route's range or below it. - assert!(start("18840").is_ok()); - assert!(start("18822").is_ok()); - assert!(start("18823").is_err()); - // The connection limit defaults, and zero is refused. - let limit = |value: &str| { - member_binding_with(( - "private_port_start = 18841", - &format!("private_port_start = 18841 -max_connections = {value}"), - )) - }; - assert_eq!( - member_binding_with(("never-matches", "")).unwrap().route.unwrap().member.unwrap().max_connections, - 128 - ); - assert_eq!(limit("7").unwrap().route.unwrap().member.unwrap().max_connections, 7); - assert!(limit("0").is_err()); - assert!(limit("65535").is_ok()); - assert!(limit("65536").is_err()); - // Zero, and a width that overflows u16. - assert!(start("0").is_err()); - assert!(start("65530").is_err()); - // The member endpoint may not be the route's stable endpoint. - assert!(member_binding_with(("http://127.0.0.1:18840", "http://127.0.0.1:18830")).is_err()); - // The same transport and address rules as the stable endpoint. - assert!(member_binding_with(("http://127.0.0.1:18840", "rudp://127.0.0.1:18840")).is_err()); - assert!(member_binding_with(("http://127.0.0.1:18840", "http://example.test:18840")).is_err()); - } - - #[test] - fn a_udp_route_cannot_carry_a_member_listener() { - let (recipe, binding) = member_listener_pair(); - let binding = binding - .replace("nginx-stream-tcp", "nginx-stream-udp") - .replace("http://127.0.0.1:18830", "rudp://127.0.0.1:18830") - .replace("http://127.0.0.1:18840", "rudp://127.0.0.1:18840"); - let recipe = TargetDeclaration::parse(&recipe).unwrap(); - assert!( - OperatorBinding::parse(&binding) - .and_then(|binding| binding.admit(&recipe)) - .is_err() - ); - } - - #[test] - fn idunn_owned_names_are_reserved_in_one_list() { - let recipe = TargetDeclaration::parse(RECIPE).unwrap(); - let owned = [ - "GAMECULT_IDUNN_RUNTIME_BUNDLE", - "GAMECULT_IDUNN_CANDIDATE_BIND", - "GAMECULT_IDUNN_MEMBER_BIND", - "GAMECULT_IDUNN_PROCESS_WRITE_LEASE", - "GAMECULT_IDUNN_ACTIVATION_CREDENTIAL", - "CREDENTIALS_DIRECTORY", - ]; - let mut listed = IDUNN_OWNED_LAUNCH_ENVIRONMENT.to_vec(); - listed.sort_unstable(); - let mut expected = owned.to_vec(); - expected.sort_unstable(); - assert_eq!(listed, expected); - - let as_environment = |name: &str| { - BINDING.replace( - "[workload.argument_bindings]", - &format!("[workload.environment]\n{name} = \"value\"\n\n[workload.argument_bindings]"), - ) - }; - let as_secret = |name: &str| { - BINDING.replace( - "[workload.secret_files]", - &format!("[workload.secret_files]\n{name} = \"/etc/gamecult/test-service/other.cc\""), - ) - }; - // The fixtures are well formed: an unowned name parses in both - // shapes, so the refusals below are about the name. - assert!(OperatorBinding::parse(&as_environment("UNOWNED_NAME")).is_ok()); - assert!(OperatorBinding::parse(&as_secret("UNOWNED_NAME")).is_ok()); - for name in owned { - // Refused when the binding is parsed... - assert!( - OperatorBinding::parse(&as_environment(name)).is_err(), - "{name} accepted as environment at parse" - ); - assert!( - OperatorBinding::parse(&as_secret(name)).is_err(), - "{name} accepted as a secret file at parse" - ); - // ...and again at admission, for a binding that did not come - // through parse. Each site reads the one list. - for secret in [false, true] { - let mut binding = OperatorBinding::parse(BINDING).unwrap(); - let WorkloadBinding::SystemdTransient(workload) = &mut binding.workload else { - panic!("the fixture is a systemd workload"); - }; - if secret { - workload.secret_files.insert(name.into(), "/etc/gamecult/test-service/other.cc".into()); - } else { - workload.environment.insert(name.into(), "forged".into()); - } - assert!( - binding.workload.validate_launch_environment().is_err(), - "{name} (secret: {secret}) accepted by the binding check" - ); - let error = binding.admit(&recipe).unwrap_err().to_string(); - assert!( - error.contains("Idunn-owned launch environment"), - "{name} (secret: {secret}) accepted by the launch-input check: {error}" - ); - } - } - } - #[test] fn state_slot_path_cannot_escape_the_operator_owned_root() { let input = RECIPE.replace( @@ -2654,17 +2261,6 @@ max_connections = {value}"), assert!(declaration.write_lease_required()); } - #[test] - fn external_directory_state_is_lease_gated_like_external_file() { - let input = RECIPE.replace("kind = \"cultcache-file\"", "kind = \"external-directory\""); - let declaration = TargetDeclaration::parse(&input).unwrap(); - let slot = &declaration.state.as_ref().unwrap().slots[0]; - assert_eq!(slot.kind, StateKind::ExternalDirectory); - assert_eq!(slot.writer, StateWriter::ProcessBoundSingleWriter); - assert_eq!(slot.startup, StateStartup::CreateOrOpenAfterWriteLease); - assert!(declaration.write_lease_required()); - } - #[test] fn state_slots_cannot_claim_overlapping_paths() { let input = RECIPE.replace( diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index eceb513..1363c48 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -734,11 +734,6 @@ pub struct ArtifactReceipt { pub sha256: String, pub size_bytes: u64, pub executable: bool, - /// Files sealed executable inside a directory artifact. Skipped when empty - /// so a receipt with no members keeps its original encoding, and so its - /// sealed release id. - #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] - pub executable_members: BTreeSet, } impl ArtifactReceipt { @@ -752,9 +747,7 @@ impl ArtifactReceipt { .find(|artifact| artifact.id == self.artifact_id) .with_context(|| format!("plan declares no artifact {}", self.artifact_id))?; ensure!( - self.destination == declared.destination - && self.executable == declared.executable - && self.executable_members == declared.executable_members, + self.destination == declared.destination && self.executable == declared.executable, "artifact receipt differs from declared output" ); if let Some(expected_sha256) = &declared.expected_sha256 { @@ -1432,7 +1425,6 @@ nodes = ["yggdrasil"] sha256: digest(b'c'), size_bytes: 42, executable: true, - executable_members: BTreeSet::new(), } } @@ -1799,113 +1791,6 @@ nodes = ["yggdrasil"] assert!(changed.validate_against(&plan).is_err()); } - fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() - } - - #[test] - fn receipt_without_members_encodes_as_before() { - // Both values were computed outside this code, from the five-field - // msgpack array the receipt encoded to at base d74ac8d. - const BASE_RECEIPT: &str = "95a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac3"; - const BASE_RELEASE_ID: &str = - "sha256-b9143d8f546ad71c493c70037813547aaec10fd87994a6f3bdc77ac3a8b3495e"; - let receipt = artifact_receipt(); - let encoded = rmp_serde::to_vec(&receipt).unwrap(); - assert_eq!(hex(&encoded), BASE_RECEIPT); - let bytes = (0..BASE_RECEIPT.len()) - .step_by(2) - .map(|at| u8::from_str_radix(&BASE_RECEIPT[at..at + 2], 16).unwrap()) - .collect::>(); - assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), receipt); - let release = SealedRelease { - schema: SEALED_RELEASE_SCHEMA.into(), - sealed_release_id: String::new(), - plan_id: digest(b'a'), - artifacts: vec![receipt], - external_inputs: vec![], - sealed_at_unix_millis: 0, - }; - assert_eq!(release.recomputed_release_id().unwrap(), BASE_RELEASE_ID); - } - - #[test] - fn sealed_release_address_covers_executable_members() { - let release_with = |members: &[&str]| SealedRelease { - schema: SEALED_RELEASE_SCHEMA.into(), - sealed_release_id: String::new(), - plan_id: digest(b'a'), - artifacts: vec![ArtifactReceipt { - executable: false, - executable_members: members.iter().map(PathBuf::from).collect(), - ..artifact_receipt() - }], - external_inputs: vec![], - sealed_at_unix_millis: 0, - }; - let none = release_with(&[]).recomputed_release_id().unwrap(); - let one = release_with(&["bin/daemon"]).recomputed_release_id().unwrap(); - let other = release_with(&["bin/other"]).recomputed_release_id().unwrap(); - let two = release_with(&["bin/daemon", "bin/other"]) - .recomputed_release_id() - .unwrap(); - let all = [&none, &one, &other, &two]; - for (index, left) in all.iter().enumerate() { - for right in &all[index + 1..] { - assert_ne!(left, right); - } - } - - // The declaration is the other half: a receipt must carry exactly the - // declared list, in a plan that declares one and in one that does not. - let plain = plan(); - let mut with_members = artifact_receipt(); - with_members.executable_members = BTreeSet::from([PathBuf::from("bin/daemon")]); - assert!( - SealedRelease::new(&plain, vec![with_members], vec![external_input_receipt()], 120) - .is_err() - ); - let recipe = RECIPE.replace("executable = true", "executable_members = [\"bin/daemon\", \"lib/tool\"]"); - let declaring = compile_deployment_plan( - recipe.as_bytes(), - BINDING.as_bytes(), - source(&recipe), - "service-incarnation-1", - Some(18001), - 110, - &[ready_odin_provider("odin", "odin-yggdrasil", 1)], - ) - .unwrap(); - let honest = ArtifactReceipt { - executable: false, - executable_members: BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]), - ..artifact_receipt() - }; - let release = SealedRelease::new( - &declaring, - vec![honest.clone()], - vec![external_input_receipt()], - 120, - ) - .unwrap(); - release.validate_against(&declaring).unwrap(); - for drifted in [ - BTreeSet::from([PathBuf::from("bin/daemon")]), - BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/other")]), - BTreeSet::new(), - ] { - let mut receipt = honest.clone(); - receipt.executable_members = drifted; - assert!( - SealedRelease::new(&declaring, vec![receipt], vec![external_input_receipt()], 120) - .is_err() - ); - } - let mut changed = release.clone(); - changed.artifacts[0].executable_members.insert(PathBuf::from("lib/extra")); - assert!(changed.validate_against(&declaring).is_err()); - } - fn binding_with_deadlines(table: &str) -> String { BINDING.replace("[placement]", &format!("{table}\n[placement]")) } diff --git a/src/drivers.rs b/src/drivers.rs index 3ea1145..fdf925c 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; @@ -22,8 +22,7 @@ use crate::control_plane::SequenceAdmittedWarming; use crate::deployment::{ ArtifactOutput, ArtifactSource, DockerRunnerBinding, GitlinkBinding, IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, - IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, - LaunchArgument, OperatorBinding, + IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, LaunchArgument, OperatorBinding, RUNTIME_PRESENCE_IDENTITY_BINDING, RUNTIME_PRESENCE_IDENTITY_FD_NAME, RouteBinding, RouteDriver, SourceSelectionPolicy, TargetDeclaration, WorkloadNetwork, }; @@ -157,24 +156,6 @@ pub struct FrozenSourceReceipt { pub snapshot_sha256: String, } -#[cfg(test)] -impl FrozenSource { - /// A frozen source over an existing tree, for tests that drive a runner. - pub(crate) fn fixture( - receipt: FrozenSourceReceipt, - facts: SourceSelectionFacts, - recipe_bytes: Vec, - root: PathBuf, - ) -> Self { - Self { - receipt, - facts, - recipe_bytes, - root, - } - } -} - impl FrozenSourceReceipt { pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> { plan.validate()?; @@ -488,9 +469,6 @@ pub struct SystemdWorkloadObservation { pub no_new_privileges: bool, pub umask: String, pub inaccessible_paths: String, - /// Retired wire slot. systemd 257 displays `LoadCredential` as `[unprintable]` for - /// every unit, so it was never evidence: written empty, never compared. Kept so - /// compact (positional) MessagePack records of earlier Idunns still decode. pub load_credential: String, pub main_pid: u32, pub process_start_time: u64, @@ -605,9 +583,7 @@ pub trait WorkloadPort { /// /// Idunn launches candidates as transient units with `Restart=no`, so a /// unit that has failed, or that systemd no longer knows, will not come - /// back on its own. Those units are collected once they fail - /// (`INCARNATION_UNIT_LIFECYCLE`), so a failed candidate usually reads as - /// forgotten rather than failed. Past the fence every error is otherwise treated as + /// back on its own. Past the fence every error is otherwise treated as /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; @@ -2172,115 +2148,6 @@ pub(crate) fn docker_run_args( Ok(args) } -/// Bytes of each output stream that a failed runner step keeps. The daemon -/// never holds more than this per stream while the step runs. -pub(crate) const RUNNER_OUTPUT_TAIL_BYTES: usize = 8192; - -/// What a runner container was running: a recipe step or an external input, -/// by the id the recipe gave it. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum RunnerSubject { - Step(String), - ExternalInput(String), -} - -impl RunnerSubject { - pub(crate) fn id(&self) -> &str { - match self { - Self::Step(id) | Self::ExternalInput(id) => id, - } - } - - fn noun(&self) -> &'static str { - match self { - Self::Step(_) => "step", - Self::ExternalInput(_) => "external input", - } - } -} - -/// A runner container that did not exit successfully: what failed, how, and -/// the last `RUNNER_OUTPUT_TAIL_BYTES` of each stream with its total size. -/// The tails are evidence for `runner-evidence.cc`. `Display` and `Debug` -/// carry none of them, so no error chain, record or log line renders output. -#[derive(Clone, PartialEq, Eq)] -pub(crate) struct RunnerStepFailure { - pub(crate) subject: RunnerSubject, - pub(crate) runner: String, - pub(crate) exit_code: Option, - pub(crate) signal: Option, - pub(crate) stdout_tail: Vec, - pub(crate) stdout_total_bytes: u64, - pub(crate) stderr_tail: Vec, - pub(crate) stderr_total_bytes: u64, -} - -impl RunnerStepFailure { - /// The status as an operator reads it. - pub(crate) fn status(&self) -> String { - runner_status(self.exit_code, self.signal) - } -} - -/// How a container ended, as an operator reads it. -pub(crate) fn runner_status(exit_code: Option, signal: Option) -> String { - match (exit_code, signal) { - (Some(code), _) => format!("exit status {code}"), - (None, Some(signal)) => format!("signal {signal}"), - (None, None) => "an unknown status".to_owned(), - } -} - -impl std::fmt::Display for RunnerStepFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - formatter, - "runner {} {} on runner {} exited with {}; its output tail is kept in runner-evidence.cc", - self.subject.noun(), - self.subject.id(), - self.runner, - self.status() - ) - } -} - -impl std::fmt::Debug for RunnerStepFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RunnerStepFailure") - .field("subject", &self.subject) - .field("runner", &self.runner) - .field("exit_code", &self.exit_code) - .field("signal", &self.signal) - .field("stdout_total_bytes", &self.stdout_total_bytes) - .field("stderr_total_bytes", &self.stderr_total_bytes) - .finish_non_exhaustive() - } -} - -impl std::error::Error for RunnerStepFailure {} - -/// Read `reader` to its end keeping only the last `cap` bytes and counting -/// every byte, so a verbose step never buffers whole in the daemon. -fn drain_tail(mut reader: impl Read, cap: usize) -> std::io::Result<(Vec, u64)> { - let mut tail = Vec::new(); - let mut total = 0u64; - let mut chunk = [0u8; 8192]; - loop { - let read = match reader.read(&mut chunk) { - Ok(0) => return Ok((tail, total)), - Ok(read) => read, - Err(error) if error.kind() == ErrorKind::Interrupted => continue, - Err(error) => return Err(error), - }; - total += read as u64; - tail.extend_from_slice(&chunk[..read]); - if tail.len() > cap { - tail.drain(..tail.len() - cap); - } - } -} - /// Docker is only a runner substrate. Recipe argv stays an argv vector, the /// operator binding selects the exact image/network/mount affordances, and the /// driver returns complete materialization receipts rather than build truth by @@ -2299,7 +2166,7 @@ impl Default for DockerRunnerDriver { } impl DockerRunnerDriver { - fn docker(&self, subject: RunnerSubject, runner_id: &str, args: I) -> Result<()> + fn docker(&self, args: I) -> Result where I: IntoIterator, S: AsRef, @@ -2308,50 +2175,23 @@ impl DockerRunnerDriver { self.docker_program.is_absolute(), "Docker runner program is not absolute" ); - let mut child = Command::new(&self.docker_program) + let output = Command::new(&self.docker_program) .args(args) .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() + .output() .with_context(|| format!("starting Docker runner {}", self.docker_program.display()))?; - let stdout = child.stdout.take().context("Docker runner has no stdout pipe")?; - let stderr = child.stderr.take().context("Docker runner has no stderr pipe")?; - let stdout = thread::spawn(move || drain_tail(stdout, RUNNER_OUTPUT_TAIL_BYTES)); - let stderr = thread::spawn(move || drain_tail(stderr, RUNNER_OUTPUT_TAIL_BYTES)); - let status = child.wait().context("waiting for the Docker runner")?; - let (stdout_tail, stdout_total_bytes) = stdout - .join() - .map_err(|_| anyhow!("the runner stdout reader panicked"))? - .context("reading the runner stdout")?; - let (stderr_tail, stderr_total_bytes) = stderr - .join() - .map_err(|_| anyhow!("the runner stderr reader panicked"))? - .context("reading the runner stderr")?; - if status.success() { - return Ok(()); + if !output.status.success() { + bail!( + "Docker runner exited with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); } - #[cfg(unix)] - let signal = std::os::unix::process::ExitStatusExt::signal(&status); - #[cfg(not(unix))] - let signal = None; - Err(RunnerStepFailure { - subject, - runner: runner_id.to_owned(), - exit_code: status.code(), - signal, - stdout_tail, - stdout_total_bytes, - stderr_tail, - stderr_total_bytes, - } - .into()) + Ok(output) } fn run_in_workspace( &self, - step_id: &str, - runner_id: &str, runner: &DockerRunnerBinding, workspace: &Path, working_directory: &Path, @@ -2368,7 +2208,8 @@ impl DockerRunnerDriver { build_machine_id_file(workspace)?; let spec = ContainerSpec::for_step(runner, required_environment, source_stamp)?; let args = docker_run_args(&spec, workspace, working_directory, argv)?; - self.docker(RunnerSubject::Step(step_id.to_owned()), runner_id, args) + self.docker(args)?; + Ok(()) } fn materialize_external_input( @@ -2410,11 +2251,7 @@ impl DockerRunnerDriver { input.url.clone(), ]; let args = docker_run_args(&spec, workspace, Path::new("."), &curl_argv)?; - self.docker( - RunnerSubject::ExternalInput(input.id.clone()), - &input.runner, - args, - )?; + self.docker(args)?; let bytes = fs::read(&destination).with_context(|| { format!( "reading materialized external input {}", @@ -2464,7 +2301,27 @@ impl DockerRunnerDriver { } ArtifactSource::WorktreeTree => (source.root.clone(), source.root.join(&artifact.source)), }; - seal_artifact(artifact, &containment_root, &source_path, staging_root) + ensure!(source_path.exists(), "declared artifact output is absent"); + let destination = staging_root.join(&artifact.destination); + ensure!( + destination.starts_with(staging_root), + "artifact destination escaped its release staging root" + ); + copy_artifact(&containment_root, &source_path, &destination)?; + let (sha256, size_bytes) = digest_artifact(&destination)?; + if let Some(expected) = &artifact.expected_sha256 { + ensure!( + &sha256 == expected, + "artifact output differs from its recipe-pinned digest" + ); + } + Ok(ArtifactReceipt { + artifact_id: artifact.id.clone(), + destination: artifact.destination.clone(), + sha256: format!("sha256-{sha256}"), + size_bytes, + executable: artifact.executable, + }) } } @@ -2536,8 +2393,6 @@ impl RunnerPort for DockerRunnerDriver { ); } self.run_in_workspace( - &step.id, - &step.runner, runner, workspace, &step.working_directory, @@ -2576,13 +2431,6 @@ impl RunnerPort for DockerRunnerDriver { } } -/// How every incarnation unit lives and dies. It runs once (`Restart=no`) -/// and systemd forgets it once it is inactive or failed (`--collect`): the -/// outcome is Idunn's to record in its own store, and a failed transient unit -/// left loaded is only clutter that marks the host degraded. -const INCARNATION_UNIT_LIFECYCLE: [&str; 3] = - ["--collect", "--property=Type=exec", "--property=Restart=no"]; - /// systemd owns process execution. This driver lowers one validated launch /// contract into a transient unit and then proves the native process and /// executable that systemd actually started. It does not decide admission, @@ -2630,6 +2478,7 @@ fn changed_systemd_workload_identity_fields( no_new_privileges, umask, inaccessible_paths, + load_credential, main_pid, process_start_time, process_uids, @@ -3029,6 +2878,7 @@ impl SystemdTransientWorkloadDriver { OsString::from("--property=NoNewPrivileges"), OsString::from("--property=UMask"), OsString::from("--property=InaccessiblePaths"), + OsString::from("--property=LoadCredential"), OsString::from("--property=OpenFile"), OsString::from("--property=WorkingDirectory"), OsString::from("--property=ControlGroup"), @@ -3140,7 +2990,38 @@ impl SystemdTransientWorkloadDriver { let umask = required_systemd_property(&values, "UMask")?.to_owned(); let inaccessible_paths = required_systemd_property(&values, "InaccessiblePaths")?.to_owned(); - require_systemd_isolation_contract(&values, &self.credential_root)?; + let load_credential = required_systemd_property(&values, "LoadCredential")?.to_owned(); + ensure!( + service_type == "exec" + && restart_policy == "no" + && kill_mode == "mixed" + // `User` is deliberately not asserted here. Idunn never emits a + // User property, and systemd reports the identity it *allocated* + // for a DynamicUser unit -- the unit name, or a generated `_du…` + // when that is too long -- so the property is never empty for a + // running workload and cannot tell an allocated identity from a + // pinned one. The invariant it was reaching for, that the workload + // does not run as a persistent named account, is carried by + // `dynamic_user` below and by Idunn never passing User at all. + && dynamic_user + && supplementary_groups.is_empty() + && capability_bounding_set.is_empty() + && ambient_capabilities.is_empty() + && private_mounts + && private_pids + && protect_proc == "invisible" + && proc_subset == "all" + && no_new_privileges + && umask == "0007" + && inaccessible_paths == self.credential_root.display().to_string() + && load_credential + == if service_credential_names.is_empty() { + "" + } else { + "[unprintable]" + }, + "systemd workload isolation properties differ from the Idunn contract" + ); let working_directory = PathBuf::from(required_systemd_property(&values, "WorkingDirectory")?); ensure!( @@ -3282,11 +3163,6 @@ impl SystemdTransientWorkloadDriver { ); Some(path) }; - require_delivered_credentials( - &process_root.join("root"), - unit, - service_credential_names, - )?; let activation_descriptor = &parent_only_file_descriptors[0]; ensure!( activation_descriptor.fd_name == IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME @@ -3394,7 +3270,7 @@ impl SystemdTransientWorkloadDriver { no_new_privileges, umask, inaccessible_paths, - load_credential: String::new(), + load_credential, main_pid, process_start_time, process_uids: process_security.uids, @@ -3501,32 +3377,10 @@ impl SystemdTransientWorkloadDriver { .is_none(), "operator binding attempts to replace the Idunn candidate bind" ); - let member_port = binding - .route - .as_ref() - .map(|binding_route| binding_route.member_candidate_port(port)) - .transpose()? - .flatten(); - match member_port { - Some(member_port) => ensure!( - environment - .insert( - IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into(), - format!("{host}:{member_port}"), - ) - .is_none(), - "operator binding attempts to replace the Idunn member bind" - ), - None => ensure!( - !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), - "workload without a member listener carries an Idunn member bind" - ), - } } None => ensure!( - !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) - && !environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT), - "unrouted workload carries an Idunn route bind" + !environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT), + "unrouted workload carries an Idunn candidate bind" ), } match &binding.process_write_lease { @@ -3608,7 +3462,12 @@ impl SystemdTransientWorkloadDriver { // workload. The same applies to `Group` when no fixed state group is // bound. In that case the numeric primary GID must match the allocated // UID; a fixed state group still requires both its exact name and GID. - let launch_contract: [(bool, &str); 26] = [ + let expected_load_credential = if regular_credential_names.is_empty() { + "" + } else { + "[unprintable]" + }; + let launch_contract: [(bool, &str); 27] = [ ( observation.unit_description == expected_description, "unit description", @@ -3648,6 +3507,10 @@ impl SystemdTransientWorkloadDriver { observation.inaccessible_paths == self.credential_root.display().to_string(), "inaccessible paths", ), + ( + observation.load_credential == expected_load_credential, + "load credential", + ), ( observation.working_directory == installed, "working directory", @@ -3773,9 +3636,8 @@ impl SystemdTransientWorkloadDriver { OsString::from("--no-ask-password"), OsString::from("--expand-environment=no"), OsString::from(format!("--unit={unit}")), - ]; - args.extend(INCARNATION_UNIT_LIFECYCLE.map(OsString::from)); - args.extend([ + OsString::from("--property=Type=exec"), + OsString::from("--property=Restart=no"), OsString::from("--property=KillMode=mixed"), OsString::from("--property=DynamicUser=yes"), OsString::from("--property=SupplementaryGroups="), @@ -3811,7 +3673,7 @@ impl SystemdTransientWorkloadDriver { OsString::from(format!("--working-directory={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", bundle.display())), - ]); + ]; for open_file in parent_only_open_files { args.push(OsString::from(format!("--property=OpenFile={open_file}"))); } @@ -4248,20 +4110,6 @@ impl CultCacheWriteLeaseDriver { fn current(&self) -> Result> { validate_root_authority_path(&self.record_path)?; - self.read_record() - } - - /// The lease as its holder reads it: the same custody, envelope, key and - /// target validation Idunn applies, without requiring the reader to be - /// root. The holder is a service uid by construction; root is who writes. - pub(crate) fn current_for_holder( - &self, - ) -> Result> { - validate_authority_path_custody(&self.record_path)?; - self.read_record() - } - - fn read_record(&self) -> Result> { if !self.record_path.exists() { return Ok(None); } @@ -4308,61 +4156,36 @@ impl CultCacheWriteLeaseDriver { expected.validate()?; activation.validate()?; lease.validate()?; + let expected_sha256 = expected.canonical_sha256()?; + let activation_sha256 = activation.canonical_sha256()?; ensure!( - self.target == expected.target, + self.target == expected.target + && lease.target == expected.target + && lease.expected_projection_sha256 == expected_sha256 + && lease.plan_id == expected.plan_id + && lease.incarnation_id == expected.incarnation_id + && lease.sealed_release_id == expected.sealed_release_id + && lease.activation_witness_sha256 == activation_sha256 + && lease.runtime_id == expected.runtime_id + && lease.runtime_instance_id == activation.runtime_instance_id + && lease.state_schema_generation + == expected + .state_schema_generation + .as_deref() + .context("write-lease Expected has no state generation")? + && lease.state_contract_sha256 + == expected + .state_contract_sha256 + .as_deref() + .context("write-lease Expected has no state contract")? + && lease.warming_presence_sha256 == warming.signed_presence_sha256() + && warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), "process write lease does not bind the exact warming candidate" ); - ensure!( - warming.runtime_instance_id() == activation.runtime_instance_id.as_str(), - "process write lease does not bind the exact warming candidate" - ); - lease_binds_candidate( - expected, - activation, - lease, - std::slice::from_ref(&warming.signed_presence_sha256().to_owned()), - ) + Ok(()) } } -/// The one rule for what makes a process write lease this incarnation's: it -/// binds this Expected, this activation and one of the given provider-signed -/// Warming proofs. Idunn grants with it and the lease holder accepts with it. -pub(crate) fn lease_binds_candidate( - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - lease: &IdunnProcessWriteLeaseRecord, - warming_presence_sha256es: &[String], -) -> Result<()> { - let expected_sha256 = expected.canonical_sha256()?; - let activation_sha256 = activation.canonical_sha256()?; - ensure!( - lease.target == expected.target - && lease.expected_projection_sha256 == expected_sha256 - && lease.plan_id == expected.plan_id - && lease.incarnation_id == expected.incarnation_id - && lease.sealed_release_id == expected.sealed_release_id - && lease.activation_witness_sha256 == activation_sha256 - && lease.runtime_id == expected.runtime_id - && lease.runtime_instance_id == activation.runtime_instance_id - && lease.state_schema_generation - == expected - .state_schema_generation - .as_deref() - .context("write-lease Expected has no state generation")? - && lease.state_contract_sha256 - == expected - .state_contract_sha256 - .as_deref() - .context("write-lease Expected has no state contract")? - && warming_presence_sha256es - .iter() - .any(|sha256| sha256 == &lease.warming_presence_sha256), - "process write lease does not bind the exact warming candidate" - ); - Ok(()) -} - impl WriteLeasePort for CultCacheWriteLeaseDriver { fn revoke_exact(&self, incumbent: Option<&IdunnProcessWriteLeaseRecord>) -> Result<()> { if let Some(incumbent) = incumbent { @@ -5379,67 +5202,50 @@ impl NginxRouteDriver { } } - /// The firewall rules this route owns, one per stable socket (the - /// presence endpoint, then the member endpoint when the binding has one): - /// inbound to the socket's address and port, its transport's protocol. - fn endpoint_rules(&self) -> Result>> { + /// The one firewall rule this route owns: inbound to the stable endpoint's + /// address and port, its transport's protocol, tagged with the route id. + fn endpoint_rule(&self) -> Result> { + let (host, port) = self.binding.stable_socket()?; let protocol = match self.binding.driver { RouteDriver::NginxStreamTcp => "tcp", RouteDriver::NginxStreamUdp => "udp", }; - let mut sockets = vec![self.binding.stable_socket()?]; - sockets.extend(self.binding.member_stable_socket()?); - Ok(sockets - .into_iter() - .map(|(host, port)| { - vec![ - OsString::from("allow"), - OsString::from("in"), - OsString::from("to"), - OsString::from(host.to_string()), - OsString::from("port"), - OsString::from(port.to_string()), - OsString::from("proto"), - OsString::from(protocol), - ] - }) - .collect()) + Ok(vec![ + OsString::from("allow"), + OsString::from("in"), + OsString::from("to"), + OsString::from(host.to_string()), + OsString::from("port"), + OsString::from(port.to_string()), + OsString::from("proto"), + OsString::from(protocol), + ]) } fn admit_endpoint(&self) -> Result<()> { - for mut args in self.endpoint_rules()? { - args.push(OsString::from("comment")); - args.push(OsString::from(format!( - "Idunn route {}", - self.binding.route_id - ))); - self.command(&self.ufw_program, args) - .context("admitting a route endpoint on the host firewall")?; - } + let mut args = self.endpoint_rule()?; + args.push(OsString::from("comment")); + args.push(OsString::from(format!( + "Idunn route {}", + self.binding.route_id + ))); + self.command(&self.ufw_program, args) + .context("admitting the stable endpoint on the host firewall")?; Ok(()) } fn withdraw_endpoint(&self) -> Result<()> { - let mut first_error = None; - for rule in self.endpoint_rules()? { - let mut args = vec![OsString::from("delete")]; - args.extend(rule); - match self.command(&self.ufw_program, args) { - Ok(_) => {} - // ufw reports a rule that is already gone as a failure; for a - // withdrawal that is the state being asked for. - Err(error) if format!("{error:#}").contains("non-existent") => {} - Err(error) => { - first_error.get_or_insert( - error.context("withdrawing a route endpoint from the host firewall"), - ); - } + let mut args = vec![OsString::from("delete")]; + args.extend(self.endpoint_rule()?); + match self.command(&self.ufw_program, args) { + Ok(_) => Ok(()), + // ufw reports a rule that is already gone as a failure; for a + // withdrawal that is the state being asked for. + Err(error) if format!("{error:#}").contains("non-existent") => Ok(()), + Err(error) => { + Err(error).context("withdrawing the stable endpoint from the host firewall") } } - match first_error { - Some(error) => Err(error), - None => Ok(()), - } } fn command(&self, program: &Path, args: I) -> Result @@ -5500,28 +5306,8 @@ impl NginxRouteDriver { RouteDriver::NginxStreamTcp => { let (stable_host, stable_port) = endpoint_host_port(&route.stable_endpoint, endpoint_prefix)?; - let member_blocks = match ( - self.binding.member_stable_socket()?, - self.binding.member_candidate_port(candidate_port)?, - ) { - (Some((member_host, member_stable_port)), Some(member_port)) => { - let member_upstream = format!("{upstream}_member"); - let max_connections = self - .binding - .member - .as_ref() - .map_or(0, |member| member.max_connections); - // One constant key: the limit is on the route's members - // together, so a flood of them can never take the - // connections the presence block needs. - format!( - "limit_conn_zone \"{member_upstream}\" zone={member_upstream}:64k;\nupstream {member_upstream} {{\n server {candidate_host}:{member_port};\n}}\nserver {{\n listen {member_host}:{member_stable_port};\n limit_conn {member_upstream} {max_connections};\n proxy_pass {member_upstream};\n}}\n" - ) - } - _ => String::new(), - }; format!( - "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n{member_blocks}" + "# Idunn Expected {expected_projection_sha256}\nupstream {upstream} {{\n server {candidate_host}:{candidate_port};\n}}\nserver {{\n listen {stable_host}:{stable_port};\n proxy_pass {upstream};\n}}\n" ) } RouteDriver::NginxStreamUdp => { @@ -6075,7 +5861,7 @@ impl Answer { } } -pub(crate) fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { +fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) .with_context(|| format!("connecting stable CultNet TCP route {target}")) .map_err(silent)?; @@ -6284,7 +6070,7 @@ fn connect_route_socket(target: SocketAddr) -> std::io::Result { } } -pub(crate) fn exact_route_snapshot_response( +fn exact_route_snapshot_response( response: CultNetMessage, message_id: &str, target: &str, @@ -6925,7 +6711,21 @@ fn hash_frozen_source_tree(root: &Path, current: &Path, hasher: &mut Sha256) -> ); hasher.update(b"\0"); hasher.update(metadata.len().to_le_bytes()); - hash_file_exact(&path, metadata.len(), hasher)?; + // S9: streamed in fixed-size chunks rather than `fs::read`ing the + // whole file, so peak RSS is bounded by the buffer, not by the + // largest file in the tree. + let mut file = fs::File::open(&path) + .with_context(|| format!("opening {} to hash it", path.display()))?; + let mut buffer = [0u8; 1 << 16]; + loop { + let read = file + .read(&mut buffer) + .with_context(|| format!("reading {} to hash it", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -6947,28 +6747,20 @@ fn frozen_source_sha256(_root: &Path) -> Result { #[cfg(unix)] fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Result<()> { - let mut executable_paths = BTreeSet::new(); - for artifact in artifacts { - let destination = root.join(&artifact.destination); - for member in &artifact.executable_members { - executable_paths.insert(destination.join(member)); - } - if artifact.executable { - executable_paths.insert(destination); - } - } - harden_root_tree(root, &mut executable_paths)?; - ensure!( - executable_paths.is_empty(), - "declared executable is missing or not a regular file" - ); - Ok(()) + let executable_paths = artifacts + .iter() + .filter(|artifact| artifact.executable) + .map(|artifact| root.join(&artifact.destination)) + .collect::>(); + harden_root_tree(root, root, &executable_paths) } -/// Each regular file the walk seals 0555 is removed from `executable_paths`; -/// whatever remains was declared executable and never sealed. #[cfg(unix)] -fn harden_root_tree(current: &Path, executable_paths: &mut BTreeSet) -> Result<()> { +fn harden_root_tree( + root: &Path, + current: &Path, + executable_paths: &std::collections::BTreeSet, +) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let metadata = fs::symlink_metadata(current)?; @@ -6980,28 +6772,48 @@ fn harden_root_tree(current: &Path, executable_paths: &mut BTreeSet) -> let mut entries = fs::read_dir(current)?.collect::>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { - harden_root_tree(&entry.path(), &mut *executable_paths)?; + harden_root_tree(root, &entry.path(), executable_paths)?; } fs::set_permissions(current, fs::Permissions::from_mode(0o555))?; } else if metadata.is_file() { fs::set_permissions( current, - fs::Permissions::from_mode(if executable_paths.remove(current) { + fs::Permissions::from_mode(if executable_paths.contains(current) { 0o555 } else { 0o444 }), )?; } else if metadata.file_type().is_symlink() { - // Left untouched: containment was decided by check_release_link in - // digest_tree, which install_release and validate_installed_release - // run over every receipt before this walk (map T16). + validate_release_symlink(root, current)?; } else { bail!("sealed release contains a special filesystem entry") } Ok(()) } +#[cfg(unix)] +fn validate_release_symlink(root: &Path, path: &Path) -> Result<()> { + let target = fs::read_link(path)?; + ensure!(target.is_relative(), "sealed release symlink is absolute"); + ensure!( + target.components().all(|component| matches!( + component, + std::path::Component::CurDir | std::path::Component::Normal(_) + )), + "sealed release symlink has a parent traversal" + ); + let destination = path + .parent() + .context("sealed release symlink has no parent")? + .join(target); + ensure!( + destination.starts_with(root), + "sealed release symlink escaped its release" + ); + Ok(()) +} + #[cfg(not(unix))] fn harden_installed_release(_root: &Path, _artifacts: &[ArtifactReceipt]) -> Result<()> { bail!("systemd workload installation requires Unix permissions") @@ -7404,44 +7216,6 @@ fn path_inside_process_root(process_root: &Path, absolute_path: &Path) -> Result Ok(process_root.join(absolute_path.strip_prefix(Path::new("/"))?)) } -/// The workload holds exactly the credentials Idunn delivered: the entries of -/// `/run/credentials/` in the MainPID's root (`process_root` is that root) -/// equal `expected`, and an absent directory is the empty set. Only entry names -/// are read; a credential is never opened here. The message names no credential. -fn require_delivered_credentials( - process_root: &Path, - unit: &str, - expected: &[String], -) -> Result<()> { - let directory = path_inside_process_root( - process_root, - &Path::new("/run/credentials").join(unit), - )?; - let mut delivered = Vec::new(); - match fs::read_dir(&directory) { - Ok(entries) => { - for entry in entries { - let name = entry - .context("reading a delivered credential entry")? - .file_name() - .into_string() - .map_err(|_| anyhow!("delivered credential name is not UTF-8"))?; - delivered.push(name); - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(error).context("reading the delivered credential directory"); - } - } - delivered.sort(); - ensure!( - delivered == expected, - "systemd delivered credentials that differ from the Idunn launch" - ); - Ok(()) -} - fn linux_process_security( status_path: &Path, expected_host_pid: u32, @@ -7657,56 +7431,6 @@ fn parse_systemd_boolean(values: &BTreeMap, name: &str) -> Resul } } -/// The isolation properties of a transient workload unit, judged from `systemctl -/// show`. `LoadCredential` is absent on purpose: systemd 257 displays it as -/// `[unprintable]` whatever the unit holds, so what the workload was given is -/// judged by `require_delivered_credentials` from the delivered directory. -fn require_systemd_isolation_contract( - values: &BTreeMap, - credential_root: &Path, -) -> Result<()> { - let service_type = required_systemd_property(values, "Type")?; - let restart_policy = required_systemd_property(values, "Restart")?; - let kill_mode = required_systemd_property(values, "KillMode")?; - let dynamic_user = parse_systemd_boolean(values, "DynamicUser")?; - let supplementary_groups = systemd_property(values, "SupplementaryGroups")?; - let capability_bounding_set = systemd_property(values, "CapabilityBoundingSet")?; - let ambient_capabilities = systemd_property(values, "AmbientCapabilities")?; - let private_mounts = parse_systemd_boolean(values, "PrivateMounts")?; - let private_pids = parse_systemd_boolean(values, "PrivatePIDs")?; - let protect_proc = required_systemd_property(values, "ProtectProc")?; - let proc_subset = required_systemd_property(values, "ProcSubset")?; - let no_new_privileges = parse_systemd_boolean(values, "NoNewPrivileges")?; - let umask = required_systemd_property(values, "UMask")?; - let inaccessible_paths = required_systemd_property(values, "InaccessiblePaths")?; - ensure!( - service_type == "exec" - && restart_policy == "no" - && kill_mode == "mixed" - // `User` is deliberately not asserted here. Idunn never emits a - // User property, and systemd reports the identity it *allocated* - // for a DynamicUser unit -- the unit name, or a generated `_du…` - // when that is too long -- so the property is never empty for a - // running workload and cannot tell an allocated identity from a - // pinned one. The invariant it was reaching for, that the workload - // does not run as a persistent named account, is carried by - // `dynamic_user` below and by Idunn never passing User at all. - && dynamic_user - && supplementary_groups.is_empty() - && capability_bounding_set.is_empty() - && ambient_capabilities.is_empty() - && private_mounts - && private_pids - && protect_proc == "invisible" - && proc_subset == "all" - && no_new_privileges - && umask == "0007" - && inaccessible_paths == credential_root.display().to_string(), - "systemd workload isolation properties differ from the Idunn contract" - ); - Ok(()) -} - fn read_process_environment(path: &Path) -> Result>> { let bytes = fs::read(path) .with_context(|| format!("reading process environment {}", path.display()))?; @@ -8279,7 +8003,7 @@ fn copy_symlink(source: &Path, destination: &Path) -> Result<()> { /// checked to contain the fully resolved `source` with the same /// whole-chain-aware resolver `validate_frozen_source_symlink` uses, before /// anything is read from it. -fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { +pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { let canonical_root = root.canonicalize().context("resolving artifact root")?; let canonical_source = source .canonicalize() @@ -8302,124 +8026,11 @@ fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { } } -/// The one seal-time owner of mode declarations for a copied artifact: a -/// directory output is never `executable` (it declares `executable_members`), -/// a file output never declares members, and every member is a regular file -/// reached through real directories, never through a symlink. Errors name the -/// rule, never the member path. -fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { - let is_dir = fs::symlink_metadata(destination) - .context("inspecting a copied artifact")? - .is_dir(); - ensure!( - !(is_dir && artifact.executable), - "directory artifact {} cannot be executable; declare executable_members", - artifact.id - ); - if !is_dir { - ensure!( - artifact.executable_members.is_empty(), - "file artifact {} cannot declare executable_members", - artifact.id - ); - return Ok(()); - } - for member in &artifact.executable_members { - ensure!(!member.as_os_str().is_empty(), "executable member is empty"); - let mut current = destination.to_path_buf(); - let mut components = member.components().peekable(); - while let Some(component) = components.next() { - let std::path::Component::Normal(name) = component else { - bail!("executable member has a forbidden path component"); - }; - current.push(name); - let metadata = match fs::symlink_metadata(¤t) { - Ok(metadata) => metadata, - Err(error) if error.kind() == ErrorKind::NotFound => { - bail!("executable member is missing") - } - Err(error) => return Err(error).context("inspecting an executable member"), - }; - if components.peek().is_some() { - ensure!( - metadata.is_dir(), - "executable member passes through a symlink or non-directory" - ); - } else { - ensure!(metadata.is_file(), "executable member is not a regular file"); - } - } - } - Ok(()) -} - -/// The one seal step for a declared artifact, for systemd and host targets -/// alike: copy it into the staging root, check its mode declarations, digest it -/// and check the recipe pin. Modes are applied by harden on systemd targets; -/// host targets (Windows, map T15) carry `executable` and `executable_members` -/// in the receipt and do not apply them. -pub(crate) fn seal_artifact( - artifact: &ArtifactOutput, - containment_root: &Path, - source_path: &Path, - staging_root: &Path, -) -> Result { - ensure!( - source_path.exists(), - "declared artifact output {} is absent", - artifact.id - ); - let destination = staging_root.join(&artifact.destination); - ensure!( - destination.starts_with(staging_root), - "artifact {} destination escaped its staging root", - artifact.id - ); - copy_artifact(containment_root, source_path, &destination)?; - check_artifact_modes(artifact, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact {} differs from its recipe-pinned digest", - artifact.id - ); - } - Ok(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - executable_members: artifact.executable_members.clone(), - }) -} - -/// Streams `path` into `hasher` in 64 KiB chunks, so peak memory is the buffer -/// and not the largest file in the tree, and refuses if the bytes read differ -/// from `expected_len`. The only owner of the length check. -pub(crate) fn hash_file_exact(path: &Path, expected_len: u64, hasher: &mut Sha256) -> Result<()> { - let mut file = fs::File::open(path).context("opening a file to hash it")?; - let mut buffer = [0u8; 1 << 16]; - let mut total = 0_u64; - loop { - let read = file.read(&mut buffer).context("reading a file to hash it")?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - total += read as u64; - } - ensure!(total == expected_len, "file changed while hashing"); - Ok(()) -} - pub(crate) fn digest_artifact(path: &Path) -> Result<(String, u64)> { let metadata = fs::symlink_metadata(path)?; if metadata.is_file() { - let mut hasher = Sha256::new(); - hash_file_exact(path, metadata.len(), &mut hasher)?; - return Ok((format!("{:x}", hasher.finalize()), metadata.len())); + let bytes = fs::read(path)?; + return Ok((raw_sha256(&bytes), bytes.len().try_into()?)); } ensure!(metadata.is_dir(), "artifact is not a file or directory"); let mut hasher = Sha256::new(); @@ -8443,16 +8054,15 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) hasher.update(b"\0"); digest_tree(root, &path, hasher, size)?; } else if metadata.is_file() { - let length = metadata.len(); + let bytes = fs::read(&path)?; hasher.update(b"file\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); - hasher.update(length.to_le_bytes()); - hash_file_exact(&path, length, hasher)?; - *size = size.saturating_add(length); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(&bytes); + *size = size.saturating_add(bytes.len().try_into()?); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; - check_release_link(path.strip_prefix(root)?, &target)?; hasher.update(b"link\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); @@ -8465,35 +8075,6 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) Ok(()) } -/// The one judge of link containment in a release tree, pure and static: -/// `link` is the link's path relative to its artifact's top and `target` what -/// it reads. A target is relative, its `..` steps all come first, and they -/// climb no higher than the link's own directory depth, so the link can only -/// land inside the artifact. Lexical normalisation of parent and target is not -/// enough: with `s -> ..` it admits `s/../x` and escapes (map T14). -fn check_release_link(link: &Path, target: &Path) -> Result<()> { - use std::path::Component; - - let depth = link.components().count().saturating_sub(1); - let mut climbs = 0_usize; - let mut named = false; - for component in target.components() { - match component { - Component::CurDir => {} - Component::RootDir | Component::Prefix(_) => { - bail!("sealed release symlink is absolute") - } - Component::ParentDir => { - ensure!(!named, "sealed release symlink climbs after a name"); - climbs += 1; - } - Component::Normal(_) => named = true, - } - } - ensure!(climbs <= depth, "sealed release symlink escapes its artifact"); - Ok(()) -} - pub(crate) fn raw_sha256(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } @@ -8746,7 +8327,7 @@ fn ensure_source_directory(path: &Path, identity: Option) -> Re } } -pub(crate) fn authority_lock_path(path: &Path) -> PathBuf { +fn authority_lock_path(path: &Path) -> PathBuf { let mut lock = path.as_os_str().to_os_string(); lock.push(".lock"); PathBuf::from(lock) @@ -8754,17 +8335,12 @@ pub(crate) fn authority_lock_path(path: &Path) -> PathBuf { #[cfg(unix)] fn validate_root_authority_path(path: &Path) -> Result<()> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + ensure!( unsafe { libc::geteuid() } == 0, "process write-lease actuation requires root Idunn" ); - validate_authority_path_custody(path) -} - -#[cfg(unix)] -fn validate_authority_path_custody(path: &Path) -> Result<()> { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - let parent = path .parent() .context("process write-lease path has no parent")?; @@ -8807,11 +8383,6 @@ fn validate_root_authority_path(_path: &Path) -> Result<()> { bail!("process write-lease actuation requires a Unix authority path") } -#[cfg(not(unix))] -fn validate_authority_path_custody(_path: &Path) -> Result<()> { - bail!("process write-lease actuation requires a Unix authority path") -} - #[cfg(unix)] fn harden_root_authority_file(path: &Path) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; @@ -9078,7 +8649,6 @@ mod tests { private_port_end: port, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), - member: None, }) .request_candidate_runtime_presence(&candidate, "candidate-probe") } @@ -9902,7 +9472,6 @@ Content-Le".to_vec()), |_| {}), private_port_end: 17998, config_path: temp.path().join("service.conf"), reload_unit: "nginx.service".into(), - member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -9952,16 +9521,6 @@ Content-Le".to_vec()), |_| {}), Ok(()) } - /// An incarnation unit runs once and is collected when it ends, so a - /// failed candidate does not stay loaded and mark the host degraded. - /// Collection is safe because a forgotten unit already reads as - /// permanently stopped (the test below). - #[test] - fn incarnation_units_run_once_and_are_collected() { - assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--collect")); - assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--property=Restart=no")); - } - /// Admission audit, claim 3: `is_permanently_stopped` now lets the /// Isolation gate skip `prove_isolation` against the incumbent. Enumerate /// what it answers `true` for, using a fake `systemctl show` that replays @@ -10201,168 +9760,6 @@ processes 9 ); } - #[cfg(unix)] - fn fixture_show_properties(text: &str) -> Result> { - use std::os::unix::fs::PermissionsExt; - - let temp = tempfile::tempdir()?; - let shown = temp.path().join("shown"); - std::fs::write(&shown, text)?; - let systemctl = temp.path().join("systemctl"); - std::fs::write( - &systemctl, - format!("#!/bin/sh\ncat '{}'\nexit 0\n", shown.display()), - )?; - std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; - let driver = SystemdTransientWorkloadDriver { - systemctl_program: systemctl, - ..Default::default() - }; - Ok(driver - .show_unit("idunn-test.service")? - .context("fixture unit was not found")? - .properties) - } - - /// systemd 257 prints `LoadCredential=[unprintable]` for every unit, so the - /// isolation contract must admit the real output of a unit with no service - /// credential as well as one with one. The inputs are real captures, not a - /// re-spelling. - #[cfg(unix)] - #[test] - fn systemd_257_isolation_contract_admits_a_presence_only_target_and_one_with_service_credentials( - ) -> Result<()> { - let root = Path::new("/run/idunn/activation-credentials"); - let vili = include_str!("../tests/fixtures/systemd-257/vili-presence-identity-only.show"); - let odin = include_str!( - "../tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show" - ); - for text in [vili, odin] { - assert!(text.contains("LoadCredential=[unprintable]")); - require_systemd_isolation_contract(&fixture_show_properties(text)?, root)?; - } - let default_proc = vili.replace("ProtectProc=invisible", "ProtectProc=default"); - assert_ne!(default_proc, vili); - let other_path = vili.replace( - "/run/idunn/activation-credentials", - "/run/idunn/elsewhere", - ); - assert_ne!(other_path, vili); - for text in [default_proc, other_path] { - let error = - require_systemd_isolation_contract(&fixture_show_properties(&text)?, root) - .expect_err("a drifted isolation property must be refused"); - assert_eq!( - error.to_string(), - "systemd workload isolation properties differ from the Idunn contract" - ); - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn delivered_credentials_are_exactly_the_launch_names() -> Result<()> { - let unit = "idunn-test.service"; - let names = |list: &[&str]| list.iter().map(|name| name.to_string()).collect::>(); - // `Some(entries)` creates the unit's directory in a fresh process root. - let check = |delivered: Option<&[&str]>, expected: &[&str]| -> Result<()> { - let temp = tempfile::tempdir()?; - if let Some(entries) = delivered { - let directory = temp.path().join("run/credentials").join(unit); - std::fs::create_dir_all(&directory)?; - for entry in entries { - std::fs::write(directory.join(entry), "canary-credential-contents")?; - } - } - let result = require_delivered_credentials(temp.path(), unit, &names(expected)); - if let Err(error) = &result { - let text = format!("{error:#}"); - for canary in ["STOWAWAY", "ODIN_TOPOLOGY_IDENTITY", "canary-credential"] { - assert!(!text.contains(canary), "error echoed {canary}"); - } - assert!(!text.contains(&*temp.path().to_string_lossy())); - } - result - }; - check(None, &[])?; - check(Some(&[]), &[])?; - check(Some(&["ODIN_TOPOLOGY_IDENTITY"]), &["ODIN_TOPOLOGY_IDENTITY"])?; - check(Some(&["B", "A"]), &["A", "B"])?; - for (delivered, expected) in [ - (None, &["ODIN_TOPOLOGY_IDENTITY"][..]), - (Some(&[][..]), &["ODIN_TOPOLOGY_IDENTITY"][..]), - ( - Some(&["ODIN_TOPOLOGY_IDENTITY", "STOWAWAY"][..]), - &["ODIN_TOPOLOGY_IDENTITY"][..], - ), - (Some(&["STOWAWAY"][..]), &[][..]), - (Some(&["A"][..]), &["B"][..]), - (Some(&["A"][..]), &["A", "B"][..]), - ] { - let error = check(delivered, expected) - .expect_err("delivered credentials differing from the launch must be refused"); - assert_eq!( - error.to_string(), - "systemd delivered credentials that differ from the Idunn launch" - ); - } - // A directory that cannot be listed is a refusal, never the empty set. - let temp = tempfile::tempdir()?; - let parent = temp.path().join("run/credentials"); - std::fs::create_dir_all(&parent)?; - std::fs::write(parent.join(unit), "")?; - let error = require_delivered_credentials(temp.path(), unit, &[]) - .expect_err("an unreadable credential directory must be refused"); - assert!(!format!("{error:#}").contains(&*temp.path().to_string_lossy())); - // The rule reads the process root it is given, never the host's /run. - let temp = tempfile::tempdir()?; - let outside = tempfile::tempdir()?; - let directory = outside.path().join("run/credentials").join(unit); - std::fs::create_dir_all(&directory)?; - std::fs::write(directory.join("A"), "")?; - require_delivered_credentials(temp.path(), unit, &[])?; - assert!(require_delivered_credentials(temp.path(), unit, &names(&["A"])).is_err()); - Ok(()) - } - - /// Live Odin and StreamPixels hold records persisted by the Idunn that - /// compared `LoadCredential`: `[unprintable]` in the retired slot. The bytes - /// are that Idunn's own output (tests/fixtures/pre-cut-*); they must decode - /// under both encodings and read as no drift against a new observation, or - /// `stop` refuses to stop the incumbents. - #[test] - fn an_incumbent_recorded_with_the_display_form_is_not_identity_drift() -> Result<()> { - let decode_hex = |text: &str| -> Vec { - (0..text.trim().len()) - .step_by(2) - .map(|at| u8::from_str_radix(&text.trim()[at..at + 2], 16).unwrap()) - .collect() - }; - let mut recorded = systemd_audit_observation("no", 61_000); - recorded.load_credential = "[unprintable]".into(); - let observed = systemd_audit_observation("no", 61_000); - assert!(observed.load_credential.is_empty()); - for hex in [ - include_str!("../tests/fixtures/pre-cut-systemd-observation.compact.hex"), - include_str!("../tests/fixtures/pre-cut-systemd-observation.named.hex"), - ] { - let decoded: WorkloadObservation = rmp_serde::from_slice(&decode_hex(hex))?; - assert_eq!(decoded, WorkloadObservation::Systemd(recorded.clone())); - let WorkloadObservation::Systemd(prior) = decoded else { - unreachable!() - }; - assert!(changed_systemd_workload_identity_fields(&observed, &prior).is_empty()); - } - let mut drifted = observed.clone(); - drifted.main_pid += 1; - assert_eq!( - changed_systemd_workload_identity_fields(&drifted, &recorded), - vec!["main_pid"] - ); - Ok(()) - } - #[test] fn host_isolation_is_two_live_distinct_processes_on_one_host() { let live = |pid: u32, created: u64| { @@ -12807,7 +12204,6 @@ processes 9 private_port_end: 17979, config_path: temp.path().join("odin.conf"), reload_unit: "nginx.service".into(), - member: None, }, nginx_program: program("nginx", "#!/bin/sh\nexit 0\n")?, systemd_run_program: program("systemd-run", "#!/bin/sh\nexit 64\n")?, @@ -12817,15 +12213,12 @@ processes 9 }; assert_eq!( driver - .endpoint_rules()? + .endpoint_rule()? .iter() - .map(|rule| rule - .iter() - .map(|value| value.to_string_lossy().into_owned()) - .collect::>() - .join(" ")) - .collect::>(), - vec!["allow in to 10.77.0.1 port 17971 proto udp"] + .map(|value| value.to_string_lossy().into_owned()) + .collect::>() + .join(" "), + "allow in to 10.77.0.1 port 17971 proto udp" ); driver.restore(None, &Unmetered)?; @@ -12885,7 +12278,6 @@ processes 9 private_port_end: 4109, config_path: temp.join("service.conf"), reload_unit: "nginx.service".into(), - member: None, }, &RouteActuators { nginx: program("nginx")?, @@ -12920,271 +12312,6 @@ processes 9 }) } - fn with_member_listener(driver: &mut NginxRouteDriver) { - driver.binding.member = Some(crate::deployment::MemberListenerBinding { - stable_endpoint: "http://127.0.0.1:4120".into(), - private_port_start: 4121, - max_connections: 200, - }); - } - - /// The fragment as the base commit rendered it, computed from its format - /// string and not from the code under test. - #[cfg(unix)] - fn base_fragment(sha: &str, candidate_port: u16) -> String { - format!( - "# Idunn Expected {sha}\nupstream idunn_service_route {{\n server 127.0.0.1:{candidate_port};\n}}\nserver {{\n listen 127.0.0.1:4103;\n proxy_pass idunn_service_route;\n}}\n" - ) - } - - /// A route without a member listener renders the bytes it always did; a - /// member listener adds a second upstream and server block to the one - /// fragment, under the one Expected comment. - #[cfg(unix)] - #[test] - fn a_member_listener_renders_a_second_block_only_when_bound() -> Result<()> { - let temp = tempfile::tempdir()?; - let (mut driver, mut candidate, _calls) = stubbed_route_driver(temp.path())?; - let sha = candidate.canonical_sha256()?; - assert_eq!( - String::from_utf8(driver.render(&candidate)?)?, - base_fragment(&sha, 4104) - ); - - with_member_listener(&mut driver); - let member_blocks = |member_port: u16| { - format!( - "limit_conn_zone \"idunn_service_route_member\" zone=idunn_service_route_member:64k;\nupstream idunn_service_route_member {{\n server 127.0.0.1:{member_port};\n}}\nserver {{\n listen 127.0.0.1:4120;\n limit_conn idunn_service_route_member 200;\n proxy_pass idunn_service_route_member;\n}}\n" - ) - }; - let rendered = String::from_utf8(driver.render(&candidate)?)?; - assert_eq!(rendered, base_fragment(&sha, 4104) + &member_blocks(4121)); - assert_eq!(rendered.matches("# Idunn Expected").count(), 1); - assert_eq!(rendered.matches("server {").count(), 2); - - candidate.route.as_mut().unwrap().candidate_endpoint = "http://127.0.0.1:4106".into(); - let sha = candidate.canonical_sha256()?; - assert_eq!( - String::from_utf8(driver.render(&candidate)?)?, - base_fragment(&sha, 4106) + &member_blocks(4123) - ); - Ok(()) - } - - /// Both stable sockets are admitted on the firewall, and both withdrawn. - #[cfg(unix)] - #[test] - fn both_route_endpoints_are_admitted_and_withdrawn() -> Result<()> { - let temp = tempfile::tempdir()?; - let (mut driver, _candidate, calls) = stubbed_route_driver(temp.path())?; - with_member_listener(&mut driver); - driver.admit_endpoint()?; - driver.withdraw_endpoint()?; - let log = fs::read_to_string(&calls)?; - let ufw: Vec<&str> = log.lines().filter(|line| line.starts_with("ufw ")).collect(); - assert_eq!( - ufw, - vec![ - "ufw allow in to 127.0.0.1 port 4103 proto tcp comment Idunn route service-route", - "ufw allow in to 127.0.0.1 port 4120 proto tcp comment Idunn route service-route", - "ufw delete allow in to 127.0.0.1 port 4103 proto tcp", - "ufw delete allow in to 127.0.0.1 port 4120 proto tcp", - ] - ); - Ok(()) - } - - /// Real nginx over the rendered two-block fragment, a real presence - /// acceptor behind the presence block and a flood of idle members on the - /// member block. Needs an `nginx` on PATH and the fixed loopback ports - /// 4103, 4104, 4120 and 4121, so it runs on demand in a container: - /// `cargo test --lib real_nginx -- --ignored`. - #[cfg(unix)] - fn challenges_answered_during_a_member_flood(max_connections: u16) -> Result { - use std::net::{TcpListener, TcpStream}; - use std::process::{Child, Command}; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; - - /// Stopped through nginx itself: killing the master would orphan its - /// workers, which keep the test's output pipe open. - struct Nginx(Child, PathBuf, PathBuf); - impl Drop for Nginx { - fn drop(&mut self) { - let _ = Command::new("nginx") - .arg("-p") - .arg(&self.1) - .arg("-c") - .arg(&self.2) - .args(["-s", "stop"]) - .status(); - let _ = self.0.wait(); - } - } - - let temp = tempfile::tempdir()?; - let (mut driver, candidate, _calls) = stubbed_route_driver(temp.path())?; - with_member_listener(&mut driver); - driver.binding.member.as_mut().unwrap().max_connections = max_connections; - let fragment = temp.path().join("route.conf"); - fs::write(&fragment, driver.render(&candidate)?)?; - // Debian builds the stream module dynamic. - let module = Path::new("/usr/lib/nginx/modules/ngx_stream_module.so"); - let load = if module.exists() { - format!("load_module {}; -", module.display()) - } else { - String::new() - }; - let conf = temp.path().join("nginx.conf"); - fs::write( - &conf, - format!( - "{load}worker_processes 1;\npid {dir}/nginx.pid;\nerror_log {dir}/error.log;\nevents {{ worker_connections 512; }}\nstream {{ include {fragment}; }}\n", - dir = temp.path().display(), - fragment = fragment.display() - ), - )?; - let halt = Arc::new(AtomicBool::new(false)); - let presence = crate::launcher::PresenceAcceptor::new( - TcpListener::bind("127.0.0.1:4104")?, - Arc::new(|request| match request { - CultNetMessage::SnapshotRequest { message_id, .. } => { - Ok(CultNetMessage::SnapshotResponseRaw { message_id, documents: vec![] }) - } - _ => bail!("not a snapshot request"), - }), - ); - let presence_halt = halt.clone(); - let presence = thread::spawn(move || presence.serve(presence_halt)); - // The member backend accepts and holds every connection. - let members = TcpListener::bind("127.0.0.1:4121")?; - members.set_nonblocking(true)?; - let member_halt = halt.clone(); - let members = thread::spawn(move || { - let mut held = Vec::new(); - while !member_halt.load(Ordering::SeqCst) { - match members.accept() { - Ok((stream, _)) => held.push(stream), - Err(_) => thread::sleep(Duration::from_millis(5)), - } - } - }); - let _nginx = Nginx( - Command::new("nginx") - .args(["-p"]) - .arg(temp.path()) - .arg("-c") - .arg(&conf) - .args(["-g", "daemon off;"]) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .spawn()?, - temp.path().to_path_buf(), - conf.clone(), - ); - let stable: SocketAddr = "127.0.0.1:4103".parse()?; - let ready = Instant::now(); - while TcpStream::connect(stable).is_err() { - ensure!(ready.elapsed() < Duration::from_secs(10), "nginx never listened"); - thread::sleep(Duration::from_millis(50)); - } - // The probe connection above reached the presence acceptor as an idle - // peer; let its deadline lapse so only the flood is measured. - thread::sleep(Duration::from_millis(1200)); - - let mut flood = Vec::new(); - for _ in 0..300 { - if let Ok(mut stream) = TcpStream::connect("127.0.0.1:4120") { - let _ = stream.write_all(b"x"); - flood.push(stream); - } - } - thread::sleep(Duration::from_millis(500)); - - let payload = encode_cultnet_message_to_vec( - &CultNetMessage::SnapshotRequest { - message_id: "flood".into(), - schema_ids: None, - record_keys: None, - }, - CultNetWireContract::CultNetSchemaV0, - )?; - let mut answered = 0; - for _ in 0..20 { - let started = Instant::now(); - if request_tcp_snapshot(stable, &payload).is_ok() - && started.elapsed() < Duration::from_secs(3) - { - answered += 1; - } - } - eprintln!("answered {answered} of 20 challenges with a member limit of {max_connections}"); - drop(flood); - halt.store(true, Ordering::SeqCst); - presence.join().unwrap(); - members.join().unwrap(); - Ok(answered) - } - - #[cfg(unix)] - #[test] - #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] - fn real_nginx_keeps_presence_answerable_under_a_member_flood() -> Result<()> { - // The rendered default limit keeps the route's members out of the - // presence block's share of a 512-connection pool. - assert_eq!(challenges_answered_during_a_member_flood(128)?, 20); - Ok(()) - } - - #[cfg(unix)] - #[test] - #[ignore = "needs nginx on PATH and fixed loopback ports; run in a container"] - fn real_nginx_flood_starves_presence_without_a_member_limit() -> Result<()> { - // The control: the same flood against a limit too high to bind (nginx allows up to 65535) starves - // presence, so the test above measures the limit and not the load. - assert!(challenges_answered_during_a_member_flood(60_000)? < 20); - Ok(()) - } - - /// Idunn sets the member bind for a bound member listener, derived from the - /// candidate, and for no other workload. - #[test] - fn the_member_bind_is_set_only_for_a_bound_member_listener() -> Result<()> { - use crate::deployment::tests::{BINDING, RECIPE, member_listener_pair}; - - let driver = SystemdTransientWorkloadDriver::default(); - let mut routed = expected(); - routed.route = Some(cultnet_rs::IdunnExpectedRoute { - route_id: "test-service-public".into(), - transport: "http".into(), - stable_endpoint: "http://127.0.0.1:18830".into(), - candidate_endpoint: "http://127.0.0.1:18833".into(), - }); - let bundle = Path::new("/etc/gamecult/test-service/runtime/bundle"); - let unit = "idunn-test-service-abc"; - - let (recipe, binding) = member_listener_pair(); - let binding = OperatorBinding::parse(&binding)?; - binding.admit(&TargetDeclaration::parse(&recipe)?)?; - let environment = driver.launch_environment(&binding, bundle, &routed, unit)?; - assert_eq!( - environment.get(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT).map(String::as_str), - Some("127.0.0.1:18843") - ); - assert_eq!( - environment.get(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT).map(String::as_str), - Some("127.0.0.1:18833") - ); - - let plain = OperatorBinding::parse(BINDING)?; - plain.admit(&TargetDeclaration::parse(RECIPE)?)?; - let environment = driver.launch_environment(&plain, bundle, &routed, unit)?; - assert!(!environment.contains_key(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT)); - assert!(environment.contains_key(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT)); - Ok(()) - } - /// A refused forward change stops before any host mutation: no program /// runs, no fragment is written, and no private-mount unit starts. #[cfg(unix)] @@ -13504,7 +12631,6 @@ processes 9 private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), - member: None, }, nginx_program: program("nginx")?, systemd_run_program: program("systemd-run")?, @@ -13565,7 +12691,6 @@ processes 9 private_port_end: 27879, config_path: config_path.clone(), reload_unit: "nginx.service".into(), - member: None, }); assert_eq!(driver.current_configuration()?, None); // What systemd leaves behind when it materializes the preflight's bind @@ -13592,7 +12717,6 @@ processes 9 private_port_end: 27879, config_path: "/etc/nginx/idunn-stream-routes/odin-rudp.conf".into(), reload_unit: "nginx.service".into(), - member: None, }); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { @@ -13654,7 +12778,6 @@ processes 9 private_port_end: 4109, config_path: config.clone(), reload_unit: "nginx.service".into(), - member: None, }; let driver = NginxRouteDriver { binding, @@ -14301,40 +13424,6 @@ processes 9 Ok(()) } - #[cfg(unix)] - #[test] - fn write_lease_grant_binds_the_exact_warming_candidate_and_the_holder_read_keeps_custody() - -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - let temp = tempfile::tempdir()?; - let path = temp.path().join("lease.cc"); - let driver = CultCacheWriteLeaseDriver::new("service", &path); - let (expected, activation, warming, _) = authenticated_warming(temp.path())?; - let good = lease(&expected, &activation, &warming); - let mut foreign = good.clone(); - foreign.warming_presence_sha256 = digest('8'); - assert!( - driver - .grant(&expected, &activation, &warming, &foreign) - .is_err() - ); - let other = CultCacheWriteLeaseDriver::new("other", temp.path().join("other.cc")); - assert!(other.grant(&expected, &activation, &warming, &good).is_err()); - assert!(driver.current_for_holder()?.is_none()); - driver.grant(&expected, &activation, &warming, &good)?; - assert_eq!( - driver.current_for_holder()?.map(|(_, held)| held), - Some(good) - ); - fs::set_permissions(&path, fs::Permissions::from_mode(0o666))?; - assert!( - driver.current_for_holder().is_err(), - "a service-writable lease store is not authority" - ); - Ok(()) - } - #[cfg(unix)] #[test] fn installed_release_is_root_owned_and_nonwritable() -> Result<()> { @@ -14354,7 +13443,6 @@ processes 9 sha256: sha256_id(b"sealed"), size_bytes: 6, executable: true, - executable_members: BTreeSet::new(), }], )?; let root_metadata = fs::symlink_metadata(&root)?; @@ -14366,554 +13454,6 @@ processes 9 Ok(()) } - #[cfg(unix)] - const MEMBER_CANARY: &str = "zzcanary"; - - #[cfg(unix)] - fn tree_output(executable: bool, members: &[&str]) -> ArtifactOutput { - ArtifactOutput { - id: "upstream".into(), - source_kind: ArtifactSource::WorktreeTree, - runner: None, - source: PathBuf::from("upstream"), - destination: PathBuf::from("upstream"), - expected_sha256: None, - executable, - executable_members: members.iter().map(PathBuf::from).collect(), - } - } - - #[cfg(unix)] - fn write_file(path: &Path, bytes: &[u8]) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - fs::create_dir_all(path.parent().context("file has no parent")?)?; - fs::write(path, bytes)?; - // Builds and git leave arbitrary exec bits; the seal must not inherit them. - fs::set_permissions(path, fs::Permissions::from_mode(0o777))?; - Ok(()) - } - - #[cfg(unix)] - fn sealed_mode(path: &Path) -> u32 { - use std::os::unix::fs::PermissionsExt; - - fs::symlink_metadata(path).unwrap().permissions().mode() & 0o777 - } - - /// `/src/upstream` is the worktree tree; `/staging` the release. - #[cfg(unix)] - fn sample_tree(root: &Path) -> Result<(PathBuf, PathBuf)> { - let source = root.join("src"); - let staging = root.join("staging"); - fs::create_dir_all(&staging)?; - let tree = source.join("upstream"); - write_file(&tree.join("bin/daemon"), b"daemon")?; - write_file(&tree.join("bin/helper"), b"helper")?; - write_file(&tree.join("lib/tool"), b"tool")?; - write_file(&tree.join("lib/data.txt"), b"data")?; - write_file(&tree.join("NOTICE"), b"notice")?; - Ok((source, staging)) - } - - #[cfg(unix)] - fn collect_tree( - source: &Path, - staging: &Path, - output: ArtifactOutput, - ) -> Result { - let mut declaration = - TargetDeclaration::parse(crate::deployment::tests::RECIPE)?; - let id = output.id.clone(); - declaration.artifacts = vec![output]; - let frozen = FrozenSource { - receipt: FrozenSourceReceipt { - transaction_id: "tx".into(), - plan_id: "plan".into(), - snapshot_sha256: "snapshot".into(), - }, - facts: crate::deployment_plan::tests::source(crate::deployment::tests::RECIPE), - recipe_bytes: Vec::new(), - root: source.to_path_buf(), - }; - DockerRunnerDriver::default().collect_artifact( - &declaration, - &frozen, - &BTreeMap::new(), - staging, - &id, - ) - } - - #[cfg(unix)] - fn refusal_text(result: Result) -> String { - let Err(error) = result else { - panic!("the release was not refused"); - }; - let text = format!("{error:#}"); - assert!( - !text.contains(MEMBER_CANARY), - "the refusal echoed a member path: {text}" - ); - text - } - - #[cfg(unix)] - #[test] - fn tree_member_sealed_0555_and_siblings_0444() -> Result<()> { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - let receipt = collect_tree( - &source, - &staging, - tree_output(false, &["bin/daemon", "lib/tool"]), - )?; - assert_eq!( - receipt.executable_members, - BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]) - ); - harden_installed_release(&staging, &[receipt])?; - let tree = staging.join("upstream"); - assert_eq!(sealed_mode(&tree.join("bin/daemon")), 0o555); - assert_eq!(sealed_mode(&tree.join("lib/tool")), 0o555); - assert_eq!(sealed_mode(&tree.join("bin/helper")), 0o444); - assert_eq!(sealed_mode(&tree.join("lib/data.txt")), 0o444); - assert_eq!(sealed_mode(&tree.join("NOTICE")), 0o444); - for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { - assert_eq!(sealed_mode(directory), 0o555); - } - Ok(()) - } - - #[test] - fn release_link_rule_is_containment() { - for (link, target) in [ - ("a/b/l", "../t/x"), - ("a/l", ".."), - ("a/l", "./../b"), - ("a/l", "b/./c"), - ("l", "python3.12"), - ("a/b/c/l", "../../x"), - ] { - check_release_link(Path::new(link), Path::new(target)) - .unwrap_or_else(|error| panic!("({link}, {target}) refused: {error:#}")); - } - for (link, target, rule) in [ - ("l", "/x", "absolute"), - ("a/l", "s/../x", "climbs after a name"), - ("a/l", "D/D/../../../etc/passwd", "climbs after a name"), - ("a/l", "../../x", "escapes its artifact"), - ("l", "..", "escapes its artifact"), - ("a/b/l", "../../../x", "escapes its artifact"), - ("a/l", "./s/../x", "climbs after a name"), - ] { - let text = format!( - "{:#}", - check_release_link(Path::new(link), Path::new(target)) - .expect_err(&format!("({link}, {target}) was admitted")) - ); - assert!(text.contains(rule), "({link}, {target}): {text}"); - } - } - - #[cfg(unix)] - #[test] - fn missing_member_refuses_the_release() -> Result<()> { - // The bad member sorts after the good one in one case and before it in - // the other, so a check that stops at the first (or last) member fails. - for members in [ - &["bin/daemon", "lib/zzcanary-missing"][..], - &["aaa-zzcanary-missing", "bin/daemon"][..], - &["zzcanary-dir/tool"][..], - ] { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - let text = refusal_text(collect_tree(&source, &staging, tree_output(false, members))); - assert!(text.contains("executable member is missing"), "{text}"); - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn symlink_member_refuses_the_release() -> Result<()> { - for link_target in ["daemon", "../zzcanary-nowhere"] { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - std::os::unix::fs::symlink( - link_target, - source.join("upstream/bin/zzcanary-link"), - )?; - let text = refusal_text(collect_tree( - &source, - &staging, - tree_output(false, &["bin/daemon", "bin/zzcanary-link"]), - )); - assert!(text.contains("not a regular file"), "{text}"); - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn member_through_symlinked_directory_refuses_the_release() -> Result<()> { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - let tree = source.join("upstream"); - write_file(&tree.join("real-bin/zzcanary-tool"), b"tool")?; - std::os::unix::fs::symlink("real-bin", tree.join("linked-bin"))?; - let text = refusal_text(collect_tree( - &source, - &staging, - tree_output(false, &["linked-bin/zzcanary-tool"]), - )); - assert!(text.contains("symlink or non-directory"), "{text}"); - // A member below a regular file is refused the same way. - let second_staging = temp.path().join("second-staging"); - fs::create_dir_all(&second_staging)?; - let text = refusal_text(collect_tree( - &source, - &second_staging, - tree_output(false, &["bin/daemon/zzcanary-below-file"]), - )); - assert!(text.contains("symlink or non-directory"), "{text}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn directory_member_refuses_the_release() -> Result<()> { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - let text = refusal_text(collect_tree( - &source, - &staging, - tree_output(false, &["bin/daemon", "lib"]), - )); - assert!(text.contains("not a regular file"), "{text}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn executable_flag_on_tree_output_refuses() -> Result<()> { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - let text = refusal_text(collect_tree(&source, &staging, tree_output(true, &[]))); - assert!(text.contains("cannot be executable"), "{text}"); - // The same flag on a file output stays admitted. - let file = temp.path().join("file-artifact"); - fs::write(&file, b"x")?; - let mut output = tree_output(true, &[]); - output.id = "file".into(); - check_artifact_modes(&output, &file)?; - Ok(()) - } - - #[cfg(unix)] - #[test] - fn members_on_file_output_refuse() -> Result<()> { - let temp = tempfile::tempdir()?; - let file = temp.path().join("file-artifact"); - fs::write(&file, b"x")?; - check_artifact_modes(&tree_output(false, &[]), &file)?; - let text = refusal_text(check_artifact_modes( - &tree_output(false, &["zzcanary"]), - &file, - )); - assert!(text.contains("cannot declare executable_members"), "{text}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn check_artifact_modes_refuses_a_member_that_is_not_a_clean_relative_path() -> Result<()> { - let temp = tempfile::tempdir()?; - let (source, _) = sample_tree(temp.path())?; - let tree = source.join("upstream"); - write_file(&temp.path().join("zzcanary-outside"), b"x")?; - for member in ["../../zzcanary-outside", "bin/../bin/daemon", "/zzcanary", ""] { - refusal_text(check_artifact_modes(&tree_output(false, &[member]), &tree)); - } - check_artifact_modes(&tree_output(false, &["bin/daemon"]), &tree)?; - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_unreadable_member_is_not_reported_as_missing() -> Result<()> { - let temp = tempfile::tempdir()?; - let (source, _) = sample_tree(temp.path())?; - // A component past NAME_MAX fails with an io error that is not NotFound. - let long = "z".repeat(300); - let text = refusal_text(check_artifact_modes( - &tree_output(false, &[long.as_str()]), - &source.join("upstream"), - )); - assert!(!text.contains("missing"), "{text}"); - assert!(text.contains("inspecting an executable member"), "{text}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn harden_fails_when_a_declared_member_is_absent() -> Result<()> { - let receipt = |members: &[&str]| ArtifactReceipt { - artifact_id: "upstream".into(), - destination: PathBuf::from("upstream"), - sha256: sha256_id(b"x"), - size_bytes: 1, - executable: false, - executable_members: members.iter().map(PathBuf::from).collect(), - }; - let installed = |root: &Path| -> Result<()> { - write_file(&root.join("upstream/bin/daemon"), b"daemon")?; - write_file(&root.join("upstream/lib/tool"), b"tool")?; - fs::create_dir_all(root.join("upstream/zzcanary-dir"))?; - std::os::unix::fs::symlink("daemon", root.join("upstream/bin/zzcanary-link"))?; - Ok(()) - }; - let temp = tempfile::tempdir()?; - installed(temp.path())?; - harden_installed_release(temp.path(), &[receipt(&["bin/daemon", "lib/tool"])])?; - for members in [ - &["bin/daemon", "lib/zzcanary-gone"][..], - &["zzcanary-gone", "bin/daemon"][..], - &["zzcanary-dir"][..], - &["bin/zzcanary-link"][..], - &["bin/daemon/zzcanary-below-file"][..], - ] { - let temp = tempfile::tempdir()?; - installed(temp.path())?; - let text = refusal_text(harden_installed_release(temp.path(), &[receipt(members)])); - assert!(text.contains("declared executable is missing"), "{text}"); - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn relative_in_tree_symlink_survives_the_seal() -> Result<()> { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - let tree = source.join("upstream"); - write_file(&tree.join("bin/python3.12"), b"python")?; - std::os::unix::fs::symlink("python3.12", tree.join("bin/python3"))?; - write_file(&tree.join("lib/terminfo/b/real"), b"terminfo")?; - fs::create_dir_all(tree.join("lib/terminfo/a"))?; - std::os::unix::fs::symlink("../b/real", tree.join("lib/terminfo/a/alias"))?; - let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/python3.12"]))?; - let (before, _) = digest_artifact(&staging.join("upstream"))?; - harden_installed_release(&staging, &[receipt])?; - let link = staging.join("upstream/bin/python3"); - assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); - assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); - assert_eq!(sealed_mode(&staging.join("upstream/bin/python3.12")), 0o555); - let alias = staging.join("upstream/lib/terminfo/a/alias"); - assert!(fs::symlink_metadata(&alias)?.file_type().is_symlink()); - assert_eq!(fs::read_link(&alias)?, PathBuf::from("../b/real")); - assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); - - // A link that escapes, is absolute or climbs after a name is refused at - // the seal, before any receipt exists. - for (target, rule) in [ - ("../../zzcanary-up", "escapes its artifact"), - ("/zzcanary/absolute", "absolute"), - ("zzcanary-s/../x", "climbs after a name"), - ] { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - std::os::unix::fs::symlink(target, source.join("upstream/lib/zzcanary-link"))?; - let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); - assert!(text.contains(rule), "{text}"); - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn links_to_existing_targets_are_refused_at_the_seal() -> Result<()> { - // Every refused shape, aimed at a target that EXISTS (file and - // directory): a judge guarded by `exists()`, `is_dir()` or a file-type - // test passes the dangling fixtures and fails these. - for kind in ["file", "dir"] { - for (shape, rule) in [ - ("absolute", "absolute"), - ("escape", "escapes its artifact"), - ("after-name", "climbs after a name"), - ] { - let temp = tempfile::tempdir()?; - let (source, staging) = sample_tree(temp.path())?; - let target = match (shape, kind) { - ("absolute", "file") => source.join("upstream/bin/daemon"), - ("absolute", _) => source.join("upstream/lib"), - ("escape", "file") => { - write_file(&source.join("outside-file"), b"outside")?; - PathBuf::from("../../outside-file") - } - ("escape", _) => { - fs::create_dir_all(source.join("outside-dir"))?; - PathBuf::from("../../outside-dir") - } - (_, "file") => PathBuf::from("../bin/../NOTICE"), - _ => PathBuf::from("../bin/../lib"), - }; - let link = source.join("upstream/lib/zzcanary-link"); - std::os::unix::fs::symlink(&target, &link)?; - assert!(fs::metadata(&link).is_ok(), "{shape} {kind} link must resolve"); - let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); - assert!(text.contains(rule), "{shape} {kind}: {text}"); - } - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn streamed_digest_is_byte_compatible() -> Result<()> { - // Not a multiple of the 64 KiB read buffer, so a short last chunk - // matters; the aligned and empty files are the other boundaries. - let big = (0..3 * 65536 + 17).map(|i| (i % 251) as u8).collect::>(); - let aligned = vec![7_u8; 65536]; - let temp = tempfile::tempdir()?; - let tree = temp.path().join("tree"); - write_file(&tree.join("a.txt"), b"alpha")?; - write_file(&tree.join("empty"), b"")?; - write_file(&tree.join("sub/aligned.bin"), &aligned)?; - write_file(&tree.join("sub/big.bin"), &big)?; - std::os::unix::fs::symlink("a.txt", tree.join("zlink"))?; - - // The encoding digest_tree had before it streamed, written out by hand. - fn add_file(hasher: &mut Sha256, name: &str, bytes: &[u8]) { - hasher.update(b"file\0"); - hasher.update(name.as_bytes()); - hasher.update(b"\0"); - hasher.update((bytes.len() as u64).to_le_bytes()); - hasher.update(bytes); - } - let mut expected = Sha256::new(); - add_file(&mut expected, "a.txt", b"alpha"); - add_file(&mut expected, "empty", b""); - let mut expected_size = 5_u64; - expected.update(b"dir\0sub\0"); - add_file(&mut expected, "sub/aligned.bin", &aligned); - add_file(&mut expected, "sub/big.bin", &big); - expected_size += (aligned.len() + big.len()) as u64; - expected.update(b"link\0zlink\0a.txt"); - expected_size += "a.txt".len() as u64; - assert_eq!( - digest_artifact(&tree)?, - (format!("{:x}", expected.finalize()), expected_size) - ); - - let single = tree.join("sub/big.bin"); - assert_eq!( - digest_artifact(&single)?, - (raw_sha256(&big), big.len() as u64) - ); - let mut hasher = Sha256::new(); - hash_file_exact(&single, big.len() as u64, &mut hasher)?; - assert_eq!(format!("{:x}", hasher.finalize()), raw_sha256(&big)); - for wrong in [big.len() as u64 + 1, big.len() as u64 - 1] { - let text = refusal_text(hash_file_exact(&single, wrong, &mut Sha256::new())); - assert!(text.contains("changed while hashing"), "{text}"); - } - Ok(()) - } - - /// Acceptance on the real trees: `IDUNN_BIG_TREE` holds `python/` and - /// `blender/`. Run on Yggdrasil as its own job (see the cut spec). - #[cfg(unix)] - #[test] - #[ignore = "needs IDUNN_BIG_TREE: a CPython+torch tree and a Blender tree"] - fn big_tree_seals_from_env() -> Result<()> { - use std::time::Instant; - - /// (links, links with a `..` step, regular files, files at 0555), never - /// following a link; every regular file must be 0444 or a member's 0555. - fn census( - root: &Path, - path: &Path, - members: &BTreeSet, - modes: bool, - totals: &mut [usize; 4], - ) -> Result<()> { - for entry in fs::read_dir(path)? { - let entry = entry?; - let metadata = fs::symlink_metadata(entry.path())?; - if metadata.is_dir() { - census(root, &entry.path(), members, modes, totals)?; - } else if metadata.file_type().is_symlink() { - totals[0] += 1; - let target = fs::read_link(entry.path())?; - if target - .components() - .any(|part| matches!(part, std::path::Component::ParentDir)) - { - totals[1] += 1; - } - } else { - totals[2] += 1; - let member = members.contains(entry.path().strip_prefix(root)?); - let mode = sealed_mode(&entry.path()); - if modes { - assert_eq!(mode, if member { 0o555 } else { 0o444 }); - totals[3] += usize::from(mode == 0o555); - } - } - } - Ok(()) - } - fn peak_rss() -> String { - fs::read_to_string("/proc/self/status") - .ok() - .and_then(|status| { - status - .lines() - .find(|line| line.starts_with("VmHWM")) - .map(str::to_owned) - }) - .unwrap_or_default() - } - - let tree = PathBuf::from( - std::env::var_os("IDUNN_BIG_TREE").context("IDUNN_BIG_TREE is not set")?, - ); - let members = ["python/bin/python3.12", "blender/blender"]; - let temp = tempfile::tempdir()?; - let staging = temp.path().join("staging"); - fs::create_dir_all(&staging)?; - let destination = staging.join("tree"); - let output = ArtifactOutput { - destination: PathBuf::from("tree"), - ..tree_output(false, &members) - }; - - let started = Instant::now(); - let receipt = seal_artifact(&output, &tree, &tree, &staging)?; - println!("seal {:?} {}", started.elapsed(), peak_rss()); - let started = Instant::now(); - let again = digest_artifact(&destination)?; - assert_eq!(format!("sha256-{}", again.0), receipt.sha256); - assert_eq!(again.1, receipt.size_bytes); - println!("digest {:?} {}", started.elapsed(), peak_rss()); - let mut before = [0; 4]; - census(&destination, &destination, &output.executable_members, false, &mut before)?; - let started = Instant::now(); - harden_installed_release(&staging, &[receipt.clone()])?; - println!("harden {:?} {}", started.elapsed(), peak_rss()); - let mut totals = [0; 4]; - census(&destination, &destination, &output.executable_members, true, &mut totals)?; - assert_eq!(totals[0], before[0], "links changed across harden"); - assert_eq!(totals[3], members.len()); - println!( - "dotdot links {} links before {} after {} files {} exec {}", - totals[1], before[0], totals[0], totals[2], totals[3] - ); - println!("bytes {}", receipt.size_bytes); - Ok(()) - } - #[cfg(unix)] #[test] fn runner_cache_is_created_once_for_one_exact_identity() -> Result<()> { @@ -16410,292 +14950,4 @@ processes 9 ); Ok(()) } - - // BEGIN runner-failure-evidence tests - // The runner decides what failed and what it kept: these drive the real - // DockerRunnerDriver over a fake docker program, never the capture alone. - - const RUNNER_CANARY: &str = "CANARY-5d2e91b7"; - - #[cfg(unix)] - fn fake_docker(dir: &Path, body: &str) -> Result { - use std::os::unix::fs::PermissionsExt; - let program = dir.join("docker"); - fs::write(&program, format!("#!/bin/sh\n{body}\n"))?; - fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?; - Ok(DockerRunnerDriver { - docker_program: program, - }) - } - - /// One recipe step through the real `run_in_workspace` path. - #[cfg(unix)] - fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> { - let workspace = dir.join("workspace"); - fs::create_dir_all(&workspace)?; - driver.run_in_workspace( - step_id, - "rust", - &fixture_docker_runner_binding(), - &workspace, - Path::new("."), - &["cargo".to_owned(), "test".to_owned()], - &std::collections::BTreeSet::new(), - ("IDUNN_STAMP", "rev"), - ) - } - - #[cfg(unix)] - fn failure_of(result: Result<()>) -> Result { - let error = result.expect_err("the step did not fail"); - error - .downcast_ref::() - .cloned() - .with_context(|| format!("the failure is not typed: {error:#}")) - } - - #[cfg(unix)] - #[test] - fn a_failed_step_is_named_with_both_tails() -> Result<()> { - let temp = tempfile::tempdir()?; - // The first step passes, the second fails: the count is the fake's. - let driver = fake_docker( - temp.path(), - &format!( - "n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\ - [ \"$n\" -lt 2 ] && exit 0\n\ - echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\ - echo 'stderr report' >&2\nexit 3" - ), - )?; - run_step(&driver, temp.path(), "first")?; - let failure = failure_of(run_step(&driver, temp.path(), "second"))?; - assert_eq!(failure.subject, RunnerSubject::Step("second".into())); - assert_eq!(failure.runner, "rust"); - assert_eq!(failure.exit_code, Some(3)); - assert_eq!(failure.signal, None); - assert!( - failure - .stdout_tail - .ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()), - "stdout was dropped: {:?}", - String::from_utf8_lossy(&failure.stdout_tail) - ); - assert_eq!(failure.stderr_tail, b"stderr report\n"); - assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> { - let temp = tempfile::tempdir()?; - let driver = fake_docker(temp.path(), "kill -9 $$")?; - let failure = failure_of(run_step(&driver, temp.path(), "build"))?; - assert_eq!((failure.exit_code, failure.signal), (None, Some(9))); - assert!(failure.to_string().contains("exited with signal 9"), "{failure}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_failed_external_input_is_named_as_one() -> Result<()> { - let temp = tempfile::tempdir()?; - let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?; - let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?; - let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?; - let input_id = declaration.external_inputs[0].id.clone(); - let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]); - fs::create_dir_all(&workspaces["rust"])?; - let error = driver - .materialize_external_input(&declaration, &binding, &workspaces, &input_id) - .expect_err("the download did not fail"); - let failure = error - .downcast_ref::() - .with_context(|| format!("the failure is not typed: {error:#}"))?; - assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone())); - assert_eq!(failure.exit_code, Some(22)); - assert!( - failure.to_string().contains(&format!("runner external input {input_id} on runner rust")), - "{failure}" - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn runner_output_never_reaches_the_error_text() -> Result<()> { - let temp = tempfile::tempdir()?; - let driver = fake_docker( - temp.path(), - &format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"), - )?; - let failure = failure_of(run_step(&driver, temp.path(), "build"))?; - assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY)); - assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY)); - let error = anyhow::Error::new(failure.clone()).context("sealing"); - for text in [ - format!("{error:#}"), - format!("{error:?}"), - format!("{failure}"), - format!("{failure:?}"), - ] { - assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}"); - } - Ok(()) - } - - /// A reader that answers from a script, so a read error the OS rarely - /// produces (an interrupted call) is produced on demand. - struct ScriptedReader(Vec>>); - - impl Read for ScriptedReader { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - if self.0.is_empty() { - return Ok(0); - } - let bytes = self.0.remove(0)?; - buffer[..bytes.len()].copy_from_slice(&bytes); - Ok(bytes.len()) - } - } - - struct BrokenReader; - - impl Read for BrokenReader { - fn read(&mut self, _: &mut [u8]) -> std::io::Result { - Err(std::io::Error::other("persistent read failure")) - } - } - - #[test] - fn an_interrupted_read_is_retried_and_loses_no_byte() { - let reader = ScriptedReader(vec![ - Ok(b"ab".to_vec()), - Err(std::io::Error::from(ErrorKind::Interrupted)), - Ok(b"cd".to_vec()), - Err(std::io::Error::from(ErrorKind::Interrupted)), - ]); - let drained = drain_tail(reader, 3).expect("an interrupted read ended the drain"); - assert_eq!(drained, (b"bcd".to_vec(), 4)); - } - - #[test] - fn a_persistent_read_error_ends_the_drain_instead_of_spinning() { - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let _ = sender.send(drain_tail(BrokenReader, 8).is_err()); - }); - assert_eq!( - receiver.recv_timeout(Duration::from_secs(5)), - Ok(true), - "a persistent read error spun or was swallowed" - ); - } - - #[test] - fn a_non_interrupted_read_error_after_data_is_returned_not_retried() { - let reader = ScriptedReader(vec![ - Ok(b"ab".to_vec()), - Err(std::io::Error::from(ErrorKind::BrokenPipe)), - Ok(b"cd".to_vec()), - ]); - let error = drain_tail(reader, 8).expect_err("a broken pipe was retried past"); - assert_eq!(error.kind(), ErrorKind::BrokenPipe); - } - - #[cfg(unix)] - #[test] - fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { - let temp = tempfile::tempdir()?; - let data = temp.path().join("data"); - for size in [ - 0, - 1, - RUNNER_OUTPUT_TAIL_BYTES - 1, - RUNNER_OUTPUT_TAIL_BYTES, - RUNNER_OUTPUT_TAIL_BYTES + 1, - 1024 * 1024, - ] { - // Distinct neighbours, so a shifted or head-keeping tail differs. - let bytes: Vec = (0..size).map(|index| (index % 251) as u8).collect(); - fs::write(&data, &bytes)?; - let driver = fake_docker( - temp.path(), - &format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()), - )?; - let failure = failure_of(run_step(&driver, temp.path(), "build"))?; - let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..]; - assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes"); - assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes"); - assert_eq!(failure.stdout_total_bytes, size as u64); - assert_eq!(failure.stderr_total_bytes, size as u64); - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> { - let temp = tempfile::tempdir()?; - run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?; - let missing = DockerRunnerDriver { - docker_program: temp.path().join("no-such-docker"), - }; - let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran"); - assert!(error.downcast_ref::().is_none()); - assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); - Ok(()) - } - - /// The 2026-10-25 vili failure: `node --test` reported on stdout alone, - /// stderr stayed empty (the runner's /tmp is a noexec tmpfs) and the step - /// exited 1. The old capture dropped stdout and printed an empty message. - #[cfg(unix)] - #[test] - fn a_step_that_reports_only_on_stdout_keeps_that_report_and_its_name() -> Result<()> { - let temp = tempfile::tempdir()?; - let driver = fake_docker( - temp.path(), - &format!("echo 'not ok 1 - daemon {RUNNER_CANARY}'\necho '# fail 1'\nexit 1"), - )?; - let error = run_step(&driver, temp.path(), "test-daemon").expect_err("the step passed"); - // The error names the step and carries no output. - let text = format!("{error:#}"); - assert!(text.contains("runner step test-daemon on runner rust exited with exit status 1"), "{text}"); - assert!(!text.contains(RUNNER_CANARY) && !text.contains("not ok"), "{text}"); - // The evidence keeps the stdout tail; the empty stderr is a recorded 0. - let failure = error - .downcast_ref::() - .context("the failure is not typed")?; - let store = temp.path().join("runner-evidence.cc"); - let record = crate::runner_evidence::RunnerFailureRecord::from_failure( - "tx-vili", - "up-vili", - "vili", - failure, - 1, - ); - crate::runner_evidence::record(&store, &record)?; - let kept = crate::runner_evidence::for_transaction(&store, "tx-vili")?.context("no evidence kept")?; - assert_eq!(kept.subject_id, "test-daemon"); - assert_eq!(kept.exit_code, Some(1)); - assert_eq!( - String::from_utf8_lossy(&kept.stdout_tail), - format!("not ok 1 - daemon {RUNNER_CANARY}\n# fail 1\n") - ); - assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); - assert!(kept.stderr_tail.is_empty()); - assert_eq!(kept.stderr_total_bytes, 0); - let mut shown = Vec::new(); - crate::runner_evidence::show_status(&mut shown, &store, "tx-vili")?; - let rendered = String::from_utf8(shown)?; - assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); - assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); - // Debug names the failure and prints no tail. - let debug = format!("{failure:?}"); - assert!(debug.starts_with("RunnerStepFailure") && !debug.contains("not ok"), "{debug}"); - Ok(()) - } - // END runner-failure-evidence tests } diff --git a/src/host.rs b/src/host.rs index 4c6cbec..5221f12 100644 --- a/src/host.rs +++ b/src/host.rs @@ -30,13 +30,13 @@ use serde::Deserialize; use crate::control_plane::read_trust_anchor; use crate::deployment::{ - HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + ArtifactSource, HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, LaunchArgument, OperatorBinding, TargetDeclaration, }; -use crate::deployment_plan::{CompiledDeploymentPlan, SealedRelease}; +use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease}; use crate::drivers::{ - HostWorkloadObservation, InstalledReleaseObservation, copy_tree, digest_artifact, - release_artifact, remove_tree_inside, seal_artifact, sha256_id, write_runtime_bundle_records, + HostWorkloadObservation, InstalledReleaseObservation, copy_artifact, copy_tree, + digest_artifact, release_artifact, remove_tree_inside, sha256_id, write_runtime_bundle_records, }; use crate::host_actuator::{ HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS, @@ -1048,12 +1048,37 @@ fn materialize( fs::create_dir_all(&staging)?; let mut artifacts = Vec::new(); for artifact in &declaration.artifacts { - artifacts.push(seal_artifact( - artifact, - &source, - &source.join(&artifact.source), - &staging, - )?); + let source_path = match artifact.source_kind { + ArtifactSource::RunnerOutput | ArtifactSource::WorktreeTree => { + source.join(&artifact.source) + } + }; + ensure!( + source_path.exists(), + "declared artifact output {} is absent", + artifact.id + ); + let destination = staging.join(&artifact.destination); + ensure!( + destination.starts_with(&staging), + "artifact destination escaped its staging root" + ); + copy_artifact(&source, &source_path, &destination)?; + let (sha256, size_bytes) = digest_artifact(&destination)?; + if let Some(expected) = &artifact.expected_sha256 { + ensure!( + &sha256 == expected, + "artifact {} differs from its recipe-pinned digest", + artifact.id + ); + } + artifacts.push(ArtifactReceipt { + artifact_id: artifact.id.clone(), + destination: artifact.destination.clone(), + sha256: format!("sha256-{sha256}"), + size_bytes, + executable: artifact.executable, + }); } let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?; Ok((release, staging)) diff --git a/src/host_actuator.rs b/src/host_actuator.rs index 8fd776c..f7bf4c7 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1151,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - if let Some(frame) = client.receive_once()? { + while let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/launcher.rs b/src/launcher.rs deleted file mode 100644 index 435f8b1..0000000 --- a/src/launcher.rs +++ /dev/null @@ -1,3238 +0,0 @@ -//! idunn-launch: the unit's process for an upstream daemon that cannot speak -//! Idunn's runtime contract itself (Synapse, MAS). -//! -//! Idunn observes the unit; the launcher, inside it and supervising the child, -//! signs the presence statement with the provider identity and the activation -//! key. The launcher binds the candidate endpoint Idunn challenges, because the -//! child cannot answer a CultNet frame, and answers presence there and nowhere -//! else: members reach the child on the member bind Idunn routes them to. It -//! starts a stateful child only after it holds the process write lease, renders -//! the child's config from a closed set of placeholders, and ends with the -//! child. -//! -//! Errors, logs and refusals name a field and a code. They never echo an -//! argument, a template line, a path or a credential. - -use std::collections::VecDeque; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, Read, Write}; -use std::net::{SocketAddr, TcpListener, TcpStream, UdpSocket}; -use std::os::fd::{AsRawFd, RawFd}; -use std::os::unix::fs::OpenOptionsExt; -use std::os::unix::process::CommandExt; -use std::path::{Component, Path, PathBuf}; -use std::process::{Child, Command, Stdio}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use std::thread; -use std::time::{Duration, Instant}; - -use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcache_rs::{DatabaseEntry, SingleFileMessagePackBackingStore}; -use cultnet_rs::{ - CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, - CultNetRudpReliableSendStatus, CultNetRudpSocketTransportConnection, - CultNetRudpSocketTransportOptions, CultNetWireContract, - GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, GameCultProviderHealthIdentity, - GameCultRuntimeCapability, GameCultRuntimePresenceHealthPurpose, - GameCultRuntimePresenceHealthRecord, IDUNN_EXPECTED_INCARNATION_SCHEMA, - IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, IDUNN_RUNTIME_ACTIVATION_SCHEMA, - IdunnExpectedIncarnationRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationRecord, - IdunnRuntimeActivationSigner, ServiceIdentitySigner, decode_cultnet_message_from_slice, - encode_cultnet_message_to_vec, encode_frame, open_service_identity_credential_reader, -}; - -use crate::deployment::{RUNTIME_PRESENCE_IDENTITY_FD_NAME, SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT}; -use crate::drivers::{CultCacheWriteLeaseDriver, authority_lock_path, lease_binds_candidate}; - -/// The route challenge is the only thing that begins with this byte: a -/// CultNet frame's 4-byte big-endian length is bounded at 1 MiB, and an HTTP -/// request starts with a method letter. -const CHALLENGE_FIRST_BYTE: u8 = 0x00; -const CHALLENGE_MAX_BYTES: usize = 1024 * 1024; -/// One whole presence exchange on the candidate endpoint, from accept to -/// answer, however the peer paces it. It is shorter than the 3 s Idunn waits -/// for a challenge answer, so one stalled peer cannot cost the next challenge -/// its reply. -const PRESENCE_DEADLINE: Duration = Duration::from_secs(1); -/// How long the presence acceptor waits after an accept error before trying -/// again. No accept error ends the acceptor. -const ACCEPT_BACKOFF: Duration = Duration::from_millis(50); -const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(1); -/// How often the lifecycle probes the child once it is Active. -const HEALTH_REFRESH: Duration = Duration::from_millis(500); -/// A probe result older than this is not evidence of health: the lifecycle -/// that refreshes it is itself stuck. -const HEALTH_STALE: Duration = Duration::from_secs(5); -const POLL: Duration = Duration::from_millis(100); -/// The largest duration a recipe may declare for the lease wait, the startup -/// wait or the heartbeat: one day. Past it a deadline is not a deadline. -const MAX_DECLARED_SECONDS: u64 = 86_400; -const MAX_RECENT_WARMING_PROOFS: usize = 64; -const TERMINATE_GRACE: Duration = Duration::from_secs(20); -const ODIN_CULTMESH_CATALOG_CONNECTION_ID: u32 = 0x0d1d_0002; -const ODIN_CAPABILITY: &str = "odin.verse-rendezvous"; - -// ---------------------------------------------------------------- contract - -/// Everything the recipe declares about the child, parsed from the launcher's -/// argv (`--flag value` pairs; `--child-arg` repeats). The release root is the -/// sealed release directory that holds the launcher itself. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct LaunchContract { - pub release_root: PathBuf, - pub child: PathBuf, - pub child_args: Vec, - /// The flag that carries the rendered config path to the child, when the - /// child takes one. - pub config_flag: Option, - pub template: PathBuf, - pub health_path: String, - pub health_contract: String, - pub state_root: Option, - pub lease_wait: Duration, - pub startup: Duration, - pub heartbeat: Duration, -} - -impl LaunchContract { - pub fn parse(release_root: PathBuf, args: impl IntoIterator) -> Result { - ensure!(release_root.is_absolute(), "the release root is not absolute"); - let mut child = None; - let mut child_args = Vec::new(); - let mut config_flag = None; - let mut template = None; - let mut health_path = None; - let mut health_contract = None; - let mut state_root = None; - let (mut lease_wait, mut startup, mut heartbeat) = (300_u64, 120_u64, 5_u64); - let mut args = args.into_iter(); - while let Some(flag) = args.next() { - let value = args.next().context("launcher argument has no value")?; - let seconds = |field: &str| -> Result { - value - .parse::() - .ok() - .filter(|seconds| (1..=MAX_DECLARED_SECONDS).contains(seconds)) - .with_context(|| { - format!("launcher argument {field} is outside 1 to {MAX_DECLARED_SECONDS} seconds") - }) - }; - match flag.as_str() { - "--child" => child = Some(release_relative(&value, "--child")?), - "--child-arg" => child_args.push(value), - "--config-flag" => config_flag = Some(value), - "--template" => template = Some(release_relative(&value, "--template")?), - "--health-path" => { - ensure!( - value.starts_with('/'), - "launcher argument --health-path is not absolute" - ); - health_path = Some(value) - } - "--health-contract" => health_contract = Some(value), - "--state-root" => state_root = Some(PathBuf::from(value)), - "--lease-wait-secs" => lease_wait = seconds("--lease-wait-secs")?, - "--startup-secs" => startup = seconds("--startup-secs")?, - "--heartbeat-secs" => heartbeat = seconds("--heartbeat-secs")?, - _ => bail!("unknown launcher argument"), - } - } - Ok(Self { - release_root, - child: child.context("launcher argument --child is required")?, - child_args, - config_flag, - template: template.context("launcher argument --template is required")?, - health_path: health_path.context("launcher argument --health-path is required")?, - health_contract: health_contract - .context("launcher argument --health-contract is required")?, - state_root, - lease_wait: Duration::from_secs(lease_wait), - startup: Duration::from_secs(startup), - heartbeat: Duration::from_secs(heartbeat), - }) - } -} - -fn release_relative(value: &str, field: &str) -> Result { - let path = PathBuf::from(value); - ensure!( - !value.is_empty() - && path - .components() - .all(|component| matches!(component, Component::Normal(_))), - "launcher argument {field} is not release-relative" - ); - Ok(path) -} - -// ---------------------------------------------------------------- authority - -/// The bundle, activation credential and provider identity Idunn hands the -/// unit, and the two signatures every presence statement needs. -pub struct RuntimeAuthority { - expected: IdunnExpectedIncarnationRecord, - expected_sha256: String, - activation: IdunnRuntimeActivationRecord, - activation_sha256: String, - activation_signer: IdunnRuntimeActivationSigner, - provider_signer: ServiceIdentitySigner, - sequence: u64, -} - -impl RuntimeAuthority { - pub fn open( - bundle: &Path, - activation_credential: impl Read, - provider_credential: impl Read, - ) -> Result { - ensure!(bundle.is_absolute(), "Idunn runtime bundle is not absolute"); - let (key, payload) = read_single_record( - &bundle.join("expected.cc"), - IdunnExpectedIncarnationRecord::TYPE, - IDUNN_EXPECTED_INCARNATION_SCHEMA, - )?; - let expected = IdunnExpectedIncarnationRecord::decode_canonical(&payload)?; - expected.validate()?; - ensure!(key == expected.target, "Expected key is substituted"); - let (key, payload) = read_single_record( - &bundle.join("activation.cc"), - IdunnRuntimeActivationRecord::TYPE, - IDUNN_RUNTIME_ACTIVATION_SCHEMA, - )?; - let activation = IdunnRuntimeActivationRecord::decode_canonical(&payload)?; - activation.validate()?; - ensure!(key == expected.target, "activation key is substituted"); - let expected_sha256 = expected.canonical_sha256()?; - ensure!( - activation.expected_projection_sha256 == expected_sha256 - && activation.runtime_id == expected.runtime_id, - "activation does not bind this Expected projection" - ); - let activation_signer = IdunnRuntimeActivationSigner::from_credential_reader( - activation_credential, - )?; - ensure!( - activation_signer.identity_id() == activation.activation_signer_identity_id - && activation_signer.public_key() == activation.activation_signer_public_key, - "activation credential does not belong to this activation" - ); - let provider_signer = - open_service_identity_credential_reader::( - provider_credential, - )?; - ensure!( - provider_signer.entry().identity_id == expected.expected_signer_identity_id, - "provider identity is not the signer selected by Expected" - ); - Ok(Self { - activation_sha256: activation.canonical_sha256()?, - expected, - expected_sha256, - activation, - activation_signer, - provider_signer, - sequence: 0, - }) - } - - pub fn expected(&self) -> &IdunnExpectedIncarnationRecord { - &self.expected - } - - pub fn activation(&self) -> &IdunnRuntimeActivationRecord { - &self.activation - } - - /// The candidate endpoint Idunn will challenge, which the launcher binds. - pub fn require_candidate_bind(&self, bind: &str) -> Result<()> { - let route = self.expected.route.as_ref().context("Expected has no route")?; - ensure!( - route.transport == "tcp" && route.candidate_endpoint == format!("tcp://{bind}"), - "candidate bind differs from the Expected route" - ); - Ok(()) - } - - /// The next presence record, signed by the provider identity and the - /// activation. The sequence is the clock: strictly increasing here and - /// above anything an earlier incarnation of this signer published. - pub fn sign( - &mut self, - state: &str, - detail: &str, - write_lease_sha256: Option, - now_millis: u64, - ) -> Result { - self.sequence = now_millis.max( - self.sequence - .checked_add(1) - .ok_or_else(|| anyhow!("runtime presence sequence exhausted"))?, - ); - let expected = &self.expected; - let mut record = GameCultRuntimePresenceHealthRecord { - schema_version: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), - target: expected.target.clone(), - expected_projection_sha256: self.expected_sha256.clone(), - plan_id: expected.plan_id.clone(), - incarnation_id: expected.incarnation_id.clone(), - sealed_release_id: expected.sealed_release_id.clone(), - activation_witness_sha256: self.activation_sha256.clone(), - state_schema_generation: expected.state_schema_generation.clone(), - state_contract_sha256: expected.state_contract_sha256.clone(), - runtime_id: expected.runtime_id.clone(), - runtime_instance_id: self.activation.runtime_instance_id.clone(), - bound_endpoint: expected - .route - .as_ref() - .map(|route| route.candidate_endpoint.clone()), - capabilities: expected - .capabilities - .iter() - .map(|capability| GameCultRuntimeCapability { - capability: capability.capability.clone(), - schema: capability.schema.clone(), - compatibility: capability.compatibility.clone(), - capacity: capability.minimum_capacity, - }) - .collect(), - health_contract: expected.health_contract.clone(), - state: state.into(), - detail: detail.into(), - write_lease_sha256, - signer_identity_id: self.provider_signer.entry().identity_id.clone(), - publisher_sequence: self.sequence, - observed_at_unix_millis: now_millis, - signature_algorithm: "ed25519".into(), - signature: Vec::new(), - activation_signer_identity_id: self.activation.activation_signer_identity_id.clone(), - activation_signature: Vec::new(), - }; - let payload = record.canonical_proof_payload()?; - record.signature = self - .provider_signer - .sign::(&payload) - .signature; - record.activation_signature = self.activation_signer.sign_presence_proof(&record)?; - record.validate()?; - Ok(record) - } -} - -fn read_single_record(path: &Path, record_type: &str, schema: &str) -> Result<(String, Vec)> { - let entries = SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot() - .context("reading a runtime bundle record")?; - let [envelope] = entries.as_slice() else { - bail!("runtime bundle record does not hold exactly one entry"); - }; - ensure!( - envelope.r#type == record_type && envelope.schema_id.as_deref() == Some(schema), - "runtime bundle record has the wrong type or schema" - ); - Ok((envelope.key.clone(), envelope.payload.clone())) -} - -/// The two authority descriptors systemd opens for the unit, in the order and -/// under the names Idunn's launch contract fixes. -pub fn inherited_authority_files() -> Result<(File, File)> { - use std::os::fd::{AsRawFd, FromRawFd}; - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - - let fds = std::env::var("LISTEN_FDS").context("systemd did not bind authority FDs")?; - let names = std::env::var("LISTEN_FDNAMES").context("systemd did not name authority FDs")?; - ensure!( - fds == "2" - && names - == format!( - "{IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME}:{RUNTIME_PRESENCE_IDENTITY_FD_NAME}" - ), - "systemd authority FDs differ from the launch contract" - ); - for fd in [3, 4] { - // SAFETY: F_GETFD observes one integer descriptor and transfers nothing. - ensure!( - unsafe { libc::fcntl(fd, libc::F_GETFD) } != -1, - "an authority descriptor is not open" - ); - } - // SAFETY: the LISTEN_* contract above assigns these two distinct, open - // descriptors to this process; this is their first and only consumer. - let (activation, provider) = unsafe { (File::from_raw_fd(3), File::from_raw_fd(4)) }; - for (file, exact) in [(&activation, Some(32_u64)), (&provider, None)] { - // OpenFile descriptors arrive without close-on-exec; the child must - // never inherit a signer. - // SAFETY: F_SETFD on a descriptor `file` owns. - ensure!( - unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) } != -1, - "an authority descriptor could not be sealed" - ); - let metadata = file.metadata()?; - ensure!( - metadata.is_file() - && metadata.uid() == 0 - && metadata.permissions().mode() & 0o777 == 0o400 - && metadata.nlink() == 1 - && exact.map_or(metadata.len() > 0, |size| metadata.len() == size), - "an authority descriptor is not one root-owned 0400 file" - ); - } - Ok((activation, provider)) -} - -// ---------------------------------------------------------------- ports - -pub trait Clock: Send + Sync { - /// Wall-clock time, for values signed or published to another party. No - /// decision about freshness, deadlines or schedules reads it. - fn now_millis(&self) -> u64; - /// Monotonic time: never steps back, unaffected by wall-clock changes. Every - /// age, deadline and schedule in the launcher reads this one source. - fn monotonic_millis(&self) -> u64; - fn sleep(&self, duration: Duration); -} - -pub struct SystemClock; - -impl Clock for SystemClock { - fn now_millis(&self) -> u64 { - u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0) - } - - fn monotonic_millis(&self) -> u64 { - static START: std::sync::OnceLock = std::sync::OnceLock::new(); - u64::try_from(START.get_or_init(std::time::Instant::now).elapsed().as_millis()) - .unwrap_or(u64::MAX) - } - - fn sleep(&self, duration: Duration) { - thread::sleep(duration); - } -} - -/// Systemd credentials as Idunn binds them: the files `NAME` under the unit's -/// credentials directory. -pub trait CredentialSource: Send + Sync { - fn contents(&self, name: &str) -> Result>; - fn path(&self, name: &str) -> Result; -} - -/// Resolves a credential only as a file directly under the credentials -/// directory systemd gives the unit (`$CREDENTIALS_DIRECTORY`, which Idunn -/// pins to `/run/credentials/`). The process environment is never a -/// credential namespace: a name is a plain file name, never a variable. -pub struct CredentialsDirectory(Option); - -impl CredentialsDirectory { - pub fn new(directory: PathBuf) -> Self { - Self(Some(directory)) - } - - /// The directory systemd exported; no directory means no credential is bound. - pub fn from_environment() -> Self { - Self(std::env::var_os(SYSTEMD_CREDENTIALS_DIRECTORY_ENVIRONMENT).map(PathBuf::from)) - } -} - -impl CredentialSource for CredentialsDirectory { - fn contents(&self, name: &str) -> Result> { - fs::read(self.path(name)?).map_err(|_| anyhow!("a credential could not be read")) - } - - fn path(&self, name: &str) -> Result { - ensure!(valid_credential_name(name), "credential name is invalid"); - let path = self - .0 - .as_ref() - .ok_or_else(|| anyhow!("a credential is not bound"))? - .join(name); - ensure!(fs::symlink_metadata(&path).is_ok(), "a credential is not bound"); - Ok(path) - } -} - -/// Where signed presence goes (Odin). -pub trait PresenceSink: Send + Sync { - fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()>; -} - -/// The lease as the launcher sees it: the current record, and a shared lock on -/// its store that Idunn's revocation cannot take while the child lives. -pub trait LeaseReader: Send + Sync { - fn read(&self) -> Result>; - fn hold_shared(&self) -> Result>; -} - -/// The shared lock on the lease store, held until dropped. Its descriptor is -/// handed to the child (`ChildSpawner::spawn`), so the lock lives as long as -/// the launcher or any process that inherited it. -pub trait LeaseLock: Send { - fn descriptor(&self) -> Option; -} - -pub trait HealthProbe: Send + Sync { - fn healthy(&self) -> bool; -} - -pub trait ChildHandle: Send { - /// The exit status once the child has exited. - fn exited(&mut self) -> Option; - /// SIGTERM, then SIGKILL after the grace period; returns once it is gone. - fn terminate(&mut self, grace: Duration); -} - -pub trait ChildSpawner: Send { - /// `inherit` is a descriptor the child keeps open across exec (the lease - /// lock); every other descriptor of the launcher is close-on-exec. - fn spawn( - &self, - program: &Path, - args: &[String], - inherit: Option, - ) -> Result>; -} - -// ---------------------------------------------------------------- production ports - -pub struct DriverLeaseReader { - driver: CultCacheWriteLeaseDriver, -} - -impl DriverLeaseReader { - pub fn new(target: &str, record_path: &Path) -> Self { - Self { - driver: CultCacheWriteLeaseDriver::new(target, record_path), - } - } -} - -struct SharedLockFile(File); - -impl LeaseLock for SharedLockFile { - fn descriptor(&self) -> Option { - Some(self.0.as_raw_fd()) - } -} - -impl LeaseReader for DriverLeaseReader { - fn read(&self) -> Result> { - self.driver - .current_for_holder() - .map(|current| current.map(|(_, lease)| lease)) - .map_err(|_| anyhow!("write-lease store refused custody, envelope or target")) - } - - /// LOCK_SH on `.lock`, the lock Idunn's revocation respects. The - /// descriptor is a plain file, so flock's lock belongs to its open file - /// description: a child that inherits it keeps the lock after the launcher - /// dies. - fn hold_shared(&self) -> Result> { - let file = File::open(authority_lock_path(&self.driver.record_path)).map_err(|_| anyhow!("write-lease lock could not be held"))?; - // SAFETY: flock on a descriptor this function owns. - ensure!( - unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0, - "write-lease lock could not be held" - ); - Ok(Box::new(SharedLockFile(file))) - } -} - -/// Odin takes presence over RUDP at the endpoint Expected's Odin dependency -/// names. -pub struct OdinRudpSink { - endpoint: SocketAddr, -} - -impl OdinRudpSink { - pub fn from_expected(expected: &IdunnExpectedIncarnationRecord) -> Result { - let dependency = expected - .dependencies - .iter() - .find(|dependency| dependency.capability == ODIN_CAPABILITY) - .context("Expected has no Odin dependency")?; - let endpoint = dependency - .provider_endpoint - .as_deref() - .context("Expected Odin dependency has no endpoint")?; - let endpoint = endpoint.strip_prefix("rudp://").unwrap_or(endpoint); - Ok(Self { - endpoint: endpoint - .parse() - .map_err(|_| anyhow!("Expected Odin endpoint is not a socket address"))?, - }) - } -} - -impl PresenceSink for OdinRudpSink { - fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { - let message = CultNetMessage::DocumentPutRaw { - message_id: format!( - "runtime-presence:{}:{}:{}", - record.target, record.runtime_instance_id, record.publisher_sequence - ), - document: presence_document( - record, - Some(vec![ - "cultnet.transport.rudp.v0".into(), - "runtime-presence".into(), - ]), - )?, - }; - let bind = if self.endpoint.is_ipv4() { - "0.0.0.0:0" - } else { - "[::]:0" - }; - let socket = UdpSocket::bind(bind)?; - socket.set_read_timeout(Some(Duration::from_millis(100)))?; - let mut transport = CultNetRudpSocketTransportConnection::new( - CultNetRudpSocketTransportOptions::client( - &record.runtime_id, - socket, - self.endpoint, - ODIN_CULTMESH_CATALOG_CONNECTION_ID, - ), - )?; - transport.connect(Vec::new())?; - let deadline = Instant::now() + Duration::from_millis(500); - while !transport.connected() { - let _ = transport.receive_once()?; - transport.poll_resends()?; - ensure!(Instant::now() < deadline, "Odin did not accept the connection"); - } - let receipt = transport.send_reliable( - "schema", - encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0)?, - )?; - let deadline = Instant::now() + Duration::from_secs(2); - loop { - match transport.reliable_send_status(&receipt) { - CultNetRudpReliableSendStatus::Acknowledged => return Ok(()), - CultNetRudpReliableSendStatus::Invalidated => { - bail!("Odin invalidated the presence acknowledgement") - } - CultNetRudpReliableSendStatus::Pending => {} - } - let _ = transport.receive_once()?; - transport.poll_resends()?; - ensure!(Instant::now() < deadline, "Odin did not acknowledge presence"); - } - } -} - -fn presence_document( - record: &GameCultRuntimePresenceHealthRecord, - tags: Option>, -) -> Result { - let payload = rmp_serde::to_vec(record).context("encoding runtime presence")?; - Ok(CultNetRawDocumentRecord { - schema_id: GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), - record_key: record.target.clone(), - stored_at: chrono::DateTime::from_timestamp_millis( - record.observed_at_unix_millis.try_into()?, - ) - .context("runtime presence timestamp is invalid")? - .to_rfc3339(), - payload_encoding: CultNetRawPayloadEncoding::Messagepack, - payload, - source_runtime_id: Some(record.runtime_id.clone()), - source_agent_id: Some(record.signer_identity_id.clone()), - source_role: Some("runtime-presence-publisher".into()), - tags, - }) -} - -/// `GET ` over TCP at the child's member bind; healthy on a 200. One -/// deadline covers connect, write and read, so a full backlog or a silent child -/// answers false within `HEALTH_PROBE_TIMEOUT`. -pub struct HttpHealthProbe { - pub member: SocketAddr, - pub path: String, -} - -impl HealthProbe for HttpHealthProbe { - fn healthy(&self) -> bool { - let deadline = Instant::now() + HEALTH_PROBE_TIMEOUT; - let Ok(mut stream) = TcpStream::connect_timeout(&self.member, HEALTH_PROBE_TIMEOUT) else { - return false; - }; - let Some(remaining) = deadline - .checked_duration_since(Instant::now()) - .filter(|left| !left.is_zero()) - else { - return false; - }; - let request = format!( - "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", - self.path - ); - let mut head = [0_u8; 12]; - stream.set_write_timeout(Some(remaining)).is_ok() - && stream.write_all(request.as_bytes()).is_ok() - && read_by(&mut stream, &mut head, deadline).is_ok() - && head.starts_with(b"HTTP/1.") - && &head[8..12] == b" 200" - } -} - -pub struct ProcessSpawner; - -struct ProcessChild(Child); - -impl ChildSpawner for ProcessSpawner { - fn spawn( - &self, - program: &Path, - args: &[String], - inherit: Option, - ) -> Result> { - let mut command = Command::new(program); - // The child gets a bare environment: no Idunn variable, no candidate - // bind, no credential path leaks into it. - command.args(args).env_clear().stdin(Stdio::null()); - for name in ["PATH", "TMPDIR", "LANG", "LC_ALL"] { - if let Some(value) = std::env::var_os(name) { - command.env(name, value); - } - } - if let Some(descriptor) = inherit { - // SAFETY: runs in the forked child before exec; fcntl is - // async-signal-safe and the parent's descriptor stays close-on-exec. - unsafe { - command.pre_exec(move || { - if libc::fcntl(descriptor, libc::F_SETFD, 0) == -1 { - return Err(io::Error::last_os_error()); - } - Ok(()) - }); - } - } - let child = command.spawn().map_err(|_| anyhow!("the child could not be spawned"))?; - Ok(Box::new(ProcessChild(child))) - } -} - -impl ChildHandle for ProcessChild { - fn exited(&mut self) -> Option { - match self.0.try_wait() { - Ok(Some(status)) => Some(status.code().unwrap_or(1)), - Ok(None) => None, - Err(_) => Some(1), - } - } - - fn terminate(&mut self, grace: Duration) { - if self.exited().is_some() { - return; - } - // SAFETY: the pid is our own live child. - unsafe { libc::kill(self.0.id() as libc::pid_t, libc::SIGTERM) }; - let deadline = Instant::now() + grace; - while Instant::now() < deadline { - if self.exited().is_some() { - return; - } - thread::sleep(Duration::from_millis(20)); - } - let _ = self.0.kill(); - let _ = self.0.wait(); - } -} - -// ---------------------------------------------------------------- config - -/// The closed placeholder set (decision D6). Any other `{{...}}` refuses. -pub struct ConfigRenderer<'a> { - pub credentials: &'a dyn CredentialSource, - /// The member bind the child serves on. - pub listen: SocketAddr, - pub state_root: Option<&'a Path>, -} - -impl ConfigRenderer<'_> { - pub fn render(&self, template: &str) -> Result { - let mut rendered = String::with_capacity(template.len()); - let mut rest = template; - while let Some(open) = rest.find("{{") { - rendered.push_str(&rest[..open]); - let after = &rest[open + 2..]; - let close = after - .find("}}") - .context("config template has an unterminated placeholder")?; - rendered.push_str(&self.expand(&after[..close])?); - rest = &after[close + 2..]; - } - rendered.push_str(rest); - Ok(rendered) - } - - fn expand(&self, key: &str) -> Result { - match key { - "child.listen" => return Ok(self.listen.to_string()), - "child.listen_host" => return Ok(self.listen.ip().to_string()), - "child.listen_port" => return Ok(self.listen.port().to_string()), - _ => {} - } - if key == "state_root" { - return Ok(self - .state_root - .context("config template needs a state root the launch did not give")? - .display() - .to_string()); - } - if let Some(name) = key.strip_prefix("credential:") { - ensure!(valid_credential_name(name), "credential placeholder name is invalid"); - let bytes = self.credentials.contents(name)?; - let text = String::from_utf8(bytes) - .map_err(|_| anyhow!("a credential is not UTF-8 text"))?; - return Ok(text.strip_suffix('\n').unwrap_or(&text).to_owned()); - } - if let Some(name) = key.strip_prefix("credential_path:") { - ensure!(valid_credential_name(name), "credential placeholder name is invalid"); - return Ok(self.credentials.path(name)?.display().to_string()); - } - bail!("config template uses a placeholder outside the closed set") - } -} - -fn valid_credential_name(name: &str) -> bool { - !name.is_empty() - && name - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') -} - -fn write_private_file(directory: &Path, name: &str, contents: &str) -> Result { - let path = directory.join(name); - let mut file = OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&path) - .map_err(|_| anyhow!("the rendered config could not be created"))?; - file.write_all(contents.as_bytes())?; - Ok(path) -} - -// ---------------------------------------------------------------- lease - -/// A write lease this launcher holds: the record binds this Expected, -/// activation and one of the launcher's own Warming proofs, and its store is -/// share-locked until the hold drops. -pub struct LeaseHold { - reader: Arc, - lease: IdunnProcessWriteLeaseRecord, - sha256: String, - lock: Box, -} - -impl LeaseHold { - pub fn acquire( - reader: Arc, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - warming_proofs: &[String], - ) -> Result { - let lease = reader.read()?.context("no write lease is granted")?; - lease.validate()?; - lease_binds_candidate(expected, activation, &lease, warming_proofs)?; - let lock = reader.hold_shared()?; - ensure!( - reader.read()?.as_ref() == Some(&lease), - "write lease changed while it was being held" - ); - Ok(Self { - sha256: lease.canonical_sha256()?, - reader, - lease, - lock, - }) - } - - pub fn sha256(&self) -> &str { - &self.sha256 - } - - /// The locked descriptor, for the child to inherit. - pub fn lock_descriptor(&self) -> Option { - self.lock.descriptor() - } - - pub fn still_current(&self) -> bool { - matches!(self.reader.read(), Ok(Some(current)) if current == self.lease) - } -} - -// ---------------------------------------------------------------- route - -#[derive(Clone, Debug, PartialEq, Eq)] -enum Phase { - Warming, - Active(Option), - Failed, -} - -/// What the launcher currently is, shared with the presence acceptor. -struct Shared { - authority: Mutex, - phase: Mutex, - lease: Mutex>, - lease_required: bool, - health: Arc, - /// The last probe result and the monotonic time it started. Only the lifecycle - /// probes; a challenge reads this and never waits on the child. - probed: Mutex>, - clock: Arc, -} - -impl Shared { - /// Probes the child now and keeps the result for presence to answer from. - fn refresh_health(&self) -> bool { - let started = self.clock.monotonic_millis(); - let healthy = self.health.healthy(); - if let Ok(mut probed) = self.probed.lock() { - *probed = Some((healthy, started)); - } - healthy - } - - /// Active only while the last probe found the child healthy and is recent. - /// A child never probed, wedged, or probed too long ago reads degraded. A - /// dead presence acceptor does not degrade presence: it fails the unit. - fn serving_state(&self) -> &'static str { - let stale = u64::try_from(HEALTH_STALE.as_millis()).unwrap_or(u64::MAX); - let now = self.clock.monotonic_millis(); - // A probe can finish between this thread reading the clock and locking - // the result, so a stamp later than `now` is a result fresher than the - // reading, age zero: monotonic time does not run backwards, so the stamp - // is not wrong. - let healthy = self - .probed - .lock() - .ok() - .and_then(|probed| *probed) - .is_some_and(|(healthy, at)| healthy && now.saturating_sub(at) <= stale); - if healthy { "active" } else { "degraded" } - } - - fn sign(&self, state: &str, detail: &str, lease: Option) -> Result { - self.authority - .lock() - .map_err(|_| anyhow!("authority lock poisoned"))? - .sign(state, detail, lease, self.clock.now_millis()) - } - - fn lease_current(&self) -> bool { - !self.lease_required - || self - .lease - .lock() - .is_ok_and(|hold| hold.as_ref().is_some_and(LeaseHold::still_current)) - } - - /// The answer to a route observation: active only if the child is healthy - /// and the lease is still ours. - fn observe(&self, request: CultNetMessage) -> Result { - let CultNetMessage::SnapshotRequest { message_id, .. } = request else { - bail!("route challenge is not a snapshot request"); - }; - let detail = format!("route-observation:{message_id}"); - let phase = self.phase.lock().map_err(|_| anyhow!("phase lock poisoned"))?.clone(); - let record = match phase { - Phase::Warming => self.sign("warming", &detail, None)?, - Phase::Failed => self.sign("failed", &detail, None)?, - Phase::Active(lease) => { - let state = if self.lease_current() { - self.serving_state() - } else { - "degraded" - }; - self.sign(state, &detail, lease)? - } - }; - Ok(CultNetMessage::SnapshotResponseRaw { - message_id, - documents: vec![presence_document(&record, None)?], - }) - } -} - -type Responder = Arc Result + Send + Sync>; - -/// Owns the candidate endpoint and answers presence there and nowhere else. -/// Members never reach it: Idunn routes them to the child's member bind. -/// -/// The endpoint is public, so it spends bounded resources on a peer: one -/// connection at a time, so a connection holds at most one descriptor; a total -/// deadline for the whole exchange; and any first byte but a challenge's closes -/// the connection unanswered. No accept error ends it: it backs off and accepts -/// again, and only `halt` stops it. -pub struct PresenceAcceptor { - listener: TcpListener, - responder: Responder, -} - -impl PresenceAcceptor { - pub fn new(listener: TcpListener, responder: Responder) -> Self { - Self { listener, responder } - } - - pub fn serve(self, halt: Arc) { - self.serve_accepting(halt, |listener| listener.accept()); - } - - fn serve_accepting( - self, - halt: Arc, - mut accept: impl FnMut(&TcpListener) -> io::Result<(TcpStream, SocketAddr)>, - ) { - // A listener that cannot be made non-blocking could never observe - // `halt`; ending here is a dead acceptor, which fails the unit. - if self.listener.set_nonblocking(true).is_err() { - return; - } - while !halt.load(Ordering::SeqCst) { - match accept(&self.listener) { - Ok((connection, _)) => { - let _ = connection.set_nonblocking(false); - let _ = self.answer(connection); - } - // An idle non-blocking accept is an error too; every one waits - // and tries again. - Err(_) => thread::sleep(ACCEPT_BACKOFF), - } - } - } - - fn answer(&self, mut connection: TcpStream) -> Result<()> { - let deadline = Instant::now() + PRESENCE_DEADLINE; - let mut first = [0_u8; 1]; - read_by(&mut connection, &mut first, deadline)?; - ensure!(first[0] == CHALLENGE_FIRST_BYTE, "not a challenge frame"); - let mut rest = [0_u8; 3]; - read_by(&mut connection, &mut rest, deadline)?; - let length = u32::from_be_bytes([CHALLENGE_FIRST_BYTE, rest[0], rest[1], rest[2]]) as usize; - ensure!( - (1..=CHALLENGE_MAX_BYTES).contains(&length), - "challenge frame is outside its bound" - ); - let mut payload = vec![0_u8; length]; - read_by(&mut connection, &mut payload, deadline)?; - let request = - decode_cultnet_message_from_slice(&payload, CultNetWireContract::CultNetSchemaV0)?; - let response = (self.responder)(request)?; - let frame = encode_frame(&encode_cultnet_message_to_vec( - &response, - CultNetWireContract::CultNetSchemaV0, - )?)?; - let remaining = deadline - .checked_duration_since(Instant::now()) - .filter(|left| !left.is_zero()) - .context("the answer was not ready in time")?; - connection.set_write_timeout(Some(remaining))?; - connection.write_all(&frame)?; - connection.flush()?; - Ok(()) - } -} - -/// Fills `buffer` or fails when `deadline` passes, however the peer paces it. -fn read_by(connection: &mut TcpStream, buffer: &mut [u8], deadline: Instant) -> Result<()> { - let mut filled = 0; - while filled < buffer.len() { - let remaining = deadline - .checked_duration_since(Instant::now()) - .filter(|left| !left.is_zero()) - .context("the peer did not send in time")?; - connection.set_read_timeout(Some(remaining))?; - match connection.read(&mut buffer[filled..]) { - Ok(0) => bail!("the peer closed early"), - Ok(count) => filled += count, - Err(error) if error.kind() == io::ErrorKind::Interrupted => {} - Err(_) => bail!("the peer did not send in time"), - } - } - Ok(()) -} - -// ---------------------------------------------------------------- lifecycle - -pub struct Ports { - pub credentials: Arc, - pub clock: Arc, - pub sink: Arc, - pub lease: Arc, - pub spawner: Box, - /// Built from the member bind the child serves on. - pub health: Box Arc + Send>, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Outcome { - /// SIGTERM: the child was stopped and waited for. - Stopped, - /// The unit must die so continuity sees it; the code names why. - Failed(&'static str), -} - -impl Outcome { - pub fn exit_code(&self) -> i32 { - match self { - Self::Stopped => 0, - Self::Failed(_) => 1, - } - } -} - -/// The member bind Idunn hands the unit: a socket address on a loopback IP, -/// other than the candidate bind. Errors name the variable, never its value. -pub fn parse_member_bind(raw: &str, candidate_bind: &str) -> Result { - let member: SocketAddr = raw - .parse() - .map_err(|_| anyhow!("the Idunn member bind is not a socket address"))?; - ensure!( - member.ip().is_loopback(), - "the Idunn member bind is not on a loopback address" - ); - let candidate: SocketAddr = candidate_bind - .parse() - .map_err(|_| anyhow!("the Idunn candidate bind is not a socket address"))?; - ensure!(member != candidate, "the Idunn member bind equals the candidate bind"); - Ok(member) -} - -/// The monotonic reading `span` after `now`; a span too long to represent is a -/// deadline that never arrives. -fn after(now: u64, span: Duration) -> u64 { - now.saturating_add(u64::try_from(span.as_millis()).unwrap_or(u64::MAX)) -} - -/// The unit's whole life: Warming, lease, child, Active, until the child or the -/// lease ends or SIGTERM arrives. `route` is the already-bound candidate -/// listener, answered for presence only; `member` is the address the child -/// serves members on; `tmp` is the unit's private temporary directory. -pub fn run( - contract: &LaunchContract, - authority: RuntimeAuthority, - route: TcpListener, - member: SocketAddr, - tmp: &Path, - ports: Ports, - stop: &AtomicBool, -) -> Outcome { - let expected = authority.expected().clone(); - let activation = authority.activation().clone(); - if expected.health_contract != contract.health_contract { - return Outcome::Failed("health-contract-mismatch"); - } - let health = (ports.health)(member); - let shared = Arc::new(Shared { - authority: Mutex::new(authority), - phase: Mutex::new(Phase::Warming), - lease: Mutex::new(None), - lease_required: expected.write_lease_required, - health: Arc::clone(&health), - probed: Mutex::new(None), - clock: Arc::clone(&ports.clock), - }); - let halt = Arc::new(AtomicBool::new(false)); - let responder: Responder = { - let shared = Arc::clone(&shared); - Arc::new(move |request| shared.observe(request)) - }; - let acceptor = PresenceAcceptor::new(route, responder); - let serving = { - let halt = Arc::clone(&halt); - thread::Builder::new() - .name("presence-acceptor".into()) - .spawn(move || acceptor.serve(halt)) - }; - let Ok(serving) = serving else { - return Outcome::Failed("presence-acceptor"); - }; - let outcome = lifecycle(contract, &expected, &activation, member, tmp, &ports, &shared, &serving, stop); - halt.store(true, Ordering::SeqCst); - let _ = serving.join(); - outcome -} - -#[allow(clippy::too_many_arguments)] -fn lifecycle( - contract: &LaunchContract, - expected: &IdunnExpectedIncarnationRecord, - activation: &IdunnRuntimeActivationRecord, - member: SocketAddr, - tmp: &Path, - ports: &Ports, - shared: &Shared, - presence: &thread::JoinHandle<()>, - stop: &AtomicBool, -) -> Outcome { - let clock = &*ports.clock; - let publish = |state: &str, detail: &str, lease: Option| -> Option { - let record = shared.sign(state, detail, lease).ok()?; - let sha = record.canonical_sha256().ok()?; - // A refused or lost publish is retried by the heartbeat; Odin may - // refuse Warming for a while after Idunn starts the unit. - let _ = ports.sink.put(&record); - Some(sha) - }; - let set_phase = |phase: Phase| { - if let Ok(mut current) = shared.phase.lock() { - *current = phase; - } - }; - let fail = |child: Option<&mut Box>, code: &'static str| { - if let Some(child) = child { - child.terminate(TERMINATE_GRACE); - } - set_phase(Phase::Failed); - publish("failed", code, None); - Outcome::Failed(code) - }; - - // 1. Warming reaches Odin before any child exists. - let mut proofs: VecDeque = VecDeque::new(); - let remember = |sha: Option, proofs: &mut VecDeque| { - if let Some(sha) = sha { - proofs.push_back(sha); - while proofs.len() > MAX_RECENT_WARMING_PROOFS { - proofs.pop_front(); - } - } - }; - remember(publish("warming", "launcher-started", None), &mut proofs); - let heartbeat = contract.heartbeat; - let mut next_heartbeat = after(clock.monotonic_millis(), heartbeat); - - // 2. A stateful child waits for the lease; a stateless one does not. - let mut lease_sha = None; - let mut lease_fd = None; - if expected.write_lease_required { - let deadline = after(clock.monotonic_millis(), contract.lease_wait); - loop { - if stop.load(Ordering::SeqCst) { - return Outcome::Stopped; - } - if presence.is_finished() { - return fail(None, "presence-acceptor"); - } - let held = LeaseHold::acquire( - Arc::clone(&ports.lease), - expected, - activation, - proofs.make_contiguous(), - ); - if let Ok(hold) = held { - lease_sha = Some(hold.sha256().to_owned()); - lease_fd = hold.lock_descriptor(); - if let Ok(mut slot) = shared.lease.lock() { - *slot = Some(hold); - } - break; - } - let now = clock.monotonic_millis(); - if now >= deadline { - return fail(None, "lease-timeout"); - } - if now >= next_heartbeat { - remember(publish("warming", "write-lease-pending", None), &mut proofs); - next_heartbeat = after(now, heartbeat); - } - clock.sleep(POLL); - } - } - - // 3. Render, spawn, and wait for health. - let Some(mut child) = start_child(contract, member, tmp, ports, lease_fd) else { - return fail(None, "child-start-refused"); - }; - let startup_deadline = after(clock.monotonic_millis(), contract.startup); - loop { - if stop.load(Ordering::SeqCst) { - child.terminate(TERMINATE_GRACE); - return Outcome::Stopped; - } - if child.exited().is_some() { - return fail(None, "child-exited"); - } - if presence.is_finished() { - return fail(Some(&mut child), "presence-acceptor"); - } - if shared.refresh_health() { - break; - } - if clock.monotonic_millis() >= startup_deadline { - return fail(Some(&mut child), "startup-timeout"); - } - clock.sleep(POLL); - } - - // 4. Active, and heartbeats while it holds. - set_phase(Phase::Active(lease_sha.clone())); - publish(shared.serving_state(), "child-healthy", lease_sha.clone()); - let mut next_probe = after(clock.monotonic_millis(), HEALTH_REFRESH); - next_heartbeat = after(clock.monotonic_millis(), heartbeat); - loop { - if stop.load(Ordering::SeqCst) { - child.terminate(TERMINATE_GRACE); - return Outcome::Stopped; - } - if child.exited().is_some() { - return fail(None, "child-exited"); - } - if presence.is_finished() { - return fail(Some(&mut child), "presence-acceptor"); - } - if !shared.lease_current() { - return fail(Some(&mut child), "lease-lost"); - } - let now = clock.monotonic_millis(); - if now >= next_probe { - shared.refresh_health(); - next_probe = after(now, HEALTH_REFRESH); - } - if now >= next_heartbeat { - publish(shared.serving_state(), "heartbeat", lease_sha.clone()); - next_heartbeat = after(now, heartbeat); - } - clock.sleep(POLL); - } -} - -fn start_child( - contract: &LaunchContract, - member: SocketAddr, - tmp: &Path, - ports: &Ports, - lease_fd: Option, -) -> Option> { - let template = fs::read_to_string(contract.release_root.join(&contract.template)).ok()?; - let rendered = ConfigRenderer { - credentials: &*ports.credentials, - listen: member, - state_root: contract.state_root.as_deref(), - } - .render(&template) - .ok()?; - let config = write_private_file(tmp, "child-config", &rendered).ok()?; - let mut args = contract.child_args.clone(); - if let Some(flag) = &contract.config_flag { - args.push(flag.clone()); - args.push(config.display().to_string()); - } - ports - .spawner - .spawn(&contract.release_root.join(&contract.child), &args, lease_fd) - .ok() -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - use std::os::unix::fs::PermissionsExt; - use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize}; - - use cultcache_rs::{CultCacheEnvelope, TryCompareExchangeSnapshotOutcome}; - use cultnet_rs::{ - IdunnExpectedCapability, IdunnExpectedRoute, IdunnRuntimeActivationLaunch, - IdunnServiceIdentity, RuntimePresenceAuthenticationContext, VerifiedRuntimeAuthority, - authenticate_runtime_presence_claim, enroll_service_identity_at, verify_runtime_authority, - }; - - use super::*; - use crate::drivers::{WriteLeasePort, exact_route_snapshot_response, request_tcp_snapshot}; - - /// The member bind Idunn would hand these fixtures' unit. - fn member() -> SocketAddr { - "127.0.0.1:18845".parse().unwrap() - } - - fn digest(byte: char) -> String { - format!("sha256-{}", byte.to_string().repeat(64)) - } - - struct Fixture { - dir: tempfile::TempDir, - expected: IdunnExpectedIncarnationRecord, - activation: IdunnRuntimeActivationRecord, - activation_credential: Vec, - provider_credential: Vec, - verified: VerifiedRuntimeAuthority, - } - - fn write_record(path: &Path, key: &str, kind: &str, schema: &str, payload: Vec) { - let envelope = CultCacheEnvelope { - key: key.into(), - r#type: kind.into(), - payload, - stored_at: chrono::Utc::now().to_rfc3339(), - schema_id: Some(schema.into()), - }; - let outcome = SingleFileMessagePackBackingStore::new(path) - .try_compare_exchange_snapshot(&[], &[envelope]) - .unwrap(); - assert!(matches!( - outcome, - TryCompareExchangeSnapshotOutcome::Exchanged - )); - } - - fn fixture(lease_required: bool) -> Fixture { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path(); - let provider_path = root.join("provider.cc"); - let provider = - enroll_service_identity_at::(&provider_path).unwrap(); - let idunn = - enroll_service_identity_at::(&root.join("idunn.cc")).unwrap(); - let expected = IdunnExpectedIncarnationRecord { - schema_version: IDUNN_EXPECTED_INCARNATION_SCHEMA.into(), - target: "synapse".into(), - plan_id: digest('1'), - incarnation_id: "incarnation-1".into(), - sealed_release_id: digest('2'), - source_repository: "github.com/GameCult/gamecult-chat".into(), - source_revision: "3".repeat(40), - recipe_sha256: digest('4'), - runtime_id: "synapse-yggdrasil".into(), - expected_signer_identity_id: provider.entry().identity_id.clone(), - health_contract: "gamecult.chat.synapse.presence-launcher.v1".into(), - artifact_sha256: digest('5'), - state_schema_generation: lease_required.then(|| "state-v1".into()), - state_contract_sha256: lease_required.then(|| digest('6')), - write_lease_required: lease_required, - route: Some(IdunnExpectedRoute { - route_id: "synapse-tcp".into(), - transport: "tcp".into(), - stable_endpoint: "tcp://127.0.0.1:18834".into(), - candidate_endpoint: "tcp://127.0.0.1:18835".into(), - }), - capabilities: vec![IdunnExpectedCapability { - capability: "gamecult.chat.homeserver".into(), - schema: "matrix.client.v3".into(), - compatibility: "v3".into(), - minimum_capacity: 1, - }], - dependencies: Vec::new(), - }; - let launch = - IdunnRuntimeActivationLaunch::issue(&expected, digest('7'), 100, &idunn).unwrap(); - let activation = launch.activation().clone(); - let mut activation_credential = Vec::new(); - launch.write_credential(&mut activation_credential).unwrap(); - let verified = verify_runtime_authority( - &expected, - &activation, - &idunn.trust_anchor().unwrap(), - &provider.entry().public_key, - ) - .unwrap(); - let bundle = root.join("bundle"); - fs::create_dir(&bundle).unwrap(); - write_record( - &bundle.join("expected.cc"), - &expected.target, - IdunnExpectedIncarnationRecord::TYPE, - IDUNN_EXPECTED_INCARNATION_SCHEMA, - expected.canonical_bytes().unwrap(), - ); - write_record( - &bundle.join("activation.cc"), - &expected.target, - IdunnRuntimeActivationRecord::TYPE, - IDUNN_RUNTIME_ACTIVATION_SCHEMA, - activation.canonical_bytes().unwrap(), - ); - let provider_credential = fs::read(&provider_path).unwrap(); - for directory in ["release/bin", "tmp"] { - fs::create_dir_all(root.join(directory)).unwrap(); - } - Fixture { - dir, - expected, - activation, - activation_credential, - provider_credential, - verified, - } - } - - impl Fixture { - fn authority(&self) -> RuntimeAuthority { - RuntimeAuthority::open( - &self.dir.path().join("bundle"), - &self.activation_credential[..], - &self.provider_credential[..], - ) - .unwrap() - } - - fn lease(&self, warming_sha256: &str) -> IdunnProcessWriteLeaseRecord { - IdunnProcessWriteLeaseRecord { - schema_version: cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA.into(), - target: self.expected.target.clone(), - expected_projection_sha256: self.expected.canonical_sha256().unwrap(), - plan_id: self.expected.plan_id.clone(), - incarnation_id: self.expected.incarnation_id.clone(), - sealed_release_id: self.expected.sealed_release_id.clone(), - activation_witness_sha256: self.activation.canonical_sha256().unwrap(), - state_schema_generation: self.expected.state_schema_generation.clone().unwrap(), - state_contract_sha256: self.expected.state_contract_sha256.clone().unwrap(), - runtime_id: self.expected.runtime_id.clone(), - runtime_instance_id: self.activation.runtime_instance_id.clone(), - warming_presence_sha256: warming_sha256.into(), - lease_epoch: 1, - issued_at_unix_millis: 200, - } - } - - fn contract(&self, template: &str) -> LaunchContract { - fs::write(self.dir.path().join("release/config.tmpl"), template).unwrap(); - LaunchContract { - release_root: self.dir.path().join("release"), - child: PathBuf::from("bin/child"), - child_args: vec!["--serve".into()], - config_flag: Some("--config".into()), - template: PathBuf::from("config.tmpl"), - health_path: "/health".into(), - health_contract: self.expected.health_contract.clone(), - state_root: Some(PathBuf::from("/var/lib/synapse")), - lease_wait: Duration::from_secs(3600), - startup: Duration::from_secs(3600), - heartbeat: Duration::from_secs(5), - } - } - } - - const WALL_EPOCH_MILLIS: i64 = 1_760_000_000_000; - - /// Virtual time runs a hundred times faster than the wall clock. - struct FastClock { - monotonic: AtomicU64, - wall_step: AtomicI64, - } - - impl FastClock { - /// The wall clock starts a Unix-epoch-sized distance from the monotonic - /// one, as the real pair does, so a deadline or schedule that reads the - /// wrong source is off by that distance and no test can pass by it. - fn new(start: u64) -> Self { - Self { monotonic: AtomicU64::new(start), wall_step: AtomicI64::new(WALL_EPOCH_MILLIS) } - } - - /// Steps the wall clock alone, as timesyncd or a VM restore would. - fn step_wall(&self, millis: i64) { - self.wall_step.fetch_add(millis, Ordering::SeqCst); - } - } - - impl Clock for FastClock { - fn now_millis(&self) -> u64 { - let wall = self.monotonic.load(Ordering::SeqCst) as i64 + self.wall_step.load(Ordering::SeqCst); - u64::try_from(wall).unwrap_or(0) - } - - fn monotonic_millis(&self) -> u64 { - self.monotonic.load(Ordering::SeqCst) - } - - fn sleep(&self, duration: Duration) { - self.monotonic - .fetch_add(duration.as_millis() as u64, Ordering::SeqCst); - thread::sleep(Duration::from_millis(1)); - } - } - - #[derive(Default)] - struct Sink(Mutex>); - - impl PresenceSink for Sink { - fn put(&self, record: &GameCultRuntimePresenceHealthRecord) -> Result<()> { - self.0.lock().unwrap().push(record.clone()); - Ok(()) - } - } - - impl Sink { - fn states(&self) -> Vec { - self.0 - .lock() - .unwrap() - .iter() - .map(|record| record.state.clone()) - .collect() - } - - fn first(&self, state: &str) -> Option { - self.0 - .lock() - .unwrap() - .iter() - .find(|record| record.state == state) - .cloned() - } - } - - #[derive(Default)] - struct FakeLease(Mutex>); - - impl LeaseReader for FakeLease { - fn read(&self) -> Result> { - Ok(self.0.lock().unwrap().clone()) - } - - fn hold_shared(&self) -> Result> { - Ok(Box::new(FakeLock)) - } - } - - /// The descriptor number a fake lock claims; the launcher must hand it on. - const FAKE_LOCK_FD: RawFd = 1234; - - struct FakeLock; - - impl LeaseLock for FakeLock { - fn descriptor(&self) -> Option { - Some(FAKE_LOCK_FD) - } - } - - struct Health(AtomicBool); - - impl HealthProbe for Health { - fn healthy(&self) -> bool { - self.0.load(Ordering::SeqCst) - } - } - - #[derive(Default)] - struct Child { - spawns: AtomicUsize, - exit: Mutex>, - terminated: AtomicBool, - args: Mutex>>, - inherited: Mutex>>, - } - - struct FakeChild(Arc); - - impl ChildHandle for FakeChild { - fn exited(&mut self) -> Option { - *self.0.exit.lock().unwrap() - } - - fn terminate(&mut self, _grace: Duration) { - self.0.terminated.store(true, Ordering::SeqCst); - *self.0.exit.lock().unwrap() = Some(143); - } - } - - struct FakeSpawner(Arc); - - impl ChildSpawner for FakeSpawner { - fn spawn( - &self, - _program: &Path, - args: &[String], - inherit: Option, - ) -> Result> { - self.0.inherited.lock().unwrap().push(inherit); - self.0.spawns.fetch_add(1, Ordering::SeqCst); - self.0.args.lock().unwrap().push(args.to_vec()); - Ok(Box::new(FakeChild(Arc::clone(&self.0)))) - } - } - - struct Credentials(BTreeMap>); - - impl CredentialSource for Credentials { - fn contents(&self, name: &str) -> Result> { - self.0.get(name).cloned().ok_or_else(|| anyhow!("unbound")) - } - - fn path(&self, name: &str) -> Result { - self.0 - .contains_key(name) - .then(|| PathBuf::from(format!("/run/credentials/unit/{name}"))) - .ok_or_else(|| anyhow!("unbound")) - } - } - - struct Run { - sink: Arc, - lease: Arc, - child: Arc, - stop: Arc, - outcome: Option>, - } - - fn start(fx: &Fixture, contract: LaunchContract, healthy: bool) -> Run { - let sink = Arc::new(Sink::default()); - let lease = Arc::new(FakeLease::default()); - let health = Arc::new(Health(AtomicBool::new(healthy))); - let child = Arc::new(Child::default()); - let stop = Arc::new(AtomicBool::new(false)); - let mut secrets = BTreeMap::new(); - secrets.insert("SYNAPSE_KEY".to_owned(), b"s3cret-value\n".to_vec()); - let ports = Ports { - credentials: Arc::new(Credentials(secrets)), - clock: Arc::new(FastClock::new(1_000_000)), - sink: sink.clone(), - lease: lease.clone(), - spawner: Box::new(FakeSpawner(child.clone())), - health: Box::new(move |_| health.clone()), - }; - let authority = fx.authority(); - let route = TcpListener::bind("127.0.0.1:0").unwrap(); - let tmp = fx.dir.path().join("tmp"); - let flag = stop.clone(); - let outcome = - thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); - Run { - sink, - lease, - child, - stop, - outcome: Some(outcome), - } - } - - impl Run { - fn finish(&mut self) -> Outcome { - self.outcome.take().unwrap().join().unwrap() - } - - fn stop(&mut self) -> Outcome { - self.stop.store(true, Ordering::SeqCst); - self.finish() - } - } - - fn wait_until(what: &str, mut condition: impl FnMut() -> bool) { - for _ in 0..4000 { - if condition() { - return; - } - thread::sleep(Duration::from_millis(5)); - } - panic!("timed out waiting for {what}"); - } - - #[test] - fn stateful_child_never_starts_before_its_lease() { - let fx = fixture(true); - let mut launch = start(&fx, fx.contract("listen {{child.listen}}"), true); - wait_until("two warming heartbeats", || { - launch.sink.states().iter().filter(|s| *s == "warming").count() >= 2 - }); - assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); - let warming = launch.sink.first("warming").unwrap(); - assert!(warming.write_lease_sha256.is_none()); - - // The lease binds the launcher's own Warming proof; only then it starts. - let lease = fx.lease(&warming.canonical_sha256().unwrap()); - *launch.lease.0.lock().unwrap() = Some(lease.clone()); - wait_until("child spawn", || { - launch.child.spawns.load(Ordering::SeqCst) == 1 - }); - wait_until("active", || launch.sink.first("active").is_some()); - let active = launch.sink.first("active").unwrap(); - assert_eq!( - active.write_lease_sha256.as_deref(), - Some(lease.canonical_sha256().unwrap().as_str()) - ); - assert_eq!(launch.stop(), Outcome::Stopped); - assert!(launch.child.terminated.load(Ordering::SeqCst)); - } - - #[test] - fn a_lease_for_another_activation_or_expected_is_not_accepted() { - let fx = fixture(true); - let foreign: [(&str, fn(&mut IdunnProcessWriteLeaseRecord)); 3] = [ - ("another runtime instance", |lease| { - lease.runtime_instance_id = "other".into() - }), - ("another Expected", |lease| { - lease.expected_projection_sha256 = digest('9') - }), - ("another warming proof", |lease| { - lease.warming_presence_sha256 = digest('9') - }), - ]; - for (name, edit) in foreign { - let mut contract = fx.contract("x"); - contract.lease_wait = Duration::from_secs(5); - let mut launch = start(&fx, contract, true); - // Installed at once; the launcher's own proofs are never digest('8'). - let mut lease = fx.lease(&digest('8')); - edit(&mut lease); - *launch.lease.0.lock().unwrap() = Some(lease); - let (child, sink) = (launch.child.clone(), launch.sink.clone()); - // Bounded: a lease deadline that never arrives fails here, not at the - // suite's outer timeout. - let outcome = within(Duration::from_secs(20), move || launch.finish()); - assert_eq!(outcome, Outcome::Failed("lease-timeout"), "{name}"); - assert_eq!(child.spawns.load(Ordering::SeqCst), 0, "{name}"); - assert_eq!(sink.states().last().unwrap(), "failed", "{name}"); - } - } - - #[test] - fn lease_loss_stops_the_child() { - let fx = fixture(true); - let mut launch = start(&fx, fx.contract("x"), true); - wait_until("warming", || launch.sink.first("warming").is_some()); - let warming = launch.sink.first("warming").unwrap(); - *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); - wait_until("active", || launch.sink.first("active").is_some()); - *launch.lease.0.lock().unwrap() = None; - let outcome = launch.finish(); - assert_eq!(outcome, Outcome::Failed("lease-lost")); - assert_ne!(outcome.exit_code(), 0); - assert!(launch.child.terminated.load(Ordering::SeqCst)); - assert_eq!(launch.sink.states().last().unwrap(), "failed"); - } - - #[test] - fn stateless_child_starts_without_a_lease() { - let fx = fixture(false); - let mut launch = start(&fx, fx.contract("x"), true); - wait_until("active", || launch.sink.first("active").is_some()); - assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 1); - assert_eq!(launch.sink.states().first().unwrap(), "warming"); - assert!( - launch - .sink - .first("active") - .unwrap() - .write_lease_sha256 - .is_none() - ); - launch.stop(); - } - - #[test] - fn child_exit_ends_the_launcher() { - for code in [0, 3] { - let fx = fixture(false); - let mut launch = start(&fx, fx.contract("x"), true); - wait_until("active", || launch.sink.first("active").is_some()); - *launch.child.exit.lock().unwrap() = Some(code); - let outcome = launch.finish(); - assert_eq!(outcome, Outcome::Failed("child-exited"), "exit {code}"); - assert_eq!(outcome.exit_code(), 1); - assert_eq!(launch.sink.states().last().unwrap(), "failed"); - } - } - - #[test] - fn closed_placeholder_set_renders_and_refuses() { - let fx = fixture(false); - let template = "listen={{child.listen}}\nhost={{child.listen_host}}\nport={{child.listen_port}}\nroot={{state_root}}\nkey={{credential:SYNAPSE_KEY}}\npath={{credential_path:SYNAPSE_KEY}}\n"; - let mut launch = start(&fx, fx.contract(template), true); - wait_until("spawn", || launch.child.spawns.load(Ordering::SeqCst) == 1); - let args = launch.child.args.lock().unwrap()[0].clone(); - assert_eq!(args[..2], ["--serve", "--config"]); - let config = PathBuf::from(&args[2]); - assert_eq!(config.parent().unwrap(), fx.dir.path().join("tmp")); - assert_eq!( - fs::metadata(&config).unwrap().permissions().mode() & 0o777, - 0o600 - ); - let rendered = fs::read_to_string(&config).unwrap(); - assert_eq!( - rendered, - "listen=127.0.0.1:18845\nhost=127.0.0.1\nport=18845\nroot=/var/lib/synapse\nkey=s3cret-value\npath=/run/credentials/unit/SYNAPSE_KEY\n" - ); - assert!( - !rendered.contains("18835"), - "the candidate bind never reaches the child" - ); - launch.stop(); - - // Any other placeholder refuses to start, and the refusal names no input. - let fx = fixture(false); - let mut launch = start(&fx, fx.contract("a={{canary_bind}}"), true); - assert_eq!(launch.finish(), Outcome::Failed("child-start-refused")); - assert_eq!(launch.child.spawns.load(Ordering::SeqCst), 0); - let credentials = Credentials(BTreeMap::new()); - let renderer = ConfigRenderer { - credentials: &credentials, - listen: member(), - state_root: None, - }; - for template in [ - "{{canary_bind}}", - "{{credential:CANARY-MISSING}}", - "{{credential:canary b}}", - "{{child.listen}} {{canary", - "{{state_root}}", - ] { - let error = format!("{:#}", renderer.render(template).unwrap_err()); - assert!(!error.to_lowercase().contains("canary"), "{error}"); - } - } - - fn shared(fx: &Fixture, health: Arc) -> Arc { - shared_on(fx, health, Arc::new(SystemClock)) - } - - fn shared_on(fx: &Fixture, health: Arc, clock: Arc) -> Arc { - Arc::new(Shared { - authority: Mutex::new(fx.authority()), - phase: Mutex::new(Phase::Active(None)), - lease: Mutex::new(None), - lease_required: false, - health, - probed: Mutex::new(None), - clock, - }) - } - - #[test] - fn route_answer_degrades_with_the_child() { - let fx = fixture(false); - let health = Arc::new(Health(AtomicBool::new(true))); - let shared = shared(&fx, health.clone()); - let state_of = |shared: &Shared| { - let request = CultNetMessage::SnapshotRequest { - message_id: "challenge-1".into(), - schema_ids: None, - record_keys: None, - }; - let CultNetMessage::SnapshotResponseRaw { documents, .. } = - shared.observe(request).unwrap() - else { - panic!("not a raw snapshot response") - }; - let record: GameCultRuntimePresenceHealthRecord = - rmp_serde::from_slice(&documents[0].payload).unwrap(); - assert_eq!(record.detail, "route-observation:challenge-1"); - record.state - }; - // Never probed: nothing says the child is healthy. - assert_eq!(state_of(&shared), "degraded"); - shared.refresh_health(); - assert_eq!(state_of(&shared), "active"); - // A challenge answers from the last probe and never runs one. - health.0.store(false, Ordering::SeqCst); - assert_eq!(state_of(&shared), "active"); - shared.refresh_health(); - assert_eq!(state_of(&shared), "degraded"); - } - - #[test] - fn route_challenge_is_answered_with_both_signatures_through_idunns_client() { - let fx = fixture(false); - let shared = shared(&fx, Arc::new(Health(AtomicBool::new(true)))); - shared.refresh_health(); - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let responder: Responder = Arc::new(move |request| shared.observe(request)); - let halt = Arc::new(AtomicBool::new(false)); - let acceptor = PresenceAcceptor::new(listener, responder); - let serving = { - let halt = halt.clone(); - thread::spawn(move || acceptor.serve(halt)) - }; - let request = CultNetMessage::SnapshotRequest { - message_id: "challenge-7".into(), - schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), - record_keys: Some(vec![fx.expected.target.clone()]), - }; - let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); - let response = request_tcp_snapshot(address, &payload).ok().unwrap(); - let message = - decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) - .unwrap(); - let answer = - exact_route_snapshot_response(message, "challenge-7", &fx.expected.target).unwrap(); - let record: GameCultRuntimePresenceHealthRecord = - rmp_serde::from_slice(&answer.canonical_presence).unwrap(); - let claim = authenticate_runtime_presence_claim( - &answer.canonical_presence, - &fx.verified, - RuntimePresenceAuthenticationContext { - trusted_received_at_unix_millis: record.observed_at_unix_millis, - maximum_age_millis: 10_000, - maximum_future_skew_millis: 10, - }, - ) - .unwrap(); - assert_eq!(claim.record().state, "active"); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap(); - } - - #[test] - fn contract_parsing_refuses_escapes_and_echoes_nothing() { - let base = |extra: &[&str]| -> Result { - let mut args: Vec = [ - "--child", - "bin/c", - "--template", - "t", - "--health-path", - "/health", - "--health-contract", - "h", - ] - .map(String::from) - .to_vec(); - args.extend(extra.iter().map(|value| value.to_string())); - LaunchContract::parse(PathBuf::from("/r"), args) - }; - assert!(base(&["--child-arg", "--flag-looking-value"]).is_ok()); - assert!(base(&["--child", "../escape"]).is_err()); - // The child serves members on Idunn's member bind: there is no - // listener to choose. - assert!(base(&["--listener", "loopback"]).is_err()); - assert!(base(&["--template", "/etc/passwd"]).is_err()); - let error = base(&["--canary-flag", "canary-value"]) - .unwrap_err() - .to_string(); - assert!(!error.contains("canary")); - } - - #[test] - fn real_lease_reader_holds_the_shared_lock_idunn_must_respect() { - let fx = fixture(true); - let path = fx.dir.path().join("lease.cc"); - let warming = digest('a'); - let lease = fx.lease(&warming); - write_record( - &path, - &lease.target, - IdunnProcessWriteLeaseRecord::TYPE, - cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, - lease.canonical_bytes().unwrap(), - ); - let reader: Arc = Arc::new(DriverLeaseReader::new(&lease.target, &path)); - let hold = LeaseHold::acquire(reader, &fx.expected, &fx.activation, &[warming]).unwrap(); - assert!(hold.still_current()); - let idunn = CultCacheWriteLeaseDriver::new(&lease.target, &path); - let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); - assert!(denial.to_string().contains("shared-lock consumer")); - drop(hold); - idunn.revoke_exact(Some(&lease)).unwrap(); - } - - #[test] - fn authority_binds_the_candidate_and_signs_strictly_increasing_sequences() { - let fx = fixture(false); - let mut authority = fx.authority(); - assert!(authority.require_candidate_bind("127.0.0.1:18835").is_ok()); - assert!(authority.require_candidate_bind("127.0.0.1:18836").is_err()); - let first = authority.sign("warming", "a", None, 5_000).unwrap(); - let second = authority.sign("warming", "b", None, 5_000).unwrap(); - assert_eq!(first.publisher_sequence, 5_000); - assert_eq!(second.publisher_sequence, 5_001); - assert_eq!( - first.bound_endpoint.as_deref(), - Some("tcp://127.0.0.1:18835") - ); - } - - #[test] - fn health_probe_is_healthy_only_on_a_200_from_the_child() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let member = listener.local_addr().unwrap(); - let probe = HttpHealthProbe { member, path: "/health".into() }; - let server = thread::spawn(move || { - for status in ["200 OK", "503 Service Unavailable"] { - let (mut stream, _) = listener.accept().unwrap(); - let mut line = String::new(); - io::BufRead::read_line(&mut io::BufReader::new(stream.try_clone().unwrap()), &mut line) - .unwrap(); - assert_eq!(line, "GET /health HTTP/1.1\r\n"); - write!(stream, "HTTP/1.1 {status}\r\nContent-Length: 2\r\n\r\nOK").unwrap(); - } - }); - assert!(probe.healthy()); - assert!(!probe.healthy()); - server.join().unwrap(); - assert!(!probe.healthy(), "no listener is not healthy"); - } - - #[test] - fn contract_parsing_reads_every_declared_field() { - let args: Vec = [ - "--child", "bin/c", "--child-arg", "-x", "--child-arg", "y", "--config-flag", "-c", - "--template", "t", "--health-path", "/h", - "--health-contract", "hc", "--state-root", "/s", "--lease-wait-secs", "11", - "--startup-secs", "22", "--heartbeat-secs", "33", - ] - .map(String::from) - .to_vec(); - let contract = LaunchContract::parse(PathBuf::from("/r"), args.clone()).unwrap(); - assert_eq!(contract.child_args, ["-x", "y"]); - assert_eq!(contract.config_flag.as_deref(), Some("-c")); - assert_eq!(contract.state_root, Some(PathBuf::from("/s"))); - assert_eq!( - (contract.lease_wait, contract.startup, contract.heartbeat), - (Duration::from_secs(11), Duration::from_secs(22), Duration::from_secs(33)) - ); - for zero in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { - let mut bad = args.clone(); - bad.extend([zero.to_owned(), "0".to_owned()]); - assert!(LaunchContract::parse(PathBuf::from("/r"), bad).is_err(), "{zero}"); - } - } - - #[test] - fn credential_names_cannot_escape_or_be_empty() { - struct Any; - impl CredentialSource for Any { - fn contents(&self, _: &str) -> Result> { - Ok(b"v".to_vec()) - } - fn path(&self, name: &str) -> Result { - Ok(PathBuf::from(name)) - } - } - let renderer = ConfigRenderer { credentials: &Any, listen: member(), state_root: None }; - for ok in ["A", "a-b_9", "Z0"] { - assert_eq!(renderer.render(&format!("{{{{credential:{ok}}}}}")).unwrap(), "v"); - } - for bad in ["", "../x", "a/b", "a b", "a.b", "a:b", "é"] { - assert!(renderer.render(&format!("{{{{credential:{bad}}}}}")).is_err(), "{bad}"); - assert!(renderer.render(&format!("{{{{credential_path:{bad}}}}}")).is_err(), "{bad}"); - } - assert_eq!(renderer.render("a{{child.listen}}b{{child.listen}}c").unwrap(), "a127.0.0.1:18845b127.0.0.1:18845c"); - } - - #[test] - fn credentials_resolve_only_as_files_under_the_credentials_directory() { - let dir = tempfile::tempdir().unwrap(); - fs::write(dir.path().join("cred"), b"abc\n").unwrap(); - let credentials = CredentialsDirectory::new(dir.path().to_owned()); - assert_eq!(credentials.path("cred").unwrap(), dir.path().join("cred")); - assert_eq!(credentials.contents("cred").unwrap(), b"abc\n"); - assert!(credentials.path("unbound").is_err()); - for bad in ["", "..", "../cred", "a/b", ".", "a.b"] { - assert!(credentials.path(bad).is_err(), "{bad}"); - assert!(credentials.contents(bad).is_err(), "{bad}"); - } - assert!(CredentialsDirectory(None).path("cred").is_err()); - } - - #[test] - fn credential_placeholders_never_read_the_process_environment() { - let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("elsewhere"); - fs::write(&file, b"canary-file\n").unwrap(); - // SAFETY: variables no other test reads. - unsafe { - std::env::set_var("GAMECULT_IDUNN_CANDIDATE_BIND", "127.0.0.1:18835"); - std::env::set_var("IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", &file); - } - let credentials = CredentialsDirectory::new(dir.path().to_owned()); - let renderer = ConfigRenderer { credentials: &credentials, listen: member(), state_root: None }; - for name in [ - "GAMECULT_IDUNN_CANDIDATE_BIND", - "IDUNN_LAUNCH_TEST_ENV_CREDENTIAL", - "../elsewhere", - "a/b", - "..", - ] { - for kind in ["credential", "credential_path"] { - let error = renderer.render(&format!("x={{{{{kind}:{name}}}}}")); - assert!(error.is_err(), "{kind}:{name}"); - assert!(!format!("{:#}", error.unwrap_err()).contains("18835")); - } - } - // The same name resolves when it is a credential file, never otherwise. - fs::write(dir.path().join("GAMECULT_IDUNN_CANDIDATE_BIND"), b"file-value\n").unwrap(); - assert_eq!( - renderer.render("{{credential:GAMECULT_IDUNN_CANDIDATE_BIND}}").unwrap(), - "file-value" - ); - } - - #[test] - fn odin_sink_takes_its_endpoint_only_from_the_odin_dependency() { - let fx = fixture(false); - let dependency = |capability: &str, endpoint: &str| cultnet_rs::IdunnExpectedDependency { - kind: "shared-infrastructure".into(), - capability: capability.into(), - schema: "s".into(), - compatibility: "v1".into(), - minimum_capacity: 1, - startup: "before-promotion".into(), - provider_id: None, - provider_authority: None, - provider_expected_projection_sha256: None, - provider_endpoint: Some(endpoint.into()), - }; - let mut expected = fx.expected.clone(); - assert!(OdinRudpSink::from_expected(&expected).is_err()); - expected.dependencies = vec![dependency("other.thing", "rudp://127.0.0.1:1")]; - assert!(OdinRudpSink::from_expected(&expected).is_err()); - expected.dependencies = vec![ - dependency("other.thing", "rudp://127.0.0.1:1"), - dependency(ODIN_CAPABILITY, "rudp://127.0.0.1:4100"), - ]; - assert_eq!( - OdinRudpSink::from_expected(&expected).unwrap().endpoint, - "127.0.0.1:4100".parse().unwrap() - ); - expected.dependencies = vec![dependency(ODIN_CAPABILITY, "rudp://not-an-address")]; - assert!(OdinRudpSink::from_expected(&expected).is_err()); - } - - - #[test] - fn a_probe_result_is_evidence_only_while_it_is_recent() { - let fx = fixture(false); - let clock = Arc::new(FastClock::new(1_000_000)); - let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); - shared.refresh_health(); - let stale = HEALTH_STALE.as_millis() as u64; - clock.monotonic.fetch_add(stale, Ordering::SeqCst); - assert_eq!(shared.serving_state(), "active", "a result exactly at the bound is still recent"); - clock.monotonic.fetch_add(1, Ordering::SeqCst); - assert_eq!(shared.serving_state(), "degraded", "a result past the bound is not evidence"); - shared.refresh_health(); - assert_eq!(shared.serving_state(), "active"); - } - - /// A lease reader that grants, at the first read once `after` Warming - /// records exist, a lease bound to the Warming record `back` places before - /// the newest. The launcher reads it from its own thread between publishes, - /// so the record it names is at a known place in the retained window. - struct WindowLease { - sink: Arc, - template: IdunnProcessWriteLeaseRecord, - after: usize, - back: usize, - granted: Mutex>, - } - - impl LeaseReader for WindowLease { - fn read(&self) -> Result> { - let mut granted = self.granted.lock().unwrap(); - if granted.is_none() { - let records = self.sink.0.lock().unwrap(); - let warming: Vec<_> = records.iter().filter(|r| r.state == "warming").collect(); - if warming.len() >= self.after { - let mut lease = self.template.clone(); - lease.warming_presence_sha256 = - warming[warming.len() - self.back].canonical_sha256().unwrap(); - *granted = Some(lease); - } - } - Ok(granted.clone()) - } - - fn hold_shared(&self) -> Result> { - Ok(Box::new(FakeLock)) - } - } - - /// Starts a stateful unit waiting on a `WindowLease` and reports whether it - /// accepted the lease (spawned the child) or retried well past the grant. - fn warming_window_accepts(back: usize) -> bool { - let fx = fixture(true); - let sink = Arc::new(Sink::default()); - let child = Arc::new(Child::default()); - let lease = Arc::new(WindowLease { - sink: sink.clone(), - template: fx.lease(""), - after: MAX_RECENT_WARMING_PROOFS + 6, - back, - granted: Mutex::new(None), - }); - let ports = Ports { - credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(FastClock::new(1_000_000)), - sink: sink.clone(), - lease: lease.clone(), - spawner: Box::new(FakeSpawner(child.clone())), - health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), - }; - let mut contract = fx.contract("x"); - contract.heartbeat = POLL; - let authority = fx.authority(); - let route = TcpListener::bind("127.0.0.1:0").unwrap(); - let tmp = fx.dir.path().join("tmp"); - let stop = Arc::new(AtomicBool::new(false)); - let flag = stop.clone(); - let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); - wait_until("the lease is granted", || lease.granted.lock().unwrap().is_some()); - let warming = || sink.states().iter().filter(|s| *s == "warming").count(); - let granted_at = warming(); - let mut spawned = false; - for _ in 0..4000 { - spawned = child.spawns.load(Ordering::SeqCst) == 1; - if spawned || warming() >= granted_at + 10 { - break; - } - thread::sleep(Duration::from_millis(5)); - } - stop.store(true, Ordering::SeqCst); - // A mutant may wedge the unit; only a unit that took the lease is joined. - if spawned { - assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); - } - spawned - } - - #[test] - fn the_warming_proof_window_keeps_exactly_the_cap() { - assert!( - warming_window_accepts(MAX_RECENT_WARMING_PROOFS), - "the oldest of the retained Warming proofs is accepted" - ); - assert!( - !warming_window_accepts(MAX_RECENT_WARMING_PROOFS + 1), - "a Warming proof past the cap is no longer accepted" - ); - } - - /// The child is healthy, then the wall clock steps by `step_millis`, then - /// the child wedges. A challenge must keep reading the truth throughout. - fn a_wall_clock_step_never_decides_freshness(step_millis: i64) { - struct Toggle { - healthy: AtomicBool, - probes: AtomicUsize, - } - impl HealthProbe for Toggle { - fn healthy(&self) -> bool { - self.probes.fetch_add(1, Ordering::SeqCst); - self.healthy.load(Ordering::SeqCst) - } - } - let fx = fixture(false); - let sink = Arc::new(Sink::default()); - let clock = Arc::new(FastClock::new(100_000_000)); - let probe = Arc::new(Toggle { healthy: AtomicBool::new(true), probes: AtomicUsize::new(0) }); - let given = probe.clone(); - let ports = Ports { - credentials: Arc::new(Credentials(BTreeMap::new())), - clock: clock.clone(), - sink: sink.clone(), - lease: Arc::new(FakeLease::default()), - spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), - health: Box::new(move |_| given.clone()), - }; - let contract = fx.contract("x"); - let authority = fx.authority(); - let route = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = route.local_addr().unwrap(); - let tmp = fx.dir.path().join("tmp"); - let stop = Arc::new(AtomicBool::new(false)); - let flag = stop.clone(); - let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); - wait_until("active", || sink.first("active").is_some()); - assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); - - clock.step_wall(step_millis); - for _ in 0..10 { - assert_eq!( - challenge_state(address, &fx).as_deref(), - Some("active"), - "a healthy child reads active across a wall-clock step of {step_millis} ms" - ); - thread::sleep(Duration::from_millis(20)); - } - - probe.healthy.store(false, Ordering::SeqCst); - let probed = probe.probes.load(Ordering::SeqCst); - let published = sink.0.lock().unwrap().len(); - let started = Instant::now(); - let mut state = None; - while started.elapsed() < Duration::from_secs(5) { - state = challenge_state(address, &fx); - if state.as_deref() == Some("degraded") { - break; - } - thread::sleep(Duration::from_millis(20)); - } - assert_eq!( - state.as_deref(), - Some("degraded"), - "a wedged child reads degraded within the freshness window after a wall-clock step of {step_millis} ms" - ); - assert!(probe.probes.load(Ordering::SeqCst) > probed, "probing continued"); - wait_until("a heartbeat after the step", || sink.0.lock().unwrap().len() > published); - stop.store(true, Ordering::SeqCst); - assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); - } - - #[test] - fn a_backward_wall_clock_step_does_not_keep_a_wedged_child_active() { - a_wall_clock_step_never_decides_freshness(-600_000); - } - - #[test] - fn a_forward_wall_clock_step_does_not_make_a_healthy_child_degraded() { - a_wall_clock_step_never_decides_freshness(600_000); - } - - #[test] - fn the_cached_result_ages_on_the_monotonic_clock_alone() { - let fx = fixture(false); - let clock = Arc::new(FastClock::new(100_000_000)); - let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); - shared.refresh_health(); - for step in [-600_000, 1_200_000] { - clock.step_wall(step); - assert_eq!(shared.serving_state(), "active", "wall step {step}"); - } - clock.monotonic.fetch_add(HEALTH_STALE.as_millis() as u64 + 1, Ordering::SeqCst); - assert_eq!(shared.serving_state(), "degraded", "monotonic age decides, whatever the wall says"); - } - - /// A monotonic clock whose read, on a thread that marks itself, returns the - /// current value and then waits to be released: the presence thread - /// descheduled right after it reads the time. - struct HeldClock { - monotonic: AtomicU64, - held: Mutex>>, - release: Mutex>>, - } - - thread_local! { - static HOLD_THIS_READ: std::cell::Cell = const { std::cell::Cell::new(false) }; - } - - impl Clock for HeldClock { - fn now_millis(&self) -> u64 { - 0 - } - - fn monotonic_millis(&self) -> u64 { - let value = self.monotonic.load(Ordering::SeqCst); - if HOLD_THIS_READ.with(std::cell::Cell::get) { - self.held.lock().unwrap().take().unwrap().send(()).unwrap(); - self.release.lock().unwrap().take().unwrap().recv().unwrap(); - } - value - } - - fn sleep(&self, _: Duration) {} - } - - #[test] - fn a_presence_read_descheduled_across_a_probe_still_sees_a_healthy_child() { - let fx = fixture(false); - let (held_tx, held_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); - let clock = Arc::new(HeldClock { - monotonic: AtomicU64::new(1_000), - held: Mutex::new(Some(held_tx)), - release: Mutex::new(Some(release_rx)), - }); - let shared = shared_on(&fx, Arc::new(Health(AtomicBool::new(true))), clock.clone()); - shared.refresh_health(); - let reader = { - let shared = shared.clone(); - thread::spawn(move || { - HOLD_THIS_READ.with(|hold| hold.set(true)); - shared.serving_state() - }) - }; - // The reader is descheduled in its clock read. The lifecycle probes a - // millisecond later and stores a result stamped after the reader's time. - held_rx.recv().unwrap(); - clock.monotonic.store(1_001, Ordering::SeqCst); - assert!(shared.refresh_health()); - release_tx.send(()).unwrap(); - assert_eq!(reader.join().unwrap(), "active"); - } - - #[test] - fn the_declared_durations_are_bounded_at_parse() { - let base: Vec = [ - "--child", "bin/c", "--template", "t", "--health-path", "/h", "--health-contract", "hc", - ] - .map(String::from) - .to_vec(); - for flag in ["--lease-wait-secs", "--startup-secs", "--heartbeat-secs"] { - for (value, accepted) in [ - ("86400".to_owned(), true), - ("86401".to_owned(), false), - (u64::MAX.to_string(), false), - ] { - let mut args = base.clone(); - args.extend([flag.to_owned(), value.clone()]); - let parsed = LaunchContract::parse(PathBuf::from("/r"), args); - assert_eq!(parsed.is_ok(), accepted, "{flag} {value}"); - if let Err(error) = parsed { - assert!(!format!("{error:#}").contains(&value), "the refusal echoes its input"); - } - } - } - } - - #[test] - fn a_span_too_long_for_the_clock_is_a_deadline_that_never_arrives() { - assert_eq!(after(10, Duration::from_millis(5)), 15); - assert_eq!(after(u64::MAX - 1, Duration::from_secs(1)), u64::MAX); - assert_eq!(after(10, Duration::MAX), u64::MAX); - } - - /// Runs `work` and returns its result, failing the test when it has not - /// finished within `limit`, so a mutant that makes it wait forever fails - /// instead of hanging the suite. - fn within(limit: Duration, work: impl FnOnce() -> T + Send + 'static) -> T { - join_within(limit, thread::spawn(work)) - } - - fn join_within(limit: Duration, handle: thread::JoinHandle) -> T { - let started = Instant::now(); - while !handle.is_finished() { - assert!(started.elapsed() < limit, "still running after {limit:?}"); - thread::sleep(Duration::from_millis(10)); - } - handle.join().unwrap() - } - - /// The state the presence endpoint signs in answer to a challenge, or None - /// when it is not answered promptly. - fn challenge_state(address: SocketAddr, fx: &Fixture) -> Option { - let request = CultNetMessage::SnapshotRequest { - message_id: "wedge".into(), - schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), - record_keys: Some(vec![fx.expected.target.clone()]), - }; - let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); - let started = Instant::now(); - let answer = request_tcp_snapshot(address, &payload).ok()?; - if started.elapsed() > Duration::from_millis(750) { - return None; - } - let CultNetMessage::SnapshotResponseRaw { documents, .. } = - decode_cultnet_message_from_slice(&answer, CultNetWireContract::CultNetSchemaV0).ok()? - else { - return None; - }; - let record: GameCultRuntimePresenceHealthRecord = - rmp_serde::from_slice(&documents.first()?.payload).ok()?; - Some(record.state) - } - - #[derive(Clone, Copy)] - enum Wedge { - /// Accepts and never answers. - Silent, - /// Never accepts; the kernel backlog fills. - FullBacklog, - /// Accepts and answers 200 after 1.1 s, past the probe's deadline. - Slow, - } - - type WedgeSwitch = Arc>>; - - /// A child that answers 200 until wedged. Returns the member address, the - /// switch, and the thread to join after `halt`. - fn serving_child(halt: Arc) -> (SocketAddr, WedgeSwitch, thread::JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - listener.set_nonblocking(true).unwrap(); - let address = listener.local_addr().unwrap(); - let wedge: WedgeSwitch = Arc::default(); - let mode = wedge.clone(); - let serving = thread::spawn(move || { - while !halt.load(Ordering::SeqCst) { - let current = *mode.lock().unwrap(); - if matches!(current, Some(Wedge::FullBacklog)) { - thread::sleep(Duration::from_millis(5)); - continue; - } - let Ok((mut stream, _)) = listener.accept() else { - thread::sleep(Duration::from_millis(5)); - continue; - }; - stream.set_nonblocking(false).unwrap(); - match current { - None => { - let mut request = [0_u8; 256]; - let _ = stream.set_read_timeout(Some(Duration::from_millis(200))); - let _ = stream.read(&mut request); - let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); - } - Some(Wedge::Slow) => { - thread::spawn(move || { - thread::sleep(Duration::from_millis(1100)); - let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"); - }); - } - _ => { - thread::spawn(move || { - thread::sleep(Duration::from_secs(8)); - drop(stream); - }); - } - } - } - }); - (address, wedge, serving) - } - - /// Whatever wedges the child, every challenge is still answered promptly, - /// and the answer turns to a signed degraded within a few probes. - fn a_wedged_child_is_answered_degraded(wedge: Wedge) { - let fx = fixture(false); - let sink = Arc::new(Sink::default()); - let halt = Arc::new(AtomicBool::new(false)); - let (member, mode, child) = serving_child(halt.clone()); - let ports = Ports { - credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(SystemClock), - sink: sink.clone(), - lease: Arc::new(FakeLease::default()), - spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), - health: Box::new(|member| Arc::new(HttpHealthProbe { member, path: "/health".into() })), - }; - let contract = fx.contract("x"); - let authority = fx.authority(); - let route = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = route.local_addr().unwrap(); - let tmp = fx.dir.path().join("tmp"); - let stop = Arc::new(AtomicBool::new(false)); - let flag = stop.clone(); - let unit = thread::spawn(move || run(&contract, authority, route, member, &tmp, ports, &flag)); - wait_until("active", || sink.first("active").is_some()); - assert_eq!(challenge_state(address, &fx).as_deref(), Some("active")); - - *mode.lock().unwrap() = Some(wedge); - // A backlog fills only once clients have connected without being - // accepted. - let mut queued = Vec::new(); - if matches!(wedge, Wedge::FullBacklog) { - for _ in 0..4096 { - match TcpStream::connect_timeout(&member, Duration::from_millis(20)) { - Ok(stream) => queued.push(stream), - Err(_) => break, - } - } - } - let started = Instant::now(); - let mut degraded = false; - while started.elapsed() < Duration::from_secs(6) { - let state = challenge_state(address, &fx); - assert!(state.is_some(), "a challenge went unanswered after {:?}", started.elapsed()); - if state.as_deref() == Some("degraded") { - degraded = true; - break; - } - thread::sleep(Duration::from_millis(100)); - } - assert!(degraded, "the wedged child was never reported degraded"); - stop.store(true, Ordering::SeqCst); - assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); - halt.store(true, Ordering::SeqCst); - join_within(Duration::from_secs(10), child); - drop(queued); - } - - #[test] - fn a_silent_child_is_answered_degraded_never_silent() { - a_wedged_child_is_answered_degraded(Wedge::Silent); - } - - #[test] - fn a_child_with_a_full_backlog_is_answered_degraded_never_silent() { - a_wedged_child_is_answered_degraded(Wedge::FullBacklog); - } - - #[test] - fn a_child_answering_after_the_probe_deadline_is_answered_degraded_never_silent() { - a_wedged_child_is_answered_degraded(Wedge::Slow); - } - - /// The virtual time (milliseconds) of each record that has the given - /// detail. - fn times_of(sink: &Sink, detail: &str) -> Vec { - sink.0 - .lock() - .unwrap() - .iter() - .filter(|record| record.detail == detail) - .map(|record| record.observed_at_unix_millis) - .collect() - } - - #[test] - fn a_child_that_never_becomes_healthy_fails_the_unit_at_the_startup_timeout() { - let fx = fixture(false); - let mut contract = fx.contract("x"); - contract.startup = Duration::from_secs(5); - let mut launch = start(&fx, contract, false); - let sink = launch.sink.clone(); - let child = launch.child.clone(); - let outcome = within(Duration::from_secs(20), move || launch.finish()); - assert_eq!(outcome, Outcome::Failed("startup-timeout")); - assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); - let started = times_of(&sink, "launcher-started")[0]; - let failed = times_of(&sink, "startup-timeout")[0]; - // Virtual time: the deadline, found within one poll of it. - let waited = failed - started; - assert!( - (5_000..=5_000 + POLL.as_millis() as u64).contains(&waited), - "failed after {waited} ms of virtual time" - ); - } - - #[test] - fn heartbeats_follow_the_contracts_cadence_while_waiting_for_the_lease_and_while_active() { - let fx = fixture(true); - let mut contract = fx.contract("x"); - contract.heartbeat = Duration::from_secs(1); - let mut launch = start(&fx, contract, true); - let cadence = 1_000..=1_000 + POLL.as_millis() as u64; - wait_until("lease-wait heartbeats", || { - times_of(&launch.sink, "write-lease-pending").len() >= 4 - }); - let warming = launch.sink.first("warming").unwrap(); - let mut beats = vec![warming.observed_at_unix_millis]; - beats.extend(times_of(&launch.sink, "write-lease-pending").into_iter().take(4)); - for pair in beats.windows(2) { - assert!(cadence.contains(&(pair[1] - pair[0])), "lease-wait beats {beats:?}"); - } - *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); - wait_until("active heartbeats", || times_of(&launch.sink, "heartbeat").len() >= 4); - let mut beats = vec![times_of(&launch.sink, "child-healthy")[0]]; - beats.extend(times_of(&launch.sink, "heartbeat").into_iter().take(4)); - for pair in beats.windows(2) { - assert!(cadence.contains(&(pair[1] - pair[0])), "active beats {beats:?}"); - } - assert_eq!(launch.stop().exit_code(), 0); - } - - - /// Healthy, and notes the virtual time of every probe. - struct RecordingProbe { - clock: Arc, - at: Mutex>, - } - - impl HealthProbe for RecordingProbe { - fn healthy(&self) -> bool { - self.at.lock().unwrap().push(self.clock.monotonic_millis()); - true - } - } - - #[test] - fn an_active_child_is_probed_at_the_refresh_cadence() { - let fx = fixture(false); - let sink = Arc::new(Sink::default()); - let clock = Arc::new(FastClock::new(1_000_000)); - let probe = Arc::new(RecordingProbe { clock: clock.clone(), at: Mutex::new(Vec::new()) }); - let seen = probe.clone(); - let ports = Ports { - credentials: Arc::new(Credentials(BTreeMap::new())), - clock, - sink: sink.clone(), - lease: Arc::new(FakeLease::default()), - spawner: Box::new(FakeSpawner(Arc::new(Child::default()))), - health: Box::new(move |_| probe.clone()), - }; - let contract = fx.contract("x"); - let authority = fx.authority(); - let route = TcpListener::bind("127.0.0.1:0").unwrap(); - let tmp = fx.dir.path().join("tmp"); - let stop = Arc::new(AtomicBool::new(false)); - let flag = stop.clone(); - let unit = thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); - wait_until("five probes", || seen.at.lock().unwrap().len() >= 5); - stop.store(true, Ordering::SeqCst); - assert_eq!(join_within(Duration::from_secs(10), unit), Outcome::Stopped); - let at = seen.at.lock().unwrap().clone(); - // The first probe ends startup; every later one follows the cadence. - let cadence = HEALTH_REFRESH.as_millis() as u64..=HEALTH_REFRESH.as_millis() as u64 + POLL.as_millis() as u64; - for pair in at.windows(2) { - assert!(cadence.contains(&(pair[1] - pair[0])), "probes at {at:?}"); - } - } - - #[test] - fn process_spawner_escalates_to_kill_when_the_child_ignores_sigterm() { - let mut stubborn = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "trap '' TERM; while :; do :; done".into()], None) - .unwrap(); - thread::sleep(Duration::from_millis(200)); - let started = Instant::now(); - stubborn.terminate(Duration::from_millis(400)); - assert!(started.elapsed() >= Duration::from_millis(400), "SIGTERM gets its grace period"); - assert!(stubborn.exited().is_some(), "then SIGKILL ends it"); - } - - #[test] - fn process_spawner_reports_exit_codes_and_terminates_a_running_child() { - let mut gone = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "exit 3".into()], None) - .unwrap(); - wait_until("child exit", || gone.exited().is_some()); - assert_eq!(gone.exited(), Some(3)); - let mut running = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "sleep 30".into()], None) - .unwrap(); - assert_eq!(running.exited(), None); - running.terminate(Duration::from_secs(5)); - assert!(running.exited().is_some()); - } - - // ------------------------------------------------------------ presence acceptor - - type Accepted = io::Result<(TcpStream, SocketAddr)>; - - fn serve_presence_accepting( - fx: &Fixture, - responder: Responder, - accept: impl FnMut(&TcpListener) -> Accepted + Send + 'static, - ) -> (SocketAddr, Arc, thread::JoinHandle<()>) { - let _ = fx; - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let acceptor = PresenceAcceptor::new(listener, responder); - let halt = Arc::new(AtomicBool::new(false)); - let flag = halt.clone(); - (address, halt, thread::spawn(move || acceptor.serve_accepting(flag, accept))) - } - - fn observing(fx: &Fixture) -> Responder { - let shared = shared(fx, Arc::new(Health(AtomicBool::new(true)))); - shared.refresh_health(); - Arc::new(move |request| shared.observe(request)) - } - - fn serve_presence(fx: &Fixture) -> (SocketAddr, Arc, thread::JoinHandle<()>) { - serve_presence_accepting(fx, observing(fx), |listener| listener.accept()) - } - - /// True when the peer closed (EOF or reset) within `within`, having sent - /// nothing. - fn closed_within(stream: &mut TcpStream, within: Duration) -> bool { - stream.set_read_timeout(Some(within)).unwrap(); - let mut byte = [0_u8; 64]; - match stream.read(&mut byte) { - Ok(0) => true, - Ok(_) => false, - Err(error) => !matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut), - } - } - - fn challenge_answered(address: SocketAddr, fx: &Fixture) -> bool { - let request = CultNetMessage::SnapshotRequest { - message_id: "bounded".into(), - schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), - record_keys: Some(vec![fx.expected.target.clone()]), - }; - let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); - request_tcp_snapshot(address, &payload).is_ok() - } - - #[test] - fn only_challenge_frames_are_answered_and_anything_else_is_closed_unanswered() { - let fx = fixture(false); - let reached = Arc::new(AtomicUsize::new(0)); - let responder: Responder = { - let (inner, reached) = (observing(&fx), reached.clone()); - Arc::new(move |request| { - reached.fetch_add(1, Ordering::SeqCst); - inner(request) - }) - }; - let (address, halt, serving) = - serve_presence_accepting(&fx, responder, |listener| listener.accept()); - // Member traffic is not presence: an HTTP request gets no byte back. - let mut member = TcpStream::connect(address).unwrap(); - member - .write_all(b"POST /_matrix/x HTTP/1.1\r\nContent-Length: 2\r\n\r\n\x00\x01") - .unwrap(); - assert!(closed_within(&mut member, Duration::from_secs(3)), "member bytes were answered"); - // A leading 0x00 with a frame length outside its bound never reaches - // the responder. - let mut malformed = TcpStream::connect(address).unwrap(); - malformed.write_all(&[0x00, 0xff, 0xff, 0xff]).unwrap(); - assert!(closed_within(&mut malformed, Duration::from_secs(3)), "a malformed frame was answered"); - assert_eq!(reached.load(Ordering::SeqCst), 0); - // A real challenge is answered, and is the only thing that reaches it. - assert!(challenge_answered(address, &fx)); - assert_eq!(reached.load(Ordering::SeqCst), 1); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap(); - } - - #[test] - fn a_challenge_up_to_the_frame_bound_is_answered_and_one_past_it_is_not() { - let fx = fixture(false); - let (address, halt, serving) = serve_presence(&fx); - // A request far larger than a few KiB, well inside the 1 MiB bound. - let request = CultNetMessage::SnapshotRequest { - message_id: "large".into(), - schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), - record_keys: Some((0..300).map(|index| format!("{index:0>40}")).collect()), - }; - let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0).unwrap(); - assert!(payload.len() > 10_000 && payload.len() < CHALLENGE_MAX_BYTES); - assert!(request_tcp_snapshot(address, &payload).is_ok()); - // One byte past the bound is closed unanswered. - let past = (CHALLENGE_MAX_BYTES as u32 + 1).to_be_bytes(); - assert_eq!(past[0], CHALLENGE_FIRST_BYTE); - let mut oversized = TcpStream::connect(address).unwrap(); - oversized.write_all(&past).unwrap(); - assert!(closed_within(&mut oversized, Duration::from_secs(3)), "an oversized frame was read"); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap(); - } - - #[test] - fn an_idle_peer_holds_the_acceptor_for_the_presence_deadline_and_no_longer() { - let fx = fixture(false); - let (address, halt, serving) = serve_presence(&fx); - let mut idle = TcpStream::connect(address).unwrap(); - let started = Instant::now(); - // Idunn waits 3 s for a challenge answer; one stalled peer ahead of it - // must not cost it that. - assert!(challenge_answered(address, &fx)); - let waited = started.elapsed(); - assert!(waited >= Duration::from_millis(800), "the idle peer was cut early: {waited:?}"); - assert!(waited < Duration::from_millis(2200), "the idle peer held the acceptor: {waited:?}"); - assert!(closed_within(&mut idle, Duration::from_secs(1)), "the idle peer stayed connected"); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap(); - } - - #[test] - fn a_challenge_trickled_a_byte_at_a_time_is_cut_off_at_the_presence_deadline() { - let fx = fixture(false); - let (address, halt, serving) = serve_presence(&fx); - let mut slow = TcpStream::connect(address).unwrap(); - slow.write_all(&[0x00, 0x00, 0x01, 0x00]).unwrap(); - let started = Instant::now(); - let mut closed = false; - while started.elapsed() < Duration::from_secs(4) { - if closed_within(&mut slow, Duration::from_millis(150)) || slow.write_all(&[0x01]).is_err() { - closed = true; - break; - } - } - assert!(closed, "the trickle was never cut off"); - let elapsed = started.elapsed(); - assert!(elapsed >= Duration::from_millis(800), "cut before its deadline: {elapsed:?}"); - assert!(elapsed < Duration::from_millis(2200), "cut after its deadline: {elapsed:?}"); - assert!(challenge_answered(address, &fx)); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap(); - } - - #[test] - fn accept_errors_never_end_the_presence_acceptor() { - let fx = fixture(false); - // Descriptor exhaustion: the first accepts fail, then descriptors free. - let failures = Arc::new(AtomicUsize::new(0)); - let (address, halt, serving) = { - let failures = failures.clone(); - serve_presence_accepting(&fx, observing(&fx), move |listener| { - if failures.fetch_add(1, Ordering::SeqCst) < 3 { - Err(io::Error::from_raw_os_error(libc::EMFILE)) - } else { - listener.accept() - } - }) - }; - assert!(challenge_answered(address, &fx), "the acceptor did not recover"); - assert!(failures.load(Ordering::SeqCst) >= 4); - assert!(!serving.is_finished()); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap(); - } - - /// A listener whose accept always fails: a datagram socket's descriptor. - fn failing_listener() -> TcpListener { - use std::os::fd::{FromRawFd, IntoRawFd}; - let datagram = UdpSocket::bind("127.0.0.1:0").unwrap(); - // SAFETY: the descriptor is owned by the new listener alone. - unsafe { TcpListener::from_raw_fd(datagram.into_raw_fd()) } - } - - #[test] - fn an_acceptor_whose_every_accept_fails_runs_until_halted() { - let fx = fixture(false); - let acceptor = PresenceAcceptor::new(failing_listener(), observing(&fx)); - let halt = Arc::new(AtomicBool::new(false)); - let flag = halt.clone(); - let serving = thread::spawn(move || acceptor.serve(flag)); - thread::sleep(Duration::from_millis(400)); - assert!(!serving.is_finished(), "the acceptor ended on an accept error"); - halt.store(true, Ordering::SeqCst); - serving.join().unwrap(); - } - - /// A clock that kills the presence thread when it reads the time: the - /// stand-in for a presence acceptor that dies while the child is fine. - /// Answering a challenge reads the cached result's age, which reads the - /// monotonic clock. - struct PresenceTripwire(FastClock); - - impl Clock for PresenceTripwire { - fn now_millis(&self) -> u64 { - self.0.now_millis() - } - - fn monotonic_millis(&self) -> u64 { - assert_ne!(thread::current().name(), Some("presence-acceptor"), "presence thread killed"); - self.0.monotonic_millis() - } - - fn sleep(&self, duration: Duration) { - self.0.sleep(duration); - } - } - - #[test] - fn a_dead_presence_acceptor_fails_the_unit() { - let fx = fixture(false); - let sink = Arc::new(Sink::default()); - let child = Arc::new(Child::default()); - let ports = Ports { - credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(PresenceTripwire(FastClock::new(1_000_000))), - sink: sink.clone(), - lease: Arc::new(FakeLease::default()), - spawner: Box::new(FakeSpawner(child.clone())), - health: Box::new(|_| Arc::new(Health(AtomicBool::new(true)))), - }; - let contract = fx.contract("x"); - let authority = fx.authority(); - let route = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = route.local_addr().unwrap(); - let tmp = fx.dir.path().join("tmp"); - let stop = Arc::new(AtomicBool::new(false)); - let flag = stop.clone(); - let handle = - thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); - wait_until("active", || sink.first("active").is_some()); - // The first challenge reaches the tripwire on the presence thread. - assert!(!challenge_answered(address, &fx)); - assert_eq!(join_within(Duration::from_secs(20), handle), Outcome::Failed("presence-acceptor")); - assert_eq!(sink.first("failed").unwrap().detail, "presence-acceptor"); - assert!(child.terminated.load(Ordering::SeqCst), "the child outlived the unit"); - } - - #[test] - fn a_live_acceptor_keeps_presence_active() { - let fx = fixture(false); - let mut launch = start(&fx, fx.contract("x"), true); - wait_until("two active heartbeats", || { - launch.sink.states().iter().filter(|state| *state == "active").count() >= 2 - }); - assert!(launch.sink.first("degraded").is_none()); - launch.stop(); - } - - /// A store with a lease for `fx`, and Idunn's own driver over it. - fn lease_store( - fx: &Fixture, - name: &str, - ) -> (DriverLeaseReader, CultCacheWriteLeaseDriver, IdunnProcessWriteLeaseRecord) { - let dir = fx.dir.path().join(name); - fs::create_dir(&dir).unwrap(); - let path = dir.join("lease.cc"); - let lease = fx.lease(&digest('a')); - write_record( - &path, - &lease.target, - IdunnProcessWriteLeaseRecord::TYPE, - cultnet_rs::IDUNN_PROCESS_WRITE_LEASE_SCHEMA, - lease.canonical_bytes().unwrap(), - ); - fs::write(dir.join("lease.cc.lock"), b"").unwrap(); - ( - DriverLeaseReader::new(&lease.target, &path), - CultCacheWriteLeaseDriver::new(&lease.target, &path), - lease, - ) - } - - #[test] - fn a_child_that_inherits_the_lease_lock_holds_it_after_the_launcher_is_gone() { - let fx = fixture(true); - let (reader, idunn, lease) = lease_store(&fx, "inherited"); - let guard = reader.hold_shared().unwrap(); - let mut child = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], guard.descriptor()) - .unwrap(); - // The launcher dies: its own descriptor goes with it. - drop(guard); - // Held for a full second: a fork of some other test could pin the lock - // for an instant, only the child under test keeps it that long. - for _ in 0..10 { - let denial = idunn.revoke_exact(Some(&lease)).unwrap_err(); - assert!(denial.to_string().contains("shared-lock consumer"), "the child does not hold the lock"); - thread::sleep(Duration::from_millis(100)); - } - child.terminate(Duration::from_secs(5)); - wait_until("child gone", || child.exited().is_some()); - idunn.revoke_exact(Some(&lease)).unwrap(); - } - - #[test] - fn a_child_spawned_without_the_descriptor_leaves_the_lock_with_the_launcher() { - let fx = fixture(true); - let (reader, idunn, lease) = lease_store(&fx, "uninherited"); - let guard = reader.hold_shared().unwrap(); - let mut child = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".into(), "exec sleep 30".into()], None) - .unwrap(); - drop(guard); - // Another test's fork may hold a copy for the instant before its exec; - // the child under test would hold it for thirty seconds. - wait_until("lock released", || idunn.revoke_exact(Some(&lease)).is_ok()); - child.terminate(Duration::from_secs(5)); - } - - #[test] - fn the_launcher_hands_its_lease_descriptor_to_the_child_it_spawns() { - let fx = fixture(true); - let mut launch = start(&fx, fx.contract("x"), true); - wait_until("warming", || launch.sink.first("warming").is_some()); - let warming = launch.sink.first("warming").unwrap(); - *launch.lease.0.lock().unwrap() = Some(fx.lease(&warming.canonical_sha256().unwrap())); - wait_until("active", || launch.sink.first("active").is_some()); - assert_eq!(*launch.child.inherited.lock().unwrap(), [Some(FAKE_LOCK_FD)]); - launch.stop(); - - let fx = fixture(false); - let mut launch = start(&fx, fx.contract("x"), true); - wait_until("active", || launch.sink.first("active").is_some()); - assert_eq!(*launch.child.inherited.lock().unwrap(), [None]); - launch.stop(); - } - - /// Unhealthy for its first `probes` calls, healthy from then on. - struct LateHealth { - calls: AtomicUsize, - probes: usize, - } - - impl HealthProbe for LateHealth { - fn healthy(&self) -> bool { - self.calls.fetch_add(1, Ordering::SeqCst) >= self.probes - } - } - - #[test] - fn a_child_that_becomes_healthy_late_is_not_killed_at_its_first_probe() { - let fx = fixture(false); - let mut contract = fx.contract("x"); - contract.startup = Duration::from_secs(60); - let sink = Arc::new(Sink::default()); - let child = Arc::new(Child::default()); - let health = Arc::new(LateHealth { calls: AtomicUsize::new(0), probes: 5 }); - let ports = Ports { - credentials: Arc::new(Credentials(BTreeMap::new())), - clock: Arc::new(FastClock::new(1_000_000)), - sink: sink.clone(), - lease: Arc::new(FakeLease::default()), - spawner: Box::new(FakeSpawner(child.clone())), - health: Box::new(move |_| health.clone()), - }; - let authority = fx.authority(); - let route = TcpListener::bind("127.0.0.1:0").unwrap(); - let tmp = fx.dir.path().join("tmp"); - let stop = Arc::new(AtomicBool::new(false)); - let flag = stop.clone(); - let handle = - thread::spawn(move || run(&contract, authority, route, member(), &tmp, ports, &flag)); - wait_until("active", || sink.first("active").is_some()); - assert!(!child.terminated.load(Ordering::SeqCst), "killed while it was still starting"); - assert_eq!(child.spawns.load(Ordering::SeqCst), 1); - assert!(sink.first("failed").is_none()); - stop.store(true, Ordering::SeqCst); - assert_eq!(handle.join().unwrap(), Outcome::Stopped); - } - - #[test] - fn a_silent_child_fails_the_probe_within_its_timeout() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let member = listener.local_addr().unwrap(); - // Accepts and never answers. - let held = thread::spawn(move || { - let _connection = listener.accept().unwrap(); - thread::sleep(Duration::from_secs(4)); - }); - let probe = HttpHealthProbe { member, path: "/health".into() }; - let started = Instant::now(); - assert!(!within(Duration::from_secs(5), move || probe.healthy())); - let elapsed = started.elapsed(); - assert!(elapsed >= Duration::from_millis(900), "gave up before its timeout: {elapsed:?}"); - assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its timeout: {elapsed:?}"); - drop(held); - } - - #[test] - fn the_probe_deadline_covers_the_whole_response() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let member = listener.local_addr().unwrap(); - // Every byte arrives inside any per-read timeout, but the head takes - // three seconds in all. - let dribbler = thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - for byte in b"HTTP/1.1 200 OK" { - if stream.write_all(&[*byte]).is_err() { - return; - } - thread::sleep(Duration::from_millis(250)); - } - }); - let probe = HttpHealthProbe { member, path: "/health".into() }; - let started = Instant::now(); - assert!(!within(Duration::from_secs(5), move || probe.healthy())); - let elapsed = started.elapsed(); - assert!(elapsed < HEALTH_PROBE_TIMEOUT + Duration::from_millis(200), "outlived its deadline: {elapsed:?}"); - join_within(Duration::from_secs(10), dribbler); - } - - #[test] - fn a_process_spawner_refuses_an_invalid_lease_descriptor_and_runs_no_child() { - let dir = tempfile::tempdir().unwrap(); - let marker = dir.path().join("ran"); - let script = format!("touch '{}'", marker.display()); - for invalid in [-1, 9999] { - let refused = ProcessSpawner.spawn( - Path::new("/bin/sh"), - &["-c".to_owned(), script.clone()], - Some(invalid), - ); - assert!(refused.is_err(), "descriptor {invalid} was accepted"); - } - thread::sleep(Duration::from_millis(300)); - assert!(!marker.exists(), "a child ran past an invalid descriptor"); - // Control: the same spawn without a descriptor runs the child. - let mut child = ProcessSpawner - .spawn(Path::new("/bin/sh"), &["-c".to_owned(), script], None) - .unwrap(); - wait_until("the control child to exit", || child.exited().is_some()); - assert!(marker.exists()); - } - - #[test] - fn the_member_bind_is_a_loopback_address_other_than_the_candidate() { - let candidate = "127.0.0.1:18835"; - assert_eq!( - parse_member_bind("127.0.0.1:18845", candidate).unwrap(), - "127.0.0.1:18845".parse().unwrap() - ); - for bad in [ - "127.0.0.1:18835", - "10.0.0.5:18845", - "0.0.0.0:18845", - "canary-not-an-address", - "", - ] { - let error = parse_member_bind(bad, candidate).unwrap_err().to_string(); - assert!(!error.contains("canary") && !error.contains("10.0.0.5"), "{error}"); - } - assert!(parse_member_bind("127.0.0.1:18845", "not-an-address").is_err()); - } -} diff --git a/src/lib.rs b/src/lib.rs index b5499a6..b96d835 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,10 +4,5 @@ pub mod deployment_plan; pub mod drivers; pub mod host; pub mod host_actuator; -#[cfg(unix)] -pub mod launcher; mod incident; -mod runner_evidence; pub mod provisioning; -#[cfg(unix)] -mod steward_orders; diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs deleted file mode 100644 index 30dd1d2..0000000 --- a/src/runner_evidence.rs +++ /dev/null @@ -1,805 +0,0 @@ -//! The output tail of a failed runner step. -//! -//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the -//! last bytes of a failed runner container's stdout and stderr, one record per -//! transaction. The engine's pre-fence failure path is its only writer, and -//! `idunn status --command` its only reader. It is never `control.cc`, which -//! refuses any type it does not know, and never `incidents.cc`, which is -//! published and carries no free text. -//! -//! The tail is raw runner output. It is kept unredacted and readable by root -//! alone (mode 0600): root can already read every secret the runner was given. -//! It never enters an error, a transaction record or the journal. The status -//! reader is a terminal, so everything it prints from this store goes through -//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped. - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result, bail, ensure}; -use cultcache_rs::{ - CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, - TryCompareExchangeSnapshotOutcome, -}; -use serde::{Deserialize, Serialize}; - -use crate::control_plane::{decode_record, require_id, typed_envelope}; -use crate::drivers::{ - LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject, - exchange_behind_private_lock, runner_status, -}; - -const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1"; - -/// Records kept; the writer drops the oldest past this, so the store needs no -/// other owner. -pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32; - -/// `runner-evidence.cc`, beside the control store. -pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf { - state_store.with_file_name("runner-evidence.cc") -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum SubjectKind { - Step, - ExternalInput, -} - -#[derive(Clone, PartialEq, Eq, DatabaseEntry)] -#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")] -pub(crate) struct RunnerFailureRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) transaction_id: String, - #[cultcache(key = 2)] - pub(crate) command_id: String, - #[cultcache(key = 3)] - pub(crate) target: String, - #[cultcache(key = 4)] - pub(crate) subject_kind: SubjectKind, - #[cultcache(key = 5)] - pub(crate) subject_id: String, - #[cultcache(key = 6)] - pub(crate) runner: String, - #[cultcache(key = 7)] - pub(crate) exit_code: Option, - #[cultcache(key = 8)] - pub(crate) signal: Option, - #[cultcache(key = 9)] - pub(crate) stdout_tail: Vec, - #[cultcache(key = 10)] - pub(crate) stdout_total_bytes: u64, - #[cultcache(key = 11)] - pub(crate) stderr_tail: Vec, - #[cultcache(key = 12)] - pub(crate) stderr_total_bytes: u64, - #[cultcache(key = 13)] - pub(crate) recorded_at_unix_millis: u64, -} - -/// Tails stay out of `Debug`, so a failed assertion or a logged record never -/// prints runner output. -impl std::fmt::Debug for RunnerFailureRecord { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RunnerFailureRecord") - .field("transaction_id", &self.transaction_id) - .field("subject_id", &self.subject_id) - .finish_non_exhaustive() - } -} - -impl RunnerFailureRecord { - pub(crate) fn from_failure( - transaction_id: &str, - command_id: &str, - target: &str, - failure: &RunnerStepFailure, - now: u64, - ) -> Self { - let (subject_kind, subject_id) = match &failure.subject { - RunnerSubject::Step(id) => (SubjectKind::Step, id), - RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id), - }; - Self { - schema_version: RUNNER_FAILURE_SCHEMA.to_owned(), - transaction_id: transaction_id.to_owned(), - command_id: command_id.to_owned(), - target: target.to_owned(), - subject_kind, - subject_id: subject_id.clone(), - runner: failure.runner.clone(), - exit_code: failure.exit_code, - signal: failure.signal, - stdout_tail: failure.stdout_tail.clone(), - stdout_total_bytes: failure.stdout_total_bytes, - stderr_tail: failure.stderr_tail.clone(), - stderr_total_bytes: failure.stderr_total_bytes, - recorded_at_unix_millis: now, - } - } - - pub(crate) fn validate(&self) -> Result<()> { - ensure!( - self.schema_version == RUNNER_FAILURE_SCHEMA, - "runner failure schema is unsupported" - ); - require_id(&self.transaction_id, "runner failure transaction")?; - require_id(&self.command_id, "runner failure command")?; - require_id(&self.target, "runner failure target")?; - require_id(&self.subject_id, "runner failure subject")?; - require_id(&self.runner, "runner failure runner")?; - for (tail, total, stream) in [ - (&self.stdout_tail, self.stdout_total_bytes, "stdout"), - (&self.stderr_tail, self.stderr_total_bytes, "stderr"), - ] { - ensure!( - tail.len() <= RUNNER_OUTPUT_TAIL_BYTES, - "runner failure {stream} tail exceeds its bound" - ); - ensure!( - tail.len() as u64 <= total, - "runner failure {stream} tail exceeds its stream" - ); - } - Ok(()) - } - - fn envelope(&self) -> Result { - self.validate()?; - typed_envelope( - &self.transaction_id, - Self::TYPE, - RUNNER_FAILURE_SCHEMA, - self, - self.recorded_at_unix_millis, - ) - } -} - -/// One read of the store: the envelopes exactly as stored, and the records -/// decoded from them in the same order. -fn snapshot(path: &Path) -> Result<(Vec, Vec)> { - let envelopes = SingleFileMessagePackBackingStore::new(path) - .pull_all_read_only_snapshot() - .context("reading the runner evidence store")?; - let mut records = Vec::new(); - for envelope in &envelopes { - ensure!( - envelope.r#type == RunnerFailureRecord::TYPE - && envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA), - "the runner evidence store holds a document that is not a runner failure" - ); - let record: RunnerFailureRecord = decode_record(envelope)?; - record.validate()?; - ensure!( - envelope.key == record.transaction_id, - "runner evidence store key differs from the record's transaction" - ); - records.push(record); - } - Ok((envelopes, records)) -} - -/// How a write left the store. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Kept { - Stored, - /// The store held bytes that do not decode; it was moved to - /// `unreadable_store_path` and the record opened a fresh one. `replaced` - /// is the last-written time (unix milliseconds) of an earlier set-aside - /// store this one displaced, which is gone: one slot, and it says so. - SetAside { replaced: Option }, -} - -/// Where an undecodable store is kept. One slot: the next one replaces it, so -/// the damage a bad store can cost is one extra file. -pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf { - path.with_file_name("runner-evidence.unreadable.cc") -} - -/// Moves the store over the previous set-aside one, root-only, and reports -/// what it displaced. Nothing of the store's content is named: a decode error -/// can quote stored values, and none is carried here. -fn set_aside_unreadable(path: &Path) -> Result> { - let aside = unreadable_store_path(path); - let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| { - earlier - .modified() - .ok() - .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) - .map_or(0, |since| since.as_millis() as u64) - }); - std::fs::rename(path, &aside) - .context("the runner evidence store is undecodable and could not be set aside")?; - keep_private(&aside)?; - Ok(replaced) -} - -/// How the writer reads the store's bytes; tests inject an I/O error here. -type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result>; - -fn unreadable_fault(error: &std::io::Error) -> anyhow::Error { - anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind()) -} - -/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the -/// oldest records go in the same exchange. The file ends at mode 0600. -/// -/// A store is set aside only when bytes that were read do not decode. An I/O -/// error leaves the store in place, keeps no evidence for that failure, and -/// the error names its kind for the journal. -pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { - record_reading(path, record, &|path| std::fs::read(path)) -} - -fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result { - let first = match read(path) { - Ok(bytes) => Some(bytes), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, - Err(error) => return Err(unreadable_fault(&error)), - }; - let (envelopes, records, how) = match snapshot(path) { - Ok((envelopes, records)) => (envelopes, records, Kept::Stored), - Err(_) => { - let Some(first) = first else { - bail!("the runner evidence store could not be decoded"); - }; - // Undecodable only if the same bytes are still there to read: a - // second read that fails, or finds other bytes, is not a verdict. - match read(path) { - Ok(second) if second == first => {} - Ok(_) => bail!("the runner evidence store changed while it was being read"), - Err(error) => return Err(unreadable_fault(&error)), - } - let replaced = set_aside_unreadable(path)?; - (Vec::new(), Vec::new(), Kept::SetAside { replaced }) - } - }; - let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes - .iter() - .zip(&records) - .filter(|(_, stored)| stored.transaction_id != record.transaction_id) - .map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone())) - .collect(); - kept.push((record.recorded_at_unix_millis, record.envelope()?)); - // Stable: records of one instant keep their stored order, so the record - // being written survives a tie. - kept.sort_by_key(|(recorded_at, _)| *recorded_at); - let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS); - let replacements: Vec<_> = kept - .into_iter() - .skip(excess) - .map(|(_, envelope)| envelope) - .collect(); - match exchange_behind_private_lock(path, &envelopes, &replacements)? { - TryCompareExchangeSnapshotOutcome::Exchanged => {} - TryCompareExchangeSnapshotOutcome::Mismatch => { - bail!("the runner evidence store changed while it was being written") - } - TryCompareExchangeSnapshotOutcome::LockContended => { - return Err(LockContended { - store: "runner evidence store", - } - .into()); - } - } - keep_private(path)?; - Ok(how) -} - -#[cfg(unix)] -fn keep_private(path: &Path) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) - .context("making the runner evidence store private") -} - -#[cfg(not(unix))] -fn keep_private(_path: &Path) -> Result<()> { - Ok(()) -} - -/// The one reader. An absent store holds nothing. -pub(crate) fn for_transaction( - path: &Path, - transaction_id: &str, -) -> Result> { - Ok(snapshot(path)? - .1 - .into_iter() - .find(|stored| stored.transaction_id == transaction_id)) -} - -/// The record as `idunn status --command` shows it, as raw lines. They are -/// not text yet: `show_status` is the only caller, and `emit` the only -/// writer. -fn render(record: &RunnerFailureRecord) -> Vec> { - let noun = match record.subject_kind { - SubjectKind::Step => "step", - SubjectKind::ExternalInput => "external input", - }; - let mut lines: Vec> = vec![ - format!(" failed {noun} {}", record.subject_id).into_bytes(), - format!(" runner {}", record.runner).into_bytes(), - format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(), - ]; - for (stream, tail, total) in [ - ("stdout", &record.stdout_tail, record.stdout_total_bytes), - ("stderr", &record.stderr_tail, record.stderr_total_bytes), - ] { - lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes()); - let text = tail.strip_suffix(b"\n").unwrap_or(tail); - if !text.is_empty() { - lines.extend(text.split(|byte| *byte == b'\n').map(|line| { - let mut shown = b" | ".to_vec(); - shown.extend_from_slice(line); - shown - })); - } - } - lines -} - -/// What `idunn status --command` prints for a transaction's runner evidence: -/// its record, or one line naming why the store is unreadable. Nothing is -/// written when the transaction has no record. -pub(crate) fn show_status( - out: &mut impl std::io::Write, - store: &Path, - transaction_id: &str, -) -> std::io::Result<()> { - let lines = match for_transaction(store, transaction_id) { - Ok(Some(record)) => render(&record), - Ok(None) => return Ok(()), - Err(error) => { - vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()] - } - }; - emit(out, &lines) -} - -/// The one sink: every byte the status read prints from the evidence store, -/// or from an error about it, is written here and nowhere else. An allowlist: -/// printable ASCII and tab pass; every other code point is a visible -/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline -/// inside a line is escaped too: this function writes the only ones. -pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { - for line in lines { - out.write_all(escape(line).as_bytes())?; - out.write_all(b"\n")?; - } - Ok(()) -} - -pub(crate) fn escape(bytes: &[u8]) -> String { - let mut shown = String::with_capacity(bytes.len()); - for chunk in bytes.utf8_chunks() { - for character in chunk.valid().chars() { - match character { - ' '..='~' | '\t' => shown.push(character), - _ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))), - } - } - for byte in chunk.invalid() { - shown.push_str(&format!("\\x{byte:02x}")); - } - } - shown -} - -#[cfg(all(test, unix))] -mod tests { - use super::*; - use std::os::unix::fs::PermissionsExt; - - fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure { - RunnerStepFailure { - subject, - runner: "rust".into(), - exit_code: Some(3), - signal: None, - stdout_tail: stdout.to_vec(), - stdout_total_bytes: stdout.len() as u64, - stderr_tail: stderr.to_vec(), - stderr_total_bytes: stderr.len() as u64, - } - } - - /// What status writes for a record, through the one sink. - fn shown(record: &RunnerFailureRecord) -> String { - let mut written = Vec::new(); - emit(&mut written, &render(record)).expect("a Vec takes any write"); - String::from_utf8(written).expect("the sink writes ASCII only") - } - - fn shown_lines(record: &RunnerFailureRecord) -> Vec { - shown(record).lines().map(str::to_owned).collect() - } - - fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { - RunnerFailureRecord::from_failure( - transaction, - "up-service", - "service", - &failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"), - now, - ) - } - - #[test] - fn the_evidence_store_keeps_the_newest_32() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - // The literal, not the constant: the spec's bound is 32. - for index in 0..=32 { - record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; - } - let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); - assert_eq!(kept.len(), 32); - assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); - assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); - assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); - // Rewriting a resident transaction replaces it and drops nothing. - let mut again = record_at("tx-05", 5000); - again.stdout_tail = b"rewritten\n".to_vec(); - again.stdout_total_bytes = 10; - record(&path, &again)?; - let kept = snapshot(&path)?.1; - assert_eq!(kept.len(), 32); - assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); - let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; - assert_eq!(stored.stdout_tail, b"rewritten\n"); - assert_eq!(for_transaction(&path, "tx-00")?, None); - Ok(()) - } - - #[test] - fn a_record_round_trips_byte_for_byte() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - let written = RunnerFailureRecord::from_failure( - "tx-a", - "up-service", - "service", - &failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""), - 77, - ); - record(&path, &written)?; - assert_eq!(for_transaction(&path, "tx-a")?, Some(written)); - assert_eq!(for_transaction(&path, "tx-b")?, None); - Ok(()) - } - - #[test] - fn the_evidence_store_is_private() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - // Whatever the umask, the mode is the store's own rule, on every write. - for index in 0..2 { - record(&path, &record_at(&format!("tx-{index}"), 10 + index))?; - assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); - } - Ok(()) - } - - #[test] - fn a_tail_over_its_bound_or_its_stream_is_refused() { - let mut oversize = record_at("tx-a", 1); - oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1]; - oversize.stdout_total_bytes = u64::MAX; - assert!(oversize.validate().is_err()); - let mut impossible = record_at("tx-a", 1); - impossible.stderr_total_bytes = 1; - assert!(impossible.validate().is_err()); - let mut exact = record_at("tx-a", 1); - exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES]; - exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64; - assert!(exact.validate().is_ok()); - } - - #[test] - fn status_renders_the_tail_with_control_bytes_escaped() { - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec(); - stored.stdout_total_bytes = 5000; - stored.stderr_tail = b"bad\r\x07".to_vec(); - stored.stderr_total_bytes = 6; - let lines = shown_lines(&stored); - let text = lines.join("\n"); - assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); - assert!(!text.contains('\u{7}')); - assert_eq!( - lines, - vec![ - " failed step build", - " runner rust", - " status exit status 3", - " stdout (last 30 of 5000 bytes)", - " | first", - " | \\u{1b}[2Jsecond\\u{0}end\tTab", - " | last", - " stderr (last 5 of 6 bytes)", - " | bad\\u{d}\\u{7}", - ] - ); - } - - #[test] - fn debug_names_the_record_and_prints_no_tail() { - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = b"SECRET-OUTPUT".to_vec(); - stored.stdout_total_bytes = 13; - let debug = format!("{stored:?}"); - assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}"); - } - - #[test] - fn an_empty_tail_renders_only_its_header() { - let mut stored = record_at("tx-a", 1); - stored.stdout_tail = Vec::new(); - stored.stdout_total_bytes = 0; - let lines = shown_lines(&stored); - assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); - assert!(!lines.iter().any(|line| line == " | ")); - } - - #[test] - fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - std::fs::write(&path, b"\xc1 not a store")?; - assert!(for_transaction(&path, "tx-a").is_err()); - Ok(()) - } - - fn files_beside(path: &Path) -> Result> { - let mut names: Vec = std::fs::read_dir(path.parent().context("no parent")?)? - .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned())) - .collect::>()?; - names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock")); - names.sort(); - Ok(names) - } - - /// A store a newer Idunn wrote: well-formed, but a schema this one does not know. - fn write_future_store(path: &Path) -> Result<()> { - let mut future = record_at("tx-future", 1); - future.schema_version = "idunn.runner_failure.v2".into(); - let envelope = typed_envelope( - &future.transaction_id, - RunnerFailureRecord::TYPE, - "idunn.runner_failure.v2", - &future, - 1, - )?; - let outcome = exchange_behind_private_lock(path, &[], &[envelope])?; - ensure!( - matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged), - "the future store was not written" - ); - Ok(()) - } - - #[test] - fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - let damage = b"\xc1 not a store \x00 damage"; - std::fs::write(&path, damage)?; - let kept = record(&path, &record_at("tx-a", 5))?; - assert_eq!(kept, Kept::SetAside { replaced: None }); - // Kept, not deleted: the damage is where the journal says, root-only. - let aside = unreadable_store_path(&path); - assert_eq!(std::fs::read(&aside)?, damage); - assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600); - // The failure's evidence is in a fresh, private store. - assert_eq!( - for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id), - Some("tx-a".to_owned()) - ); - assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); - // A healthy store is not set aside again. - assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored); - assert_eq!(std::fs::read(&aside)?, damage); - Ok(()) - } - - #[test] - fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - write_future_store(&path)?; - let before = std::fs::read(&path)?; - assert!(for_transaction(&path, "tx-future").is_err()); - assert_eq!( - record(&path, &record_at("tx-a", 5))?, - Kept::SetAside { replaced: None } - ); - assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); - assert!(for_transaction(&path, "tx-a")?.is_some()); - assert_eq!(for_transaction(&path, "tx-future")?, None); - Ok(()) - } - - #[test] - fn at_most_one_unreadable_store_is_kept() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"]; - for (index, damage) in damages.into_iter().enumerate() { - std::fs::write(&path, damage)?; - record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?; - assert_eq!( - std::fs::read(unreadable_store_path(&path))?, - damage, - "the newest damage is the one kept" - ); - } - assert_eq!( - files_beside(&path)?, - vec![ - "runner-evidence.cc".to_owned(), - "runner-evidence.unreadable.cc".to_owned() - ] - ); - Ok(()) - } - - #[test] - fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - // A directory where the store should be: unreadable, not undecodable. - std::fs::create_dir(&path)?; - assert!(record(&path, &record_at("tx-a", 1)).is_err()); - assert!(path.is_dir()); - assert!(!unreadable_store_path(&path).exists()); - Ok(()) - } - - #[test] - fn every_code_point_but_printable_ascii_is_a_visible_escape() { - // The allowlist, exhaustively: no class list decides anything. - for code in 0..=0x10ffffu32 { - let Some(character) = char::from_u32(code) else { - continue; - }; - let text = character.to_string(); - let shown = escape(text.as_bytes()); - if matches!(character, ' '..='~' | '\t') { - assert_eq!(shown, text, "U+{code:04X} should pass"); - } else { - assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}"); - } - } - // Invalid UTF-8 is escaped byte by byte; valid text round it passes. - assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82"); - assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}"); - } - - #[test] - fn a_newline_inside_a_line_cannot_forge_a_line() { - let mut stored = record_at("tx-a", 1); - stored.subject_id = "build\n status exit status 0".into(); - let lines = shown_lines(&stored); - assert_eq!(lines[0], " failed step build\\u{a} status exit status 0"); - assert_eq!( - lines.iter().filter(|line| line.starts_with(" status ")).count(), - 1 - ); - } - - /// A store whose first record is well-formed except that its subject kind - /// holds `marker`: the decode error quotes it. - fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> { - record(path, &record_at("tx-a", 1))?; - let mut bytes = std::fs::read(path)?; - let at = bytes - .windows(4) - .position(|window| window == b"step") - .context("no step marker")?; - bytes[at..at + 4].copy_from_slice(marker); - std::fs::write(path, bytes)?; - Ok(()) - } - - #[test] - fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - store_with_hostile_subject_kind(&path, b"\x1b[2J")?; - let mut written = Vec::new(); - show_status(&mut written, &path, "tx-a")?; - assert!(!written.contains(&0x1b), "an escape byte reached the terminal"); - assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n'))); - let text = String::from_utf8(written)?; - assert!(text.starts_with(" runner evidence unreadable: "), "{text}"); - assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}"); - Ok(()) - } - - #[test] - fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - store_with_hostile_subject_kind(&path, b"CNRY")?; - let damage = std::fs::read(&path)?; - // Sanity: this damage decodes to an error that quotes the canary. - assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY")); - // A non-empty directory where the aside goes: rename fails even as root. - let aside = unreadable_store_path(&path); - std::fs::create_dir(&aside)?; - std::fs::write(aside.join("occupied"), b"x")?; - let error = record(&path, &record_at("tx-b", 2)).unwrap_err(); - for text in [format!("{error:#}"), format!("{error:?}")] { - assert!(text.contains("could not be set aside"), "{text}"); - assert!(!text.contains("CNRY"), "stored content reached the error: {text}"); - } - // The failure cost nothing: the store is as it was, with no evidence - // written over it. - assert_eq!(std::fs::read(&path)?, damage); - assert!(aside.join("occupied").exists()); - Ok(()) - } - - #[test] - fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - record(&path, &record_at("tx-a", 1))?; - let before = std::fs::read(&path)?; - let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied); - // The first read fails on a healthy store. - let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err(); - assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); - assert_eq!(std::fs::read(&path)?, before, "the store changed"); - assert!(!unreadable_store_path(&path).exists()); - assert!(for_transaction(&path, "tx-b")?.is_none()); - // The second read, the one that confirms the bytes do not decode, fails. - std::fs::write(&path, b"\xc1 not a store")?; - let calls = std::cell::Cell::new(0usize); - let read = |path: &Path| { - calls.set(calls.get() + 1); - if calls.get() == 2 { - return Err(denied()); - } - std::fs::read(path) - }; - let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err(); - assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); - assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); - assert!(!unreadable_store_path(&path).exists()); - // Bytes that changed between the reads are no verdict either. - let calls = std::cell::Cell::new(0usize); - let read = |path: &Path| -> std::io::Result> { - calls.set(calls.get() + 1); - let mut bytes = std::fs::read(path)?; - bytes.push(calls.get() as u8); - Ok(bytes) - }; - assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err()); - assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); - assert!(!unreadable_store_path(&path).exists()); - Ok(()) - } - - #[test] - fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = runner_evidence_store_path(&temp.path().join("control.cc")); - std::fs::write(&path, b"\xc1 first")?; - assert_eq!( - record(&path, &record_at("tx-a", 1))?, - Kept::SetAside { replaced: None } - ); - std::fs::write(&path, b"\xc1 second")?; - let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else { - bail!("a displaced set-aside store went unreported"); - }; - assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second"); - Ok(()) - } -} diff --git a/src/steward_orders.rs b/src/steward_orders.rs deleted file mode 100644 index d4be1ee..0000000 --- a/src/steward_orders.rs +++ /dev/null @@ -1,4079 +0,0 @@ -//! Typed orders from Idunn's steward. -//! -//! `idunn orders serve` is a second root unit from the same binary as -//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints -//! order ids, decides acceptance and refusal, actuates, and records outcomes. -//! The survival daemon takes no part: `control.cc` never learns an order type. -//! -//! An order is a closed enum, never a command string: install packages, act on -//! a systemd unit, or deploy a target through the same commit primitive -//! `idunn up` uses. Every unit is allowed except the always-refused set, which -//! is compiled in here and read from nowhere: no binding field, record or flag -//! holds it. The `#operator` post is a retried projection of the store. -//! -//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the -//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and -//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE -//! `mind-control check`; START record before actuation, at most once; RUN; -//! RESULT outcome; POST accepted and result. -//! -//! No refusal or error here carries a value the issuer supplied: it names the -//! field and a code. A refused unit that a record names was resolved by -//! systemd and passed the unit grammar. - -use std::fmt; -use std::io::{Read, Write}; -use std::os::unix::fs::{MetadataExt, PermissionsExt}; -use std::os::unix::net::{UnixListener, UnixStream}; -use std::os::unix::process::ExitStatusExt; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::sync::Arc; -use std::sync::mpsc; -use std::thread; -use std::time::Duration; - -use anyhow::{Context, Result, anyhow, bail, ensure}; -use cultcache_rs::{ - CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, -}; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use crate::control_plane::{ - command_state, commit_deployment_command, decode_record, default_state_store, now_millis, - path_value, require_id, require_selector, string_value, typed_envelope, -}; -use crate::runner_evidence::{emit, escape}; - -const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; -const START_SCHEMA: &str = "idunn.steward_order_start.v1"; -const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; -const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; - -const REQUEST_FRAME_LIMIT: usize = 16 * 1024; -const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; -const TAIL_BYTES: usize = 8 * 1024; -const REASON_LIMIT: usize = 300; -const PACKAGE_LIMIT: usize = 16; -const UNIT_LIMIT: usize = 200; -const LIST_LIMIT: usize = 50; -const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; - -/// The always-refused set: Idunn's own units and the minds' own. Matched on -/// every name a unit answers to, ASCII case-insensitively (systemd units are -/// case-sensitive, so the fold only widens the refusal). -fn is_refused_name(name: &str) -> bool { - let name = name.to_ascii_lowercase(); - ["idunn-", "mind-self@", "mind-persona@"] - .iter() - .any(|prefix| name.starts_with(prefix)) -} - -/// Units whose binaries suspend, hibernate or open recovery: no property -/// reveals it, so they are refused by name, on every name they answer to. -const HOST_NAMES: [&str; 6] = [ - "systemd-suspend.service", - "systemd-hibernate.service", - "systemd-hybrid-sleep.service", - "systemd-suspend-then-hibernate.service", - "rescue.service", - "emergency.service", -]; - -fn is_host_name(name: &str) -> bool { - HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) -} - -// ---------------------------------------------------------------- grammar -- - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Code { - Empty, - TooLong, - TooMany, - BadCharacter, - BadShape, - Malformed, - Unauthorized, - Unknown, - Unavailable, -} - -/// A refusal names the field and a code. It never carries the offending value. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct Refusal { - pub(crate) field: String, - pub(crate) code: Code, -} - -impl Refusal { - fn new(field: &'static str, code: Code) -> Self { - Self { - field: field.to_owned(), - code, - } - } -} - -impl fmt::Display for Refusal { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "{} refused: {:?}", self.field, self.code) - } -} - -impl std::error::Error for Refusal {} - -fn validate_package(spec: &str) -> Result<(), Refusal> { - const FIELD: &str = "package"; - if spec.is_empty() { - return Err(Refusal::new(FIELD, Code::Empty)); - } - if spec.len() > UNIT_LIMIT { - return Err(Refusal::new(FIELD, Code::TooLong)); - } - let (name, version) = match spec.split_once('=') { - Some((name, version)) => (name, Some(version)), - None => (spec, None), - }; - let bytes = name.as_bytes(); - if bytes.len() < 2 { - return Err(Refusal::new(FIELD, Code::BadShape)); - } - let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); - let rest = bytes[1..].iter().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') - }); - if !head || !rest { - return Err(Refusal::new(FIELD, Code::BadCharacter)); - } - // apt reads `name-` as a removal and `name.deb` as a local file. - if name.ends_with('-') || name.ends_with(".deb") { - return Err(Refusal::new(FIELD, Code::BadShape)); - } - if let Some(version) = version { - if version.is_empty() { - return Err(Refusal::new(FIELD, Code::BadShape)); - } - if !version.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') - }) { - return Err(Refusal::new(FIELD, Code::BadCharacter)); - } - } - Ok(()) -} - -fn validate_unit(unit: &str) -> Result<(), Refusal> { - const FIELD: &str = "unit"; - if unit.is_empty() { - return Err(Refusal::new(FIELD, Code::Empty)); - } - if unit.len() > UNIT_LIMIT { - return Err(Refusal::new(FIELD, Code::TooLong)); - } - let allowed = unit.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') - }); - if !allowed || unit.starts_with('-') { - return Err(Refusal::new(FIELD, Code::BadCharacter)); - } - match UNIT_SUFFIXES - .iter() - .find_map(|suffix| unit.strip_suffix(suffix)) - { - Some(stem) if !stem.is_empty() => Ok(()), - _ => Err(Refusal::new(FIELD, Code::BadShape)), - } -} - -fn validate_reason(reason: &str) -> Result<(), Refusal> { - const FIELD: &str = "reason"; - if reason.is_empty() { - return Err(Refusal::new(FIELD, Code::Empty)); - } - if reason.len() > REASON_LIMIT { - return Err(Refusal::new(FIELD, Code::TooLong)); - } - if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { - return Err(Refusal::new(FIELD, Code::BadCharacter)); - } - Ok(()) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Verb { - Start, - Stop, - Restart, - Reload, -} - -impl Verb { - fn as_str(self) -> &'static str { - match self { - Verb::Start => "start", - Verb::Stop => "stop", - Verb::Restart => "restart", - Verb::Reload => "reload", - } - } - - fn parse(word: &str) -> Option { - [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] - .into_iter() - .find(|verb| verb.as_str() == word) - } -} - -/// The closed set of things an order can ask for. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Operation { - PackageInstall { packages: Vec }, - UnitAction { verb: Verb, unit: String }, - Deploy { selector: String }, -} - -/// One fixed argv for `systemd-run`, built from the typed operation. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Step { - pub(crate) argv: Vec, - pub(crate) environment: Vec<(String, String)>, -} - -impl Operation { - fn validate(&self) -> Result<(), Refusal> { - match self { - Operation::PackageInstall { packages } => { - if packages.is_empty() { - return Err(Refusal::new("package", Code::Empty)); - } - if packages.len() > PACKAGE_LIMIT { - return Err(Refusal::new("package", Code::TooMany)); - } - packages.iter().try_for_each(|package| validate_package(package)) - } - Operation::UnitAction { unit, .. } => validate_unit(unit), - Operation::Deploy { selector } => require_selector(selector) - .map_err(|_| Refusal::new("selector", Code::BadShape)), - } - } - - fn steps(&self) -> Vec { - let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); - let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; - match self { - Operation::PackageInstall { packages } => { - // Never a removal, never a pattern: each name selects exactly - // itself, and apt aborts rather than displace an installed package. - let mut install: Vec = - words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); - let mut present: Vec = words(&["dpkg", "-s", "--"]); - for package in packages { - let (name, version) = package.split_once('=').unwrap_or((package, "")); - install.push(match version { - "" => format!("?exact-name({name})"), - version => format!("?exact-name({name})={version}"), - }); - present.push(name.to_owned()); - } - // An exact name that matches nothing installs nothing and exits 0, - // so the last step fails unless every named package is installed. - vec![ - Step { - argv: words(&["apt-get", "update", "-q"]), - environment: apt_environment.clone(), - }, - Step { - argv: install, - environment: apt_environment, - }, - Step { - argv: present, - environment: Vec::new(), - }, - ] - } - Operation::UnitAction { verb, unit } => vec![Step { - argv: vec![ - "systemctl".into(), - verb.as_str().into(), - "--job-mode=ignore-requirements".into(), - "--".into(), - unit.clone(), - ], - environment: Vec::new(), - }], - Operation::Deploy { .. } => Vec::new(), - } - } - - fn describe(&self) -> String { - match self { - Operation::PackageInstall { packages } => { - format!("package-install {}", packages.join(" ")) - } - Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), - Operation::Deploy { selector } => format!("deploy {selector}"), - } - } -} - -// ---------------------------------------------------------------- records -- - -#[derive(Clone, PartialEq, Eq, DatabaseEntry)] -#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] -pub(crate) struct OrderRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) order_id: String, - #[cultcache(key = 2)] - pub(crate) principal: String, - #[cultcache(key = 3)] - pub(crate) issuer_uid: u32, - #[cultcache(key = 4)] - pub(crate) issuer_unit: String, - #[cultcache(key = 5)] - pub(crate) issuer_invocation: String, - #[cultcache(key = 6)] - pub(crate) issued_at_unix_millis: u64, - #[cultcache(key = 7)] - pub(crate) operation: Operation, - #[cultcache(key = 8)] - pub(crate) reason: String, -} - -impl fmt::Debug for OrderRecord { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("OrderRecord") - .field("order_id", &self.order_id) - .finish_non_exhaustive() - } -} - -#[derive(Clone, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.steward_order_start", - schema = "idunn.steward_order_start.v1" -)] -pub(crate) struct StartRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) order_id: String, - #[cultcache(key = 2)] - pub(crate) started_at_unix_millis: u64, - /// The transient unit name, or the deployment command id. - #[cultcache(key = 3)] - pub(crate) actuation: String, -} - -impl fmt::Debug for StartRecord { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("StartRecord") - .field("order_id", &self.order_id) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Via { - Name, - Alias, - Host, - OnEnd, - Bound, - Upholds, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Refused { - AlwaysRefused { unit: String, via: Via }, - /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload - /// of the wrapper exits 0 and leaves the member alone. - Wrapper { member: String }, - Braked, - /// The order's principal or issuing uid is not the steward bound now. - NotSteward, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Outcome { - Ran, - Failed { - exit_code: Option, - signal: Option, - }, - Refused(Refused), - Lost, - Submitted { command_id: String }, -} - -impl Outcome { - fn describe(&self) -> String { - match self { - Outcome::Ran => "ran (exit 0)".into(), - Outcome::Failed { - exit_code: Some(code), - .. - } => format!("failed (exit {code})"), - Outcome::Failed { - signal: Some(signal), - .. - } => format!("failed (signal {signal})"), - Outcome::Failed { .. } => "failed (not run to an exit)".into(), - Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( - "refused: always-refused unit {} reached by {}", - escape(unit.as_bytes()), - match via { - Via::Name => "name", - Via::Alias => "alias", - Via::Host => "host", - Via::OnEnd => "on-end", - Via::Bound => "bound", - Via::Upholds => "upholds", - } - ), - Outcome::Refused(Refused::Wrapper { member }) => format!( - "refused: wrapper unit, it only wraps {}", - escape(member.as_bytes()) - ), - Outcome::Refused(Refused::Braked) => "refused: braked".into(), - Outcome::Refused(Refused::NotSteward) => { - "refused: the issuer is not the current steward".into() - } - Outcome::Lost => "lost: the orders service restarted mid-run".into(), - Outcome::Submitted { command_id } => { - format!("submitted deployment {}", escape(command_id.as_bytes())) - } - } - } -} - -#[derive(Clone, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.steward_order_outcome", - schema = "idunn.steward_order_outcome.v1" -)] -pub(crate) struct OutcomeRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - #[cultcache(key = 1)] - pub(crate) order_id: String, - #[cultcache(key = 2)] - pub(crate) ended_at_unix_millis: u64, - #[cultcache(key = 3)] - pub(crate) outcome: Outcome, - #[cultcache(key = 4)] - pub(crate) stdout_tail: Vec, - #[cultcache(key = 5)] - pub(crate) stderr_tail: Vec, -} - -/// Tails stay out of `Debug`: they are raw program output. -impl fmt::Debug for OutcomeRecord { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("OutcomeRecord") - .field("order_id", &self.order_id) - .field("outcome", &self.outcome) - .finish_non_exhaustive() - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum PostKind { - Accepted, - Result, -} - -impl PostKind { - fn as_str(self) -> &'static str { - match self { - PostKind::Accepted => "accepted", - PostKind::Result => "result", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache( - type = "idunn.steward_order_post", - schema = "idunn.steward_order_post.v1" -)] -pub(crate) struct PostRecord { - #[cultcache(key = 0)] - pub(crate) schema_version: String, - /// `:`. - #[cultcache(key = 1)] - pub(crate) post_key: String, - #[cultcache(key = 2)] - pub(crate) event_id: String, - #[cultcache(key = 3)] - pub(crate) posted_at_unix_millis: u64, -} - -// ------------------------------------------------------------------ store -- - -/// `orders.cc`: root-only, written by `idunn orders serve` alone. -pub(crate) struct OrderStore { - directory: PathBuf, -} - -#[derive(Default)] -struct Ledger { - orders: Vec, - starts: Vec, - outcomes: Vec, - posts: Vec, -} - -impl Ledger { - fn start_of(&self, order_id: &str) -> Option<&StartRecord> { - self.starts.iter().find(|start| start.order_id == order_id) - } - - fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { - self.outcomes - .iter() - .find(|outcome| outcome.order_id == order_id) - } - - fn posted(&self, order_id: &str, kind: PostKind) -> bool { - let key = post_key(order_id, kind); - self.posts.iter().any(|post| post.post_key == key) - } -} - -fn post_key(order_id: &str, kind: PostKind) -> String { - format!("{order_id}:{}", kind.as_str()) -} - -impl OrderStore { - pub(crate) fn new(directory: &Path) -> Self { - Self { - directory: directory.to_owned(), - } - } - - fn path(&self) -> PathBuf { - self.directory.join("orders.cc") - } - - fn prepare(&self) -> Result<()> { - std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; - std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) - .context("making the orders directory private") - } - - /// Insert a record whose (type, key) is absent. `false` means it was there. - fn put(&self, key: &str, schema: &str, value: &T) -> Result { - self.prepare()?; - let path = self.path(); - let now = now_millis()?; - let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: T::TYPE.into(), - key: key.into(), - current: None, - }], - &[typed_envelope(key, T::TYPE, schema, value, now)?], - )?; - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) - .context("making the orders store private")?; - Ok(inserted) - } - - fn ledger(&self) -> Result { - let path = self.path(); - let mut ledger = Ledger::default(); - if !path.exists() { - return Ok(ledger); - } - let envelopes = SingleFileMessagePackBackingStore::new(&path) - .pull_all_read_only_snapshot() - .context("reading the orders store")?; - for envelope in &envelopes { - let schema = envelope.schema_id.as_deref(); - let key = envelope.key.as_str(); - match envelope.r#type.as_str() { - OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { - let record: OrderRecord = decode_record(envelope)?; - ensure!(record.order_id == key, "an order is stored under another key"); - ledger.orders.push(record); - } - StartRecord::TYPE if schema == Some(START_SCHEMA) => { - let record: StartRecord = decode_record(envelope)?; - ensure!(record.order_id == key, "a start is stored under another key"); - ledger.starts.push(record); - } - OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { - let record: OutcomeRecord = decode_record(envelope)?; - ensure!(record.order_id == key, "an outcome is stored under another key"); - ledger.outcomes.push(record); - } - PostRecord::TYPE if schema == Some(POST_SCHEMA) => { - let record: PostRecord = decode_record(envelope)?; - ensure!(record.post_key == key, "a post is stored under another key"); - ledger.posts.push(record); - } - _ => bail!("the orders store holds a document that is not an order record"), - } - } - ledger - .orders - .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); - Ok(ledger) - } -} - -// ------------------------------------------------------------------ ports -- - -/// The properties naming what systemd does when a unit ends or its job times -/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). -const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; - -/// The properties naming units systemd starts when a unit ends. -const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; - -/// What `systemctl show` says about one unit. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct UnitFacts { - pub(crate) id: String, - pub(crate) names: Vec, - pub(crate) not_found: bool, - /// The properties that make the unit act when it ends: each emergency - /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. - pub(crate) on_end: Vec, - /// The units bound to this one at runtime (`BoundBy=`): systemd stops - /// them when this unit stops, whatever the job mode. - pub(crate) bound_by: Vec, - /// The units this one upholds (`Upholds=`): systemd starts them in a full - /// transaction whenever this unit is active, whatever the job mode. - pub(crate) upholds: Vec, - /// The units this one groups (`ConsistsOf=`): the verbs reach them only - /// through propagation, which the job mode removes. - pub(crate) consists_of: Vec, -} - -impl UnitFacts { - fn identities(&self) -> impl Iterator { - std::iter::once(&self.id).chain(self.names.iter()) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct RunReport { - pub(crate) exit_code: Option, - pub(crate) signal: Option, - pub(crate) stdout_tail: Vec, - pub(crate) stderr_tail: Vec, -} - -pub(crate) trait Systemd: Send + Sync { - fn show(&self, unit: &str) -> Result; - fn invocation(&self, unit: &str) -> Option; - /// Run one step as the transient unit `unit`, to its end. - fn run(&self, unit: &str, step: &Step) -> Result; -} - -pub(crate) struct Peer { - pub(crate) uid: u32, - pub(crate) issuer_unit: String, -} - -pub(crate) trait PeerCredentials: Send + Sync { - fn peer(&self, stream: &UnixStream) -> Result; -} - -pub(crate) trait BrakeCheck: Send + Sync { - /// `true` only when the instance's brake check exits 0. - fn clear(&self, principal: &str) -> bool; -} - -pub(crate) trait Homeserver: Send + Sync { - /// Idempotent by `txn_id`; returns the event id. - fn post(&self, txn_id: &str, text: &str) -> Result; -} - -pub(crate) struct Ctx { - pub(crate) store: OrderStore, - pub(crate) state_store: PathBuf, - pub(crate) steward_uid: u32, - pub(crate) principal: String, - pub(crate) systemd: Arc, - pub(crate) peers: Arc, - pub(crate) brake: Arc, - pub(crate) homeserver: Arc, -} - -// ------------------------------------------------------------------ scope -- - -fn refused(unit: &str, via: Via) -> Refused { - Refused::AlwaysRefused { - unit: unit.to_owned(), - via, - } -} - -/// Would `verb` on `unit` be refused. systemd builds the transaction and the -/// order tells it to build one job (`--job-mode=ignore-requirements`), so this -/// reads only what that job mode cannot remove from the one named unit: the -/// name it answers to, what fires when it ends, who is bound to it, what it upholds and -/// what it only wraps. -fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { - if is_refused_name(unit) { - return Ok(Some(refused(unit, Via::Name))); - } - let facts = systemd.show(unit)?; - ensure!(!facts.not_found, "systemd has no such unit to check"); - for name in facts.identities() { - if is_refused_name(name) { - return Ok(Some(refused(name, Via::Alias))); - } - if is_host_name(name) { - return Ok(Some(refused(name, Via::Host))); - } - } - if verb != Verb::Reload && !facts.on_end.is_empty() { - return Ok(Some(refused(unit, Via::OnEnd))); - } - if matches!(verb, Verb::Stop | Verb::Restart) - && let Some(bound) = facts.bound_by.first() - { - return Ok(Some(refused(bound, Via::Bound))); - } - if matches!(verb, Verb::Start | Verb::Restart) && !facts.upholds.is_empty() { - return Ok(Some(refused(unit, Via::Upholds))); - } - if verb != Verb::Start - && let Some(member) = facts.consists_of.first() - { - return Ok(Some(Refused::Wrapper { - member: member.clone(), - })); - } - Ok(None) -} - -// -------------------------------------------------------------- execution -- - -fn tail(bytes: &[u8]) -> Vec { - bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() -} - -fn conclude( - ctx: &Ctx, - order_id: &str, - outcome: Outcome, - stdout_tail: &[u8], - stderr_tail: &[u8], -) -> Result<()> { - ctx.store.put( - order_id, - OUTCOME_SCHEMA, - &OutcomeRecord { - schema_version: OUTCOME_SCHEMA.into(), - order_id: order_id.into(), - ended_at_unix_millis: now_millis()?, - outcome, - stdout_tail: tail(stdout_tail), - stderr_tail: tail(stderr_tail), - }, - )?; - Ok(()) -} - -fn order_uuid(order_id: &str) -> &str { - order_id.strip_prefix("ord-").unwrap_or(order_id) -} - -pub(crate) fn requester(principal: &str, order_id: &str) -> String { - format!("steward:{principal} order {order_id}") -} - -/// Run one accepted order. A start record is written before anything is -/// actuated and an order that has one is never run again. -pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { - let ledger = ctx.store.ledger()?; - let order = ledger - .orders - .iter() - .find(|order| order.order_id == order_id) - .context("the order is not in the store")?; - if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { - return Ok(()); - } - // One gate for a fresh order and a recovered one: who may order is what the - // binding says now, not what it said when the order was accepted. - if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { - return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); - } - if order.operation.validate().is_err() { - return conclude( - ctx, - order_id, - Outcome::Failed { - exit_code: None, - signal: None, - }, - b"", - b"the stored order no longer satisfies the grammar", - ); - } - if let Operation::UnitAction { verb, unit } = &order.operation { - match refusal_for(ctx.systemd.as_ref(), unit, *verb) { - Ok(None) => {} - Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), - Err(_) => { - return conclude( - ctx, - order_id, - Outcome::Failed { - exit_code: None, - signal: None, - }, - b"", - b"the unit could not be resolved, so it was not acted on", - ); - } - } - } - if !ctx.brake.clear(&ctx.principal) { - return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); - } - let uuid = order_uuid(order_id); - let command_id = format!("up-{uuid}"); - let steps = order.operation.steps(); - let actuation = match &order.operation { - Operation::Deploy { .. } => command_id.clone(), - _ => format!("idunn-order-{uuid}"), - }; - let started = StartRecord { - schema_version: START_SCHEMA.into(), - order_id: order_id.into(), - started_at_unix_millis: now_millis()?, - actuation, - }; - if !ctx.store.put(order_id, START_SCHEMA, &started)? { - return Ok(()); - } - if let Operation::Deploy { selector } = &order.operation { - let outcome = match commit_deployment_command( - &ctx.state_store, - &command_id, - selector, - &requester(&order.principal, order_id), - ) { - Ok(_) => Outcome::Submitted { command_id }, - Err(_) => Outcome::Failed { - exit_code: None, - signal: None, - }, - }; - return conclude(ctx, order_id, outcome, b"", b""); - } - let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); - let mut outcome = Outcome::Ran; - for (index, step) in steps.iter().enumerate() { - let unit = if steps.len() == 1 { - format!("idunn-order-{uuid}") - } else { - format!("idunn-order-{uuid}-{}", index + 1) - }; - match ctx.systemd.run(&unit, step) { - Ok(report) => { - stdout = report.stdout_tail; - stderr = report.stderr_tail; - if report.exit_code != Some(0) { - outcome = Outcome::Failed { - exit_code: report.exit_code, - signal: report.signal, - }; - break; - } - } - Err(_) => { - outcome = Outcome::Failed { - exit_code: None, - signal: None, - }; - stderr = b"the transient unit could not be started".to_vec(); - break; - } - } - } - conclude(ctx, order_id, outcome, &stdout, &stderr) -} - -/// On service start: an order with a start record and no outcome ran, or began -/// to, in a life that ended. It is Lost and never run again. Returns the -/// orders that never started, oldest first. -pub(crate) fn recover(ctx: &Ctx) -> Result> { - let ledger = ctx.store.ledger()?; - for start in &ledger.starts { - if ledger.outcome_of(&start.order_id).is_none() { - conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; - } - } - Ok(ledger - .orders - .iter() - .filter(|order| { - ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() - }) - .map(|order| order.order_id.clone()) - .collect()) -} - -// ---------------------------------------------------------------- posting -- - -struct Due { - order_id: String, - kind: PostKind, - text: String, -} - -fn quoted(reason: &str) -> String { - format!("\"{}\"", escape(reason.as_bytes())) -} - -fn accepted_text(order: &OrderRecord) -> String { - format!( - "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", - order.order_id, - escape(order.operation.describe().as_bytes()), - escape(order.issuer_unit.as_bytes()), - escape(order.issuer_invocation.as_bytes()), - quoted(&order.reason) - ) -} - -/// A deploy's command has finished once its state is neither queued nor running. -fn deploy_finished(state: &str) -> bool { - !matches!(state, "queued" | "running") -} - -/// What is owed to the room. `deploy_state` is a deployment command's derived -/// state and detail, read from the control store by the caller. -fn due_posts( - ledger: &Ledger, - deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, -) -> Vec { - let mut due = Vec::new(); - for order in &ledger.orders { - let id = &order.order_id; - if !ledger.posted(id, PostKind::Accepted) { - due.push(Due { - order_id: id.clone(), - kind: PostKind::Accepted, - text: accepted_text(order), - }); - } - if ledger.posted(id, PostKind::Result) { - continue; - } - let Some(record) = ledger.outcome_of(id) else { - continue; - }; - let mut line = record.outcome.describe(); - if let Outcome::Submitted { command_id } = &record.outcome { - match deploy_state(command_id) { - Ok((state, detail)) if deploy_finished(state) => { - line = format!("{line}: {state} {}", escape(detail.as_bytes())); - } - _ => continue, - } - } - due.push(Due { - order_id: id.clone(), - kind: PostKind::Result, - text: format!( - "Steward order {id}: {line}\n{}", - escape(order.operation.describe().as_bytes()) - ), - }); - } - due -} - -/// Post what is due, in order, stopping at the first failure. Returns how many -/// went out. -pub(crate) fn post_pass(ctx: &Ctx) -> Result { - let ledger = ctx.store.ledger()?; - let mut sent = 0; - let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); - for due in due_posts(&ledger, &deploy_state) { - let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); - let event_id = ctx.homeserver.post(&transaction, &due.text)?; - ctx.store.put( - &post_key(&due.order_id, due.kind), - POST_SCHEMA, - &PostRecord { - schema_version: POST_SCHEMA.into(), - post_key: post_key(&due.order_id, due.kind), - event_id, - posted_at_unix_millis: now_millis()?, - }, - )?; - sent += 1; - } - Ok(sent) -} - -// ----------------------------------------------------------------- frames -- - -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct OrderRequest { - pub(crate) operation: Operation, - pub(crate) reason: String, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ShowRequest { - pub(crate) id: String, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Request { - Order(OrderRequest), - Show(ShowRequest), - List, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct OrderView { - pub(crate) order: OrderRecord, - pub(crate) start: Option, - pub(crate) outcome: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct OrderLine { - pub(crate) order_id: String, - pub(crate) status: String, - pub(crate) summary: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Response { - Accepted { order_id: String }, - Refused(Refusal), - Show(Box), - List(Vec), -} - -fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { - let bytes = rmp_serde::to_vec_named(value)?; - stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; - stream.write_all(&bytes)?; - Ok(stream.flush()?) -} - -fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { - let mut length = [0u8; 4]; - stream.read_exact(&mut length)?; - let length = u32::from_be_bytes(length) as usize; - ensure!(length <= limit, "a frame exceeds its limit"); - let mut bytes = vec![0u8; length]; - stream.read_exact(&mut bytes)?; - Ok(bytes) -} - -fn status_word(view: &OrderView) -> &'static str { - match (&view.outcome, &view.start) { - (Some(record), _) => match record.outcome { - Outcome::Ran => "ran", - Outcome::Failed { .. } => "failed", - Outcome::Refused(_) => "refused", - Outcome::Lost => "lost", - Outcome::Submitted { .. } => "submitted", - }, - (None, Some(_)) => "running", - (None, None) => "accepted", - } -} - -fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { - OrderView { - order: order.clone(), - start: ledger.start_of(&order.order_id).cloned(), - outcome: ledger.outcome_of(&order.order_id).cloned(), - } -} - -fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { - write_frame(stream, &response) -} - -fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { - answer(stream, Response::Refused(Refusal::new(field, code))) -} - -/// Serve one connection. The peer is authenticated before its frame is read. -pub(crate) fn handle( - ctx: &Ctx, - stream: &mut UnixStream, - queue: &mpsc::Sender, -) -> Result<()> { - stream.set_read_timeout(Some(Duration::from_secs(5)))?; - stream.set_write_timeout(Some(Duration::from_secs(5)))?; - let peer = ctx.peers.peer(stream)?; - // The steward may order and read; root, the operator, may only read. - let steward = peer.uid == ctx.steward_uid; - if !steward && peer.uid != 0 { - eprintln!("steward socket: refused a peer that is not the steward"); - return refuse(stream, "peer", Code::Unauthorized); - } - let request = read_frame(stream, REQUEST_FRAME_LIMIT) - .ok() - .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); - let Some(request) = request else { - eprintln!("steward socket: refused a malformed frame"); - return refuse(stream, "frame", Code::Malformed); - }; - match request { - Request::Order(_) if !steward => { - eprintln!("steward socket: refused an order from a peer that is not the steward"); - refuse(stream, "peer", Code::Unauthorized) - } - Request::Order(request) => { - let checked = - validate_reason(&request.reason).and_then(|()| request.operation.validate()); - if let Err(refusal) = checked { - eprintln!("steward socket: refused an order at the grammar"); - return answer(stream, Response::Refused(refusal)); - } - let order = OrderRecord { - schema_version: ORDER_SCHEMA.into(), - order_id: format!("ord-{}", Uuid::new_v4()), - principal: ctx.principal.clone(), - issuer_uid: peer.uid, - issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), - issuer_unit: peer.issuer_unit, - issued_at_unix_millis: now_millis()?, - operation: request.operation, - reason: request.reason, - }; - match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { - Ok(true) => {} - _ => return refuse(stream, "store", Code::Unavailable), - } - queue - .send(order.order_id.clone()) - .map_err(|_| anyhow!("the executor has stopped"))?; - answer( - stream, - Response::Accepted { - order_id: order.order_id, - }, - ) - } - Request::Show(request) => { - if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) - { - return refuse(stream, "id", Code::BadShape); - } - let ledger = ctx.store.ledger()?; - match ledger.orders.iter().find(|order| order.order_id == request.id) { - Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), - None => refuse(stream, "id", Code::Unknown), - } - } - Request::List => { - let ledger = ctx.store.ledger()?; - let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); - let lines = ledger - .orders - .iter() - .skip(skip) - .map(|order| { - let view = view_of(&ledger, order); - OrderLine { - order_id: order.order_id.clone(), - status: status_word(&view).into(), - summary: order.operation.describe(), - } - }) - .collect(); - answer(stream, Response::List(lines)) - } - } -} - -// ------------------------------------------------------------- real ports -- - -/// `systemctl` and `systemd-run`, found by name. -pub(crate) struct HostSystemd; - -/// The names in one `systemctl show` list value. systemd quotes a name that -/// holds a backslash and doubles the backslash inside the quotes. -fn parse_names(value: &str) -> Vec { - let mut names = Vec::new(); - let mut current = String::new(); - let (mut quoted, mut open, mut escaped) = (false, false, false); - for character in value.chars() { - match character { - _ if escaped => { - current.push(character); - escaped = false; - } - '\\' if quoted => escaped = true, - '"' => { - quoted = !quoted; - open = true; - } - _ if character.is_whitespace() && !quoted => { - if open { - names.push(std::mem::take(&mut current)); - open = false; - } - } - _ => { - current.push(character); - open = true; - } - } - } - if open { - names.push(current); - } - names -} - -/// The properties one `systemctl show` is asked for: identity, what fires at -/// the unit's end, who is bound to it, and what it upholds or groups. -fn show_properties() -> String { - ["Id", "Names", "LoadState"] - .into_iter() - .chain(ACTIONS) - .chain(ON_END) - .chain(["BoundBy", "Upholds", "ConsistsOf"]) - .collect::>() - .join(",") -} - -fn parse_show(output: &str) -> Result { - let mut facts = UnitFacts::default(); - for line in output.lines() { - let Some((key, value)) = line.split_once('=') else { - continue; - }; - match key { - "Id" => facts.id = value.to_owned(), - "Names" => facts.names = parse_names(value), - "LoadState" => facts.not_found = value == "not-found", - "BoundBy" => facts.bound_by = parse_names(value), - "Upholds" => facts.upholds = parse_names(value), - "ConsistsOf" => facts.consists_of = parse_names(value), - _ if ACTIONS.contains(&key) => { - if !matches!(value, "" | "none") { - facts.on_end.push(key.to_owned()); - } - } - _ if ON_END.contains(&key) => { - facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); - } - _ => {} - } - } - ensure!(!facts.id.is_empty(), "systemctl named no unit"); - Ok(facts) -} - -impl Systemd for HostSystemd { - fn show(&self, unit: &str) -> Result { - let output = Command::new("systemctl") - .args(["show", "--no-pager"]) - .arg(format!("--property={}", show_properties())) - .args(["--", unit]) - .stdin(Stdio::null()) - .output() - .context("running systemctl show")?; - ensure!(output.status.success(), "systemctl show failed"); - parse_show(&String::from_utf8_lossy(&output.stdout)) - } - - fn invocation(&self, unit: &str) -> Option { - let output = Command::new("systemctl") - .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) - .arg(unit) - .stdin(Stdio::null()) - .output() - .ok()?; - let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - (output.status.success() && !value.is_empty()).then_some(value) - } - - fn run(&self, unit: &str, step: &Step) -> Result { - let mut command = Command::new("systemd-run"); - command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); - command.args(["-p", "RuntimeMaxSec=15min"]); - for (name, value) in &step.environment { - command.arg(format!("--setenv={name}={value}")); - } - let output = command - .arg("--") - .args(&step.argv) - .stdin(Stdio::null()) - .output() - .context("running systemd-run")?; - Ok(RunReport { - exit_code: output.status.code(), - signal: output.status.signal(), - stdout_tail: tail(&output.stdout), - stderr_tail: tail(&output.stderr), - }) - } -} - -/// A system service or scope names the issuer: the first unit under -/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices -/// above it are the manager's, and what a delegated subtree adds below it is -/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. -fn unit_from_cgroup(text: &str) -> String { - let line = text - .lines() - .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) - .unwrap_or_default(); - let path = line.rsplit_once(':').map_or("", |(_, path)| path); - let unit = path.strip_prefix("/system.slice/").and_then(|under| { - for segment in under.split('/') { - if segment.ends_with(".slice") { - continue; - } - let unit = segment.ends_with(".service") || segment.ends_with(".scope"); - return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); - } - None - }); - unit.map_or_else(|| "unknown".to_owned(), str::to_owned) -} - -/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. -pub(crate) struct KernelPeers; - -impl PeerCredentials for KernelPeers { - fn peer(&self, stream: &UnixStream) -> Result { - use std::os::fd::AsRawFd; - let mut credentials = libc::ucred { - pid: 0, - uid: 0, - gid: 0, - }; - let mut length = std::mem::size_of::() as libc::socklen_t; - // SAFETY: the descriptor is a live socket and the pointer and length - // describe `credentials`, which outlives the call. - let status = unsafe { - libc::getsockopt( - stream.as_raw_fd(), - libc::SOL_SOCKET, - libc::SO_PEERCRED, - (&mut credentials as *mut libc::ucred).cast(), - &mut length, - ) - }; - ensure!( - status == 0 && length as usize == std::mem::size_of::(), - "peer credentials are unavailable" - ); - let cgroup = - std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); - Ok(Peer { - uid: credentials.uid, - issuer_unit: unit_from_cgroup(&cgroup), - }) - } -} - -/// `mind-control --instance check`; a missing binary is a held brake. -pub(crate) struct MindControl { - pub(crate) program: PathBuf, -} - -impl BrakeCheck for MindControl { - fn clear(&self, principal: &str) -> bool { - Command::new(&self.program) - .args(["--instance", principal, "check"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .is_ok_and(|status| status.success()) - } -} - -/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is -/// never chunked. The token is read from its credential file for each post. -pub(crate) struct Synapse { - pub(crate) address: std::net::SocketAddr, - pub(crate) room: String, - pub(crate) token_file: PathBuf, - /// The whole wait for a reply, however slowly it drips. - pub(crate) reply_within: Duration, -} - -fn percent_encode(text: &str) -> String { - let mut encoded = String::new(); - for byte in text.bytes() { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { - encoded.push(char::from(byte)); - } else { - encoded.push_str(&format!("%{byte:02X}")); - } - } - encoded -} - -impl Homeserver for Synapse { - fn post(&self, txn_id: &str, text: &str) -> Result { - let token = std::fs::read_to_string(&self.token_file) - .context("the homeserver token is unreadable")?; - let token = token.trim(); - ensure!( - !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), - "the homeserver token is not a bearer token" - ); - let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; - let head = format!( - "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ - Host: {}\r\nAuthorization: Bearer {token}\r\n\ - Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", - percent_encode(&self.room), - self.address, - body.len() - ); - let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) - .context("the homeserver is unreachable")?; - socket.set_write_timeout(Some(self.reply_within))?; - socket.write_all(head.as_bytes())?; - socket.write_all(&body)?; - let deadline = std::time::Instant::now() + self.reply_within; - let mut reply = Vec::new(); - let mut chunk = [0u8; 4096]; - while reply.len() < RESPONSE_FRAME_LIMIT { - let remaining = deadline - .checked_duration_since(std::time::Instant::now()) - .filter(|left| !left.is_zero()) - .context("the homeserver reply was too slow")?; - socket.set_read_timeout(Some(remaining))?; - let count = socket - .read(&mut chunk) - .context("the homeserver reply was cut short")?; - if count == 0 { - break; - } - reply.extend_from_slice(&chunk[..count]); - } - let split = reply - .windows(4) - .position(|window| window == b"\r\n\r\n") - .context("the homeserver reply has no body")?; - let status = String::from_utf8_lossy(&reply[..split]); - ensure!( - status.split_whitespace().nth(1) == Some("200"), - "the homeserver refused the post" - ); - #[derive(Deserialize)] - struct Sent { - event_id: String, - } - let sent: Sent = serde_json::from_slice(&reply[split + 4..]) - .map_err(|_| anyhow!("the homeserver reply has no event id"))?; - Ok(sent.event_id) - } -} - -// ---------------------------------------------------------------- binding -- - -/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. -/// It lists no unit of any kind, and any field outside this set is refused. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct StewardBinding { - pub(crate) principal: String, - pub(crate) user: String, - pub(crate) mind_control: PathBuf, - pub(crate) room: String, - pub(crate) homeserver: String, - pub(crate) token_credential: String, -} - -impl StewardBinding { - pub(crate) fn parse(text: &str) -> Result { - let binding: Self = toml::from_str(text) - .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; - binding.validate()?; - Ok(binding) - } - - fn validate(&self) -> Result<()> { - require_id(&self.principal, "steward principal")?; - ensure!( - !self.principal.starts_with('-') && !self.principal.contains('/'), - "steward principal is invalid" - ); - let user_ok = !self.user.is_empty() - && self.user.len() <= 32 - && self.user.bytes().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') - }) - && !self.user.starts_with(|first: char| first.is_ascii_digit() || first == '-'); - ensure!(user_ok && self.user != "root", "steward user is invalid or root"); - ensure!( - self.mind_control.is_absolute(), - "mind_control must be an absolute path" - ); - let room_ok = self.room.starts_with('!') - && self.room.len() <= 255 - && self.room.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') - }); - ensure!(room_ok, "room must be a Matrix room id"); - self.address()?; - let credential_ok = !self.token_credential.is_empty() - && !self.token_credential.starts_with('.') - && self.token_credential.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') - }); - ensure!(credential_ok, "token_credential is not a credential name"); - Ok(()) - } - - fn address(&self) -> Result { - let address: std::net::SocketAddr = self - .homeserver - .strip_prefix("http://") - .and_then(|rest| rest.parse().ok()) - .context("homeserver must be http://:")?; - ensure!(address.ip().is_loopback(), "homeserver must be loopback"); - Ok(address) - } -} - -/// Root-admitted: owned by root and writable by no one else. -fn admitted(owner: u32, mode: u32) -> bool { - owner == 0 && mode & 0o022 == 0 -} - -fn load_binding(path: &Path) -> Result { - let metadata = std::fs::metadata(path).context("reading the steward binding")?; - ensure!( - metadata.is_file() && admitted(metadata.uid(), metadata.mode()), - "the steward binding is not a root-owned file closed to writes" - ); - let text = std::fs::read_to_string(path).context("reading the steward binding")?; - StewardBinding::parse(&text) -} - -fn user_uid(name: &str) -> Result { - let name = std::ffi::CString::new(name)?; - // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in - // `buffer`, which outlives every read of them below (none are read). - let (status, found, uid) = unsafe { - let mut passwd: libc::passwd = std::mem::zeroed(); - let mut buffer = vec![0 as libc::c_char; 4096]; - let mut result: *mut libc::passwd = std::ptr::null_mut(); - let status = libc::getpwnam_r( - name.as_ptr(), - &mut passwd, - buffer.as_mut_ptr(), - buffer.len(), - &mut result, - ); - (status, !result.is_null(), passwd.pw_uid) - }; - ensure!(status == 0 && found, "the steward user is unknown"); - ensure!(uid != 0, "the steward user resolves to root"); - Ok(uid) -} - -// ------------------------------------------------------------------ serve -- - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct ServeOptions { - pub(crate) state_store: PathBuf, - pub(crate) store: PathBuf, - pub(crate) steward_binding: PathBuf, - pub(crate) socket: PathBuf, -} - -const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; -const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; -const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; - -pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { - ensure!( - args.next().as_deref() == Some("serve"), - "idunn orders takes the subcommand serve" - ); - let mut options = ServeOptions { - state_store: default_state_store(), - store: DEFAULT_STORE.into(), - steward_binding: DEFAULT_BINDING.into(), - socket: DEFAULT_SOCKET.into(), - }; - while let Some(argument) = args.next() { - match argument.as_str() { - "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, - "--store" => options.store = path_value(&mut args, "--store")?, - "--steward-binding" => { - options.steward_binding = path_value(&mut args, "--steward-binding")? - } - "--socket" => options.socket = path_value(&mut args, "--socket")?, - _ => bail!("an orders serve option is not part of the grammar"), - } - } - Ok(options) -} - -/// Seconds between post passes: two when the last pass went through, and -/// doubling for each failed pass in a row up to a minute. -fn post_pause(failures: u32) -> u64 { - if failures == 0 { - 2 - } else { - (2u64 << failures.min(5)).min(60) - } -} - -/// Run a thread whose end ends the service: systemd restarts it, and the -/// start recovery settles what it left. -fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { - struct ExitOnDrop(&'static str); - impl Drop for ExitOnDrop { - fn drop(&mut self) { - eprintln!("steward orders: the {} thread ended", self.0); - std::process::exit(1); - } - } - thread::Builder::new() - .name(name.into()) - .spawn(move || { - let _guard = ExitOnDrop(name); - work(); - }) - .expect("a thread can be spawned"); -} - -/// The socket belongs to the steward and no one else may open it; the peer -/// credential check on every connection stays the lock. -fn bind_socket(path: &Path, steward_uid: u32) -> Result { - if let Ok(metadata) = std::fs::symlink_metadata(path) { - use std::os::unix::fs::FileTypeExt; - ensure!( - metadata.file_type().is_socket(), - "the socket path holds something that is not a socket" - ); - std::fs::remove_file(path).context("removing a stale socket")?; - } - let listener = UnixListener::bind(path).context("binding the steward socket")?; - std::os::unix::fs::chown(path, Some(steward_uid), None) - .context("giving the steward socket to the steward")?; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) - .context("setting the steward socket mode")?; - Ok(listener) -} - -/// Files this service creates in Idunn's own stores (a deploy order writes -/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. -const IDUNN_UMASK: libc::mode_t = 0o027; - -pub(crate) fn serve(options: ServeOptions) -> Result<()> { - // SAFETY: umask only sets the process file-mode mask; it cannot fail. - unsafe { libc::umask(IDUNN_UMASK) }; - let binding = load_binding(&options.steward_binding)?; - let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") - .map(PathBuf::from) - .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; - let ctx = Arc::new(Ctx { - store: OrderStore::new(&options.store), - state_store: options.state_store, - steward_uid: user_uid(&binding.user)?, - principal: binding.principal.clone(), - systemd: Arc::new(HostSystemd), - peers: Arc::new(KernelPeers), - brake: Arc::new(MindControl { - program: binding.mind_control.clone(), - }), - homeserver: Arc::new(Synapse { - address: binding.address()?, - room: binding.room.clone(), - token_file: credentials.join(&binding.token_credential), - reply_within: Duration::from_secs(10), - }), - }); - ctx.store.prepare()?; - let pending = recover(&ctx)?; - let listener = bind_socket(&options.socket, ctx.steward_uid)?; - let (queue, orders) = mpsc::channel::(); - for order_id in pending { - queue.send(order_id)?; - } - let executor = Arc::clone(&ctx); - vital("executor", move || { - for order_id in orders { - if let Err(error) = execute(&executor, &order_id) { - eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); - } - } - }); - let poster = Arc::clone(&ctx); - vital("poster", move || { - let mut failures = 0u32; - loop { - match post_pass(&poster) { - Ok(_) => failures = 0, - Err(error) => { - failures = failures.saturating_add(1); - eprintln!("steward orders: posting is delayed: {error:#}"); - } - } - thread::sleep(Duration::from_secs(post_pause(failures))); - } - }); - for stream in listener.incoming() { - match stream { - Ok(mut stream) => { - if let Err(error) = handle(&ctx, &mut stream, &queue) { - eprintln!("steward socket: a connection ended in error: {error:#}"); - } - } - Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), - } - } - bail!("the steward socket stopped accepting") -} - -// ----------------------------------------------------------------- client -- - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum OrderAction { - Issue { operation: Operation, reason: String }, - Show { id: String }, - List, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct OrderCommand { - pub(crate) socket: PathBuf, - pub(crate) action: OrderAction, -} - -pub(crate) fn parse_order(mut args: impl Iterator) -> Result { - let kind = args.next().context("idunn order requires an operation")?; - let mut socket = PathBuf::from(DEFAULT_SOCKET); - let mut reason = None; - let mut positional = Vec::new(); - while let Some(argument) = args.next() { - match argument.as_str() { - "--socket" => socket = path_value(&mut args, "--socket")?, - "--reason" => reason = Some(string_value(&mut args, "--reason")?), - word if word.starts_with('-') => bail!("an order option is not part of the grammar"), - _ => positional.push(argument), - } - } - let action = match kind.as_str() { - "list" => { - ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); - OrderAction::List - } - "show" => { - ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); - OrderAction::Show { - id: positional.remove(0), - } - } - "package-install" | "unit" | "deploy" => { - let operation = match kind.as_str() { - "package-install" => Operation::PackageInstall { - packages: positional, - }, - "unit" => { - ensure!(positional.len() == 2, "unit takes a verb and a unit"); - let verb = Verb::parse(&positional[0]) - .context("the unit verb is not start, stop, restart or reload")?; - Operation::UnitAction { - verb, - unit: positional.remove(1), - } - } - _ => { - ensure!(positional.len() == 1, "deploy takes one selector"); - Operation::Deploy { - selector: positional.remove(0), - } - } - }; - let reason = reason.context("an order requires --reason")?; - validate_reason(&reason)?; - operation.validate()?; - OrderAction::Issue { operation, reason } - } - _ => bail!("the order operation is not part of the grammar"), - }; - Ok(OrderCommand { socket, action }) -} - -fn call(socket: &Path, request: &Request) -> Result { - let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; - stream.set_read_timeout(Some(Duration::from_secs(30)))?; - write_frame(&mut stream, request)?; - let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; - rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) -} - -fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { - if bytes.is_empty() { - return; - } - lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); - let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); - lines.extend(text.split(|byte| *byte == b'\n').map(|line| { - let mut shown = b" | ".to_vec(); - shown.extend_from_slice(line); - shown - })); -} - -/// Raw lines for the terminal; `emit` is their only way out. -fn render_view(view: &OrderView) -> Vec> { - let order = &view.order; - let mut lines: Vec> = vec![ - format!("order {}", order.order_id).into_bytes(), - format!("status {}", status_word(view)).into_bytes(), - format!("operation {}", order.operation.describe()).into_bytes(), - format!( - "issuer {} (invocation {}, uid {})", - order.issuer_unit, order.issuer_invocation, order.issuer_uid - ) - .into_bytes(), - format!("reason {}", quoted(&order.reason)).into_bytes(), - ]; - if let Some(start) = &view.start { - lines.push(format!("started {}", start.actuation).into_bytes()); - } - if let Some(record) = &view.outcome { - lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); - render_tail(&mut lines, "stdout", &record.stdout_tail); - render_tail(&mut lines, "stderr", &record.stderr_tail); - } - lines -} - -pub(crate) fn run_order(command: OrderCommand) -> Result<()> { - let request = match command.action { - OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), - OrderAction::Show { id } => Request::Show(ShowRequest { id }), - OrderAction::List => Request::List, - }; - let mut out = std::io::stdout().lock(); - match call(&command.socket, &request)? { - Response::Accepted { order_id } => writeln!(out, "{order_id}")?, - Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), - Response::Show(view) => emit(&mut out, &render_view(&view))?, - Response::List(lines) => { - let lines: Vec> = lines - .iter() - .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) - .collect(); - emit(&mut out, &lines)? - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - use std::sync::Mutex; - use std::sync::atomic::{AtomicBool, Ordering}; - - use tempfile::TempDir; - - use super::*; - - const STEWARD: u32 = 1000; - const CANARY: &str = "CANARY-7f3a91"; - - /// What `systemctl show` prints for a loaded unit that acts on nothing, - /// plus the given properties (a given key replaces its default). - fn show_text(id: &str, properties: &[(&str, &str)]) -> String { - let mut text = format!("Id={id}\n"); - for (key, default) in [("Names", id), ("LoadState", "loaded")] { - if !properties.iter().any(|(given, _)| *given == key) { - text += &format!("{key}={default}\n"); - } - } - for (key, value) in properties { - text += &format!("{key}={value}\n"); - } - text - } - - struct FakeSystemd { - units: Mutex>, - runs: Mutex>, - shows: Mutex>, - report: Mutex, - } - - impl Systemd for FakeSystemd { - fn show(&self, unit: &str) -> Result { - self.shows.lock().unwrap().push(unit.into()); - let known = self.units.lock().unwrap().get(unit).cloned(); - parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) - } - - fn invocation(&self, _unit: &str) -> Option { - Some("invocation-1".into()) - } - - fn run(&self, unit: &str, step: &Step) -> Result { - self.runs.lock().unwrap().push((unit.into(), step.clone())); - Ok(self.report.lock().unwrap().clone()) - } - } - - struct FakePeers(u32); - - impl PeerCredentials for FakePeers { - fn peer(&self, _stream: &UnixStream) -> Result { - Ok(Peer { - uid: self.0, - issuer_unit: "mind-eureka-self.service".into(), - }) - } - } - - /// Whether the brake is clear, and the principals it was asked about. - struct FakeBrake(AtomicBool, Mutex>); - - impl BrakeCheck for FakeBrake { - fn clear(&self, principal: &str) -> bool { - self.1.lock().unwrap().push(principal.into()); - self.0.load(Ordering::SeqCst) - } - } - - #[derive(Default)] - struct FakeHome { - failing: AtomicBool, - sent: Mutex>, - } - - impl Homeserver for FakeHome { - fn post(&self, txn_id: &str, text: &str) -> Result { - ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); - let mut sent = self.sent.lock().unwrap(); - sent.push((txn_id.into(), text.into())); - Ok(format!("$event{}", sent.len())) - } - } - - struct Harness { - directory: TempDir, - ctx: Ctx, - systemd: Arc, - brake: Arc, - home: Arc, - queue: mpsc::Sender, - _orders: mpsc::Receiver, - } - - fn harness_for(peer_uid: u32) -> Harness { - let directory = TempDir::new().unwrap(); - let systemd = Arc::new(FakeSystemd { - units: Mutex::new(BTreeMap::new()), - runs: Mutex::new(Vec::new()), - shows: Mutex::new(Vec::new()), - report: Mutex::new(RunReport { - exit_code: Some(0), - ..RunReport::default() - }), - }); - let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); - let home = Arc::new(FakeHome::default()); - let (queue, orders) = mpsc::channel(); - let ctx = Ctx { - store: OrderStore::new(&directory.path().join("orders")), - state_store: directory.path().join("state").join("control.cc"), - steward_uid: STEWARD, - principal: "eureka".into(), - systemd: systemd.clone(), - peers: Arc::new(FakePeers(peer_uid)), - brake: brake.clone(), - homeserver: home.clone(), - }; - Harness { - directory, - ctx, - systemd, - brake, - home, - queue, - _orders: orders, - } - } - - fn harness() -> Harness { - harness_for(STEWARD) - } - - impl Harness { - fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { - let (mut client, mut server) = UnixStream::pair().unwrap(); - client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); - write(&mut client); - handle(&self.ctx, &mut server, &self.queue).unwrap(); - let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); - rmp_serde::from_slice(&bytes).unwrap() - } - - fn ask(&self, request: &Request) -> Response { - self.exchange(|client| write_frame(client, request).unwrap()) - } - - fn issue(&self, operation: Operation) -> Response { - self.ask(&Request::Order(OrderRequest { - operation, - reason: "probe".into(), - })) - } - - fn issue_unit(&self, verb: Verb, unit: &str) -> Response { - self.issue(Operation::UnitAction { - verb, - unit: unit.into(), - }) - } - - fn accept(&self, response: Response) -> String { - match response { - Response::Accepted { order_id } => order_id, - other => panic!("expected an accepted order, got {other:?}"), - } - } - - fn outcome(&self, order_id: &str) -> Outcome { - self.ctx - .store - .ledger() - .unwrap() - .outcome_of(order_id) - .expect("the order has an outcome") - .outcome - .clone() - } - - fn runs(&self) -> Vec<(String, Step)> { - self.systemd.runs.lock().unwrap().clone() - } - - /// Tell the fake `systemctl show` what it prints for `name`. - fn describe_unit(&self, name: &str, text: String) { - self.systemd.units.lock().unwrap().insert(name.into(), text); - } - - fn describe(&self, name: &str, properties: &[(&str, &str)]) { - self.describe_unit(name, show_text(name, properties)); - } - } - - fn all_verbs() -> [Verb; 4] { - [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] - } - - #[test] - fn foreign_uid_is_refused_at_the_door() { - let h = harness_for(STEWARD + 1); - let response = h.issue_unit(Verb::Restart, "nginx.service"); - assert_eq!( - response, - Response::Refused(Refusal::new("peer", Code::Unauthorized)) - ); - let response = h.ask(&Request::List); - assert_eq!( - response, - Response::Refused(Refusal::new("peer", Code::Unauthorized)) - ); - assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); - assert!(h.runs().is_empty()); - } - - #[derive(Serialize)] - struct ForgedBody { - operation: Operation, - reason: String, - #[serde(skip_serializing_if = "Option::is_none")] - order_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - requested_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - command: Option, - } - - #[derive(Serialize)] - struct ForgedFrame { - order: ForgedBody, - } - - #[test] - fn order_id_and_requester_are_minted_not_declared() { - let h = harness(); - let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { - let frame = ForgedFrame { - order: ForgedBody { - operation: Operation::UnitAction { - verb: Verb::Restart, - unit: "nginx.service".into(), - }, - reason: "probe".into(), - order_id: order_id.map(str::to_owned), - requested_by: requested_by.map(str::to_owned), - command: command.map(str::to_owned), - }, - }; - h.exchange(|client| write_frame(client, &frame).unwrap()) - }; - // The same frame without the extra field is a good order. - let honest = h.accept(forge(None, None, None)); - for forged in [ - forge(Some("ord-declared"), None, None), - forge(None, Some("steward:someone-else order x"), None), - forge(None, None, Some("reboot-now")), - ] { - assert_eq!( - forged, - Response::Refused(Refusal::new("frame", Code::Malformed)) - ); - } - let ledger = h.ctx.store.ledger().unwrap(); - assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); - let order = &ledger.orders[0]; - assert_eq!(order.order_id, honest); - let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); - assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); - assert_eq!(order.issuer_unit, "mind-eureka-self.service"); - assert_eq!(order.issuer_invocation, "invocation-1"); - assert_eq!(order.principal, "eureka"); - assert_eq!(order.issuer_uid, STEWARD); - assert_eq!( - requester(&order.principal, &order.order_id), - format!("steward:eureka order {}", order.order_id) - ); - } - - #[test] - fn package_names_outside_grammar_are_refused() { - let h = harness(); - let bad = [ - "-y", - "--allow-unauthenticated", - "-", - "a", - "", - "Foo", - "a/b", - "/tmp/x", - "./x", - "x.deb", - "foo.deb", - "foo bar", - "foo;reboot", - "foo$(id)", - "foo`id`", - "foo|bar", - "foo&bar", - "foo\nbar", - "foo*", - "foo?", - "pkg:amd64", - "pkg/stable", - "libc6-", - "foo=", - "foo=1 2", - "foo=1;2", - "foo=1=2", - "fo\u{ff0f}o", - "f\u{43e}o", - ]; - for name in bad { - let response = h.issue(Operation::PackageInstall { - packages: vec!["git-lfs".into(), name.into()], - }); - assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); - } - let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); - assert_eq!( - h.issue(Operation::PackageInstall { packages: many }), - Response::Refused(Refusal::new("package", Code::TooMany)) - ); - assert_eq!( - h.issue(Operation::PackageInstall { packages: vec![] }), - Response::Refused(Refusal::new("package", Code::Empty)) - ); - assert!(!h.ctx.store.path().exists(), "no refusal left a record"); - - let good = [ - "git-lfs", - "matrix-synapse-py3", - "libnss-systemd", - "bubblewrap", - "socat", - "g++", - "libstdc++6", - "python3.11", - "nginx=1.24.0-2ubuntu7", - "foo=1:2.3-4+b1~rc1", - ]; - let id = h.accept(h.issue(Operation::PackageInstall { - packages: good.iter().map(|name| (*name).to_owned()).collect(), - })); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran); - let runs = h.runs(); - assert_eq!(runs.len(), 3); - assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); - // Each name selects exactly itself and apt may remove nothing. - let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; - install.extend(["--no-remove".to_owned(), "--".into()]); - install.extend(good.iter().map(|spec| match spec.split_once('=') { - Some((name, version)) => format!("?exact-name({name})={version}"), - None => format!("?exact-name({spec})"), - })); - assert_eq!(runs[1].1.argv, install); - assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); - assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); - // The last step fails unless every named package is installed. - let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; - present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); - assert_eq!(runs[2].1.argv, present); - for (unit, step) in &runs { - assert!(unit.starts_with("idunn-order-")); - let apt = step.argv[0] == "apt-get"; - assert_eq!( - step.environment, - if apt { - vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] - } else { - Vec::new() - } - ); - } - assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); - assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); - - // A package whose name apt would read as a regex or a conflict is - // exactly itself and never a removal. - for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { - let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); - assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); - } - // A step that failed ends the order: the absent package is a failure. - *h.systemd.report.lock().unwrap() = RunReport { - exit_code: Some(1), - ..RunReport::default() - }; - let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); - execute(&h.ctx, &absent).unwrap(); - assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); - assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); - } - - #[test] - fn any_unit_outside_the_refused_set_runs() { - let h = harness(); - let units = [ - "wg-quick@wg0.service", - "nginx.service", - "matrix-synapse.service", - "ssh.socket", - "huginn@eureka.service", - "claude-connector.service", - "persona-matrix-mouth.service", - "mind-heartbeat.service", - "apt-daily.timer", - // Near misses: the prefix is the rule, not a substring or a stem. - "midunn-x.service", - "not-idunn-x.service", - "idunn.service", - "mind-self.service", - "mind-selfx@a.service", - "mind-persona.service", - "mind-personal@a.service", - ]; - for verb in all_verbs() { - for unit in units { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); - let (transient, step) = h.runs().pop().unwrap(); - assert!(transient.starts_with("idunn-order-")); - assert_eq!( - step.argv, - ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] - ); - } - } - assert_eq!(h.runs().len(), units.len() * 4); - } - - #[test] - fn refused_set_is_refused_by_name_and_alias() { - let h = harness(); - h.describe_unit( - "backup-name.service", - show_text( - "idunn-yggdrasil.service", - &[("Names", "idunn-yggdrasil.service backup-name.service")], - ), - ); - h.describe( - "quiet-alias.service", - &[("Names", "quiet-alias.service mind-persona@void.service")], - ); - let direct = [ - "idunn-yggdrasil.service", - "idunn-orders.service", - "idunn-vili-x.service", - "idunn-order-3f2c.service", - "idunn-yggdrasil.timer", - "idunn-x.socket", - "idunn-.service", - "idunn-yggdrasil.service.service", - "idunn-x@y.service", - "mind-self@eureka.service", - "mind-persona@void.service", - "mind-self@.service", - "mind-persona@a@b.service", - // Case folds: a lookalike is still refused, never run. - "IDUNN-yggdrasil.service", - "Idunn-Orders.service", - "MIND-SELF@eureka.service", - "Mind-Persona@Void.service", - ]; - for verb in all_verbs() { - for unit in direct { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - match h.outcome(&id) { - Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { - assert_eq!(hit, unit, "{verb:?} {unit}"); - assert_eq!(via, Via::Name); - } - other => panic!("{verb:?} {unit} was {other:?}"), - } - } - } - for verb in all_verbs() { - for (unit, hit) in [ - ("backup-name.service", "idunn-yggdrasil.service"), - ("quiet-alias.service", "mind-persona@void.service"), - ] { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - assert_eq!( - h.outcome(&id), - Outcome::Refused(Refused::AlwaysRefused { - unit: hit.into(), - via: Via::Alias - }), - "{verb:?} {unit}" - ); - } - } - assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); - // Every refusal is recorded and has a result post due. - let ledger = h.ctx.store.ledger().unwrap(); - let refusals = ledger - .outcomes - .iter() - .filter(|record| matches!(record.outcome, Outcome::Refused(_))) - .count(); - assert_eq!(refusals, direct.len() * 4 + 8); - let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); - let results = due - .iter() - .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) - .count(); - assert_eq!(results, refusals); - // A refused order never started. - assert!(ledger.starts.is_empty()); - } - - fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { - let id = h.accept(h.issue_unit(verb, unit)); - execute(&h.ctx, &id).unwrap(); - h.outcome(&id) - } - - fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { - match outcome { - Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { - Some(unit.as_str()) - } - _ => None, - } - } - - #[test] - fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { - let h = harness(); - // systemctl show quotes a name with a backslash and doubles the backslash. - h.describe( - "huginn.service", - &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], - ); - let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); - assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); - h.describe( - "quiet.service", - &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], - ); - let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); - assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); - assert!(h.runs().is_empty()); - } - - #[test] - fn unit_orders_run_as_one_job() { - let h = harness(); - for verb in all_verbs() { - let id = h.accept(h.issue_unit(verb, "nginx.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); - let (_, step) = h.runs().pop().unwrap(); - assert_eq!( - step.argv, - ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] - ); - } - } - - #[test] - fn the_rule_reads_only_the_named_unit() { - let h = harness(); - // Whatever the unit lists, no other unit is looked at. - h.describe( - "plain.service", - &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], - ); - h.describe("other.service", &[("LoadState", "not-found")]); - for verb in all_verbs() { - assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); - } - assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); - } - - #[test] - fn a_unit_that_acts_when_it_ends_is_refused() { - let h = harness(); - let properties = [ - ("SuccessAction", "reboot"), - ("FailureAction", "poweroff-force"), - ("StartLimitAction", "soft-reboot-immediate"), - ("JobTimeoutAction", "exit"), - ("FailureAction", "a-spelling-nobody-knows"), - ("OnFailure", "rescue.service"), - ("OnSuccess", "cleanup.service"), - ]; - for (property, value) in properties { - h.describe("ender.service", &[(property, value)]); - for verb in [Verb::Start, Verb::Stop, Verb::Restart] { - assert_eq!( - outcome_of_unit(&h, verb, "ender.service"), - Outcome::Refused(Refused::AlwaysRefused { - unit: "ender.service".into(), - via: Via::OnEnd - }), - "{verb:?} {property}={value}" - ); - } - assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); - } - // `none` and an empty list say the unit does nothing at its end. - h.describe( - "calm.service", - &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], - ); - for verb in all_verbs() { - assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); - } - assert_eq!(h.runs().len(), 11); - } - - #[test] - fn host_names_are_refused() { - let h = harness(); - let names = [ - "systemd-suspend.service", - "systemd-hibernate.service", - "systemd-hybrid-sleep.service", - "systemd-suspend-then-hibernate.service", - "rescue.service", - "emergency.service", - ]; - h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); - for verb in all_verbs() { - for name in names { - assert_eq!( - outcome_of_unit(&h, verb, name), - Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), - "{verb:?} {name}" - ); - } - assert_eq!( - outcome_of_unit(&h, verb, "sleepy.service"), - Outcome::Refused(Refused::AlwaysRefused { - unit: "systemd-hibernate.service".into(), - via: Via::Host - }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); - } - - #[test] - fn a_unit_others_are_bound_to_only_starts() { - let h = harness(); - h.describe("base.service", &[("BoundBy", "first.service second.service")]); - for verb in [Verb::Stop, Verb::Restart] { - assert_eq!( - outcome_of_unit(&h, verb, "base.service"), - Outcome::Refused(Refused::AlwaysRefused { - unit: "first.service".into(), - via: Via::Bound - }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - for verb in [Verb::Start, Verb::Reload] { - assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); - } - } - - #[test] - fn a_unit_that_upholds_others_is_refused_where_it_would_start() { - let h = harness(); - h.describe("holder.service", &[("Upholds", "held.service")]); - for verb in [Verb::Start, Verb::Restart] { - assert_eq!( - outcome_of_unit(&h, verb, "holder.service"), - Outcome::Refused(Refused::AlwaysRefused { - unit: "holder.service".into(), - via: Via::Upholds - }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - for verb in [Verb::Stop, Verb::Reload] { - assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); - } - } - - #[test] - fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { - let h = harness(); - h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); - for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { - assert_eq!( - outcome_of_unit(&h, verb, "wrapper.service"), - Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); - } - - #[test] - fn empty_dependency_lists_name_no_one() { - let h = harness(); - // systemctl prints every property it is asked for, empty or not. - h.describe( - "alone.service", - &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], - ); - for verb in all_verbs() { - assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); - } - } - - #[test] - fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { - let h = harness(); - h.describe("base.service", &[("BoundBy", "first.service")]); - h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); - assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); - assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); - h.systemd.shows.lock().unwrap().clear(); - assert_eq!( - hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), - Some("idunn-ghost.service") - ); - assert!(h.systemd.shows.lock().unwrap().is_empty()); - } - - #[test] - fn each_refusal_is_posted_under_its_own_reason() { - for (via, word) in [ - (Via::Name, "name"), - (Via::Alias, "alias"), - (Via::Host, "host"), - (Via::OnEnd, "on-end"), - (Via::Bound, "bound"), - (Via::Upholds, "upholds"), - ] { - assert_eq!( - Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) - .describe(), - format!("refused: always-refused unit u.service reached by {word}") - ); - } - assert_eq!( - Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), - "refused: wrapper unit, it only wraps m.service" - ); - } - - #[test] - fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { - let h = harness(); - h.describe("ghost.service", &[("LoadState", "not-found")]); - for verb in all_verbs() { - let id = h.accept(h.issue_unit(verb, "ghost.service")); - execute(&h.ctx, &id).unwrap(); - assert!( - matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), - "{verb:?}" - ); - } - assert!(h.runs().is_empty()); - } - - #[test] - fn globs_slices_scopes_and_targets_are_not_units() { - let h = harness(); - let bad = [ - "idunn*", - "idunn-*.service", - "idunn-?ggdrasil.service", - "mind-?elf@eureka.service", - "mind-self@*.service", - "mind-[s]elf@eureka.service", - "system.slice", - "poweroff.target", - "multi-user.target", - "user.slice", - "session-1.scope", - "init.scope", - "idunn-yggdrasil", - "mind-self@eureka", - "nginx", - ".service", - "", - "-idunn-x.service", - "-x.service", - "/etc/systemd/system/idunn-yggdrasil.service", - "../idunn-yggdrasil.service", - "idunn-yggdrasil.service\n", - "idunn-yggdrasil.service ", - " idunn-yggdrasil.service", - "idunn-yggdrasil.service\0", - "idunn-yggdrasil.service;reboot", - "idunn\\x2dyggdrasil.service", - "idunn%2dyggdrasil.service", - // Unicode lookalikes of a refused name. - "\u{131}dunn-yggdrasil.service", - "idunn\u{2010}yggdrasil.service", - "idunn\u{ff0d}yggdrasil.service", - "idunn-yggdrasil.service\u{200b}", - "\u{200b}idunn-yggdrasil.service", - "idunn-yggdrasil\u{2024}service", - "mind-self\u{ff20}eureka.service", - ]; - for unit in bad { - for verb in all_verbs() { - assert!( - matches!(h.issue_unit(verb, unit), Response::Refused(_)), - "{unit:?} was not refused by the grammar" - ); - } - } - let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); - assert_eq!( - h.issue_unit(Verb::Start, &long), - Response::Refused(Refusal::new("unit", Code::TooLong)) - ); - assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); - } - - #[test] - fn binding_with_a_unit_list_is_refused() { - let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ - mind_control = \"/usr/local/bin/mind-control\"\n\ - room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ - token_credential = \"idunn-matrix-token\"\n"; - let binding = StewardBinding::parse(valid).unwrap(); - assert_eq!(binding.principal, "eureka"); - assert_eq!(binding.address().unwrap().port(), 8008); - for extra in [ - "units = [\"nginx.service\"]", - "allow = [\"nginx.service\"]", - "deny = [\"idunn-*\"]", - "denylist = [\"idunn-*\"]", - "refused = [\"idunn-*\"]", - "unit = \"nginx.service\"", - "[units]\nnginx = true", - ] { - assert!( - StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), - "{extra} loaded" - ); - } - let swap = |from: &str, to: &str| valid.replace(from, to); - for broken in [ - swap("user = \"mind-eureka-self\"", "user = \"root\""), - swap("user = \"mind-eureka-self\"", "user = \"\""), - swap("/usr/local/bin/mind-control", "mind-control"), - swap("!abc:example.org", "#alias:example.org"), - swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), - swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), - swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), - swap("idunn-matrix-token", "../token"), - swap("principal = \"eureka\"\n", ""), - ] { - assert!(StewardBinding::parse(&broken).is_err()); - } - assert!(admitted(0, 0o100600)); - assert!(admitted(0, 0o100644)); - assert!(!admitted(0, 0o100664)); - assert!(!admitted(0, 0o100666)); - assert!(!admitted(0, 0o100602)); - assert!(!admitted(1000, 0o100600)); - } - - #[test] - fn held_brake_refuses_at_start() { - let h = harness(); - h.brake.0.store(false, Ordering::SeqCst); - let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); - assert!(h.runs().is_empty()); - // The deploy path is gated the same way and writes no command. - let deploy = h.accept(h.issue(Operation::Deploy { - selector: "vili".into(), - })); - execute(&h.ctx, &deploy).unwrap(); - assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); - assert!(!h.ctx.state_store.exists()); - // A brake that cannot be asked is a held brake. - let directory = TempDir::new().unwrap(); - assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); - } - - #[test] - fn brake_check_is_the_exit_status_of_mind_control_check() { - let directory = TempDir::new().unwrap(); - let program = |name: &str, body: &str| { - let path = directory.path().join(name); - std::fs::write(&path, body).unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); - MindControl { program: path } - }; - let recorder = directory.path().join("argv"); - let clear = program( - "clear", - &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), - ); - assert!(clear.clear("eureka")); - assert_eq!( - std::fs::read_to_string(&recorder).unwrap().trim(), - "--instance eureka check" - ); - assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); - } - - #[test] - fn started_order_is_never_rerun() { - let h = harness(); - let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); - // The first order began and its service died: a start with no outcome. - h.ctx - .store - .put( - &id, - START_SCHEMA, - &StartRecord { - schema_version: START_SCHEMA.into(), - order_id: id.clone(), - started_at_unix_millis: 1, - actuation: format!("idunn-order-{}", order_uuid(&id)), - }, - ) - .unwrap(); - let pending = recover(&h.ctx).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Lost); - assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); - execute(&h.ctx, &id).unwrap(); - assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); - // Recovery is idempotent, and a finished order is not run again. - assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); - execute(&h.ctx, &waiting).unwrap(); - execute(&h.ctx, &waiting).unwrap(); - assert_eq!(h.runs().len(), 1); - assert!(recover(&h.ctx).unwrap().is_empty()); - } - - #[test] - fn deploy_order_uses_the_up_primitive() { - let h = harness(); - let id = h.accept(h.issue(Operation::Deploy { - selector: "vili".into(), - })); - execute(&h.ctx, &id).unwrap(); - let command_id = format!("up-{}", order_uuid(&id)); - assert_eq!( - h.outcome(&id), - Outcome::Submitted { - command_id: command_id.clone() - } - ); - let ledger = h.ctx.store.ledger().unwrap(); - assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); - assert!(h.runs().is_empty(), "a deploy runs no transient unit"); - let mut shown = Vec::new(); - crate::control_plane::status( - &mut shown, - &h.ctx.state_store, - &h.directory.path().join("incidents.cc"), - Some(&command_id), - ) - .unwrap(); - let shown = String::from_utf8(shown).unwrap(); - assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); - assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); - // Re-executing writes no second command. - execute(&h.ctx, &id).unwrap(); - let mut again = Vec::new(); - crate::control_plane::status( - &mut again, - &h.ctx.state_store, - &h.directory.path().join("incidents.cc"), - None, - ) - .unwrap(); - assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); - // The result post waits for the transactions: queued is not finished. - let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); - assert_eq!(due.len(), 1); - assert_eq!(due[0].kind, PostKind::Accepted); - assert!(!deploy_finished("queued") && !deploy_finished("running")); - assert!(deploy_finished("succeeded") && deploy_finished("failed")); - assert!(deploy_finished("complete")); - } - - #[test] - fn chat_outage_delays_posts_not_orders() { - let h = harness(); - h.home.failing.store(true, Ordering::SeqCst); - let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Ran); - assert_eq!(h.runs().len(), 1); - assert!(post_pass(&h.ctx).is_err()); - assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); - h.home.failing.store(false, Ordering::SeqCst); - assert_eq!(post_pass(&h.ctx).unwrap(), 2); - assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); - let sent = h.home.sent.lock().unwrap().clone(); - assert_eq!(sent.len(), 2); - assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); - assert_eq!(sent[1].0, format!("idunn-{id}-result")); - assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); - assert!(sent[0].1.contains("mind-eureka-self.service")); - assert!(sent[0].1.contains("reason: \"probe\"")); - assert!(sent[1].1.contains("ran (exit 0)")); - } - - #[test] - fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { - let h = harness(); - *h.systemd.report.lock().unwrap() = RunReport { - exit_code: Some(100), - signal: None, - stdout_tail: vec![b'o'; 3 * TAIL_BYTES], - stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), - }; - let id = h.accept(h.issue(Operation::PackageInstall { - packages: vec!["git-lfs".into()], - })); - execute(&h.ctx, &id).unwrap(); - assert_eq!( - h.outcome(&id), - Outcome::Failed { - exit_code: Some(100), - signal: None - } - ); - assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); - let ledger = h.ctx.store.ledger().unwrap(); - let record = ledger.outcome_of(&id).unwrap(); - assert_eq!(record.stdout_tail.len(), TAIL_BYTES); - assert!(!format!("{record:?}").contains("secret-looking")); - // Neither the post nor the journal text carries a tail. - let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) - .into_iter() - .filter(|due| due.kind == PostKind::Result) - .collect(); - assert_eq!(results.len(), 1); - assert!(results[0].text.contains("failed (exit 100)")); - assert!(!results[0].text.contains("secret-looking")); - // The steward reads them through the escape sink and nothing raw. - let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { - panic!("expected a show") - }; - let mut shown = Vec::new(); - emit(&mut shown, &render_view(&view)).unwrap(); - let shown = String::from_utf8(shown).unwrap(); - assert!(shown.contains("secret-looking")); - assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); - assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); - assert!(shown.contains("status failed")); - // A show of an unknown or malformed id names the field only. - assert_eq!( - h.ask(&Request::Show(ShowRequest { - id: format!("ord-{}", Uuid::new_v4()) - })), - Response::Refused(Refusal::new("id", Code::Unknown)) - ); - assert_eq!( - h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), - Response::Refused(Refusal::new("id", Code::BadShape)) - ); - let Response::List(lines) = h.ask(&Request::List) else { - panic!("expected a list") - }; - assert_eq!(lines.len(), 1); - assert_eq!(lines[0].status, "failed"); - } - - #[test] - fn a_refusal_or_error_never_echoes_an_input_value() { - let h = harness(); - let mut seen: Vec> = Vec::new(); - let mut note = |response: Response| { - let bytes = rmp_serde::to_vec_named(&response).unwrap(); - if let Response::Refused(refusal) = &response { - seen.push(refusal.to_string().into_bytes()); - } - seen.push(bytes); - }; - for unit in [ - format!("{CANARY}*.service"), - format!("{CANARY}/../idunn-orders.service"), - format!("idunn-{CANARY};x.service"), - CANARY.to_string(), - ] { - note(h.issue_unit(Verb::Restart, &unit)); - } - note(h.issue(Operation::PackageInstall { - packages: vec![format!("{CANARY}/x")], - })); - note(h.issue(Operation::PackageInstall { - packages: vec![format!("pkg={CANARY} 1")], - })); - note(h.issue(Operation::Deploy { - selector: format!("{CANARY} bad"), - })); - note(h.ask(&Request::Order(OrderRequest { - operation: Operation::UnitAction { - verb: Verb::Start, - unit: "nginx.service".into(), - }, - reason: format!("{CANARY}\n"), - }))); - note(h.ask(&Request::Order(OrderRequest { - operation: Operation::UnitAction { - verb: Verb::Start, - unit: "nginx.service".into(), - }, - reason: CANARY.repeat(40), - }))); - note(h.ask(&Request::Show(ShowRequest { - id: format!("ord-{CANARY}"), - }))); - // A frame whose own keys and values are the canary, and plain garbage. - let named: std::collections::BTreeMap> = - [( - CANARY.to_owned(), - [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), - )] - .into_iter() - .collect(); - note(h.exchange(|client| write_frame(client, &named).unwrap())); - note(h.exchange(|client| { - let garbage = format!("{CANARY}{CANARY}"); - client - .write_all(&(garbage.len() as u32).to_be_bytes()) - .unwrap(); - client.write_all(garbage.as_bytes()).unwrap(); - })); - note(h.exchange(|client| { - client.write_all(&u32::MAX.to_be_bytes()).unwrap(); - })); - // The client grammar fails the same way. - let mut messages = Vec::new(); - let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); - for line in [ - words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), - words(&["unit", CANARY, "nginx.service", "--reason", "r"]), - words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), - words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), - words(&[CANARY]), - words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), - words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), - words(&["show"]), - ] { - messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); - } - for line in [ - words(&[CANARY]), - words(&["serve", &format!("--{CANARY}")]), - words(&["serve", "--store"]), - ] { - messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); - } - for bytes in &seen { - assert!( - !String::from_utf8_lossy(bytes).contains(CANARY), - "a response carried an input value" - ); - } - for message in &messages { - assert!(!message.contains(CANARY), "an error carried an input value: {message}"); - } - assert!(!h.ctx.store.path().exists()); - } - - #[test] - fn cgroup_names_the_issuer_unit() { - assert_eq!( - unit_from_cgroup("0::/system.slice/claude-connector.service\n"), - "claude-connector.service" - ); - // What a delegated subtree adds below the unit is the peer's to name. - assert_eq!( - unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), - "huginn@eureka.service" - ); - assert_eq!( - unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), - "getty@tty1.service" - ); - assert_eq!( - unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), - "a.scope" - ); - // A user manager's units, and paths no manager made, name no one. - for path in [ - "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", - "0::/user.slice/user-1000.slice/session-3.scope\n", - "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", - "0::/system.slice/\n", - "0::/\n", - "", - ] { - assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); - } - } - - #[test] - fn systemctl_show_output_is_read_into_facts() { - let facts = parse_show( - "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ - SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ - OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ - Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", - ) - .unwrap(); - assert_eq!(facts.id, "a.service"); - assert_eq!(facts.names, ["a.service", "b.service"]); - assert!(!facts.not_found); - assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); - // A quoted name holds one backslash, not two and not a quote mark. - assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); - assert_eq!(facts.upholds, ["u.service"]); - assert_eq!(facts.consists_of, ["m.service", "n.service"]); - assert!(parse_show("Names=a.service\n").is_err()); - // The request asks for exactly the properties the parse reads. - assert_eq!( - show_properties(), - "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ - OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" - ); - // A backslash outside quotes is a backslash; inside, it escapes one character. - assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); - assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); - assert!(parse_names(" ").is_empty()); - let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); - assert!(gone.not_found); - } - - #[test] - fn the_store_refuses_a_second_record_and_a_foreign_document() { - let h = harness(); - let order = OrderRecord { - schema_version: ORDER_SCHEMA.into(), - order_id: "ord-x".into(), - principal: "eureka".into(), - issuer_uid: 1, - issuer_unit: "u.service".into(), - issuer_invocation: String::new(), - issued_at_unix_millis: 1, - operation: Operation::Deploy { - selector: "vili".into(), - }, - reason: "r".into(), - }; - assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); - assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); - assert_eq!( - std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, - 0o600 - ); - let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); - SingleFileMessagePackBackingStore::new(h.ctx.store.path()) - .compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: "idunn.other".into(), - key: "k".into(), - current: None, - }], - &[foreign], - ) - .unwrap(); - assert!(h.ctx.store.ledger().is_err()); - } - - #[test] - fn synapse_post_is_one_idempotent_put_on_loopback() { - let directory = TempDir::new().unwrap(); - let token_file = directory.path().join("token"); - std::fs::write(&token_file, "tok-123\n").unwrap(); - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let (sent, requests) = mpsc::channel(); - let server = thread::spawn(move || { - for reply in [ - "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", - "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", - ] { - let (mut socket, _) = listener.accept().unwrap(); - let mut received = Vec::new(); - let mut chunk = [0u8; 1024]; - while !received.windows(4).any(|window| window == b"\r\n\r\n") - || !String::from_utf8_lossy(&received).contains("\"body\"") - { - let count = socket.read(&mut chunk).unwrap(); - if count == 0 { - break; - } - received.extend_from_slice(&chunk[..count]); - } - sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); - socket.write_all(reply.as_bytes()).unwrap(); - } - }); - let synapse = Synapse { - address, - room: "!room:example.org".into(), - token_file, - reply_within: Duration::from_secs(10), - }; - assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); - let request = requests.recv().unwrap(); - assert!(request.starts_with( - "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" - )); - assert!(request.contains("Authorization: Bearer tok-123\r\n")); - assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); - let error = synapse.post("txn-2", "x").unwrap_err(); - assert!(!format!("{error:#}").contains("tok-123")); - server.join().unwrap(); - assert!( - Synapse { - address, - room: "!r:e".into(), - token_file: directory.path().join("absent"), - reply_within: Duration::from_secs(10), - } - .post("t", "x") - .is_err() - ); - } - - #[test] - fn grammar_limits_are_exact_at_their_edges() { - let h = harness(); - let order = |operation: Operation, reason: &str| { - h.ask(&Request::Order(OrderRequest { - operation, - reason: reason.into(), - })) - }; - let packages = |names: Vec| Operation::PackageInstall { packages: names }; - let unit = |name: String| Operation::UnitAction { - verb: Verb::Start, - unit: name, - }; - // Package: two characters is the shortest name, 200 bytes the longest. - assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); - assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); - assert_eq!( - order(packages(vec!["a".repeat(201)]), "r"), - Response::Refused(Refusal::new("package", Code::TooLong)) - ); - assert_eq!( - order(packages(vec!["a".into()]), "r"), - Response::Refused(Refusal::new("package", Code::BadShape)) - ); - let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); - assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); - // Unit: 200 bytes with its suffix is the longest. - let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); - assert_eq!(longest.len(), UNIT_LIMIT); - assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); - let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); - assert_eq!( - order(unit(too_long), "r"), - Response::Refused(Refusal::new("unit", Code::TooLong)) - ); - // Reason: 300 bytes is the longest, and it must be there. - assert!(matches!( - order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), - Response::Accepted { .. } - )); - assert_eq!( - order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), - Response::Refused(Refusal::new("reason", Code::TooLong)) - ); - assert_eq!( - order(unit("nginx.service".into()), ""), - Response::Refused(Refusal::new("reason", Code::Empty)) - ); - assert_eq!( - order(unit("nginx.service".into()), "caf\u{e9}"), - Response::Refused(Refusal::new("reason", Code::BadCharacter)) - ); - } - - #[test] - fn a_concluded_order_is_never_run_later() { - let h = harness(); - h.brake.0.store(false, Ordering::SeqCst); - let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - execute(&h.ctx, &id).unwrap(); - assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - // Someone else clears the brake; the refused order stays refused. - h.brake.0.store(true, Ordering::SeqCst); - execute(&h.ctx, &id).unwrap(); - assert!(h.runs().is_empty()); - let ledger = h.ctx.store.ledger().unwrap(); - assert!(ledger.start_of(&id).is_none()); - assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); - } - - #[test] - fn the_orders_cli_has_no_command_string() { - let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); - let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); - assert_eq!( - parsed.action, - OrderAction::Issue { - operation: Operation::UnitAction { - verb: Verb::Restart, - unit: "nginx.service".into() - }, - reason: "why".into() - } - ); - assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); - assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); - assert!(parse(&["show", "ord-x"]).is_ok()); - assert!(parse(&["list"]).is_ok()); - for refused in [ - vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], - vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], - vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], - vec!["run", "sh", "--reason", "r"], - vec!["exec", "--reason", "r"], - vec!["unit", "enable", "nginx.service", "--reason", "r"], - vec!["unit", "restart", "--reason", "r"], - vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], - vec!["unit", "restart", "nginx.service"], - vec!["unit", "restart", "nginx*", "--reason", "r"], - vec!["package-install", "--reason", "r"], - vec!["package-install", "-y", "x", "--reason", "r"], - vec!["deploy", "--reason", "r"], - vec!["deploy", "a", "b", "--reason", "r"], - vec!["list", "extra"], - vec!["show"], - vec![], - ] { - assert!(parse(&refused).is_err(), "{refused:?} parsed"); - } - } - - #[test] - fn an_order_runs_only_under_the_steward_bound_now() { - let mut h = harness(); - let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); - // The binding changes while both wait: another uid, then another - // principal with the old uid. Each is refused with a record, before - // the brake is asked and before anything runs. - h.ctx.steward_uid = STEWARD + 7; - execute(&h.ctx, &first).unwrap(); - h.ctx.steward_uid = STEWARD; - h.ctx.principal = "void".into(); - execute(&h.ctx, &second).unwrap(); - for id in [&first, &second] { - assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); - assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); - } - assert!(h.runs().is_empty()); - assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); - // The refusal is posted like any other. - let ledger = h.ctx.store.ledger().unwrap(); - let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); - assert!(due.iter().any(|due| due.kind == PostKind::Result - && due.text.contains("refused: the issuer is not the current steward"))); - // The other direction: an order the bound steward issued after the - // rebinding, and a recovered one under an unchanged binding, run and - // are gated by the bound principal's brake. - let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); - let resumed = harness(); - let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); - assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); - execute(&resumed.ctx, &waiting).unwrap(); - execute(&h.ctx, &third).unwrap(); - assert_eq!(resumed.outcome(&waiting), Outcome::Ran); - assert_eq!(h.outcome(&third), Outcome::Ran); - assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); - assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); - } - - #[test] - fn root_reads_orders_and_never_issues_one() { - let mut h = harness(); - let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); - h.ctx.peers = Arc::new(FakePeers(0)); - let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); - assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); - let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { - panic!("root could not read an order"); - }; - assert_eq!(view.order.order_id, id); - let Response::List(lines) = h.ask(&Request::List) else { - panic!("root could not list orders"); - }; - assert_eq!(lines.len(), 1, "root's refused order left no record"); - h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); - assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); - } - - #[test] - fn every_order_posts_once_in_its_turn() { - let h = harness(); - let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] - .into_iter() - .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) - .collect(); - for id in &ids { - execute(&h.ctx, id).unwrap(); - } - assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); - assert_eq!(post_pass(&h.ctx).unwrap(), 0); - let sent = h.home.sent.lock().unwrap().clone(); - let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); - keys.sort(); - let mut expected: Vec = ids - .iter() - .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) - .collect(); - expected.sort(); - assert_eq!(keys, expected); - // An order accepted later is posted by the next pass alone. - let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); - assert_eq!(post_pass(&h.ctx).unwrap(), 1); - execute(&h.ctx, &late).unwrap(); - assert_eq!(post_pass(&h.ctx).unwrap(), 1); - } - - #[test] - fn a_deploys_result_is_posted_once_its_command_has_finished() { - let h = harness(); - let id = h.accept(h.issue(Operation::Deploy { - selector: "vili".into(), - })); - execute(&h.ctx, &id).unwrap(); - let ledger = h.ctx.store.ledger().unwrap(); - let results = |state: Option<(&'static str, &str)>| { - let owed = due_posts(&ledger, &|_| match state { - Some((state, detail)) => Ok((state, detail.to_owned())), - None => Err(anyhow!("the command cannot be read")), - }); - owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() - }; - assert!(results(Some(("queued", ""))).is_empty()); - assert!(results(Some(("running", "vili:Sealing"))).is_empty()); - assert!(results(None).is_empty()); - for state in ["succeeded", "failed"] { - let owed = results(Some((state, "vili"))); - assert_eq!(owed.len(), 1, "{state}"); - assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); - assert!(owed[0].text.contains("submitted deployment up-")); - } - } - - #[test] - fn intake_gives_up_on_a_peer_that_sends_no_frame() { - let h = harness(); - let (mut client, mut server) = UnixStream::pair().unwrap(); - client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); - // Half a length prefix, then nothing: the connection must end by itself. - client.write_all(&[0, 0]).unwrap(); - let (done, ended) = mpsc::channel(); - let worker = thread::spawn(move || { - let result = handle(&h.ctx, &mut server, &h.queue); - let _ = done.send(result.is_ok()); - h - }); - let answered = ended - .recv_timeout(Duration::from_secs(20)) - .expect("intake never gave up on a silent peer"); - assert!(answered, "a silent peer is refused as malformed, not left as an error"); - let h = worker.join().unwrap(); - assert_eq!( - rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), - Response::Refused(Refusal::new("frame", Code::Malformed)) - ); - assert!(!h.ctx.store.path().exists()); - } - - #[test] - fn a_record_under_another_schema_is_not_read() { - let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { - "order" => h.ctx.store.put( - key, - schema, - &OrderRecord { - schema_version: schema.into(), - order_id: key.into(), - principal: "eureka".into(), - issuer_uid: 1, - issuer_unit: "u.service".into(), - issuer_invocation: String::new(), - issued_at_unix_millis: 1, - operation: Operation::Deploy { - selector: "vili".into(), - }, - reason: "r".into(), - }, - ), - "start" => h.ctx.store.put( - key, - schema, - &StartRecord { - schema_version: schema.into(), - order_id: key.into(), - started_at_unix_millis: 1, - actuation: "idunn-order-x".into(), - }, - ), - "outcome" => h.ctx.store.put( - key, - schema, - &OutcomeRecord { - schema_version: schema.into(), - order_id: key.into(), - ended_at_unix_millis: 1, - outcome: Outcome::Lost, - stdout_tail: Vec::new(), - stderr_tail: Vec::new(), - }, - ), - _ => h.ctx.store.put( - key, - schema, - &PostRecord { - schema_version: schema.into(), - post_key: key.into(), - event_id: "$e".into(), - posted_at_unix_millis: 1, - }, - ), - }; - for (kind, current, key) in [ - ("order", ORDER_SCHEMA, "ord-a"), - ("start", START_SCHEMA, "ord-a"), - ("outcome", OUTCOME_SCHEMA, "ord-a"), - ("post", POST_SCHEMA, "ord-a:accepted"), - ] { - let h = harness(); - assert!(put(&h, kind, current, key).unwrap()); - assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); - let h = harness(); - let other = current.replace(".v1", ".v0"); - assert!(put(&h, kind, &other, key).unwrap()); - assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); - } - } - - #[test] - fn posts_are_retried_after_a_widening_pause() { - let pauses: Vec = (0..9).map(post_pause).collect(); - assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); - } - - #[test] - fn serve_options_are_read_from_every_flag() { - let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); - let defaults = parse(&["serve"]).unwrap(); - assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); - assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); - assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); - let given = parse(&[ - "serve", - "--state-store", - "/a/control.cc", - "--store", - "/b", - "--steward-binding", - "/c.toml", - "--socket", - "/d.sock", - ]) - .unwrap(); - assert_eq!( - given, - ServeOptions { - state_store: "/a/control.cc".into(), - store: "/b".into(), - steward_binding: "/c.toml".into(), - socket: "/d.sock".into(), - } - ); - assert!(parse(&["serve", "--store"]).is_err()); - assert!(parse(&["serve", "--other", "/x"]).is_err()); - assert!(parse(&["start"]).is_err()); - } - - #[test] - fn the_steward_user_is_a_real_user_and_never_root() { - assert!(user_uid("root").is_err()); - assert!(user_uid("no-such-user-here").is_err()); - assert!(user_uid("bad\0name").is_err()); - assert_eq!(user_uid("nobody").unwrap(), 65534); - } - - #[test] - fn the_orders_directory_is_made_private() { - let directory = TempDir::new().unwrap(); - let store = OrderStore::new(&directory.path().join("orders")); - store.prepare().unwrap(); - let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; - assert_eq!(mode, 0o700); - } - - #[test] - fn the_socket_belongs_to_the_steward_alone() { - let directory = TempDir::new().unwrap(); - let path = directory.path().join("steward.sock"); - let me = std::fs::metadata(directory.path()).unwrap().uid(); - let _listener = bind_socket(&path, me).unwrap(); - let metadata = std::fs::metadata(&path).unwrap(); - assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); - // A stale socket is replaced; anything else in the way is not. - let _again = bind_socket(&path, me).unwrap(); - let other = directory.path().join("other"); - std::fs::write(&other, "x").unwrap(); - assert!(bind_socket(&other, me).is_err()); - } - - #[test] - fn synapse_gives_up_on_a_reply_that_never_ends() { - let directory = TempDir::new().unwrap(); - let token_file = directory.path().join("token"); - std::fs::write(&token_file, "tok-123\n").unwrap(); - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - thread::spawn(move || { - // The first connection stays silent; the second drips a byte at a time. - for (index, stream) in listener.incoming().enumerate() { - let Ok(mut socket) = stream else { continue }; - thread::spawn(move || { - let mut chunk = [0u8; 4096]; - let _ = socket.read(&mut chunk); - for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { - if index == 0 || socket.write_all(&[*byte]).is_err() { - break; - } - thread::sleep(Duration::from_millis(40)); - } - thread::sleep(Duration::from_secs(5)); - }); - } - }); - let synapse = Synapse { - address, - room: "!r:e".into(), - token_file, - reply_within: Duration::from_millis(600), - }; - let synapse = Arc::new(synapse); - for _ in 0..2 { - let started = std::time::Instant::now(); - let (done, ended) = mpsc::channel(); - let poster = Arc::clone(&synapse); - thread::spawn(move || { - let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); - }); - let error = ended - .recv_timeout(Duration::from_secs(4)) - .expect("the post never gave up") - .unwrap_err(); - assert!(started.elapsed() < Duration::from_secs(3), "{error}"); - assert!(!error.contains("tok-123")); - } - } - - /// The whole service, as root, against fake `systemctl`, `systemd-run` - /// and a loopback homeserver, with the steward connecting from another - /// uid. It runs only where the test process is root (the verify container). - #[test] - fn serve_end_to_end_as_root() { - // SAFETY: geteuid reads the process credentials. - if unsafe { libc::geteuid() } != 0 { - eprintln!("serve_end_to_end_as_root: skipped, not root"); - return; - } - let directory = TempDir::new().unwrap(); - let root = directory.path().to_owned(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); - let bin = root.join("bin"); - std::fs::create_dir(&bin).unwrap(); - let systemctl = format!( - "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ - case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", - root.display() - ); - let run = format!( - "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", - root.display() - ); - for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { - let path = bin.join(name); - std::fs::write(&path, text).unwrap(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); - } - let credentials = root.join("cred"); - std::fs::create_dir(&credentials).unwrap(); - std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - let (posted, posts) = mpsc::channel::(); - thread::spawn(move || { - for (index, stream) in listener.incoming().enumerate() { - let Ok(mut socket) = stream else { continue }; - let mut received = Vec::new(); - let mut chunk = [0u8; 4096]; - loop { - let text = String::from_utf8_lossy(&received).into_owned(); - if let Some(split) = text.find("\r\n\r\n") { - let length = text - .lines() - .find_map(|line| line.strip_prefix("Content-Length: ")) - .and_then(|value| value.trim().parse::().ok()) - .unwrap_or(0); - if received.len() >= split + 4 + length { - break; - } - } - let count = socket.read(&mut chunk).unwrap_or(0); - if count == 0 { - break; - } - received.extend_from_slice(&chunk[..count]); - } - let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); - let _ = socket.write_all( - format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), - ); - } - }); - let binding = root.join("steward.toml"); - std::fs::write( - &binding, - format!( - "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ - room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ - token_credential = \"idunn-matrix-token\"\n" - ), - ) - .unwrap(); - std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); - let run_directory = root.join("run"); - std::fs::create_dir(&run_directory).unwrap(); - std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); - let socket = run_directory.join("steward.sock"); - let store = root.join("orders"); - let control = root.join("state").join("control.cc"); - // SAFETY: the only test that reads these variables is this one; PATH - // keeps its old entries. - unsafe { - std::env::set_var( - "PATH", - format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), - ); - std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); - } - let options = ServeOptions { - state_store: control.clone(), - store: store.clone(), - steward_binding: binding, - socket: socket.clone(), - }; - // The unit's umask must not decide the modes of what the service writes. - // SAFETY: umask only sets the process file-mode mask. - let before = unsafe { libc::umask(0o117) }; - let (ended, serve_end) = mpsc::channel::(); - thread::spawn(move || { - let result = serve(options); - let _ = ended.send(format!("{result:?}")); - }); - for _ in 0..100 { - if socket.exists() || serve_end.try_recv().is_ok() { - break; - } - thread::sleep(Duration::from_millis(100)); - } - assert!(socket.exists(), "serve never bound its socket"); - let metadata = std::fs::metadata(&socket).unwrap(); - assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); - // Root is not the steward: it may read, it may not order. - let unit = |name: &str| { - Request::Order(OrderRequest { - operation: Operation::UnitAction { - verb: Verb::Restart, - unit: name.into(), - }, - reason: "end to end".into(), - }) - }; - let as_root = |request: &Request| { - let mut stream = UnixStream::connect(&socket).unwrap(); - stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); - write_frame(&mut stream, request).unwrap(); - rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() - }; - assert_eq!( - as_root(&unit("nginx.service")), - Response::Refused(Refusal::new("peer", Code::Unauthorized)) - ); - assert_eq!(as_root(&Request::List), Response::List(Vec::new())); - let orders = OrderStore::new(&store); - let finished = |order_id: &str| { - for _ in 0..200 { - if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { - return record.clone(); - } - thread::sleep(Duration::from_millis(100)); - } - panic!("{order_id} never concluded"); - }; - // The steward is uid 65534 with gid 0: a service that read the gid for - // the uid would take it for root. - let first = steward_asks(&socket, &unit("nginx.service")); - let Response::Accepted { order_id: first } = first else { - panic!("the steward was not accepted: {first:?}"); - }; - assert_eq!(finished(&first).outcome, Outcome::Ran); - std::fs::write(root.join("fail"), "").unwrap(); - let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { - panic!("the second order was not accepted"); - }; - let failed = finished(&second); - assert_eq!( - failed.outcome, - Outcome::Failed { - exit_code: Some(3), - signal: None - } - ); - assert_eq!(failed.stderr_tail, b"boom\n"); - let deploy = Request::Order(OrderRequest { - operation: Operation::Deploy { - selector: "vili".into(), - }, - reason: "end to end".into(), - }); - let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { - panic!("the deploy was not accepted"); - }; - assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); - let ledger = orders.ledger().unwrap(); - let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); - assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); - // An invocation id is what a succeeding systemctl prints, if anything. - assert_eq!(HostSystemd.invocation("quiet.service"), None); - assert_eq!(HostSystemd.invocation("broken.service"), None); - assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); - // What the service called, exactly. - let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); - assert!( - run_log.contains(&format!( - "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", - order_uuid(&first) - )), - "{run_log}" - ); - let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); - assert!( - systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), - "{systemctl_log}" - ); - // Every record is posted; the deploy's result waits for its command. - let mut seen = Vec::new(); - while seen.len() < 5 { - seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); - } - for (id, kinds) in [ - (&first, ["accepted", "result"].as_slice()), - (&second, &["accepted", "result"]), - (&third, &["accepted"]), - ] { - for kind in kinds { - let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); - assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); - } - } - // Modes: the store is private, and the deploy wrote control.cc under - // Idunn's umask, not the 0117 the process was started with. - let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; - assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); - let control_mode = mode(&control); - // SAFETY: umask only sets the process file-mode mask. - unsafe { libc::umask(before) }; - assert_eq!(control_mode, 0o640); - } - - /// One request as the steward: uid 65534 and gid 0, from a forked child - /// that only makes system calls. - fn steward_asks(socket: &Path, request: &Request) -> Response { - let body = rmp_serde::to_vec_named(request).unwrap(); - let mut frame = (body.len() as u32).to_be_bytes().to_vec(); - frame.extend_from_slice(&body); - // SAFETY: plain data. - let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; - address.sun_family = libc::AF_UNIX as libc::sa_family_t; - for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { - address.sun_path[index] = *byte as libc::c_char; - } - let mut fds = [0i32; 2]; - // SAFETY: `fds` holds two descriptors. - assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); - // SAFETY: the child allocates nothing and ends with _exit. - let pid = unsafe { libc::fork() }; - if pid == 0 { - // SAFETY: only system calls on buffers made before the fork. - unsafe { - libc::close(fds[0]); - libc::alarm(10); - if libc::setgroups(0, std::ptr::null()) != 0 - || libc::setgid(0) != 0 - || libc::setuid(65534) != 0 - { - libc::_exit(2); - } - let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); - if libc::connect( - fd, - (&address as *const libc::sockaddr_un).cast(), - std::mem::size_of::() as libc::socklen_t, - ) != 0 - { - libc::_exit(3); - } - let mut offset = 0usize; - while offset < frame.len() { - let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); - if count <= 0 { - libc::_exit(4); - } - offset += count as usize; - } - let mut buffer = [0u8; 4096]; - loop { - let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); - if count <= 0 { - break; - } - libc::write(fds[1], buffer.as_ptr().cast(), count as usize); - } - libc::_exit(0); - } - } - // SAFETY: the parent's copy of the write end. - unsafe { libc::close(fds[1]) }; - let mut reply = Vec::new(); - // SAFETY: `fds[0]` is an open descriptor this function now owns. - let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; - pipe.read_to_end(&mut reply).unwrap(); - let mut status = 0; - // SAFETY: `pid` is our child. - unsafe { libc::waitpid(pid, &mut status, 0) }; - assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); - rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() - } - - fn host_refusal(verb: Verb, unit: &str) -> Option { - refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) - } - - #[test] - #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] - fn real_host_unit_set() { - for name in [ - "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", - "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", - "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", - "wg-quick@wg-gamecult", "cron", "apt-daily", - ] { - let unit = format!("{name}.service"); - for verb in all_verbs() { - assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); - } - } - for name in [ - "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", - "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", - "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", - "systemd-storagetm", - ] { - let unit = format!("{name}.service"); - for verb in [Verb::Start, Verb::Stop, Verb::Restart] { - assert_eq!( - host_refusal(verb, &unit), - Some(refused(&unit, Via::OnEnd)), - "{verb:?} {unit}" - ); - } - } - for unit in HOST_NAMES { - for verb in all_verbs() { - assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); - } - } - for unit in [ - "systemd-networkd.service", - "user-runtime-dir@1000.service", - "user-runtime-dir@105.service", - "user-runtime-dir@978.service", - ] { - assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); - assert!( - matches!( - host_refusal(Verb::Stop, unit), - Some(Refused::AlwaysRefused { via: Via::Bound, .. }) - ), - "{unit}" - ); - } - let listing = Command::new("systemctl") - .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) - .output() - .unwrap(); - let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); - let units: Vec<&str> = listing - .lines() - .filter_map(|line| line.split_whitespace().next()) - .filter(|unit| validate_unit(unit).is_ok()) - .collect(); - for verb in all_verbs() { - let mut counts = BTreeMap::new(); - for unit in &units { - let key = match refusal_for(&HostSystemd, unit, verb) { - Ok(None) => "ok".to_owned(), - Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), - Ok(Some(other)) => format!("{other:?}"), - Err(_) => "unresolved".to_owned(), - }; - *counts.entry(key).or_insert(0usize) += 1; - } - println!("{verb:?} over {} units: {counts:?}", units.len()); - } - } - - #[test] - #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] - fn one_job_argv_against_a_real_manager() { - let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); - std::fs::create_dir_all(&directory).unwrap(); - let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); - let body = "[Service]\nExecStart=/bin/sleep 600\n"; - std::fs::write(directory.join(a), body).unwrap(); - std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); - let user = |args: &[&str]| { - let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); - (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) - }; - let outcome = std::panic::catch_unwind(|| { - let one_job = |verb: Verb, unit: &str| { - let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); - let mut argv = step.argv.clone(); - argv.insert(1, "--user".into()); - let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); - assert!(status.success(), "{verb:?} {unit}"); - }; - assert!(user(&["daemon-reload"]).0); - // A start of b leaves a running, though b conflicts with it. - assert!(user(&["start", a]).0); - one_job(Verb::Start, b); - assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); - assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); - // A restart of b, with both running, leaves a running. - one_job(Verb::Restart, b); - assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); - assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); - }); - user(&["stop", a, b]); - std::fs::remove_file(directory.join(a)).unwrap(); - std::fs::remove_file(directory.join(b)).unwrap(); - user(&["daemon-reload"]); - outcome.unwrap(); - } -} diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index b9f2637..70060bf 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -10,19 +10,3 @@ It is checked in because the TypeScript runtime emits a camelCase MessagePack map while Rust `DatabaseEntry` derives tuple payloads. Idunn must prove it can decode the producer's real wire shape rather than round-tripping a fixture it serialized itself. - -# systemd 257 `systemctl show` captures - -`systemd-257/*.show` are the verbatim output of systemd 257.13 for the 27 -properties `show_unit` requested at `112a7a5`, captured read-only on Yggdrasil -on 2026-10-24: `vili-presence-identity-only.show` (a failed idunn-vili unit whose -only credential is the presence identity) and -`odin-presence-and-one-service-credential.show` (the live Odin unit). systemd 257 -prints `LoadCredential=[unprintable]` in both. Tests read these bytes; do not edit. - -# Pre-cut systemd observation records - -`pre-cut-systemd-observation.{compact,named}.hex` are the two MessagePack -encodings of a `SystemdWorkloadObservation` with `load_credential` set to -`[unprintable]`, produced by the code at `112a7a5` (the form live incumbents -persisted). They pin that later Idunns still decode such records. diff --git a/tests/fixtures/pre-cut-systemd-observation.compact.hex b/tests/fixtures/pre-cut-systemd-observation.compact.hex deleted file mode 100644 index e7d09fb..0000000 --- a/tests/fixtures/pre-cut-systemd-observation.compact.hex +++ /dev/null @@ -1 +0,0 @@ -dc0033d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365aa4964756e6e2074657374a3696e7601a465786563a26e6fa56d69786564c3a6753631303030a6753631303030a0a0a0c3c3a9696e76697369626c65a3616c6cc3a430303037a0ad5b756e7072696e7461626c655dcd10920194cdee48cdee48cdee48cdee4894cdee48cdee48cdee48cdee4891cdee480000000000c391010708b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d3232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323290d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190 \ No newline at end of file diff --git a/tests/fixtures/pre-cut-systemd-observation.named.hex b/tests/fixtures/pre-cut-systemd-observation.named.hex deleted file mode 100644 index 8898adf..0000000 --- a/tests/fixtures/pre-cut-systemd-observation.named.hex +++ /dev/null @@ -1 +0,0 @@ -de0033a4756e6974d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365b0756e69745f6465736372697074696f6eaa4964756e6e2074657374ad696e766f636174696f6e5f6964a3696e76d923657865635f6d61696e5f73746172745f74696d657374616d705f6d6f6e6f746f6e696301ac736572766963655f74797065a465786563ae726573746172745f706f6c696379a26e6fa96b696c6c5f6d6f6465a56d69786564ac64796e616d69635f75736572c3ac73797374656d645f75736572a6753631303030ad73797374656d645f67726f7570a6753631303030b4737570706c656d656e746172795f67726f757073a0b76361706162696c6974795f626f756e64696e675f736574a0b4616d6269656e745f6361706162696c6974696573a0ae707269766174655f6d6f756e7473c3ac707269766174655f70696473c3ac70726f746563745f70726f63a9696e76697369626c65ab70726f635f737562736574a3616c6cb16e6f5f6e65775f70726976696c65676573c3a5756d61736ba430303037b2696e61636365737369626c655f7061746873a0af6c6f61645f63726564656e7469616cad5b756e7072696e7461626c655da86d61696e5f706964cd1092b270726f636573735f73746172745f74696d6501ac70726f636573735f7569647394cdee48cdee48cdee48cdee48ac70726f636573735f6769647394cdee48cdee48cdee48cdee48ae70726f636573735f67726f75707391cdee48b770726f636573735f6361705f696e686572697461626c6500b570726f636573735f6361705f7065726d697474656400b570726f636573735f6361705f65666665637469766500b470726f636573735f6361705f626f756e64696e6700b370726f636573735f6361705f616d6269656e7400b970726f636573735f6e6f5f6e65775f70726976696c65676573c3b670726f636573735f6e616d6573706163655f706964739101b26d6f756e745f6e616d6573706163655f696407b07069645f6e616d6573706163655f696408aa65786563757461626c65b52f6f70742f746573742f62696e2f73657276696365b165786563757461626c655f64657669636501b065786563757461626c655f696e6f646501b165786563757461626c655f736861323536d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131b372756e74696d655f696e7374616e63655f6964d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161b1776f726b696e675f6469726563746f7279a92f6f70742f74657374ae72756e74696d655f62756e646c65a92f72756e2f74657374b3636f6d6d616e645f6c696e655f736861323536d9477368613235362d32323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232b1656e7669726f6e6d656e745f6e616d657390bb656e7669726f6e6d656e745f636f6e74726163745f736861323536d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333ad636f6e74726f6c5f67726f7570d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365b563726564656e7469616c735f6469726563746f7279c0bc706172656e745f6f6e6c795f66696c655f64657363726970746f727390bd61637469766174696f6e5f7369676e65725f6964656e746974795f6964aa61637469766174696f6ebc61637469766174696f6e5f7369676e65725f7075626c69635f6b6579dc00200101010101010101010101010101010101010101010101010101010101010101b3736572766963655f63726564656e7469616c7390 \ No newline at end of file diff --git a/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show deleted file mode 100644 index 21e4573..0000000 --- a/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show +++ /dev/null @@ -1,28 +0,0 @@ -Type=exec -Restart=no -MainPID=2571 -OpenFile=/run/idunn/activation-credentials/sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.credential:gamecult-idunn-runtime-activation-key:read-only -OpenFile=/etc/gamecult/odin/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only -ExecMainStartTimestampMonotonic=20630000 -ControlGroup=/system.slice/idunn-odin-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.service -UMask=0007 -WorkingDirectory=/srv/odin/releases/sha256-e5c7d0b2b9c0a8ffba368aeb08abdbe37ae0d17484a785e60a66ec4a62b2f18c -CapabilityBoundingSet= -AmbientCapabilities= -User=_duc23369f1b8b167db -Group=odin -DynamicUser=yes -LoadCredential=[unprintable] -SupplementaryGroups= -InaccessiblePaths=/run/idunn/activation-credentials -PrivateMounts=yes -PrivatePIDs=yes -NoNewPrivileges=yes -ProtectProc=invisible -ProcSubset=all -KillMode=mixed -Description=Idunn odin sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8 -LoadState=loaded -ActiveState=active -SubState=running -InvocationID=ada48921043449f39dacc87b14a1bf70 diff --git a/tests/fixtures/systemd-257/vili-presence-identity-only.show b/tests/fixtures/systemd-257/vili-presence-identity-only.show deleted file mode 100644 index fc78fd1..0000000 --- a/tests/fixtures/systemd-257/vili-presence-identity-only.show +++ /dev/null @@ -1,28 +0,0 @@ -Type=exec -Restart=no -MainPID=0 -OpenFile=/run/idunn/activation-credentials/sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041.credential:gamecult-idunn-runtime-activation-key:read-only -OpenFile=/etc/gamecult/vili/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only -ExecMainStartTimestampMonotonic=520083420000 -ControlGroup= -UMask=0007 -WorkingDirectory=/srv/vili-idunn/releases/sha256-aeaee4e1669742572c45a3f26bc86893c44b2a4149520975eb7d8a28aa931906 -CapabilityBoundingSet= -AmbientCapabilities= -User=_du8c06c563aac6d205 -Group=vili-state -DynamicUser=yes -LoadCredential=[unprintable] -SupplementaryGroups= -InaccessiblePaths=/run/idunn/activation-credentials -PrivateMounts=yes -PrivatePIDs=yes -NoNewPrivileges=yes -ProtectProc=invisible -ProcSubset=all -KillMode=mixed -Description=Idunn vili sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041 -LoadState=loaded -ActiveState=failed -SubState=failed -InvocationID=8f7a349f2f8f49fe87ac5d89239260b4