diff --git a/Cargo.lock b/Cargo.lock index dfc3087..44c9457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "libc", "rmp-serde", "serde", + "serde_json", "sha2", "tempfile", "toml", diff --git a/Cargo.toml b/Cargo.toml index dcaca2e..ee797ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ serde = { version = "1", features = ["derive"] } ed25519-dalek = "2.2" toml = "0.8" uuid = { version = "1", features = ["v4"] } +serde_json = "1" [dev-dependencies] tempfile = "3" diff --git a/README.md b/README.md index db728a9..8c6935c 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,64 @@ idunn status --command the bindings, so `idunn up profile:full-gamecult` brings up everything bound into that profile. +### Steward orders + +A second root unit from the same binary, `idunn-orders` (`idunn orders serve`), +takes typed orders from one steward Mind over `/run/idunn-orders/steward.sock`. +It has its own store, `orders.cc`, and its own binding, `steward.toml`; the +survival daemon takes no order. There is still no command string: + +```bash +idunn order package-install [=version]... --reason TEXT # apt-get update, install named packages, check them +idunn order unit --reason TEXT +idunn order deploy --reason TEXT # the same commit as `idunn up` +idunn order show # status and output tails +idunn order list +``` + +A unit order submits exactly one job, for the unit it names: every verb runs +`systemctl --job-mode=ignore-requirements -- `, so systemd builds a +transaction of one job. It does not start what the unit requires or restart what +is part of it: order the dependency first, and order `postgresql@16-main.service` +rather than the `postgresql.service` wrapper. Targets are not orderable. The job is one; what the unit does afterwards is +not: systemd still runs full transactions of its own for `Restart=` after a +failure, for a `StopWhenUnneeded=` dependency left without a user, and for a +`Conflicts=` partner of a unit that fails, so an ordered stop of a unit +that something restarts does not last, and a start of a `Restart=always` +unit can report failure and be active seconds later. A unit +order may act on any unit except Idunn's own (`idunn-*`) and the minds' own +(`mind-self@*`, `mind-persona@*`), and the units that end or take the host +(`systemd-suspend`, `systemd-hibernate`, `systemd-hybrid-sleep`, +`systemd-suspend-then-hibernate`, `rescue`, `emergency`). That set is compiled +into the binary and read from no configuration. An order is refused, recorded +and posted when the ordered unit, or an alias it resolves from, is one of them; +when it has an emergency action (`SuccessAction=`, `FailureAction=`, +`StartLimitAction=`, `JobTimeoutAction=`) other than `none` or an `OnFailure=` +or `OnSuccess=` unit (not for a reload); or, for a stop or restart, when another +unit is `BoundBy=` it, because systemd stops that unit with it whatever the job +mode; for a start or restart, when it has an `Upholds=` unit, which systemd +starts in a full transaction whatever the job mode; or, for a stop, restart or +reload, when it has a `ConsistsOf=` member, a wrapper such as +`postgresql.service` whose verbs would exit 0 and touch nothing. A unit +`systemctl show` cannot resolve fails the order. Slices, scopes, +targets and globs are not units an order can name. + +A package order installs the packages it names and nothing else: apt runs with +`--no-remove`, each name is passed as an exact-name pattern so `.` and `+` +match themselves, and a last step fails the order unless every named package +is installed. + +An order runs once, after the steward's uid is checked and the instance's +brake check (`mind-control --instance check`) exits 0. The check +that the order's principal and uid are the bound steward's runs again at every +execution, so an order accepted before `steward.toml` changed is refused and +recorded, not run. The socket belongs to the steward user alone (mode 0600); +root may `show` and `list` orders but never issue one. An order that had +started when the service died is recorded as lost and never run again. No order releases a brake. Acceptance and result are posted to +`#operator` as a retried projection of `orders.cc`: a chat outage delays the +posts and not the orders. A refusal names the field and a code, never the value +it refused. + ## Authority **Owner.** Deployment admission, and the continuity of installed bodies. @@ -181,6 +239,32 @@ required_environment = [ ] ``` +A directory artifact is sealed read-only: its files are 0444 and its +directories 0555, whatever mode the build or git gave them. A file the tree +must run, such as the launcher's `upstream/bin/daemon` child, is declared in +`executable_members` and sealed 0555. The members are part of the sealed release +address, so changing the list makes a new release. Idunn refuses the release if a +member is absolute, climbs with `..`, is missing, is a symlink or sits behind +one, or is not a regular file; `executable = true` on a directory output and +`executable_members` on a file output are refused too. + +A tree may hold relative symlinks whose `..` steps all come first and climb no +higher than the artifact's top (`share/terminfo/4/4025ex -> ../t/tek4025ex` is +kept). An absolute link, a `..` after a name, or a climb out of the artifact +refuses the release when it is sealed. Host targets (Windows) record +`executable` and `executable_members` in the receipt and do not apply them. + +```toml +# recipe fragment — the upstream artifact declares the child it launches +[[artifacts]] +id = "upstream" +source_kind = "runner-output" +runner = "build" +source = "dist/upstream" +destination = "upstream" +executable_members = ["bin/daemon"] +``` + ```toml # binding fragment — members connect to the member endpoint, which Idunn # forwards to the private port derived from the candidate: member diff --git a/deploy/idunn-orders.service b/deploy/idunn-orders.service new file mode 100644 index 0000000..f2681c2 --- /dev/null +++ b/deploy/idunn-orders.service @@ -0,0 +1,47 @@ +[Unit] +Description=Idunn steward orders: typed package, unit and deploy orders from the steward +# Deliberately no After= or Requires= on any managed target, Synapse or +# idunn-yggdrasil: orders are how an operator repairs those, so none of them +# may be a condition of this unit starting. Posts are a retried projection and +# wait for Synapse on their own. + +[Service] +Type=simple + +# Root, from the same binary as idunn-yggdrasil, with its own store and socket. +# idunn-yggdrasil keeps its no-imperative-input invariant and its write set: +# no order code runs in `idunn serve`. +# +# The unit name matches idunn-*, which is in the always-refused set, so no +# order can stop or restart this service. +User=root +# No Group= and no UMask=: the service gives its socket to the steward user +# (mode 0600) itself and sets Idunn's own umask for the files it writes, so +# nothing here reaches control.cc. The steward user is the only peer the +# service answers: it checks SO_PEERCRED on every connection. +ExecStart=/usr/local/bin/idunn orders serve \ + --store /var/lib/gamecult/idunn-orders \ + --state-store /var/lib/gamecult/idunn/control.cc \ + --steward-binding /etc/gamecult/idunn/steward.toml \ + --socket /run/idunn-orders/steward.sock +Restart=always +RestartSec=5 +TimeoutStopSec=20 + +RuntimeDirectory=idunn-orders +RuntimeDirectoryMode=0755 + +# The Synapse access token of @idunn, copied from a root-only file, and read +# through $CREDENTIALS_DIRECTORY; the service names it in steward.toml. +LoadCredential=idunn-matrix-token:/etc/gamecult/idunn/matrix-token + +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +# Its own order store, and the control store a deploy order commits a command +# to. Transient units started with systemd-run run in the manager's namespace, +# so packages and units are not limited by this list. +ReadWritePaths=/var/lib/gamecult/idunn-orders /var/lib/gamecult/idunn + +[Install] +WantedBy=multi-user.target diff --git a/deploy/idunn-yggdrasil.service b/deploy/idunn-yggdrasil.service index da82bd2..c7407ca 100644 --- a/deploy/idunn-yggdrasil.service +++ b/deploy/idunn-yggdrasil.service @@ -45,7 +45,6 @@ ExecStart=/usr/local/bin/idunn serve \ --idunn-identity-store /var/lib/gamecult/idunn/idunn-service-identity.cc \ --odin-correlation-store /var/lib/gamecult/odin/topology.cc \ --odin-trust-anchor /etc/gamecult/idunn/odin-topology-anchor.cc \ - --deployment-brake-operator-anchor /etc/gamecult/idunn/deployment-brake-operator-anchor.cc \ --host-actuator-bind 10.77.0.1:17890 \ --source-uid 986 \ --source-gid 978 @@ -104,6 +103,11 @@ ReadWritePaths=/etc/gamecult/ghostlight/runtime /var/lib/gamecult/ghostlight-wor ReadWritePaths=/etc/gamecult/streampixels-service/runtime /var/lib/gamecult/streampixels-service /srv/streampixels-idunn/releases/service # StreamPixels web is stateless; it needs only its runtime bundle and releases. ReadWritePaths=/etc/gamecult/streampixels-web/runtime /srv/streampixels-idunn/releases/web +# Vili (motion worker): runtime bundle, state, and releases. Prefixed "-" so a +# target whose roots are not created yet cannot stop Idunn itself from starting +# (Idunn startup never depends on a managed target). A root created after Idunn +# starts is writable only from Idunn's next restart. +ReadWritePaths=-/etc/gamecult/vili/runtime -/var/lib/gamecult/vili -/srv/vili-idunn/releases [Install] WantedBy=multi-user.target diff --git a/deploy/legacy/idunn-yggdrasil b/deploy/legacy/idunn-yggdrasil index 26970c0..62a18eb 100644 --- a/deploy/legacy/idunn-yggdrasil +++ b/deploy/legacy/idunn-yggdrasil @@ -16,7 +16,6 @@ target="${2:-}" compose=/srv/compose/yggdrasil-apps.yaml brake_store=/var/lib/gamecult/idunn-authority/deployment-brake.cc brake_lock=/var/lib/gamecult/idunn-authority/deployment-brake.cc.lock -brake_anchor=/etc/gamecult/idunn/deployment-brake-operator-anchor.cc generation_lock=/var/lib/gamecult/idunn-authority/idunn-generation.lock idunn_provision=/usr/local/bin/idunn-provision @@ -154,7 +153,7 @@ deployment_target_policy() { esac } -require_signed_deployment_grant() { +require_deployment_admission() { source_commit="$3" deployment_request_id="$4" release_authority_id="$5" @@ -180,28 +179,17 @@ require_signed_deployment_grant() { [ -z "$release_authority_sha256" ] || { echo "source-observed release cannot claim Bifrost authority digest" >&2; exit 1; } require_root_owned_resolved_executable "$idunn_provision" - require_root_owned_regular_path "$brake_store" - require_root_owned_regular_path "$brake_lock" - require_root_owned_regular_path "$brake_anchor" - # This validation is the deployment-admission point. The signed expiry is a - # start-before deadline: it denies a later transaction, but does not become a - # timer-shaped veto over one already admitted. The root actuator remains the - # sole shared-lock holder through the synchronous consequence, so an explicit - # engage/release transition cannot race or rewrite that transaction. - exec 9<"$brake_lock" + # This validation is the deployment-admission point. An absent or clear stop + # admits; an engaged or unreadable one makes the status verb exit nonzero. + # The root actuator remains the sole shared-lock holder through the + # synchronous consequence, so an engage or clear cannot race or rewrite it. + # The lock file is created here when no store has ever been written. + exec 9>>"$brake_lock" /usr/bin/flock --shared 9 - now_unix_millis="$(/usr/bin/date +%s%3N)" - case "$now_unix_millis" in - ''|*[!0-9]*) echo "root actuator clock did not produce Unix milliseconds" >&2; exit 1 ;; - esac "$idunn_provision" deployment-brake-status \ --store "$brake_store" \ - --operator-anchor "$brake_anchor" \ - --runtime-id yggdrasil \ - --release-id "$source_commit" \ - --deployment-id "$deployment_request_id" \ - --now-unix-millis "$now_unix_millis" + --runtime-id yggdrasil if [ "$minimum_source_revision_required" = true ]; then require_root_owned_resolved_executable /usr/local/bin/idunn @@ -322,7 +310,7 @@ case "$action:$target" in manifest="/srv/odin/deploy-manifests/$target" require_root_owned_regular_path "$manifest" [ -x "$manifest" ] || { echo "root actuator manifest is not executable: $manifest" >&2; exit 1; } - require_signed_deployment_grant "$@" + require_deployment_admission "$@" export IDUNN_ACTUATOR=1 export IDUNN_COMMAND_AUTHORITY=idunn-daemon export IDUNN_SOURCE_COMMIT="$source_commit" diff --git a/docs/authority-map.md b/docs/authority-map.md index c8a8546..001b644 100644 --- a/docs/authority-map.md +++ b/docs/authority-map.md @@ -240,7 +240,7 @@ Fencing retries its phase forward on a capped backoff and never stops its live candidate, since that is the only running copy of the admitted body; the deadline opens the same incident. The one exception is yielding: a continuity whose route proof has not succeeded gives way, through its own post-fencing -abort, to a deployment the deployment brake has released and that awaits the +abort, to a deployment the deployment stop admits and that awaits the handover; a continuity that has proven its route never yields. The `ContinuityExhausted` incident (six charged attempts in an hour, `restarts_page`) pages and gates nothing. When continuity next acts is derived @@ -257,9 +257,14 @@ held until a lift succeeds. It is a migration with a deletion line (follow-up still owe dependency failures not growing the streak and the dead-generation route refresh. -The deployment brake gates changes to source, release, configuration, schema, -unit, or authority binding. The lifecycle brake separately gates restart of an -already-admitted incarnation. Neither target brake may gate Idunn itself or an +The deployment brake is a stop, not a per-deploy gate: it gates changes to +source, release, configuration, schema, unit, or authority binding only while +engaged. Its one reader and writer is `src/brake.rs` (`idunn.deployment_stop.v1`: +engaged or clear, who, why, when, sequence). An absent or clear store admits; +a store that is present but unreadable, foreign, for another runtime, a symlink +or owned by another uid fails closed. Nothing is signed, bound to a release or +consumed, and one clear admits every deploy until it is engaged again. The +lifecycle brake separately gates restart of an already-admitted incarnation. Neither target brake may gate Idunn itself or an unrelated service. If Odin is unavailable, Idunn preserves existing routes and continuity for diff --git a/docs/deploy-provenance-map.md b/docs/deploy-provenance-map.md new file mode 100644 index 0000000..95c0eee --- /dev/null +++ b/docs/deploy-provenance-map.md @@ -0,0 +1,394 @@ +# Deployment provenance: map + +Status: map, Imagination, 2026-10-23 (campaign `idunn-deploy-provenance`, +session `self-2026-10-23-eureka-b`). The ends are in +[`deploy-provenance-target.md`](deploy-provenance-target.md). Questions, +rulings, cut specs and follow-ups live in Eureka's mind; this page holds what +has no kind: body facts, rationale and sequencing. Line numbers in T1-T12 are +at `d74ac8d` (origin/main on 2026-10-23); T4, T9 and T13-T16 are at `fd040f0`, +the head of Hands attempt 1, which revision 2 of the spec builds on. + +## Cut `tree-executable-members` + +Spec `idunn-deploy-provenance:cut_spec:cut-tree-executable-members.r2` +(revision 2, after Soul pass 1 found the real interpreter tree refused). It +serves `motion-pipeline:follow_up:idunn-tree-executable-members` (Vili's +admission waits on it) and will serve gamecult-chat C3 too: the presence +launcher runs a child such as `upstream/bin/daemon` from inside a tree artifact +(README, launcher fragment), and Synapse needs a Python interpreter there. + +### Body facts + +- T1. `ArtifactOutput` (`src/deployment.rs:106-117`) carries one `executable` + bool. `ArtifactReceipt` (`src/deployment_plan.rs:731-737`) copies it and + `validate` (`:740-760`) requires it to equal the declaration. +- T2. `harden_installed_release` (`src/drivers.rs:6848-6855`) builds its + executable set from `root.join(receipt.destination)` only. `harden_root_tree` + (`:6858-6892`) seals every directory 0555 and every file 0444 unless that + exact path is in the set. A directory artifact with `executable = true` is + therefore silently meaningless today: its files are all 0444. Nothing refuses + it. No live recipe declares one (CodexConnector, Ghostlight, Heimdall, Odin, + StreamPixels service and web surveyed): `executable = true` is on files only. +- T3. A tree output is copied by `copy_artifact`/`copy_tree` + (`src/drivers.rs:8053-8126`), which recreate symlinks verbatim. The digest + (`digest_tree`, `:8142-8175`) covers names, file bytes and link targets, not + mode bits. So runner or git exec bits never reach the release: harden is the + only mode authority. +- T4. `validate_release_symlink` (`src/drivers.rs:6897-6916`) keeps a symlink + only if its target is relative with no `..`. It runs only inside + `harden_root_tree`, at install, so a tree with a `..` or absolute link copies, + digests and seals, then fails at `install_release` (Soul s1: copying and + digesting the real tree twice took 160 s before harden refused it). The rule + is sound but too narrow: it refuses the real interpreter tree (T13). The + frozen-source path has a physical resolver (`resolve_frozen_source_symlink`, + `:6617`) that walks the filesystem with a 40-link budget and a memo. Its + first version was exponential (Soul S4-1, recorded in `d562a12`), and it needs + the tree on disk to answer. Revision 2 does not reuse it (T14). +- T5. The sealed release id hashes `(schema, plan_id, artifacts, + external_inputs)` (`src/deployment_plan.rs:1000-1011`), and `plan_id` hashes + the whole plan including `recipe_blob`. A changed member list is therefore + already a new plan and a new release. The receipt must still carry the list, + because `harden_installed_release` reads receipts only; its two callers + (`install_release` `:2788`, `validate_installed_release` `:2822`) re-harden + every installed release on each observation. +- T6. Encoding is `rmp_serde::to_vec`, positional. A trailing field with + `#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]` keeps every + existing receipt's bytes and release id: the plan's `deadlines` field + (`:574-577`) is the precedent. +- T7. `digest_artifact` and `digest_tree` (`:8128-8175`) `fs::read` each file + whole. A torch CPU tree's largest file (`libtorch_cpu.so`, hundreds of MB) + is held in Idunn's memory three times per install (seal, install, every + validate). `hash_frozen_source_tree` (`:6788-6840`) already streams in 64 KiB + chunks (S9). The cut collapses both onto one streaming helper. +- T8. `require_leaf_path` (`src/deployment.rs:1951-1958`) only keeps the + artifact destination a single filename. Members are paths inside that + destination, so it does not change. `require_relative_path` (`:1936-1949`) + with `allow_current = false` already refuses absolute, `..`, root and prefix + components. +- T9. The host actuator re-spells the seal (`src/host.rs:1050-1086`): copy, + mode check, digest, pinned-digest check and receipt, the same steps as + `collect_artifact` (`src/drivers.rs:2305-2326`). Soul s1: deleting its mode + check survives every test. Revision 2 collapses both onto one `seal_artifact` + in `drivers.rs`. Each caller keeps only how it finds the source path. +- T10. The transient unit runs with `ProtectSystem=strict` and the release in + `ReadOnlyPaths` (`src/drivers.rs:3675-3698`), `/tmp` mounted `noexec`, and no + `MemoryDenyWriteExecute`. Executing a 0555 file from the release works. + Shared libraries need no exec bit. +- T11. Live body on Yggdrasil: `/usr/local/bin/idunn` sha256 `44cf8a14...`, + installed 2026-10-20 02:34 CEST, before `6e2a31c` (02:40 CEST). `src/` is + identical between `8760a4f` and `6e2a31c`, so the live code is the + reincarnation c2 merge. `origin/main` is 26 commits ahead (+4194/-112 in + `src/`: the presence launcher and member route). Installing this cut ships all + of them. Idunn's own body changes only through gamecult-ops + `scripts/install-idunn-yggdrasil-release.sh` (staged binaries, unit, + bindings). The stage at `/home/gamecultadmin/idunn-stage` is stale (2026-09-11). +- T13. Link census of the real interpreter tree (Yggdrasil probe of + `cpython-3.12.15+20261021-x86_64-unknown-linux-gnu-install_only.tar.gz`, + sha256 `5ff9f060...77191124`; torch adds no links, as Soul s1 counted 1,049 + after pip): 1,049 links. 0 are absolute, 749 have no `..`, and 300 have all + their `..` components before the first name (every one is + `share/terminfo// -> ..//`, one `..` each). 0 have a `..` + after a name, 0 climb above the tree, 0 dangle and 0 resolve outside it. + Blender 5.2.2 has 79 links, none absolute or with `..` (Soul s1). +- T14. Lexically normalising `/` does not check + containment. The kernel resolves a target one component at a time, so a `..` + after a symlinked name climbs from where that link points, not from where its + name sits. Probe on Yggdrasil: `root/a/s -> ..` and `root/a/L -> s/../x` + normalise to `root` and `root/a/x`, both inside, yet `L` reads the file `x` + beside `root`. `root/a/d -> .` with `root/a/M -> d/../../x` escapes the same + way. Idunn has met this before: verify-transaction-cut F5 (`D -> .`, + `L -> D/D/../../../etc/passwd` passed a lexical check). + The sound lexical rule constrains the target's shape: all its `..` components + come before its first name, and they number no more than the link's depth + below the artifact's top. The climb then starts from the link's own + directory, which the walk reached through real directories only (copy and + digest never descend through a link), so it lands on a real ancestor inside + the artifact. After that the target only descends, and any link it passes + through obeys the same rule. By induction on the kernel's hops (capped at 40, + then ELOOP), every hop stays inside. The check is one pass over the target's + components with no filesystem reads, so its cost is linear in the tree and no + resolver is needed. The shapes it refuses (`s/../x`) occur in neither real + tree (T13). + Prior art, from documentation as known to Imagination (high confidence for Go + and securejoin, medium for the tar details): + - Go `path/filepath.IsLocal` (1.20) is lexical and says so: it does not account + for symlinks. It would admit `s/../x`. Used alone, it is the T14 bug. + - Go `os.Root` (1.24) and `cyphar/filepath-securejoin` (used by runc, + containerd and umoci) resolve each component inside the root when a path is + opened, refusing or clamping an escape. They guard one later access, need + the tree on disk, and check again on every open. Idunn instead certifies a + sealed tree once, so that the kernel's own resolution stays inside on every + later access. + - Zip-slip defences and archive extractors check entry names lexically and + protect against link targets by refusing to write through links + (libarchive's `ARCHIVE_EXTRACT_SECURE_SYMLINKS`, and `SECURE_NODOTDOT` for + names; GNU tar creates such links last). They protect the extraction, not + what a kept link points at afterwards. + - OCI layer extraction keeps link targets verbatim, absolute ones included, + because the runtime resolves them inside the container's root. Idunn runs a + release in the host's mount namespace, so absolute targets stay refused. + - Nix keeps link targets verbatim and tracks dependencies by scanning for + references. The whole store is its root, so it has no per-artifact + containment to compare with. + None of these is a static per-link rule for a tree that runs unconfined. The + T14 shape rule fits that case. +- T15. Host targets: `idunn-host` observes, starts and stops processes on + Windows only (the non-Windows `platform` module in `src/host.rs` bails), and + `host.rs` sets no file modes. On Windows a file runs by its type, not by a + mode bit, so `executable` and `executable_members` have nothing to set there. + The file `executable` flag has always been carried and never applied on host + targets. A recipe does not know which workload kind its binding will choose. +- T16. The release root holds only artifact destinations. Runner workspaces, + where external inputs land, are removed before `SealedRelease::new` + (`src/drivers.rs:2421-2423`). `install_release` (`:2781-2788`) and + `validate_installed_release` (`:2815-2822`) digest every receipt's + destination before calling harden. So a rule in the digest walk runs at seal, + at install and at every validate, over every link harden would see. +- T12. Disk: `/` on Yggdrasil has 1.3 TB free. A tree release costs about three + copies (runner workspace, staging, installed). Nothing collects old releases or + staging (`ghostlight-verse:follow_up:idunn-generation-retention`), so every + Vili deploy leaves about 7 GiB behind. + +### Contract, corrected against the body + +- Any directory artifact may declare `executable_members` (a set of relative + paths). The follow-up limited it to runner-output; that limit protects no + invariant, and the launcher's worktree or fetched trees need the same feature. +- `executable = true` names a file artifact. On a directory output the release + is refused, and `executable_members` on a file output is refused (T2). +- A member is refused at recipe parse if it is empty, absolute, or holds `.`, + `..`, a root or a prefix. At seal it is refused if it is missing, is a + symlink, passes through a symlink or non-directory, or is not a regular file. + Refusal happens before `SealedRelease::new`, so no sealed release can name a + member it cannot seal. +- Harden seals each member 0555 and fails if any declared executable path was + not visited as a regular file. "Every declared executable is sealed 0555, or + the install fails" holds structurally, not by a lookup that can miss. +- Symlinks: a link in a tree artifact is kept when its target is relative, its + `..` components all come before its first name, and they number no more than + the link's depth below the artifact's top (T14). One function, + `check_release_link`, decides this. Only the digest walk calls it, so a bad + link refuses the release at seal, before `SealedRelease::new`, and again at + install and validate (T16). Harden no longer judges links. The climb is + bounded by the artifact's top, not the release root: no live release has a + `..` link (the old rule refused them all), so nothing installed is newly + refused, and each artifact's digest describes a closed tree. +- Host targets carry `executable` and `executable_members` in the receipt, + shape-checked at seal by the shared `seal_artifact`, and do not apply them + (T15). Refusing members there would make a recipe's validity depend on its + binding. +- Vili's trees must + not come from `python -m venv`: its `bin/python` is an absolute link and its + `pyvenv.cfg` names the build workspace. Install packages into the relocatable + interpreter tree instead (`motion-pipeline:follow_up:vili-relocatable-python-tree`). + +### Authority + +Owner: the seal (`seal_artifact`, called by `collect_artifact` for systemd +targets and by `materialize` in `host.rs` for host targets) decides +membership from the recipe and writes it into the receipt. Harden enforces the +receipt and decides nothing. Runner exec bits and git `100755` modes are dead +inputs for release modes. Link containment belongs to `check_release_link`, +which only `digest_tree` calls. `validate_release_symlink` is deleted, and +harden's symlink arm leaves links as they are. + +### Acceptance sizing + +The big-tree job (spec `verification.operator`) needs about 12 GiB of scratch +on Yggdrasil: a 3.5 GiB tree, its staging copy, downloads and pip cache. It +removes it at the end. Revision 2 runs it on the trees as shipped, with the 300 +terminfo links (T13). If a tree holds a link the T14 rule refuses, the job +prints it and stops, and the link is reported to Self, not deleted. + +## Cut `isolation-credential-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-cut-isolation-credential-evidence.r1`, +at `112a7a5` (origin/main, live on Yggdrasil since 2026-10-24 11:29 UTC). It +serves Vili's admission (`motion-pipeline:cut_report:cut-vili-admission.h4`, +first deviation): every `idunn up vili` is refused before fencing. + +### Body facts (probed read-only on Yggdrasil, 2026-10-24) + +- I1. `idunn up vili` runs 6 and 7 failed with + `systemd workload isolation properties differ from the Idunn contract`, the + message of `observe_unit` (`src/drivers.rs:2975-3005`). Of its conjuncts, + only one differs on the failed Vili unit: `LoadCredential`. Idunn expects + `""` because Vili has no service credential besides the presence identity + (`service_credential_names` empty); systemd shows `[unprintable]`. Every + other conjunct matches, InaccessiblePaths included (one value across all + five idunn-vili/odin units). +- I2. systemd 257.13 `systemctl show -p LoadCredential` prints + `LoadCredential=[unprintable]` for every unit, whatever the credentials: + `ssh.service` and `systemd-journald.service` (not DynamicUser, zero + LoadCredential), the four failed `idunn-vili-*` units (zero), and the Odin + unit (one). The display form carries no information, so the conjunct was + vacuous whenever credentials existed and always false when none did. Odin + and StreamPixels passed only because each carries a regular credential. +- I3. The exact value is readable: D-Bus + `org.freedesktop.systemd1.Service.LoadCredential` (`busctl get-property`) + answers `a(ss) 0` for the Vili units and ssh, `a(ss) 1 + "ODIN_TOPOLOGY_IDENTITY" ""` for Odin. +- I4. What the workload can actually read is the delivered directory. On the + host, `/run/credentials/` holds a directory only for units with credentials + (getty, journald, Odin, StreamPixels service and web). Inside Odin's MainPID + root, `/run/credentials//` holds exactly `ODIN_TOPOLOGY_IDENTITY` + (`-r--r-----+ root root`). The presence identity is a parent-only `OpenFile` + descriptor, never a credential. +- I5. The rule arrived whole in `1fc0dad` (2026-09-03, "Rebuild Idunn + deployment transaction authority"), with its twin clause in + `validate_launch_observation` (`:3468-3472`, `:3513-3516`); no later commit + touched it. No ruling or finding in any campaign concerns workload isolation + verification; gamecult-chat's foreign-workloads cut touched + `CREDENTIALS_DIRECTORY` reservation, not this check. +- I6. `SystemdWorkloadObservation` is persisted by previous Idunns in both + MessagePack encodings, compact (positional) included + (`persisted_systemd_observations_decode_as_the_systemd_variant`, + `:9915-9930`), so deleting the `load_credential` field shifts every later + slot of a live incumbent's record. The field is also in the drift list + (`:2462`) that `stop` (`:4064-4070`) uses to refuse stopping a changed + workload: changing what the field holds without leaving that list would make + the next Idunn refuse to stop the live Odin and StreamPixels incumbents. +- I7. Fixtures: the verbatim 257.13 `systemctl show` output of base's 27 + properties for a Vili unit (presence identity only) and the Odin unit + (presence identity plus one service credential), captured 2026-10-24 into + this Imagination's scratchpad `fixtures/`. +- I8. The new rule holds for every live systemd incumbent: inside each + MainPID root the delivered entries equal its binding's `secret_files` keys + minus `GAMECULT_RUNTIME_PRESENCE_IDENTITY` (Odin 1, StreamPixels service 11, + StreamPixels web 1; key names compared, no values read). An Idunn restart onto + this cut re-observes them (`:3955-3970`) and passes. + +### Decision + +A property systemd cannot display does not belong in the comparison. The +invariant it reached for, that the workload holds exactly the credentials Idunn +delivered, is observed where it is decided: the entries of +`/run/credentials/` in the MainPID's root equal Idunn's service +credential names, absent directory equal to none. That covers +`LoadCredential`, `SetCredential`, `ImportCredential` and drop-ins alike, and +needs no new program. The D-Bus read (I3) is exact but sees only one of those +properties and adds a second systemd client and parser; rejected. + +### Authority + +Owner: `observe_unit` decides the credential rule, from the delivered +directory. `LoadCredential`'s display form is no longer read or compared; the +`load_credential` field is a retired wire slot, written empty, out of the drift +list, kept only so persisted records decode. Forbidden writers: the +`observe_unit` conjunct and the `validate_launch_observation` clause. Dropping +the slot waits on the next schema revision of the observation. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). After it, Vili's Hands reruns from step 4 of +`motion-pipeline:cut_report:cut-vili-admission.h4`'s undone list. + +## Cut `runner-failure-evidence` + +Spec `idunn-deploy-provenance:cut_spec:cut-runner-failure-evidence.r1`, at +`804ed75` (origin/main, live on Yggdrasil). It serves the StreamPixels-area +Self: `idunn up vili` (command `up-4866467c-ee98-4a2b-b4a9-a008d7c44acd`, +transaction `tx-a04b68c7-c742-4d7d-8220-8aa08b9febae`, 2026-10-25 00:50-00:55 +UTC) failed in Sealing with `Docker runner exited with exit status: 1: ` and +nobody could tell which recipe step failed. + +### Body facts (source read at `804ed75`; host read-only, 2026-10-25) + +- R1. The runner is `docker run --rm` (`src/drivers.rs:2088-2090`), launched by + `DockerRunnerDriver::docker` (`:2173-2195`) with stdin null and + `Command::output()`: stdout and stderr are each buffered whole in the + daemon's memory, unbounded. On a non-zero exit the error is + `"Docker runner exited with {status}: {stderr}"`; stdout is dropped. The + container is gone the moment it exits, so `docker ps -a` and `docker logs` + hold nothing. +- R2. The empty message: the failing process wrote nothing to stderr, and + stdout, the only stream that carried its report, is discarded at + `:2182-2193`. The same interpolation echoes whatever a step does write to + stderr into `last_error`, `pre_fencing_abort.error` and + `completion.error` (`bounded_detail`, 2048 bytes), and from there into + `idunn up`'s `deployment failed:` line and `idunn status`. +- R3. No step identity reaches the error. `materialize` (`:2368-2391`) loops + over `declaration.steps` and calls `run_in_workspace` with argv only; + `step.id` is used only in the environment precondition. External inputs + (`curl` in the runner, `:2247-2258`) fail through the same helper with the + same anonymous message. +- R4. Which step failed on 2026-10-25, inferred from the failed transaction's + staging root, which survives (926 MiB, + `/var/lib/gamecult/idunn/staging/tx-a04b68c7-.../`): in `.runner-node-build`, + `node_modules/.package-lock.json` was written at 00:51:24 (so + `install-vili-dependencies`, `npm ci`, finished), nothing outside + `node_modules` is newer, and `release/vili` (the next step's output) is + absent; the python workspace holds only fetched inputs. The failing step was + `test-daemon` (`node --test ... scripts/test-vili-daemon.mjs`), whose test + reporter writes to stdout: exactly the stream R1 discards. Inference from + residue, not from output. +- R5. The failure path: `resume_one_transaction` (`src/control_plane.rs:6291-6320`) + sends a pre-fence error to `begin_pre_fencing_abort` (`:9916-9935`), which + persists `bounded_detail(format!("{error:#}"))`. Nothing reaches the journal + on this path: the idunn-yggdrasil journal for 00:45-01:10 holds only + provider-exclusion lines. +- R6. Stores. `control.cc` (root 0640) holds commands, transactions + (`idunn.deployment_transaction.v4`, keys 0-36), generations, supervision and + expiry requests; `ControlSnapshot::read` refuses any other type + (`:3420`). Records must re-encode byte-exactly (`decode_record`, + `:3717-3727`), so a new transaction field is a schema bump and a v4 lift + (`:4100-4110`). Terminal transactions move verbatim to `history.cc` + (`:4366-4396`; 39.8 MB on the host), which status reads for `--command`. + `incidents.cc` lives in the published projection directory and by its own + doc (`src/incident.rs:1-11`) carries no free text and is never `control.cc`, + because a type added there stops continuity for every target on a rollback. +- R7. The daemon runs as root (`User=root`). Runner secrets reach the + container as read-only file mounts named by env (`:2041-2047`), so any text + a step derives from them can land in its output. +- R8. Staging residue: 475 transaction roots, 55 GiB under + `/var/lib/gamecult/idunn/staging`; a failed Sealing leaves its workspaces + because `materialize` returns before `remove_tree_inside`. + +### Decision + +The failure becomes a typed value at the runner, `RunnerStepFailure`: the +subject (recipe step or external input, by id), the runner id, the exit code or +signal, and the last 8 KiB of stdout and of stderr with each stream's total +byte count. Its `Display` carries the subject and status, never output. +Capture streams both pipes through bounded tails, so a verbose build no longer +buffers whole in the daemon. + +The tail lives in a new private store, `runner-evidence.cc`, beside +`control.cc` (mode 0600), one record per transaction +(`idunn.runner_failure.v1`), newest 32 kept by the writer. Not in the +transaction record: that is a v5 bump whose rollback makes every older Idunn +refuse its control store (R6), the scar `incident.rs` already names. Not in +`incidents.cc`: published, and free text is barred there. Not in the journal: +journal readers are a wider set than root, so the journal line names the +transaction, the subject, the status and the store, and no output. + +No redaction. Matching secret bytes would make the daemon read every secret, +and still misses any encoding, prefix or transform of one; a claimed redaction +that leaks is worse than a stated rule. The rule instead is the reader set: the +store's only readers (root) can already read every secret the runner was +given. + +Keeping the failed container (dropping `--rm`) is worse: its output then lives +in Docker's json-file log, unbounded, untyped, outside Idunn's state and +readable by the docker group; containers accumulate with no owner to reap them; +`docker system prune` erases the evidence; and nothing ties a container to the +transaction in `idunn status`. + +### Authority + +Owner: the runner decides what failed and keeps the bounded tail; the engine's +pre-fence failure path is the one writer of `runner-evidence.cc`. +`last_error`, `pre_fencing_abort.error` and `completion.error` keep only the +failure's `Display`. Forbidden writers: the `stderr` interpolation in +`DockerRunnerDriver::docker`. Retention is by count inside the writer, so the +store needs no external owner; staging roots (R8) stay with +`ghostlight-verse:follow_up:idunn-generation-retention`. + +### Deploy + +Idunn's body changes only through gamecult-ops +`scripts/install-idunn-yggdrasil-release.sh` (stage `idunn` only; unit and +bindings unchanged). The next failed Sealing then shows its step and tail on +`sudo idunn status --command `. diff --git a/docs/deploy-provenance-target.md b/docs/deploy-provenance-target.md index 74015c7..e1086e8 100644 --- a/docs/deploy-provenance-target.md +++ b/docs/deploy-provenance-target.md @@ -4,6 +4,15 @@ Status: target, Self, 2026-10-19 (campaign `idunn-deploy-provenance` in Eureka's mind). It states the ends. The map will live in `deploy-provenance-map.md`. +Live as of cut `brake-is-a-stop` (2026-11-13): the deployment brake is the +stop of invariant 3 and the signed per-deploy release is gone. `src/brake.rs` +reads and writes it; `idunn-provision deployment-brake-engage|clear|status` +take only `--store`, `--runtime-id` and `--reason`; the writer's login name and +the verb's clock are recorded; an absent or clear store admits. The +transaction stamps `deployment_admitted_at_unix_millis` (record v6). Still +planned: the typed authority provenance on the transaction (invariant 2), and a +steward-scoped store. + ## Why Operator, 2026-10-19: @@ -53,6 +62,11 @@ In: (`heimdall-discord-launch.md`, `idunn-host-raven.md`, the StreamPixels runbook). +- A static release (a sealed tree published by repointing a link, with no + workload): its transaction records `flip` (release, previous release, the + sha256 of the carried per-site file) beside the brake bytes. See + `deployment-authority.md`, "Static release". + Out: - `idunn-reincarnation`. This campaign is mapped after that campaign's cut 2 diff --git a/docs/deployment-authority.md b/docs/deployment-authority.md index bdc7efc..048e784 100644 --- a/docs/deployment-authority.md +++ b/docs/deployment-authority.md @@ -48,8 +48,8 @@ and health. CultMesh carries typed projections between them. scaling, restart, host reboot recovery, and crash recovery use the same source-to-recipe-to-artifact provenance and the same distinct write-lease and route-membership primitives. - Deployment may change that chain only while the deployment brake admits the - exact transaction. Continuity may restart only the already-admitted chain and + Deployment may change that chain only while the deployment stop admits it + (absent or clear; an engaged stop halts). Continuity may restart only the already-admitted chain and is governed by a separate lifecycle brake. - **Deletion line:** hard-coded target copies, raw `deploy_command` and `restart_command` state, `sh -c` actuation, root Git inspection, staged @@ -171,6 +171,96 @@ different semantic capability. `gamecult-ops` publishes these fleet bindings and provisions host substrate. It does not own application build or deployment programs. +## Static release + +A target whose product is a sealed tree, with no process to run (a built web +front end), declares `[static_release] artifact = ""` in place of +`[service]`, and its binding declares `driver = "static-release"` with a +`release_root`, a `current_link` and one `[workload.carry]`. It is the same +transaction as any deployment, in three phases: Sealing (source, build, seal, +install: unchanged), Starting (the one external change, behind the deployment +brake) and Complete. It never enters Warming through Committing, has no +Expected projection, no activation and no workload, publishes nothing to +Odin, and its Complete creates no admitted generation, so continuity never +supervises it. It declares no state, capabilities, dependencies, conflicts, +route or write lease, and its rollout is `candidate-then-promote` with at least +two retained releases. + +What Starting does, in three durable steps recorded as `flip` on the +transaction: the release to name and the release the link names now are +written first; then Idunn copies the operator's carry source into +`//` (mode 0444) and +renames a fresh symlink over `current_link`; then it observes the link naming +the release and completes. A failure before the rename ends the transaction +through the ordinary pre-fencing abort with `current_link` untouched. A crash +resumes at the recorded intent and repeats the flip, which is harmless. + +The carry is a copy of one per-site file (for example `config.json`) taken at +flip time. It sits beside the artifacts, never inside one, so it is outside +every sealed digest: changing the site file changes no release id, and the +sha256 of the bytes carried is provenance on the transaction +(`flip.carried_sha256`), beside the brake bytes and the requester. The +destination is a single leaf name that no artifact uses. The web server reads +`current_link/` for the tree and `current_link/` for the carried file (a one-time root edit of the vhost, an +`alias` for the carried file beside the root for the tree). A missing or +non-regular carry source fails the deployment before anything moves. + +What `current_link` names is observed from the symlink and confirmed by the +latest Admitted static transaction's `flip.release_id`; it is stored nowhere +else. A rollback is a flip back to `flip.previous_release_id`: release +directories stay, and Idunn prunes none. The deployment stop is read exactly as for a +service, by the binding's runtime id. Release ids are per transaction, so +redeploying an unchanged tarball with a changed site file is a new release that +carries the new file, never an in-place re-carry of an existing one: the carry +is part of a release. If `current_link` is replaced by a directory or by a link +that names nothing under the release root after the flip was recorded, the +transaction keeps retrying with backoff: `idunn status --command ` shows the +error as "waiting on ...", operator cancel is refused (Starting, not held), and +the fix is by hand (restore the link or remove the directory); it then +completes. A bounded retry is follow-up work. A new +upstream tarball enters as a recipe `external_input` with a pinned url and +sha256, so each release is a recipe commit whose pin is its provenance. + +### Rollback + +`idunn rollback [--state-store PATH] [--no-wait]` writes a +`rollback-` command and nothing else. The daemon freezes it into one new +transaction (`CommandKind::Rollback`) that reuses the previous release exactly: +the plan, sealed release and installed release are copied from the transaction +that flipped it, so there is no source, no build and no source cleanup, and it +creates no admitted generation. It starts at Sealing with that evidence and +reaches the same deployment stop as a deploy, which admits it unless engaged; +then `advance_static` flips, with the same three durable steps, the same +nothing-aborts-after-the-rename rule and the same stuck-flip behavior. + +The previous release is derived, never stored: the newest Admitted static +transaction for the target (control.cc and history.cc) names the release +`current` must point at (`flip.release_id`) and the one it displaced +(`flip.previous_release_id`); the newest transaction that flipped that release +supplies the evidence. A rollback of a rollback therefore flips forward again. +The command is refused, recorded as a rejected transaction, when the target has +no completed static release, when the newest release is the first one, when +`current` does not name the newest flip, when the previous release directory is +gone, or when an artifact in it no longer digests to its sealed receipt. A +failed check at flip time ends the transaction before fencing with `current` +untouched. + +The rollback reuses the old plan verbatim, so its binding snapshot, brake path +and carry source are the old ones: if the operator moved the site file the +rollback fails before the link moves. The carry is re-applied from the site file +as it is at rollback time, and its sha256 is recorded in the rollback's own +flip evidence (it may differ from the original release's; the copy inside the +rolled-back-to release is replaced). Nothing prunes release directories. + +`CommandKind::Rollback` is stored in the existing v5 command and transaction +records with no schema change; an Idunn at 9f15cee or earlier cannot decode +them, so this is forward-only, like v5 itself. The newest flip is ordered by +`flipped_at` wall clock, and the observed-link check makes a clock step refuse +the rollback rather than mis-flip. A rollback cannot be cancelled, like a static +deploy. A rollback is also refused when the previous release was deployed under +a different release root or current link than the newest one. + ## Typed deployment state Mutable control-plane state will be CultCache. Runtime integration must use @@ -209,6 +299,82 @@ names both content addresses and the executable artifact digest. Present and Ready remain Odin-owned correlations over independently verified runtime evidence; the deployment-plan module deliberately does not model them. +## The deployment stop + +`src/brake.rs` is the only reader and writer of the stop. A stop halts work +that has not crossed an irreversible step, and it is read again at each such +step: Sealing (`advance_sealing`), the static flip intent (`advance_static`) and +the entry into Fencing for a deploy (`advance_warming`). An engaged stop there +makes the transaction wait with a note naming who engaged it and why; nothing +aborts and nothing moves. After the flip intent is recorded, or once a deploy +has fenced, the transaction runs to its end: a stop halts before the +irreversible step, never mid-fence or mid-flip. Continuity is never gated. +The sequence of a stop record is positive and advances by one per verb; nothing +reads it for authority, so it is not validated further. + +Absent means the store file is missing in an existing directory. Anything else +Idunn cannot judge (a symlink, a zero-length or undecodable file, another +runtime's record, another owner, an unsearchable or non-directory parent) reads +as unreadable and fails closed; the wait note carries the +`idunn-provision deployment-brake-clear` command that replaces it. + +A binding may name a second stop, the target's steward: + +```toml +[brakes.steward] +user = "mind-streampixels" # a login, never root +store = "/var/lib/gamecult/idunn-steward/streampixels/deployment-brake.cc" +``` + +Deploys admit only when every stop admits. The steward's store is read as owned +by that user's uid: the owner uid proves who wrote the file, and the wait note +names the stop by the binding's steward user, never by the record's `by` (which +is the verb's own text). Idunn never writes the steward directory and no key, +group or ACL is involved. The operator's store is a separate stop the steward +cannot clear, and the wait note names each holding stop. + +The steward owns its directory, so it could unlink any file in it. Binding +validation therefore refuses a steward store whose directory is shared with, +inside or around the directory of any other Idunn authority store (deployment +and lifecycle brakes, trust anchor, write-lease record, release authority). +Validation cannot see the file system: the root who provisions the directory +makes it contain the steward's store and nothing else. + +A steward store owned by another uid, or a steward user that does not resolve +to a non-root uid, is unreadable and fails closed. Only the steward (the +owner uid) can repair it: a clear run as root leaves a root-owned file that is +still unreadable. + +### Provisioning a steward (root, once per target) + +1. Create the store directory, owned by the steward and holding nothing else: + `install -d -o mind-streampixels -g mind-streampixels -m 0755 + /var/lib/gamecult/idunn-steward/streampixels`. It must not be, contain or sit + inside any other authority directory, and no workload `ReadWritePaths` may + cover it. Idunn (root) reads it without further grants; no group or ACL. +2. Add the stanza above to the target's operator binding, after `[brakes]`, + with the steward's real login. Deploys planned afterwards see it. +3. The steward engages and clears as itself, never through root: + `idunn-provision deployment-brake-engage --store --runtime-id + --reason ` and `deployment-brake-clear` with the same flags. +4. The operator reads it with `idunn-provision deployment-brake-status --store + --runtime-id --owner-user mind-streampixels`. + +The binding is sealed into each plan. A deploy already planned keeps the +binding it was sealed with: editing the stanza does not apply to it. If the +steward's login is deleted or its directory is lost, that deploy stays +unreadable-held at Sealing (or the static flip intent). `idunn cancel` withdraws +only a deploy held for a readiness disagreement before Fencing, and `idunn +expire` only one in Warming through Committing, so neither frees a deploy held +by the stop. Recovery is to restore the steward's login or recreate the +directory owned by it (and have the steward clear its store). + +Record v6 (`deployment_admitted_at_unix_millis`, key 38) is forward-only: an +Idunn that predates v6 cannot read a migrated `control.cc`, so, as for v5 (see +`route-continuity-cut.md`, "An older Idunn refuses the v5 store"), take a +byte-compared pre-install backup of `control.cc` and `history.cc` with Idunn +stopped, and roll back by restoring the backup, never by swapping the binary. + ## Initial drivers and replaceable boundary The first Yggdrasil implementation uses: @@ -247,7 +413,7 @@ For a singleton incarnation: declaration, binding, launch contract, and input digests. 5. Derive the release-bound Expected incarnation from the validated sealed release and its exact compiled plan. Publish it only once the deployment - brake has admitted this exact transaction, under the candidate's own + stop has admitted this transaction (stamping `deployment_admitted_at_unix_millis`), under the candidate's own incarnation key, beside whatever the target's admitted incarnation projects. 6. Issue one opaque runtime-instance activation bound to the Expected digest, pass the exact Expected and activation documents through the actuator's @@ -270,8 +436,8 @@ For a singleton incarnation: and stop the incumbent. Sealing resolves the source, freezes it, runs every recipe step, and installs -the release before the deployment brake is consulted at all (`advance_sealing`). -The brake gates step 5, the first Verse-visible change: publishing the +the release before the deployment stop is consulted at all (`advance_sealing`). +The stop gates step 5, the first Verse-visible change: publishing the candidate's Expected incarnation on the move into `Starting`. Build and test already run unbraked, inside the sealing phase. @@ -302,7 +468,7 @@ identity, and route membership. Every restart receives a new runtime-instance activation and, for a state writer, a new process-bound lease only after the new warming presence is observed. A lease from the dead process is never reused. A continuity restart does not fetch a branch, rebuild, rerun a migration, alter -the launch contract, or consume the deployment brake. An explicit lifecycle +the launch contract, or pass the deployment stop. An explicit lifecycle brake may suspend this actuation without changing deployment authority. Continuity never stops. Six charged attempts in an hour only open the @@ -335,7 +501,7 @@ the admitted body. Both operator cases open an `operator-required` incident on. The one yield: a continuity whose route proof has not succeeded gives way to a -deployment the deployment brake has released that awaits the handover of the +deployment the deployment stop admits that awaits the handover of the target. It begins its own post-fencing abort and the deployment fences what is admitted. A continuity that has proven its route is serving never yields. diff --git a/docs/guide.md b/docs/guide.md index e48bc85..fcf806b 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -112,22 +112,12 @@ cargo run --release --bin idunn -- --daemon demo --rudp-health-bind &'static str { + match self { + Self::Engaged => "engaged", + Self::Clear => "clear", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.deployment_stop", schema = "idunn.deployment_stop.v1")] +pub(crate) struct DeploymentStop { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) runtime_id: String, + #[cultcache(key = 2)] + pub(crate) state: StopState, + /// The login name of the user who ran the verb, read from the process uid. + #[cultcache(key = 3)] + pub(crate) by: String, + #[cultcache(key = 4)] + pub(crate) reason: String, + /// The verb's clock. + #[cultcache(key = 5)] + pub(crate) at_unix_millis: u64, + /// The previous record's sequence plus one; 1 for the first. + #[cultcache(key = 6)] + pub(crate) sequence: u64, +} + +fn bounded(value: &str, label: &str, max: usize) -> Result<()> { + ensure!( + !value.trim().is_empty() && value.len() <= max && !value.chars().any(char::is_control), + "deployment stop {label} is empty, oversized, or contains control characters" + ); + Ok(()) +} + +impl DeploymentStop { + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == DEPLOYMENT_STOP_SCHEMA, + "deployment stop schema is unsupported" + ); + bounded(&self.runtime_id, "runtime id", 256)?; + bounded(&self.by, "author", 256)?; + bounded(&self.reason, "reason", 1024)?; + ensure!( + self.at_unix_millis > 0 && self.sequence > 0, + "deployment stop time or sequence is zero" + ); + Ok(()) + } + + fn envelope(&self) -> Result { + self.validate()?; + let stored_at = chrono::DateTime::from_timestamp_millis(i64::try_from(self.at_unix_millis)?) + .ok_or_else(|| anyhow!("deployment stop time is out of range"))? + .to_rfc3339(); + Ok(CultCacheEnvelope { + key: DEPLOYMENT_STOP_KEY.into(), + r#type: Self::TYPE.into(), + payload: rmp_serde::to_vec(self)?, + stored_at, + schema_id: Some(DEPLOYMENT_STOP_SCHEMA.into()), + }) + } + + fn decode(envelope: &CultCacheEnvelope) -> Result { + ensure!( + envelope.r#type == Self::TYPE + && envelope.key == DEPLOYMENT_STOP_KEY + && envelope.schema_id.as_deref() == Some(DEPLOYMENT_STOP_SCHEMA), + "deployment stop store holds a foreign record" + ); + let record: Self = rmp_serde::from_slice(&envelope.payload)?; + ensure!( + rmp_serde::to_vec(&record)? == envelope.payload, + "deployment stop is noncanonical" + ); + record.validate()?; + Ok(record) + } +} + +/// What one store says about a runtime's deployments. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Stop { + /// No store, or an empty one: nothing halts. + Absent, + Clear(DeploymentStop), + Engaged(DeploymentStop), + /// Present but not a record this runtime can trust. Does not admit. + Unreadable(String), +} + +impl Stop { + pub(crate) fn admits(&self) -> bool { + matches!(self, Self::Absent | Self::Clear(_)) + } +} + +/// One stop a binding names, with how to name it in a wait note. +pub(crate) struct NamedStop { + pub(crate) label: String, + /// The login the file system proves wrote the store: the steward user, whose + /// uid owns the file. None for the operator's store, whose record carries + /// only the verb's own claim of `by`. + pub(crate) proven_writer: Option, + pub(crate) stop: Stop, +} + +impl NamedStop { + /// The wait-note line for a stop that does not admit; None when it admits. + /// A steward stop is named by the proven owner, never by the record's `by`. + pub(crate) fn note(&self) -> Option { + let label = &self.label; + match &self.stop { + Stop::Absent | Stop::Clear(_) => None, + Stop::Engaged(held) => Some(match &self.proven_writer { + Some(_) => format!("{label} engaged: {}", held.reason), + None => format!("{label} engaged by {}: {}", held.by, held.reason), + }), + Stop::Unreadable(why) => Some(format!("{label} is unreadable and fails closed: {why}")), + } + } +} + +/// Every stop a binding names, each with the words that identify its store in +/// a wait note: the operator's, then the steward's when the binding names one. +/// Deployments admit iff every one admits. The operator's store is read as +/// owned by this process's uid; the steward's as owned by the steward user's +/// uid, so the file system proves who wrote it. A steward that does not +/// resolve to a non-root uid is an unreadable stop, never an absent one. +pub(crate) fn read_stops(brakes: &BrakeBinding, runtime_id: &str) -> Vec { + let mut stops = vec![NamedStop { + label: "deployment brake".to_string(), + proven_writer: None, + stop: read_stop(&brakes.deployment_store, runtime_id, process_uid()), + }]; + if let Some(steward) = &brakes.steward { + let label = format!("steward {} deployment brake", steward.user); + let stop = match crate::steward_orders::user_uid(&steward.user) { + Ok(uid) => read_stop_with_remedy(&steward.store, runtime_id, uid, |path| { + // Only the owner uid can mend it: a clear run as root leaves a + // root-owned file, which this reader refuses again. + format!( + "only {} (the store's owner uid) can repair it, by running as that user: \ + idunn-provision deployment-brake-clear --store {} --runtime-id {runtime_id} \ + --reason ; a clear run as root stays unreadable", + steward.user, + path.display() + ) + }), + Err(why) => Stop::Unreadable(format!( + "{why:#}; the binding's steward user {} must be a non-root login", + steward.user + )), + }; + stops.push(NamedStop { + label, + proven_writer: Some(steward.user.clone()), + stop, + }); + } + stops +} + +#[cfg(unix)] +pub(crate) fn process_uid() -> u32 { + // SAFETY: geteuid has no failure mode and no arguments. + unsafe { libc::geteuid() } +} + +#[cfg(not(unix))] +pub(crate) fn process_uid() -> u32 { + 0 +} + +#[cfg(unix)] +pub(crate) fn login_name(uid: u32) -> Result { + // SAFETY: getpwuid returns null or a pointer into libc's static buffer, + // copied before any other call; the verbs are single threaded. + unsafe { + let entry = libc::getpwuid(uid); + ensure!(!entry.is_null(), "uid {uid} has no login name"); + Ok(std::ffi::CStr::from_ptr((*entry).pw_name) + .to_string_lossy() + .into_owned()) + } +} + +#[cfg(not(unix))] +pub(crate) fn login_name(_: u32) -> Result { + std::env::var("USERNAME").context("USERNAME is unset") +} + +/// Open without following a final symlink and judge the file by fstat before +/// anything is read from it. +#[cfg(unix)] +fn check_file(path: &Path, expected_owner_uid: u32) -> Result<()> { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + .with_context(|| format!("opening deployment stop store {}", path.display()))?; + let metadata = file.metadata()?; + ensure!( + metadata.is_file(), + "deployment stop store is not a regular file" + ); + ensure!( + metadata.uid() == expected_owner_uid, + "deployment stop store is owned by uid {}, not {expected_owner_uid}", + metadata.uid() + ); + Ok(()) +} + +#[cfg(not(unix))] +fn check_file(path: &Path, _: u32) -> Result<()> { + ensure!( + std::fs::metadata(path)?.is_file(), + "deployment stop store is not a regular file" + ); + Ok(()) +} + +/// Absent means exactly this: the final component does not exist and its +/// parent is an existing directory. Every other failure to look (permission, +/// a parent that is a file, a symlink loop) is a store this runtime cannot +/// judge, and fails closed. +fn is_absent(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let parent = path.parent().filter(|p| !p.as_os_str().is_empty()); + let parent = parent.unwrap_or_else(|| Path::new(".")); + ensure!( + std::fs::metadata(parent).is_ok_and(|m| m.is_dir()), + "deployment stop store directory {} does not exist", + parent.display() + ); + Ok(true) + } + Err(error) => Err(error) + .with_context(|| format!("looking for deployment stop store {}", path.display())), + } +} + +fn read_checked(path: &Path, runtime_id: &str, owner: u32) -> Result> { + if is_absent(path)? { + return Ok(None); + } + check_file(path, owner)?; + ensure!( + std::fs::metadata(path)?.len() > 0, + "deployment stop store is zero length" + ); + let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + match entries.as_slice() { + [] => bail!("deployment stop store holds no record"), + [entry] => { + let record = DeploymentStop::decode(entry)?; + ensure!( + record.runtime_id == runtime_id, + "deployment stop belongs to runtime {}", + record.runtime_id + ); + Ok(Some(record)) + } + _ => bail!("deployment stop store is ambiguous"), + } +} + +pub(crate) fn read_stop(path: &Path, runtime_id: &str, expected_owner_uid: u32) -> Stop { + read_stop_with_remedy(path, runtime_id, expected_owner_uid, |path| { + format!( + "to replace it run: idunn-provision deployment-brake-clear \ + --store {} --runtime-id {runtime_id} --reason (or deployment-brake-engage)", + path.display() + ) + }) +} + +fn read_stop_with_remedy( + path: &Path, + runtime_id: &str, + expected_owner_uid: u32, + remedy: impl FnOnce(&Path) -> String, +) -> Stop { + match read_checked(path, runtime_id, expected_owner_uid) { + Ok(None) => Stop::Absent, + Ok(Some(record)) if record.state == StopState::Clear => Stop::Clear(record), + Ok(Some(record)) => Stop::Engaged(record), + Err(error) => Stop::Unreadable(format!("{error:#}; {}", remedy(path))), + } +} + +/// Write the next record under a compare-exchange of the whole store, so a +/// concurrent verb or a store that moved since it was read is refused. The +/// author is the login name of the process uid and the sequence is the +/// predecessor's plus one (1 when there is none or it cannot be decoded); +/// neither is an argument. +/// +/// This is also the operator's break-glass. A predecessor that is a retired v1 +/// brake, another runtime's record, ambiguous, or zero length is replaced. A +/// file that is not a CultCache store at all (junk bytes) is replaced only +/// when it is a regular file owned by the caller, by removing it first. A +/// symlink or another user's file is refused. The sequence is not validated +/// beyond being positive: nothing reads it for authority. +pub(crate) fn replace_stop( + path: &Path, + runtime_id: &str, + state: StopState, + reason: &str, + now_unix_millis: u64, +) -> Result { + let store = SingleFileMessagePackBackingStore::new(path); + let before = match store.pull_all_read_only_snapshot() { + Ok(entries) => entries, + Err(undecodable) => { + check_file(path, process_uid()).with_context(|| { + format!("deployment stop store is undecodable ({undecodable:#}) and not the caller's to replace") + })?; + std::fs::remove_file(path)?; + Vec::new() + } + }; + let sequence = match before.as_slice() { + [entry] => match DeploymentStop::decode(entry) { + Ok(previous) => previous.sequence.saturating_add(1), + Err(_) => 1, + }, + _ => 1, + }; + let record = DeploymentStop { + schema_version: DEPLOYMENT_STOP_SCHEMA.into(), + runtime_id: runtime_id.into(), + state, + by: login_name(process_uid())?, + reason: reason.into(), + at_unix_millis: now_unix_millis, + sequence, + }; + let next = record.envelope()?; + ensure!( + store.compare_exchange_snapshot(&before, &[next])?, + "deployment stop changed during transition" + ); + Ok(record) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn engage(path: &Path, reason: &str, at: u64) -> Result { + replace_stop(path, "runtime", StopState::Engaged, reason, at) + } + + #[test] + fn absent_and_empty_stores_admit_a_clear_one_admits_an_engaged_one_does_not() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + let me = process_uid(); + assert_eq!(read_stop(&path, "runtime", me), Stop::Absent); + engage(&path, "freeze for the audit", 100)?; + let Stop::Engaged(held) = read_stop(&path, "runtime", me) else { + bail!("an engaged stop did not read as engaged") + }; + assert_eq!(held.reason, "freeze for the audit"); + assert_eq!(held.by, login_name(me)?); + assert!(!Stop::Engaged(held).admits()); + replace_stop(&path, "runtime", StopState::Clear, "audit done", 200)?; + let cleared = read_stop(&path, "runtime", me); + assert!(matches!(&cleared, Stop::Clear(record) if record.sequence == 2)); + assert!(cleared.admits()); + // Clear is not consumed: the same record admits again. + assert!(read_stop(&path, "runtime", me).admits()); + Ok(()) + } + + #[test] + fn corrupt_foreign_other_runtime_symlinked_or_foreign_owned_stores_stop() -> Result<()> { + let dir = tempfile::tempdir()?; + let me = process_uid(); + let path = dir.path().join("stop.cc"); + engage(&path, "held", 100)?; + // Another runtime's record. + assert!(!read_stop(&path, "other", me).admits()); + // Owned by someone other than the expected uid: even a CLEAR record + // from a foreign owner must not admit. + let cleared = dir.path().join("cleared.cc"); + replace_stop(&cleared, "runtime", StopState::Clear, "thaw", 100)?; + assert!(read_stop(&cleared, "runtime", me).admits()); + assert!(matches!(read_stop(&cleared, "runtime", me + 1), Stop::Unreadable(_))); + // A symlink to a valid, clear store. + let real = dir.path().join("real.cc"); + replace_stop(&real, "runtime", StopState::Clear, "clear", 100)?; + let link = dir.path().join("link.cc"); + #[cfg(unix)] + std::os::unix::fs::symlink(&real, &link)?; + #[cfg(unix)] + assert!(matches!(read_stop(&link, "runtime", me), Stop::Unreadable(_))); + // Bytes that are no store at all. + let junk = dir.path().join("junk.cc"); + std::fs::write(&junk, b"not msgpack at all")?; + assert!(matches!(read_stop(&junk, "runtime", me), Stop::Unreadable(_))); + // A record of another type under another key (the retired v1 brake). + let foreign = dir.path().join("foreign.cc"); + let store = SingleFileMessagePackBackingStore::new(&foreign); + let envelope = CultCacheEnvelope { + key: "deployment".into(), + r#type: "idunn.deployment_brake".into(), + payload: vec![0x90], + stored_at: "1970-01-01T00:00:00+00:00".into(), + schema_id: Some("idunn.deployment_brake.v1".into()), + }; + assert!(store.compare_exchange_snapshot(&[], &[envelope])?); + assert!(matches!(read_stop(&foreign, "runtime", me), Stop::Unreadable(_))); + // A noncanonical payload under the right identity. + let mut stop = DeploymentStop::decode(&store_entry(&path)?)?; + stop.sequence = 9; + let mut loose = stop.envelope()?; + loose.payload.push(0xc0); + let noncanonical = dir.path().join("noncanonical.cc"); + assert!(SingleFileMessagePackBackingStore::new(&noncanonical) + .compare_exchange_snapshot(&[], &[loose])?); + assert!(matches!(read_stop(&noncanonical, "runtime", me), Stop::Unreadable(_))); + Ok(()) + } + + /// A binding whose steward is the user running the test, so the steward's + /// store can really be owned by the steward. None when run as root, which + /// no steward may be. + fn binding_with_steward(dir: &Path, user: &str) -> BrakeBinding { + BrakeBinding { + deployment_store: dir.join("operator.cc"), + lifecycle_store: dir.join("lifecycle.cc"), + steward: Some(crate::deployment::StewardBrakeBinding { + user: user.into(), + store: dir.join("steward.cc"), + }), + } + } + + fn admits_all(stops: &[NamedStop]) -> bool { + stops.iter().all(|named| named.stop.admits()) + } + + #[test] + fn the_steward_store_is_a_second_stop_and_the_operator_store_is_one_it_cannot_clear() -> Result<()> { + let me = process_uid(); + if me == 0 { + eprintln!("SKIPPED: a steward is never root, and this run is root"); + return Ok(()); + } + let dir = tempfile::tempdir()?; + let binding = binding_with_steward(dir.path(), &login_name(me)?); + let (operator, steward) = (&binding.deployment_store, &binding.steward.as_ref().unwrap().store); + // Both absent: admits. + assert!(admits_all(&read_stops(&binding, "runtime"))); + // Steward engaged alone: halts, and the stop names the steward's store. + replace_stop(steward, "runtime", StopState::Engaged, "steward freeze", 10)?; + let stops = read_stops(&binding, "runtime"); + assert!(!admits_all(&stops)); + assert!(matches!(&stops[1].stop, Stop::Engaged(held) if held.reason == "steward freeze")); + // The note names the proven owner, and never the record's claimed `by`. + assert_eq!( + stops[1].note().unwrap(), + format!("steward {} deployment brake engaged: steward freeze", login_name(me)?) + ); + // Both engaged, then the steward clears its own: the operator's holds. + replace_stop(operator, "runtime", StopState::Engaged, "operator hold", 11)?; + replace_stop(steward, "runtime", StopState::Clear, "steward thaw", 12)?; + let stops = read_stops(&binding, "runtime"); + assert!(matches!(&stops[0].stop, Stop::Engaged(_)) && matches!(&stops[1].stop, Stop::Clear(_))); + assert!(!admits_all(&stops)); + // The other order: operator clear, steward engaged still halts. + replace_stop(operator, "runtime", StopState::Clear, "operator thaw", 13)?; + replace_stop(steward, "runtime", StopState::Engaged, "steward again", 14)?; + assert!(!admits_all(&read_stops(&binding, "runtime"))); + // Both clear: admits. + replace_stop(steward, "runtime", StopState::Clear, "steward thaw", 15)?; + assert!(admits_all(&read_stops(&binding, "runtime"))); + Ok(()) + } + + #[test] + fn steward_store_owned_by_another_uid_stops() -> Result<()> { + let me = process_uid(); + let dir = tempfile::tempdir()?; + // Steward is "nobody" (a real non-root user that is not the writer); + // the file is written by this process, so even a Clear record fails. + if me == 65534 || user_missing("nobody") { + eprintln!("SKIPPED: needs a user `nobody` other than the caller"); + return Ok(()); + } + let binding = binding_with_steward(dir.path(), "nobody"); + let steward = &binding.steward.as_ref().unwrap().store; + replace_stop(steward, "runtime", StopState::Clear, "forged clear", 10)?; + let stops = read_stops(&binding, "runtime"); + assert!(matches!(&stops[1].stop, Stop::Unreadable(why) if why.contains("is owned by uid") + && why.contains("only nobody (the store's owner uid) can repair it") + && why.contains("stays unreadable") + && why.contains("idunn-provision deployment-brake-clear"))); + // The operator's stop keeps the operator's remedy. + replace_stop(&binding.deployment_store, "other-runtime", StopState::Clear, "x", 11)?; + assert!(matches!(&read_stops(&binding, "runtime")[0].stop, Stop::Unreadable(why) + if why.contains("to replace it run:") && !why.contains("only "))); + assert!(!admits_all(&stops)); + Ok(()) + } + + fn user_missing(name: &str) -> bool { + crate::steward_orders::user_uid(name).is_err() + } + + #[test] + fn an_unresolvable_or_root_steward_user_is_an_unreadable_stop_not_an_absent_one() -> Result<()> { + let dir = tempfile::tempdir()?; + for user in ["no-such-login-zz", "root"] { + let binding = binding_with_steward(dir.path(), user); + let stops = read_stops(&binding, "runtime"); + assert!(matches!(&stops[1].stop, Stop::Unreadable(_)), "{user}"); + assert!(!admits_all(&stops)); + } + Ok(()) + } + + #[test] + fn a_binding_without_a_steward_reads_only_the_operator_store() -> Result<()> { + let dir = tempfile::tempdir()?; + let mut binding = binding_with_steward(dir.path(), "nobody"); + binding.steward = None; + let stops = read_stops(&binding, "runtime"); + assert_eq!(stops.len(), 1); + assert!(admits_all(&stops)); + Ok(()) + } + + fn store_entry(path: &Path) -> Result { + let mut entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; + entries.pop().context("no entry") + } + + #[test] + fn a_verb_replaces_a_retired_brake_record_and_the_sequence_advances_by_one() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + let retired = CultCacheEnvelope { + key: "deployment".into(), + r#type: "idunn.deployment_brake".into(), + payload: vec![0x90], + stored_at: "1970-01-01T00:00:00+00:00".into(), + schema_id: Some("idunn.deployment_brake.v1".into()), + }; + assert!(store.compare_exchange_snapshot(&[], &[retired])?); + assert_eq!(engage(&path, "first", 10)?.sequence, 1); + assert_eq!(store.pull_all_read_only_snapshot()?.len(), 1); + assert_eq!(engage(&path, "second", 11)?.sequence, 2); + assert_eq!( + replace_stop(&path, "runtime", StopState::Clear, "third", 12)?.sequence, + 3 + ); + assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "third"); + Ok(()) + } + + #[test] + fn a_stale_exchange_is_refused_and_a_record_without_a_reason_is_not_written() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + engage(&path, "held", 10)?; + let store = SingleFileMessagePackBackingStore::new(&path); + let stale = store.pull_all_read_only_snapshot()?; + engage(&path, "moved", 11)?; + let next = DeploymentStop { + schema_version: DEPLOYMENT_STOP_SCHEMA.into(), + runtime_id: "runtime".into(), + state: StopState::Clear, + by: "someone".into(), + reason: "late".into(), + at_unix_millis: 12, + sequence: 2, + } + .envelope()?; + assert!(!store.compare_exchange_snapshot(&stale, &[next])?); + assert!(replace_stop(&path, "runtime", StopState::Clear, " ", 13).is_err()); + assert_eq!(DeploymentStop::decode(&store_entry(&path)?)?.reason, "moved"); + Ok(()) + } + + #[test] + fn only_a_missing_file_in_an_existing_directory_is_absent() -> Result<()> { + let dir = tempfile::tempdir()?; + let me = process_uid(); + // Parent is a regular file: ENOTDIR, not absence. + let file = dir.path().join("plain"); + std::fs::write(&file, b"x")?; + assert!(matches!(read_stop(&file.join("stop.cc"), "runtime", me), Stop::Unreadable(_))); + // Parent does not exist. + assert!(matches!( + read_stop(&dir.path().join("nope/stop.cc"), "runtime", me), + Stop::Unreadable(_) + )); + // Zero-length file. + let empty = dir.path().join("empty.cc"); + std::fs::write(&empty, b"")?; + assert!(matches!(read_stop(&empty, "runtime", me), Stop::Unreadable(_))); + // An existing directory with no store is absent. + assert_eq!(read_stop(&dir.path().join("stop.cc"), "runtime", me), Stop::Absent); + // A parent that cannot be searched (skipped as root, which can). + #[cfg(unix)] + if me != 0 { + use std::os::unix::fs::PermissionsExt; + let locked = dir.path().join("locked"); + std::fs::create_dir(&locked)?; + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000))?; + let reading = read_stop(&locked.join("stop.cc"), "runtime", me); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700))?; + assert!(matches!(reading, Stop::Unreadable(_)), "{reading:?}"); + } + Ok(()) + } + + #[test] + fn the_author_is_the_uid_login_never_the_environment() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("stop.cc"); + // SAFETY: no other test reads USER, LOGNAME or LNAME. + unsafe { + std::env::set_var("USER", "spoof"); + std::env::set_var("LOGNAME", "spoof"); + } + let written = engage(&path, "held", 10)?; + assert_eq!(written.by, login_name(process_uid())?); + assert_ne!(written.by, "spoof"); + Ok(()) + } + + #[test] + fn unreadable_names_the_command_and_a_break_glass_clear_replaces_junk_and_v1() -> Result<()> { + let dir = tempfile::tempdir()?; + let me = process_uid(); + let path = dir.path().join("stop.cc"); + std::fs::write(&path, b"junk that is no store")?; + let Stop::Unreadable(why) = read_stop(&path, "runtime", me) else { + bail!("junk admitted") + }; + assert!(why.contains("idunn-provision deployment-brake-clear --store"), "{why}"); + assert!(why.contains("--runtime-id runtime"), "{why}"); + let cleared = replace_stop(&path, "runtime", StopState::Clear, "operator break-glass", 50)?; + assert_eq!(cleared.sequence, 1); + assert!(read_stop(&path, "runtime", me).admits()); + // A symlink is not the caller's to replace. + let real = dir.path().join("real.cc"); + std::fs::write(&real, b"junk")?; + let link = dir.path().join("link.cc"); + #[cfg(unix)] + { + std::os::unix::fs::symlink(&real, &link)?; + assert!(replace_stop(&link, "runtime", StopState::Clear, "x", 60).is_err()); + assert_eq!(std::fs::read(&real)?, b"junk"); + } + Ok(()) + } +} diff --git a/src/control_plane.rs b/src/control_plane.rs index 9597d43..83a266b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -14,22 +14,23 @@ use cultcache_rs::{ }; use cultnet_rs::{ AuthenticatedOdinRuntimeTopologyCorrelation, GameCultProviderHealthIdentity, - IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, - IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, - IdunnDeploymentBrakeRecord, IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, + IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_PROCESS_WRITE_LEASE_SCHEMA, + IdunnExpectedIncarnationRecord, IdunnLifecycleBrakeObservation, IdunnLifecycleBrakeRecord, IdunnProcessWriteLeaseRecord, IdunnRuntimeActivationLaunch, IdunnRuntimeActivationRecord, IdunnServiceIdentity, OdinTopologyAuthenticationContext, OdinTopologyDisagreement, OdinTopologyIdentity, RuntimePresenceAuthenticationContext, ServiceIdentityProfile, ServiceIdentitySigner, ServiceIdentityTrustAnchor, authenticate_odin_runtime_topology_correlation, authenticate_runtime_presence_claim, correlate_runtime_presence_claim, derive_service_identity_id, - evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, open_service_identity_at, - verify_idunn_deployment_brake_authorization, verify_runtime_authority, + evaluate_idunn_continuity_restart, open_service_identity_at, verify_runtime_authority, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; +use crate::brake::{NamedStop, read_stops}; +#[cfg(test)] +use crate::brake::StopState; use crate::deployment::{ DependencyKind, OperatorBinding, RolloutStrategy, RouteBinding, WorkloadBinding, capability_compatible, @@ -39,21 +40,26 @@ use crate::deployment_plan::{ SealedRelease, WARMING_DEADLINE_SECONDS, compile_deployment_plan, }; use crate::drivers::{ - ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, - GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, + ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FilesystemStaticRelease, + FlipRefused, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, LockContended, ProcessIdentity, RouteActuation, RouteActuationGate, RouteActuationRefused, RouteActuators, - RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, - SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, - WriteLeasePort, + RouteObservation, RoutePreflightReceipt, RunnerPort, RunnerStepFailure, SourcePort, + StaticFlip, StaticReleasePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, + WriteLeasePort, require_sha256_id, }; use crate::incident::{CloseReason, IncidentCondition, IncidentStore, render_incidents}; +use crate::runner_evidence::{ + self, Kept, RunnerFailureRecord, runner_evidence_store_path, unreadable_store_path, +}; use crate::host_actuator::{ HostActuatorAccess, HostActuatorHub, HostActuatorRunnerDriver, HostActuatorWorkloadDriver, HostUnobservable, IdunnHostActuatorIdentity, SharedHostActuatorHub, spawn_hub_service, }; const DEPLOYMENT_COMMAND_SCHEMA: &str = "idunn.deployment_command.v2"; -const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v4"; +const DEPLOYMENT_TRANSACTION_SCHEMA: &str = "idunn.deployment_transaction.v6"; +const DEPLOYMENT_TRANSACTION_SCHEMA_V5: &str = "idunn.deployment_transaction.v5"; +const DEPLOYMENT_TRANSACTION_SCHEMA_V4: &str = "idunn.deployment_transaction.v4"; const DEPLOYMENT_TRANSACTION_SCHEMA_V3: &str = "idunn.deployment_transaction.v3"; const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2"; const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v4"; @@ -139,9 +145,11 @@ struct DeploymentCommand { #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] -enum CommandKind { +pub(crate) enum CommandKind { Deploy, Continuity, + /// Flip a static release's current link back to the release it replaced. + Rollback, } impl DeploymentCommand { @@ -159,8 +167,13 @@ impl DeploymentCommand { ); match self.kind { CommandKind::Deploy => ensure!( - !self.command_id.starts_with("continuity-"), - "operator deployment uses the continuity command namespace" + !self.command_id.starts_with("continuity-") + && !self.command_id.starts_with("rollback-"), + "operator deployment uses a reserved command namespace" + ), + CommandKind::Rollback => ensure!( + self.command_id.starts_with("rollback-") && !self.selector.starts_with("profile:"), + "rollback command identity or selector is invalid" ), CommandKind::Continuity => ensure!( self.command_id.starts_with("continuity-") @@ -232,8 +245,12 @@ enum DeploymentPhase { Complete, } +/// The signed brake release a Deploy once consumed. Never written and never +/// verified now: the deployment brake is a stop (`brake.rs`). It stays so the +/// terminal records that hold one in key 16 still decode. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] +#[allow(dead_code)] struct DeploymentAuthorization { authorization_id: String, brake_sha256: String, @@ -241,27 +258,6 @@ struct DeploymentAuthorization { authorized_at_unix_millis: u64, } -impl DeploymentAuthorization { - fn validate_shape(&self) -> Result<()> { - require_id(&self.authorization_id, "deployment authorization id")?; - ensure!( - !self.canonical_brake_bytes.is_empty() - && self.brake_sha256 == sha256_id(&self.canonical_brake_bytes) - && self.authorized_at_unix_millis > 0, - "deployment authorization receipt is incomplete" - ); - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&self.canonical_brake_bytes)?; - record.validate()?; - ensure!( - rmp_serde::to_vec(&record)? == self.canonical_brake_bytes - && record.authorization_id.as_deref() == Some(&self.authorization_id), - "deployment authorization receipt is noncanonical or mismatched" - ); - Ok(()) - } -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct TopologyEvidence { @@ -632,6 +628,50 @@ impl RoutingEvidence { } } +/// The one external change a static release makes, recorded before it runs and +/// completed after it lands. `release_id` is the sealed release the current link +/// is to name and `previous_release_id` the one it named before (the release a +/// rollback deploys again). Both land in the record before the link moves, so a +/// crash between the rename and the Complete write resumes at this record, +/// repeats the harmless flip and observes the link. `carried_sha256` and +/// `flipped_at_unix_millis` are set together, once the carry and the link landed. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FlipEvidence { + release_id: String, + previous_release_id: Option, + carry_source: PathBuf, + carried_sha256: Option, + flipped_at_unix_millis: Option, +} + +impl FlipEvidence { + fn landed(&self) -> bool { + self.carried_sha256.is_some() + } + + fn validate_shape(&self) -> Result<()> { + require_sha256_id(&self.release_id, "flip release")?; + if let Some(previous) = &self.previous_release_id { + require_sha256_id(previous, "flip previous release")?; + ensure!(previous != &self.release_id, "flip names its own release as previous"); + } + ensure!( + self.carry_source.is_absolute(), + "flip carry source is not absolute" + ); + match (&self.carried_sha256, self.flipped_at_unix_millis) { + (None, None) => {} + (Some(sha256), Some(at)) => { + require_sha256_id(sha256, "flip carried digest")?; + ensure!(at > 0, "flip has no time"); + } + _ => bail!("flip carries a digest or a time but not both"), + } + Ok(()) + } +} + /// The states a route-proof candidate may answer its Warming challenge in. A /// stateful one is warming until it holds a lease it cannot hold yet, and the /// lease binds that warming presence. A stateless one may have finished @@ -1462,7 +1502,7 @@ mod cleanup_evidence { } } - /// The schema migration's one entry point: read a stored v2, v3 or v4 + /// The schema migration's one entry point: read a stored v2, v3, v4 or v5 /// transaction (lifting older ones) and return its key and current-shape /// envelope. The record may carry the marker the lift just set. pub(super) fn migrate_transaction_record( @@ -1623,7 +1663,7 @@ impl PostCommitCleanup { #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache( type = "idunn.deployment_transaction", - schema = "idunn.deployment_transaction.v4" + schema = "idunn.deployment_transaction.v6" )] struct DeploymentTransaction { #[cultcache(key = 0)] @@ -1705,6 +1745,298 @@ struct DeploymentTransaction { phase_deadline: Option, #[cultcache(key = 36)] lease_adoption: Option, + /// A static release's carry and flip. Absent on every record that + /// predates it, which decodes as None: no schema bump (precedent key 34). + #[cultcache(key = 37, default)] + flip: Option, + /// When the deployment stop admitted this Deploy or Rollback at Sealing: + /// the stamp that replaced the signed release in key 16. Absent on every + /// record that predates it. + #[cultcache(key = 38, default)] + deployment_admitted_at_unix_millis: Option, +} + +/// The v5 transaction layout: the one before `deployment_admitted_at_unix_millis` +/// (key 38), which replaced the signed brake release. Records are decoded by it +/// and lifted, because the control store refuses a record whose bytes differ +/// from its re-encoding. Migration to v6 is forward-only: an Idunn that predates +/// v6 cannot read a migrated store. `history.cc` is never migrated, so this +/// struct lives until history is pruned. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_transaction", + schema = "idunn.deployment_transaction.v5" +)] +struct LegacyDeploymentTransactionV5 { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + transaction_id: String, + #[cultcache(key = 2)] + command_id: String, + #[cultcache(key = 3)] + command_kind: CommandKind, + #[cultcache(key = 4)] + target: String, + #[cultcache(key = 5)] + ordinal: u32, + #[cultcache(key = 6)] + phase: DeploymentPhase, + #[cultcache(key = 7)] + created_at_unix_millis: u64, + #[cultcache(key = 8)] + updated_at_unix_millis: u64, + #[cultcache(key = 9)] + incumbent_generation_id: Option, + #[cultcache(key = 10)] + plan: Option, + #[cultcache(key = 11)] + frozen_source: Option, + #[cultcache(key = 12)] + sealed_release: Option, + #[cultcache(key = 13)] + installed_release: Option, + #[cultcache(key = 14)] + expected: Option, + #[cultcache(key = 15)] + expected_publication_sha256: Option, + #[cultcache(key = 16)] + deployment_authorization: Option, + #[cultcache(key = 17)] + lifecycle_authorized_at_unix_millis: Option, + #[cultcache(key = 18)] + activation: Option, + #[cultcache(key = 19)] + workload: Option, + #[cultcache(key = 20)] + activation_publication_sha256: Option, + #[cultcache(key = 21)] + latest_odin_observation: Option, + #[cultcache(key = 22)] + warming: Option, + #[cultcache(key = 23)] + route_preflight: Option, + #[cultcache(key = 24)] + isolation: Option, + #[cultcache(key = 25)] + fencing: Option, + #[cultcache(key = 26)] + leasing: Option, + #[cultcache(key = 27)] + ready: Option, + #[cultcache(key = 28)] + routing: Option, + #[cultcache(key = 29)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 30)] + last_error: Option, + #[cultcache(key = 31)] + completion: Option, + #[cultcache(key = 32)] + pre_fencing_abort: Option, + #[cultcache(key = 33)] + post_commit_cleanup: Option, + // because every transaction already durable predates this field: + // an absent key 34 decodes as None, which is exactly "this transaction has + // no post-fencing abort". Without it the daemon refuses to read its own + // control store and crashloops. + #[cultcache(key = 34, default)] + post_fencing_abort: Option, + /// Set by `transition` on entry to each post-fencing phase. + #[cultcache(key = 35)] + phase_deadline: Option, + #[cultcache(key = 36)] + lease_adoption: Option, + /// A static release's carry and flip. Absent on every record that + /// predates it, which decodes as None: no schema bump (precedent key 34). + #[cultcache(key = 37, default)] + flip: Option, +} + + +impl LegacyDeploymentTransactionV5 { + fn into_current(self) -> DeploymentTransaction { + DeploymentTransaction { + schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), + transaction_id: self.transaction_id, + command_id: self.command_id, + command_kind: self.command_kind, + target: self.target, + ordinal: self.ordinal, + phase: self.phase, + created_at_unix_millis: self.created_at_unix_millis, + updated_at_unix_millis: self.updated_at_unix_millis, + incumbent_generation_id: self.incumbent_generation_id, + plan: self.plan, + frozen_source: self.frozen_source, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + expected_publication_sha256: self.expected_publication_sha256, + deployment_authorization: self.deployment_authorization, + lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, + activation: self.activation, + workload: self.workload, + activation_publication_sha256: self.activation_publication_sha256, + latest_odin_observation: self.latest_odin_observation, + warming: self.warming, + route_preflight: self.route_preflight, + isolation: self.isolation, + fencing: self.fencing, + leasing: self.leasing, + ready: self.ready, + routing: self.routing, + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + last_error: self.last_error, + completion: self.completion, + pre_fencing_abort: self.pre_fencing_abort, + post_commit_cleanup: self.post_commit_cleanup, + post_fencing_abort: self.post_fencing_abort, + phase_deadline: self.phase_deadline, + lease_adoption: self.lease_adoption, + flip: self.flip, + deployment_admitted_at_unix_millis: None, + } + } +} + +/// The v4 transaction layout: the current one before a static release's +/// `flip` (key 37). Records are decoded by it and lifted, because the control +/// store refuses a record whose bytes differ from its re-encoding. Migration +/// to v5 is forward-only: an Idunn that predates v5 cannot read a migrated +/// store. `history.cc` is never migrated, so this struct lives until history +/// is pruned. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.deployment_transaction", + schema = "idunn.deployment_transaction.v4" +)] +struct LegacyDeploymentTransactionV4 { + #[cultcache(key = 0)] + schema_version: String, + #[cultcache(key = 1)] + transaction_id: String, + #[cultcache(key = 2)] + command_id: String, + #[cultcache(key = 3)] + command_kind: CommandKind, + #[cultcache(key = 4)] + target: String, + #[cultcache(key = 5)] + ordinal: u32, + #[cultcache(key = 6)] + phase: DeploymentPhase, + #[cultcache(key = 7)] + created_at_unix_millis: u64, + #[cultcache(key = 8)] + updated_at_unix_millis: u64, + #[cultcache(key = 9)] + incumbent_generation_id: Option, + #[cultcache(key = 10)] + plan: Option, + #[cultcache(key = 11)] + frozen_source: Option, + #[cultcache(key = 12)] + sealed_release: Option, + #[cultcache(key = 13)] + installed_release: Option, + #[cultcache(key = 14)] + expected: Option, + #[cultcache(key = 15)] + expected_publication_sha256: Option, + #[cultcache(key = 16)] + deployment_authorization: Option, + #[cultcache(key = 17)] + lifecycle_authorized_at_unix_millis: Option, + #[cultcache(key = 18)] + activation: Option, + #[cultcache(key = 19)] + workload: Option, + #[cultcache(key = 20)] + activation_publication_sha256: Option, + #[cultcache(key = 21)] + latest_odin_observation: Option, + #[cultcache(key = 22)] + warming: Option, + #[cultcache(key = 23)] + route_preflight: Option, + #[cultcache(key = 24)] + isolation: Option, + #[cultcache(key = 25)] + fencing: Option, + #[cultcache(key = 26)] + leasing: Option, + #[cultcache(key = 27)] + ready: Option, + #[cultcache(key = 28)] + routing: Option, + #[cultcache(key = 29)] + odin_publisher_sequence_cursor: u64, + #[cultcache(key = 30)] + last_error: Option, + #[cultcache(key = 31)] + completion: Option, + #[cultcache(key = 32)] + pre_fencing_abort: Option, + #[cultcache(key = 33)] + post_commit_cleanup: Option, + // because every transaction already durable predates this field: + // an absent key 34 decodes as None, which is exactly "this transaction has + // no post-fencing abort". Without it the daemon refuses to read its own + // control store and crashloops. + #[cultcache(key = 34, default)] + post_fencing_abort: Option, + /// Set by `transition` on entry to each post-fencing phase. + #[cultcache(key = 35)] + phase_deadline: Option, + #[cultcache(key = 36)] + lease_adoption: Option, +} + +impl LegacyDeploymentTransactionV4 { + fn into_current(self) -> DeploymentTransaction { + DeploymentTransaction { + schema_version: DEPLOYMENT_TRANSACTION_SCHEMA.into(), + transaction_id: self.transaction_id, + command_id: self.command_id, + command_kind: self.command_kind, + target: self.target, + ordinal: self.ordinal, + phase: self.phase, + created_at_unix_millis: self.created_at_unix_millis, + updated_at_unix_millis: self.updated_at_unix_millis, + incumbent_generation_id: self.incumbent_generation_id, + plan: self.plan, + frozen_source: self.frozen_source, + sealed_release: self.sealed_release, + installed_release: self.installed_release, + expected: self.expected, + expected_publication_sha256: self.expected_publication_sha256, + deployment_authorization: self.deployment_authorization, + lifecycle_authorized_at_unix_millis: self.lifecycle_authorized_at_unix_millis, + activation: self.activation, + workload: self.workload, + activation_publication_sha256: self.activation_publication_sha256, + latest_odin_observation: self.latest_odin_observation, + warming: self.warming, + route_preflight: self.route_preflight, + isolation: self.isolation, + fencing: self.fencing, + leasing: self.leasing, + ready: self.ready, + routing: self.routing, + odin_publisher_sequence_cursor: self.odin_publisher_sequence_cursor, + last_error: self.last_error, + completion: self.completion, + pre_fencing_abort: self.pre_fencing_abort, + post_commit_cleanup: self.post_commit_cleanup, + post_fencing_abort: self.post_fencing_abort, + phase_deadline: self.phase_deadline, + lease_adoption: self.lease_adoption, + flip: None, + deployment_admitted_at_unix_millis: None, + } + } } /// The note a deployment keeps while another transaction owns its target. @@ -1918,8 +2250,8 @@ impl DeploymentTransaction { let issued = leasing.and_then(LeasingEvidence::issued_lease).is_some(); match self.command_kind { CommandKind::Continuity => PhaseEnd::RetryForward, - CommandKind::Deploy if issued => PhaseEnd::OperatorRequired, - CommandKind::Deploy => PhaseEnd::Abort { + CommandKind::Deploy | CommandKind::Rollback if issued => PhaseEnd::OperatorRequired, + CommandKind::Deploy | CommandKind::Rollback => PhaseEnd::Abort { lease_not_adopted: granted && self.lease_adoption.is_none(), }, } @@ -1936,6 +2268,36 @@ impl DeploymentTransaction { } } + /// Whether this transaction publishes a static release: it has no workload, + /// no Expected and no admitted generation, and its one external change is + /// the flip. Derived from the plan's binding, never stored. + fn is_static(&self) -> bool { + self.plan + .as_ref() + .and_then(|plan| plan.parsed_inputs().ok()) + .is_some_and(|(_, binding)| matches!(binding.workload, WorkloadBinding::StaticRelease(_))) + } + + /// Whether `error` can no longer end this transaction before fencing: a + /// static flip is recorded as intended and only `FlipRefused` (the link did + /// not move) says it did not happen. Any other error after the intent may + /// follow the rename, so the step is retried, and a retry observes the link. + fn flip_may_have_moved_current(&self, error: &anyhow::Error) -> bool { + self.flip.is_some() && error.downcast_ref::().is_none() + } + + /// The runtime and the sealed release this transaction names, for the + /// status listing. One reading for both kinds of target: the binding's + /// runtime identity and the sealed release. + fn brake_subject(&self) -> Result<(String, String)> { + let plan = required(&self.plan, "transaction plan")?; + let release = required(&self.sealed_release, "sealed release")?; + Ok(( + plan.parsed_inputs()?.1.runtime_identity.runtime_id, + release.sealed_release_id.clone(), + )) + } + /// Whether this transaction's binding declares stop-then-start. A /// transaction without a plan yet cannot, so it reads as false. fn rollout_stops_incumbent_first(&self) -> bool { @@ -1991,6 +2353,8 @@ impl DeploymentTransaction { post_commit_cleanup: None, phase_deadline: None, lease_adoption: None, + flip: None, + deployment_admitted_at_unix_millis: None, }; transaction.validate()?; Ok(transaction) @@ -2011,13 +2375,25 @@ impl DeploymentTransaction { Ok(transaction) } + /// A rollback: the release that was flipped before, reused exactly (its + /// plan, sealed release and installed release are the prior transaction's), + /// with no source to freeze and nothing to build. + fn from_rollback(command: &DeploymentCommand, prior: &DeploymentTransaction, now: u64) -> Result { + let mut transaction = Self::new(command, prior.target.clone(), 0, None, now)?; + transaction.plan = prior.plan.clone(); + transaction.sealed_release = prior.sealed_release.clone(); + transaction.installed_release = prior.installed_release.clone(); + transaction.validate()?; + Ok(transaction) + } + /// Whether an abort owes the projection a reconciliation. A deployment /// published its own Expected key. A continuity shares the admitted key, /// whose Expected is never withdrawn, so it owes only the demotion of the /// activation it issued. fn abort_topology_reconciliation(&self) -> CleanupEvidence { let owes = match self.command_kind { - CommandKind::Deploy => self.expected_publication_sha256.is_some(), + CommandKind::Deploy | CommandKind::Rollback => self.expected_publication_sha256.is_some(), CommandKind::Continuity => self.activation.is_some(), }; if owes { @@ -2157,6 +2533,93 @@ impl DeploymentTransaction { self.readiness_disagreement().filter(|_| !finishing) } + /// What a static release transaction may hold. It never runs a workload, so + /// every piece of evidence of one is absent, and it uses three phases: + /// Sealing, Starting (the flip, behind the brake) and Complete. + /// Whether the deployment stop admitted this transaction: stamped at + /// Sealing now, or a legacy record that holds a consumed signed release. + fn deployment_admitted(&self) -> bool { + self.deployment_admitted_at_unix_millis.is_some() || self.deployment_authorization.is_some() + } + + fn validate_static(&self) -> Result<()> { + ensure!( + self.command_kind != CommandKind::Continuity, + "a static release has no continuity" + ); + // A deploy sealed its own source; a rollback reuses a release already + // sealed and installed, so it never froze one. + ensure!( + self.command_kind != CommandKind::Rollback || self.frozen_source.is_none(), + "a rollback never freezes a source" + ); + ensure!( + self.expected.is_none() + && self.expected_publication_sha256.is_none() + && self.lifecycle_authorized_at_unix_millis.is_none() + && self.activation.is_none() + && self.workload.is_none() + && self.activation_publication_sha256.is_none() + && self.latest_odin_observation.is_none() + && self.warming.is_none() + && self.route_preflight.is_none() + && self.isolation.is_none() + && self.fencing.is_none() + && self.leasing.is_none() + && self.ready.is_none() + && self.routing.is_none() + && self.lease_adoption.is_none(), + "a static release transaction carries evidence of a running workload" + ); + ensure!( + matches!( + self.phase, + DeploymentPhase::Sealing | DeploymentPhase::Starting | DeploymentPhase::Complete + ), + "a static release never enters {:?}", + self.phase + ); + if self.phase >= DeploymentPhase::Starting { + ensure!( + self.plan.is_some() + && self.sealed_release.is_some() + && self.installed_release.is_some() + && (self.command_kind == CommandKind::Rollback || self.frozen_source.is_some()) + && self.deployment_admitted(), + "static release entered Starting without sealed release evidence and the stop's admission" + ); + } else { + ensure!( + self.flip.is_none() && !self.deployment_admitted(), + "static release flipped or was admitted by the stop before Starting" + ); + } + if self.phase == DeploymentPhase::Complete { + ensure!( + self.flip.as_ref().is_some_and(FlipEvidence::landed), + "completed static release has no landed flip" + ); + ensure!( + self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_some(), + "admitted transaction lacks exclusive post-commit cleanup evidence" + ); + ensure!( + matches!( + &self.completion, + Some(TransactionCompletion::Admitted { generation_id }) + if generation_id == &format!("generation-{}", self.transaction_id) + ), + "successful Complete transaction lacks its admission receipt" + ); + } else { + ensure!( + self.completion.is_none() && self.post_commit_cleanup.is_none(), + "non-Complete transaction has completion or post-commit state" + ); + } + Ok(()) + } + fn validate(&self) -> Result<()> { ensure!( self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, @@ -2340,11 +2803,24 @@ impl DeploymentTransaction { "lease adoption does not name the granted lease" ); } - if let Some(authorization) = &self.deployment_authorization { - authorization.validate_shape()?; + let is_static = self.is_static(); + if let Some(flip) = &self.flip { + ensure!(is_static, "a flip belongs to a static release transaction"); + flip.validate_shape()?; + if let Some(release) = &self.sealed_release { + ensure!( + flip.release_id == release.sealed_release_id, + "flip names another release than the sealed one" + ); + } } if let Some(abort) = &self.pre_fencing_abort { require_detail(&abort.error, "pre-fencing abort error")?; + ensure!( + !self.flip.as_ref().is_some_and(FlipEvidence::landed), + "a transaction whose flip landed cannot fail before fencing" + ); + ensure!( self.post_commit_cleanup.is_none(), "aborted transaction also carries post-commit cleanup" @@ -2375,7 +2851,7 @@ impl DeploymentTransaction { ( CommandKind::Deploy, CleanupEvidence::Pending | CleanupEvidence::Complete - ) | (CommandKind::Continuity, CleanupEvidence::Skipped) + ) | (CommandKind::Continuity | CommandKind::Rollback, CleanupEvidence::Skipped) ) || (self.command_kind == CommandKind::Deploy && abort.source_cleanup == CleanupEvidence::Skipped && self.plan.is_none() @@ -2461,111 +2937,117 @@ impl DeploymentTransaction { ), "non-failure branch carries failure completion" ); - if self.phase >= DeploymentPhase::Starting { - ensure!( - self.plan.is_some() - && self.sealed_release.is_some() - && self.installed_release.is_some() - && self.expected.is_some(), - "Starting transaction lacks sealed release evidence" - ); - // Expected is published as Starting's first step, after the - // brake; from the activation onward it must be there. - ensure!( - self.expected_publication_sha256.is_some() - || (self.activation.is_none() && self.phase == DeploymentPhase::Starting), - "started transaction has not published its Expected" - ); - match self.command_kind { - CommandKind::Deploy => ensure!( - self.frozen_source.is_some() && self.deployment_authorization.is_some(), - "deployment entered Starting without frozen source and brake consumption" - ), - CommandKind::Continuity => ensure!( - self.lifecycle_authorized_at_unix_millis.is_some() - && self.deployment_authorization.is_none(), - "continuity entered Starting without its lifecycle gate" - ), - } - } - if self.phase >= DeploymentPhase::Warming { - ensure!( - self.activation.is_some() - && self.workload.is_some() - && self.activation_publication_sha256.is_some(), - "Warming transaction lacks observed activation" - ); - } - if self.phase >= DeploymentPhase::Fencing { - ensure!( - self.warming.is_some() && self.isolation.is_some(), - "Fencing transaction lacks warming or isolation evidence" - ); - ensure!( - self.expected - .as_ref() - .and_then(|expected| expected.route.as_ref()) - .is_some() - == self.route_preflight.is_some(), - "route preflight does not match routed Expected" - ); - if let Some(preflight) = &self.route_preflight { - preflight.validate()?; - } - } - if self.phase >= DeploymentPhase::Leasing { - ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence"); - } - if self.phase >= DeploymentPhase::AwaitingReady { - ensure!( - matches!( - self.leasing.as_ref(), - Some(LeasingEvidence::SkippedStateless) - | Some(LeasingEvidence::Granted { .. }) - ), - "AwaitingReady lacks finalized lease evidence" - ); - } - if self.phase >= DeploymentPhase::Routing { - ensure!(self.ready.is_some(), "Routing lacks Ready evidence"); - } - if self.phase >= DeploymentPhase::Committing { - ensure!(self.routing.is_some(), "Committing lacks routing evidence"); - } - if let Some(RoutingEvidence::Promoted { - observation, - promoted_at_unix_millis, - }) = &self.routing - { - observation.validate()?; - ensure!(*promoted_at_unix_millis > 0, "route promotion has no time"); - } - if self.phase == DeploymentPhase::Complete { - ensure!( - matches!( - self.completion, - Some(TransactionCompletion::Admitted { .. }) - ), - "successful Complete transaction lacks admission receipt" - ); - ensure!( - self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_some(), - "admitted transaction lacks exclusive post-commit cleanup evidence" - ); - let TransactionCompletion::Admitted { generation_id } = - self.completion.as_ref().unwrap() - else { - unreachable!() - }; - ensure!( - generation_id == &format!("generation-{}", self.transaction_id), - "admission completion names another transaction generation" - ); + if is_static { + self.validate_static()?; } else { - ensure!( - self.completion.is_none() && self.post_commit_cleanup.is_none(), - "non-Complete transaction has completion or post-commit state" - ); + if self.phase >= DeploymentPhase::Starting { + ensure!( + self.plan.is_some() + && self.sealed_release.is_some() + && self.installed_release.is_some() + && self.expected.is_some(), + "Starting transaction lacks sealed release evidence" + ); + // Expected is published as Starting's first step, after the + // brake; from the activation onward it must be there. + ensure!( + self.expected_publication_sha256.is_some() + || (self.activation.is_none() && self.phase == DeploymentPhase::Starting), + "started transaction has not published its Expected" + ); + match self.command_kind { + CommandKind::Deploy => ensure!( + self.frozen_source.is_some() && self.deployment_admitted(), + "deployment entered Starting without frozen source and the stop's admission" + ), + CommandKind::Rollback => bail!("a rollback is a static release transaction"), + CommandKind::Continuity => ensure!( + self.lifecycle_authorized_at_unix_millis.is_some() + && !self.deployment_admitted(), + "continuity entered Starting without its lifecycle gate" + ), + } + } + if self.phase >= DeploymentPhase::Warming { + ensure!( + self.activation.is_some() + && self.workload.is_some() + && self.activation_publication_sha256.is_some(), + "Warming transaction lacks observed activation" + ); + } + if self.phase >= DeploymentPhase::Fencing { + ensure!( + self.warming.is_some() && self.isolation.is_some(), + "Fencing transaction lacks warming or isolation evidence" + ); + ensure!( + self.expected + .as_ref() + .and_then(|expected| expected.route.as_ref()) + .is_some() + == self.route_preflight.is_some(), + "route preflight does not match routed Expected" + ); + if let Some(preflight) = &self.route_preflight { + preflight.validate()?; + } + } + if self.phase >= DeploymentPhase::Leasing { + ensure!(self.fencing.is_some(), "Leasing lacks fencing evidence"); + } + if self.phase >= DeploymentPhase::AwaitingReady { + ensure!( + matches!( + self.leasing.as_ref(), + Some(LeasingEvidence::SkippedStateless) + | Some(LeasingEvidence::Granted { .. }) + ), + "AwaitingReady lacks finalized lease evidence" + ); + } + if self.phase >= DeploymentPhase::Routing { + ensure!(self.ready.is_some(), "Routing lacks Ready evidence"); + } + if self.phase >= DeploymentPhase::Committing { + ensure!(self.routing.is_some(), "Committing lacks routing evidence"); + } + if let Some(RoutingEvidence::Promoted { + observation, + promoted_at_unix_millis, + }) = &self.routing + { + observation.validate()?; + ensure!(*promoted_at_unix_millis > 0, "route promotion has no time"); + } + if self.phase == DeploymentPhase::Complete { + ensure!( + matches!( + self.completion, + Some(TransactionCompletion::Admitted { .. }) + ), + "successful Complete transaction lacks admission receipt" + ); + ensure!( + self.pre_fencing_abort.is_none() && self.post_commit_cleanup.is_some(), + "admitted transaction lacks exclusive post-commit cleanup evidence" + ); + let TransactionCompletion::Admitted { generation_id } = + self.completion.as_ref().unwrap() + else { + unreachable!() + }; + ensure!( + generation_id == &format!("generation-{}", self.transaction_id), + "admission completion names another transaction generation" + ); + } else { + ensure!( + self.completion.is_none() && self.post_commit_cleanup.is_none(), + "non-Complete transaction has completion or post-commit state" + ); + } + } } if let Some(cleanup) = &self.post_commit_cleanup { @@ -2592,7 +3074,7 @@ impl DeploymentTransaction { (CommandKind::Deploy, SourceCleanupEvidence::Pending) | (CommandKind::Deploy, SourceCleanupEvidence::Complete) | ( - CommandKind::Continuity, + CommandKind::Continuity | CommandKind::Rollback, SourceCleanupEvidence::SkippedContinuity ) ), @@ -2859,7 +3341,6 @@ struct RuntimeOptions { odin_correlation_store: PathBuf, odin_trust_anchor: PathBuf, idunn_identity_store: PathBuf, - deployment_brake_operator_anchor: PathBuf, source_identity: Option, topology_maximum_age_millis: u64, topology_maximum_future_skew_millis: u64, @@ -2889,9 +3370,6 @@ impl Default for RuntimeOptions { idunn_identity_store: PathBuf::from( "/var/lib/gamecult/idunn/idunn-service-identity.cc", ), - deployment_brake_operator_anchor: PathBuf::from( - "/etc/gamecult/idunn/deployment-brake-operator-anchor.cc", - ), source_identity: None, topology_maximum_age_millis: DEFAULT_TOPOLOGY_MAXIMUM_AGE_MILLIS, topology_maximum_future_skew_millis: DEFAULT_TOPOLOGY_MAXIMUM_FUTURE_SKEW_MILLIS, @@ -2906,6 +3384,7 @@ impl Default for RuntimeOptions { enum Command { Serve(RuntimeOptions), Up { + kind: CommandKind, selector: String, requested_by: String, state_store: PathBuf, @@ -2931,12 +3410,17 @@ enum Command { recipe: PathBuf, binding: Option, }, + #[cfg(unix)] + OrdersServe(crate::steward_orders::ServeOptions), + #[cfg(unix)] + Order(crate::steward_orders::OrderCommand), } pub fn run(args: impl Iterator) -> Result<()> { match parse(args)? { Command::Serve(options) => serve(options), Command::Up { + kind, selector, requested_by, state_store, @@ -2944,6 +3428,7 @@ pub fn run(args: impl Iterator) -> Result<()> { timeout_seconds, } => submit( &state_store, + kind, &selector, &requested_by, wait, @@ -2970,6 +3455,10 @@ pub fn run(args: impl Iterator) -> Result<()> { requested_by, } => expire(&state_store, &command_id, &requested_by), Command::Validate { recipe, binding } => validate(&recipe, binding.as_deref()), + #[cfg(unix)] + Command::OrdersServe(options) => crate::steward_orders::serve(options), + #[cfg(unix)] + Command::Order(order) => crate::steward_orders::run_order(order), } } @@ -3017,11 +3506,16 @@ fn parse(args: impl Iterator) -> Result { let command = args.next().ok_or_else(|| anyhow!(usage()))?; match command.as_str() { "serve" => parse_serve(args), - "up" => parse_up(args), + "up" => parse_up(args, CommandKind::Deploy), "status" => parse_status(args), + "rollback" => parse_up(args, CommandKind::Rollback), "cancel" => parse_cancel(args), "expire" => parse_expire(args), "validate" => parse_validate(args), + #[cfg(unix)] + "orders" => crate::steward_orders::parse_serve(args).map(Command::OrdersServe), + #[cfg(unix)] + "order" => crate::steward_orders::parse_order(args).map(Command::Order), "--help" | "-h" | "help" => bail!(usage()), _ => bail!("unknown Idunn command {command:?}\n\n{}", usage()), } @@ -3046,9 +3540,6 @@ fn parse_serve(mut args: impl Iterator) -> Result { "--idunn-identity-store" => { options.idunn_identity_store = path_value(&mut args, &argument)? } - "--deployment-brake-operator-anchor" => { - options.deployment_brake_operator_anchor = path_value(&mut args, &argument)? - } "--topology-maximum-age-millis" => { options.topology_maximum_age_millis = u64_value(&mut args, &argument)? } @@ -3088,11 +3579,17 @@ fn parse_serve(mut args: impl Iterator) -> Result { Ok(Command::Serve(options)) } -fn parse_up(mut args: impl Iterator) -> Result { +/// `idunn up ` and `idunn rollback `: both only write a command. +fn parse_up(mut args: impl Iterator, kind: CommandKind) -> Result { + let verb = if kind == CommandKind::Rollback { "rollback" } else { "up" }; let selector = args .next() - .ok_or_else(|| anyhow!("idunn up requires a service or profile selector"))?; + .ok_or_else(|| anyhow!("idunn {verb} requires a service{} selector", if kind == CommandKind::Rollback { "" } else { " or profile" }))?; require_selector(&selector)?; + ensure!( + kind != CommandKind::Rollback || !selector.starts_with("profile:"), + "a rollback names one target, not a profile" + ); let mut state_store = RuntimeOptions::default().state_store; let mut requested_by = env::var("SUDO_USER") .or_else(|_| env::var("USER")) @@ -3107,12 +3604,13 @@ fn parse_up(mut args: impl Iterator) -> Result { "--no-wait" => wait = false, "--timeout-seconds" => timeout_seconds = u64_value(&mut args, &argument)?, "--help" | "-h" => bail!(usage()), - _ => bail!("unknown Idunn up option {argument:?}"), + _ => bail!("unknown Idunn {verb} option {argument:?}"), } } ensure!(timeout_seconds > 0, "--timeout-seconds must be positive"); require_value(&requested_by, "deployment requester")?; Ok(Command::Up { + kind, selector, requested_by, state_store, @@ -3489,7 +3987,7 @@ impl ControlSnapshot { "multiple transactions own the incumbent slot of one target" ); let (seen, kind) = match transaction.value.command_kind { - CommandKind::Deploy => (&mut live_deploys, "deployments"), + CommandKind::Deploy | CommandKind::Rollback => (&mut live_deploys, "deployments"), CommandKind::Continuity => (&mut live_continuities, "continuity transactions"), }; ensure!(seen.insert(target), "multiple live {kind} for one target"); @@ -3540,17 +4038,6 @@ impl ControlSnapshot { } } } - let mut authorizations = BTreeSet::new(); - for authorization in self - .transactions - .iter() - .filter_map(|stored| stored.value.deployment_authorization.as_ref()) - { - ensure!( - authorizations.insert(authorization.authorization_id.as_str()), - "deployment authorization was consumed more than once" - ); - } Ok(()) } @@ -3867,6 +4354,8 @@ impl LegacyDeploymentTransaction { post_fencing_abort: self.post_fencing_abort, phase_deadline: None, lease_adoption: None, + flip: None, + deployment_admitted_at_unix_millis: None, } } } @@ -4075,10 +4564,28 @@ impl LegacyAdmittedGenerationV3 { } } -/// Decode a v2 or v3 transaction and lift it to the current shape, without +/// Decode a v2, v3, v4 or v5 transaction and lift it to the current shape, without /// validating it. History reads stop here; the control store validates. fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result { let schema = match envelope.schema_id.as_deref() { + Some(schema @ DEPLOYMENT_TRANSACTION_SCHEMA_V5) => { + let value: LegacyDeploymentTransactionV5 = rmp_serde::from_slice(&envelope.payload) + .with_context(|| format!("decoding a {schema} deployment transaction"))?; + ensure!( + value.schema_version == schema, + "stored transaction schema differs from its envelope" + ); + return Ok(value.into_current()); + } + Some(schema @ DEPLOYMENT_TRANSACTION_SCHEMA_V4) => { + let value: LegacyDeploymentTransactionV4 = rmp_serde::from_slice(&envelope.payload) + .with_context(|| format!("decoding a {schema} deployment transaction"))?; + ensure!( + value.schema_version == schema, + "stored transaction schema differs from its envelope" + ); + return Ok(value.into_current()); + } Some(schema @ (DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3)) => { schema } @@ -4097,7 +4604,7 @@ fn lift_legacy_transaction(envelope: &CultCacheEnvelope) -> Result Result { (envelope.r#type == DeploymentTransaction::TYPE && matches!( schema, - Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) + Some( + DEPLOYMENT_TRANSACTION_SCHEMA_V2 + | DEPLOYMENT_TRANSACTION_SCHEMA_V3 + | DEPLOYMENT_TRANSACTION_SCHEMA_V4 + | DEPLOYMENT_TRANSACTION_SCHEMA_V5 + ) )) || (envelope.r#type == AdmittedGeneration::TYPE && matches!( @@ -4530,7 +5042,12 @@ fn decode_history_transactions( .filter(|envelope| envelope.r#type == DeploymentTransaction::TYPE) { let decoded = match envelope.schema_id.as_deref() { - Some(DEPLOYMENT_TRANSACTION_SCHEMA_V2 | DEPLOYMENT_TRANSACTION_SCHEMA_V3) => { + Some( + DEPLOYMENT_TRANSACTION_SCHEMA_V2 + | DEPLOYMENT_TRANSACTION_SCHEMA_V3 + | DEPLOYMENT_TRANSACTION_SCHEMA_V4 + | DEPLOYMENT_TRANSACTION_SCHEMA_V5 + ) => { lift_legacy_transaction(&envelope) } _ => rmp_serde::from_slice::(&envelope.payload) @@ -4727,21 +5244,23 @@ fn resolve_selector( Ok(targets) } -fn submit( +/// The one commit primitive for a deployment command: `idunn up` and a steward +/// deploy order both write theirs here. Returns the commit time. +pub(crate) fn commit_deployment_command( store_path: &Path, + kind: CommandKind, + command_id: &str, selector: &str, requested_by: &str, - wait: bool, - timeout_seconds: u64, -) -> Result<()> { +) -> Result { if let Some(parent) = store_path.parent() { fs::create_dir_all(parent)?; } let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: format!("up-{}", Uuid::new_v4()), - kind: CommandKind::Deploy, + command_id: command_id.into(), + kind, selector: selector.into(), requested_by: requested_by.into(), requested_at_unix_millis: now, @@ -4758,7 +5277,35 @@ fn submit( )?, "deployment command id collided" ); - println!("{}", command.command_id); + Ok(now) +} + +/// A command's present state as `idunn status --command` derives it, for the +/// steward order poster: `queued` and `running` are not terminal. +#[cfg(unix)] +pub(crate) fn command_state( + state_store: &Path, + command_id: &str, +) -> Result<(&'static str, String)> { + let snapshot = ControlSnapshot::read(state_store)?; + let archived = read_history_transactions(state_store); + let mut transactions = snapshot.transaction_for_command(command_id); + transactions.extend(archived.iter().filter(|value| value.command_id == command_id)); + Ok(derived_command_status(&transactions)) +} + +fn submit( + store_path: &Path, + kind: CommandKind, + selector: &str, + requested_by: &str, + wait: bool, + timeout_seconds: u64, +) -> Result<()> { + let prefix = if kind == CommandKind::Rollback { "rollback" } else { "up" }; + let command_id = format!("{prefix}-{}", Uuid::new_v4()); + let now = commit_deployment_command(store_path, kind, &command_id, selector, requested_by)?; + println!("{command_id}"); if !wait { return Ok(()); } @@ -4769,10 +5316,10 @@ fn submit( snapshot .commands .iter() - .any(|stored| stored.value.command_id == command.command_id), + .any(|stored| stored.value.command_id == command_id), "submitted deployment command disappeared" ); - let transactions = snapshot.transaction_for_command(&command.command_id); + let transactions = snapshot.transaction_for_command(&command_id); if !transactions.is_empty() && transactions.iter().all(|value| value.is_terminal()) { if let Some(error) = transactions @@ -4797,7 +5344,7 @@ fn submit( } } -fn status( +pub(crate) fn status( out: &mut impl Write, store_path: &Path, incident_store: &Path, @@ -4849,9 +5396,9 @@ fn status( " target {} phase {:?}", transaction.target, transaction.phase )?; - if let Some(expected) = &transaction.expected { - writeln!(out, " runtime {}", expected.runtime_id)?; - writeln!(out, " release {}", expected.sealed_release_id)?; + if let Ok((runtime_id, sealed_release_id)) = transaction.brake_subject() { + writeln!(out, " runtime {runtime_id}")?; + writeln!(out, " release {sealed_release_id}")?; } writeln!( out, @@ -4877,6 +5424,13 @@ fn status( } if let Some(abort) = &transaction.pre_fencing_abort { writeln!(out, " pre-fencing abort {abort:?}")?; + // The step that failed and its output tail live in their + // own private store. An unreadable store costs its lines. + runner_evidence::show_status( + out, + &runner_evidence_store_path(store_path), + &transaction.transaction_id, + )?; } if let Some(abort) = &transaction.post_fencing_abort { writeln!(out, " post-fencing abort {abort:?}")?; @@ -4971,14 +5525,23 @@ fn render_supervision( } if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { lines.push(format!( - " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", - match route.degraded_since_unix_millis { + " route {} consecutive-failures {} last-challenge-at {}{} next-challenge-at {}{}", match route.degraded_since_unix_millis { Some(since) => format!("degraded-since {since}"), None => "healthy".to_owned(), }, route.consecutive_failures, at(route.last_challenge_at_unix_millis), - at(route.next_challenge_at_unix_millis) + route + .last_challenge_at_unix_millis + .map_or(String::new(), |last| format!(" ({} ms ago)", now.saturating_sub(last))), + at(route.next_challenge_at_unix_millis), + // Status sees the stored schedule, not why it is not being kept + // (a refused write, a transaction, a daemon that is down): it + // says only how overdue the challenge is. + route + .next_challenge_at_unix_millis + .filter(|next| *next < now) + .map_or(String::new(), |next| format!(" (past due {} ms)", now - next)) )); } if generation.is_none_or(|value| value.route_supervision.is_some()) @@ -5115,15 +5678,26 @@ struct Engine { source: Arc, docker_runner: DockerRunnerDriver, systemd_workload: Arc, + /// The one external change a static release makes. + static_release: Arc, host_actuators: Option, /// A test world that serves no host actuator runs a host-actuator /// binding's workload through this port instead. #[cfg(test)] host_workload: Mutex>>, + /// A test world that cannot assign runner workspaces (that wants root) + /// seals through this runner instead of the Docker one. + #[cfg(test)] + runner_stand_in: Mutex>>, history_report: ReportOnce, /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. fault_reports: Mutex>, + /// The key of every `report_once` line that reached stderr: what a test + /// counts, since stderr is not observable from inside it, and a key's + /// own entry is dropped when its fault clears. + #[cfg(test)] + printed_reports: Mutex>, /// Retry pacing for a transaction whose step keeps failing, by /// transaction id: consecutive failures and the earliest next attempt. /// Process-local on purpose: a restart retries at once, and no schema @@ -5161,7 +5735,7 @@ struct EngineRouteGate<'a> { impl RouteActuationGate for EngineRouteGate<'_> { fn admit(&self, requested: RouteActuation) -> Result<()> { let kind = match self.command { - CommandKind::Deploy => requested, + CommandKind::Deploy | CommandKind::Rollback => requested, CommandKind::Continuity => RouteActuation::Survival, }; self.engine.charge_route_actuation(self.target, kind) @@ -5243,11 +5817,16 @@ impl Engine { source, docker_runner: DockerRunnerDriver::default(), systemd_workload, + static_release: Arc::new(FilesystemStaticRelease), host_actuators, #[cfg(test)] host_workload: Mutex::default(), + #[cfg(test)] + runner_stand_in: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), + #[cfg(test)] + printed_reports: Mutex::default(), resume_backoff: Mutex::default(), lift_held: Mutex::default(), lift_unsettled: std::sync::atomic::AtomicBool::new(false), @@ -5366,6 +5945,8 @@ impl Engine { .or_default() .offer(Some(message)); if let Some(message) = offered { + #[cfg(test)] + self.printed_reports.lock().unwrap().push(key.to_string()); eprintln!("{message}"); } } @@ -5374,6 +5955,32 @@ impl Engine { IncidentStore::new(&self.options.incident_store) } + /// A failed runner step keeps its output tail in the runner evidence + /// store and says so on the journal, naming no output. The store write is + /// best effort: its failure is named in the line and never stops the + /// abort that follows. + fn keep_runner_failure( + &self, + transaction: &DeploymentTransaction, + error: &anyhow::Error, + ) -> Option { + let failure = error.downcast_ref::()?; + let path = runner_evidence_store_path(&self.options.state_store); + let recorded = now_millis().and_then(|now| { + runner_evidence::record( + &path, + &RunnerFailureRecord::from_failure( + &transaction.transaction_id, + &transaction.command_id, + &transaction.target, + failure, + now, + ), + ) + }); + Some(runner_failure_journal_line(transaction, failure, &path, &recorded)) + } + /// Settle the outcome of one incident operation. A failure never leaves /// this function: it is traced once per episode, under the operation and /// the subject it was decided for (`*` for a pass over the whole store), @@ -5470,8 +6077,26 @@ impl Engine { /// is of one kind; the binding validated that. fn runner_for(&self, plan: &CompiledDeploymentPlan) -> Result> { let (_, binding) = plan.parsed_inputs()?; + #[cfg(test)] + if let Some(runner) = self.runner_stand_in.lock().expect("runner mutex").clone() { + struct Shared(Arc); + impl RunnerPort for Shared { + fn materialize( + &self, + source: &crate::drivers::FrozenSource, + plan: &CompiledDeploymentPlan, + staging_root: &Path, + sealed_at_unix_millis: u64, + ) -> Result { + self.0.materialize(source, plan, staging_root, sealed_at_unix_millis) + } + } + return Ok(Box::new(Shared(runner))); + } Ok(match &binding.workload { - WorkloadBinding::SystemdTransient(_) => Box::new(self.docker_runner.clone()), + WorkloadBinding::SystemdTransient(_) | WorkloadBinding::StaticRelease(_) => { + Box::new(self.docker_runner.clone()) + } WorkloadBinding::HostActuator(_) => Box::new(HostActuatorRunnerDriver { access: self.host_access()?, }), @@ -5482,7 +6107,11 @@ impl Engine { fn workload_for(&self, plan: &CompiledDeploymentPlan) -> Result> { let (_, binding) = plan.parsed_inputs()?; Ok(match &binding.workload { - WorkloadBinding::SystemdTransient(_) => Arc::clone(&self.systemd_workload), + // A static release has no workload to run: the systemd driver + // serves its install, and nothing else is ever asked of it. + WorkloadBinding::SystemdTransient(_) | WorkloadBinding::StaticRelease(_) => { + Arc::clone(&self.systemd_workload) + } WorkloadBinding::HostActuator(_) => { #[cfg(test)] if let Some(port) = self.host_workload.lock().expect("host workload mutex").clone() { @@ -5552,42 +6181,15 @@ impl Engine { /// and authorize nothing further, so re-proving them only creates ways for /// history to refuse a boot. On yggdrasil that is 259 of 260 records. /// - /// The operator anchor is read on demand rather than up front. Reading it - /// unconditionally meant an absent brake artifact refused startup against - /// an *empty* store -- a deployment brake gating Idunn itself, which - /// `F:\Projects\CLAUDE.md` forbids outright: a target brake "may never gate - /// Idunn itself or unrelated service lifecycle". + /// The deployment stop is not durable authority: it is read live at + /// Sealing and gates nothing already admitted, so no brake artifact is + /// read here and none can refuse a boot. fn validate_durable_authority(&self, snapshot: &ControlSnapshot) -> Result<()> { - let mut operator_anchor = None; for stored in &snapshot.transactions { let transaction = &stored.value; if transaction.is_terminal() { continue; } - if let Some(authorization) = &transaction.deployment_authorization { - authorization.validate_shape()?; - let record: IdunnDeploymentBrakeRecord = - rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; - if operator_anchor.is_none() { - operator_anchor = - Some(read_trust_anchor::( - &self.options.deployment_brake_operator_anchor, - )?); - } - verify_idunn_deployment_brake_authorization( - &record, - operator_anchor.as_ref().expect("read directly above"), - )?; - let expected = required(&transaction.expected, "authorized Expected projection")?; - ensure!( - record.authorized_release_id.as_deref() - == Some(expected.sealed_release_id.as_str()) - && record.authorized_deployment_id.as_deref() - == Some(transaction.transaction_id.as_str()) - && record.runtime_id == expected.runtime_id, - "durable deployment authorization names another release or transaction" - ); - } let lease = transaction .leasing .as_ref() @@ -5864,16 +6466,11 @@ impl Engine { { return LiftDecision::KeepAborting; } - match now_millis() { - Ok(now) => { - if self - .deploy_the_continuity_yields_to(snapshot, transaction, now) - .is_some() - { - return LiftDecision::KeepAborting; - } - } - Err(error) => return LiftDecision::Undecided(error), + if self + .deploy_the_continuity_yields_to(snapshot, transaction) + .is_some() + { + return LiftDecision::KeepAborting; } let lifted = (|| -> Result { let plan = required(&transaction.plan, "transaction plan")?; @@ -6151,10 +6748,23 @@ impl Engine { } fn clear_fault(&self, transaction_id: &str) { - self.fault_reports - .lock() - .expect("fault report mutex") - .remove(transaction_id); + let mut reports = self.fault_reports.lock().expect("fault report mutex"); + reports.remove(transaction_id); + reports.remove(&route_proof_report_key(transaction_id)); + } + + /// Say why the stable route's proof failed. The Routing phase retries the + /// proof until its deadline, and the deadline's own cause names only the + /// clock, so this line is where an operator learns what the listener + /// actually did. Said once per distinct reason while the fault lasts. + fn report_route_proof_failure(&self, transaction_id: &str, error: &anyhow::Error) { + self.report_once( + &route_proof_report_key(transaction_id), + format!( + "Idunn route proof for {transaction_id} failed: {}", + route_proof_reason(error) + ), + ); } /// Write `detail` as the record's `last_error`, touching nothing else and @@ -6314,7 +6924,11 @@ impl Engine { // phase deadline, not an abort, bounds the wait. if latest.value.pre_fencing_abort.is_none() && error.downcast_ref::().is_none() + && !latest.value.flip_may_have_moved_current(&error) { + if let Some(line) = self.keep_runner_failure(&latest.value, &error) { + eprintln!("{line}"); + } self.begin_pre_fencing_abort(latest, error)?; } else { self.record_resumable_error(latest, &error)?; @@ -6386,7 +7000,7 @@ impl Engine { .commands .iter() .filter(|stored| { - stored.value.kind == CommandKind::Deploy + stored.value.kind != CommandKind::Continuity && !live_commands.contains(&stored.value.command_id) }) .collect::>(); @@ -6456,6 +7070,9 @@ impl Engine { snapshot: &ControlSnapshot, command: &Stored, ) -> Result { + if command.value.kind == CommandKind::Rollback { + return self.freeze_rollback(snapshot, command); + } let bindings = match load_bindings(&self.options.bindings_dir) { Ok(bindings) => bindings, Err(error) => { @@ -6490,7 +7107,7 @@ impl Engine { .transactions .iter() .filter(|stored| { - stored.value.command_kind == CommandKind::Deploy && stored.value.is_live() + stored.value.command_kind != CommandKind::Continuity && stored.value.is_live() }) .map(|stored| stored.value.target.as_str()) .collect::>(); @@ -6531,6 +7148,106 @@ impl Engine { Ok(true) } + /// Freeze a rollback command into its one transaction, or record why it + /// cannot be one. The previous release is derived from what was flipped + /// (control.cc and history.cc) and the link as observed, never stored. + fn freeze_rollback( + &self, + snapshot: &ControlSnapshot, + command: &Stored, + ) -> Result { + let target = command.value.selector.as_str(); + if snapshot + .on_target(target) + .any(|stored| stored.value.command_kind != CommandKind::Continuity && stored.value.is_live()) + { + return Ok(false); + } + let Some(history) = self.history_for_decision() else { + return Ok(false); + }; + let now = now_millis()?; + let transaction = match self.rollback_source(snapshot, &history, target) { + Ok(prior) => DeploymentTransaction::from_rollback(&command.value, &prior, now)?, + Err(error) => DeploymentTransaction::rejected(&command.value, error, now)?, + }; + ensure!( + SingleFileMessagePackBackingStore::new(&self.options.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }], + &[transaction_envelope(&transaction, now)?], + )?, + "rollback command lost its transaction creation CAS" + ); + Ok(true) + } + + /// The transaction that flipped the release before the newest one, when the + /// link still names the newest and that previous release is intact. + fn rollback_source( + &self, + snapshot: &ControlSnapshot, + history: &[DeploymentTransaction], + target: &str, + ) -> Result { + let mut flipped = snapshot + .transactions + .iter() + .map(|stored| &stored.value) + .chain(history) + .filter(|transaction| { + transaction.target == target + && matches!(transaction.completion, Some(TransactionCompletion::Admitted { .. })) + && transaction.flip.as_ref().is_some_and(FlipEvidence::landed) + && transaction.is_static() + }) + .collect::>(); + flipped.sort_by_key(|transaction| { + ( + transaction.flip.as_ref().and_then(|flip| flip.flipped_at_unix_millis), + transaction.created_at_unix_millis, + ) + }); + let newest = flipped + .last() + .with_context(|| format!("target {target} has no completed static release to roll back from"))?; + let flip = newest.flip.as_ref().expect("filtered to landed flips"); + let previous = flip.previous_release_id.clone().with_context(|| { + format!("no previous release: {} is the first release {target} was flipped to", flip.release_id) + })?; + let binding = required(&newest.plan, "newest release plan")?.parsed_inputs()?.1; + let workload = binding.workload.static_release()?; + let observed = self.static_release.observe_current(workload)?; + ensure!( + observed.as_deref() == Some(flip.release_id.as_str()), + "current names {} but the newest flip was to {}; rollback does not guess", + observed.as_deref().unwrap_or("nothing"), + flip.release_id + ); + let prior = flipped + .iter() + .rev() + .find(|transaction| transaction.flip.as_ref().is_some_and(|f| f.release_id == previous)) + .with_context(|| format!("no record of a transaction that flipped the previous release {previous}"))?; + // The prior plan is reused verbatim, but the link was observed through + // the newest plan's binding: they must be the same release root and link. + let prior_binding = required(&prior.plan, "previous release plan")?.parsed_inputs()?.1; + let prior_workload = prior_binding.workload.static_release()?; + ensure!( + prior_workload.release_root == workload.release_root + && prior_workload.current_link == workload.current_link, + "the previous release was deployed under a different release root or current link than the newest; rollback does not cross bindings" + ); + self.static_release.verify_installed( + workload, + required(&prior.sealed_release, "previous sealed release")?, + )?; + Ok((*prior).clone()) + } + fn supervise_one_admitted_generation(&self) -> Result { let snapshot = ControlSnapshot::read(&self.options.state_store)?; // A wait belongs to the failed record it spaces. Once any write for its @@ -6567,37 +7284,18 @@ impl Engine { || snapshot.on_target(¤t.value.target).any(|stored| { stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity }); - match self.supervise_admitted_route(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn rejected admitted {} route continuity: {error:#}", - current.value.target - ), - } - match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved admitted {} after rejecting topology observation: {error:#}", - current.value.target - ), - } match self.retire_stale_incarnations(&snapshot, current) { Ok(true) => { progressed = true; continue; } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved a stale {} incarnation projection: {error:#}", - current.value.target + Ok(false) => self.clear_fault(&format!("stale-incarnation:{}", current.value.target)), + Err(error) => self.report_once( + &format!("stale-incarnation:{}", current.value.target), + format!( + "Idunn preserved a stale {} incarnation projection: {error:#}", + current.value.target + ), ), } // Restore the admitted Expected before anything is asked of the @@ -6626,11 +7324,16 @@ impl Engine { } Ok(()) })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target - ); + let repair_key = format!("projection-repair:{}", current.value.target); + match repair { + Ok(()) => self.clear_fault(&repair_key), + Err(error) => self.report_once( + &repair_key, + format!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ), + ), } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -6641,6 +7344,7 @@ impl Engine { ) }) { Ok(observation) => { + self.clear_fault(&format!("observe:{}", current.value.target)); let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { let lease_health = (|| -> Result { let lease_path = current @@ -6661,6 +7365,10 @@ impl Engine { } bail!("write-lease store contains unexpected authority") })(); + let lease_key = format!("write-lease:{}", current.value.target); + if lease_health.is_ok() { + self.clear_fault(&lease_key); + } match lease_health { Ok(true) => false, Ok(false) => { @@ -6669,9 +7377,12 @@ impl Engine { true } Err(error) => { - eprintln!( - "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", - current.value.target + self.report_once( + &lease_key, + format!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target + ), ); continue; } @@ -6691,13 +7402,20 @@ impl Engine { // silence, not a death; counting it as one burned every // continuity attempt in the reattach window on // 2026-09-11 and left a healthy Muninn demoted. - eprintln!( - "Idunn cannot observe admitted {} this tick: {error:#}", - current.value.target + // Said once while it lasts, and cleared the moment the host + // answers: a reattach window is not a line per tick. + self.report_once( + &format!("observe:{}", current.value.target), + format!( + "Idunn cannot observe admitted {} this tick: {error:#}", + current.value.target + ), ); continue; } Err(error) => { + // The host answered, with a death: no longer silence. + self.clear_fault(&format!("observe:{}", current.value.target)); operational_error = Some(error); None } @@ -6736,6 +7454,42 @@ impl Engine { ) }), ); + // Route and topology supervision are about a running incarnation; + // workload liveness decides whether there is one (route-continuity-cut F8, + // resolved by this order). Topology cannot bind without an observed + // activation, so a HostUnobservable tick skips both too. + let route_key = format!("route-supervision:{}", current.value.target); + match self.supervise_admitted_route(&snapshot, current) { + Ok(true) => { + self.clear_fault(&route_key); + progressed = true; + continue; + } + Ok(false) => self.clear_fault(&route_key), + Err(error) => self.report_once( + &route_key, + format!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + ), + } + let topology_key = format!("topology:{}", current.value.target); + match self.refresh_admitted_topology(&snapshot, current) { + Ok(true) => { + self.clear_fault(&topology_key); + progressed = true; + continue; + } + Ok(false) => self.clear_fault(&topology_key), + Err(error) => self.report_once( + &topology_key, + format!( + "Idunn preserved admitted {} after rejecting topology observation: {error:#}", + current.value.target + ), + ), + } continue; } let workload_error = operational_error @@ -6873,10 +7627,17 @@ impl Engine { // could not resolve. So a failed demotion mints nothing. The // failure is written to the generation, and the next attempt is // not before the deferral ends. + let demotion_key = format!("demotion:{}", current.value.target); + if demotion.is_ok() { + self.clear_fault(&demotion_key); + } if let Err(error) = demotion { - eprintln!( - "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", - current.value.target + self.report_once( + &demotion_key, + format!( + "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", + current.value.target + ), ); let mut deferred = supervision.clone(); deferred.defer_continuity(now, &bounded_detail(&format!("{error:#}"))); @@ -7003,17 +7764,13 @@ impl Engine { "admitted route receipt names another incarnation" ); let binding = current.value.plan.parsed_inputs()?.1; - let lease_driver = if let Some(lease) = current.value.leasing.lease() { + let lease_driver = if current.value.leasing.lease().is_some() { let lease_path = &binding .process_write_lease .as_ref() .context("stateful admitted generation has no write-lease binding")? .record_path; let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - ensure!( - driver.observe_exact(lease)?, - "admitted process write lease is no longer exact" - ); Some(driver) } else { None @@ -7274,8 +8031,12 @@ impl Engine { &providers, )?; // Idunn infers no way to prove readiness. A recipe that declares - // none is refused here, before anything is frozen, built or installed. - plan.readiness_class()?; + // none is refused here, before anything is frozen, built or + // installed. A static release proves nothing to anyone: it runs + // nothing, and its flip is observed on disk. + if !matches!(loaded.binding.workload, WorkloadBinding::StaticRelease(_)) { + plan.readiness_class()?; + } return self.persist_same_phase(current, |next| { next.plan = Some(plan); next.updated_at_unix_millis = now; @@ -7298,9 +8059,10 @@ impl Engine { }); } + let is_static = current.value.is_static(); if current.value.sealed_release.is_none() || current.value.installed_release.is_none() - || current.value.expected.is_none() + || (current.value.expected.is_none() && !is_static) { ensure!( current.value.command_kind == CommandKind::Deploy, @@ -7317,11 +8079,13 @@ impl Engine { now, )?; let installed = self.workload_for(plan)?.install(plan, &materialized)?; - let expected = materialized.release.expected_projection(plan)?; + let expected = (!is_static) + .then(|| materialized.release.expected_projection(plan)) + .transpose()?; return self.persist_same_phase(current, |next| { next.sealed_release = Some(materialized.release); next.installed_release = Some(installed); - next.expected = Some(expected); + next.expected = expected; next.updated_at_unix_millis = now; next.last_error = None; Ok(()) @@ -7334,14 +8098,11 @@ impl Engine { let now = now_millis()?; let mut next = current.value.clone(); match current.value.command_kind { - CommandKind::Deploy => { - let Some(authorization) = self.deployment_authorization(current, now)? else { - return self.record_gate_wait( - current, - "deployment brake has not released this exact transaction", - ); - }; - next.deployment_authorization = Some(authorization); + CommandKind::Deploy | CommandKind::Rollback => { + if let Some(note) = self.stop_note(current)? { + return self.record_gate_wait(current, ¬e); + } + next.deployment_admitted_at_unix_millis = Some(now); } CommandKind::Continuity => { if !self.lifecycle_allows(current, now)? { @@ -7365,6 +8126,9 @@ impl Engine { } fn advance_starting(&self, current: &Stored) -> Result<()> { + if current.value.is_static() { + return self.advance_static(current); + } let expected = required(¤t.value.expected, "Expected projection")?; let plan = required(¤t.value.plan, "transaction plan")?; let release = required(¤t.value.sealed_release, "sealed release")?; @@ -7450,6 +8214,90 @@ impl Engine { replace_transaction(&self.options.state_store, current, &next) } + /// A static release's Starting: its one external change, in three durable + /// steps. The intent (the release to name and the one named now) lands + /// first; then the carry and the flip, whose facts land second; then the + /// link is observed naming the release and the transaction completes. The + /// flip is one rename and repeating it is harmless, so a crash anywhere + /// resumes at the record, and a failure before the rename leaves the + /// current link exactly as it was, which is what the pre-fencing abort + /// this error takes says. + fn advance_static(&self, current: &Stored) -> Result<()> { + let plan = required(¤t.value.plan, "transaction plan")?; + let release = required(¤t.value.sealed_release, "sealed release")?; + let binding = plan.parsed_inputs()?.1; + let workload = binding.workload.static_release()?; + let now = now_millis()?; + let Some(flip) = ¤t.value.flip else { + // Nothing has moved yet, so a refusal here ends the transaction + // honestly: the release is still what was sealed, or it is not. + // The flip intent is the last point a stop can halt a static + // release: once it is recorded the flip runs to its end. + if let Some(note) = self.stop_note(current)? { + return self.record_gate_wait(current, ¬e); + } + self.static_release.verify_installed(workload, release)?; + let previous_release_id = self.static_release.observe_current(workload)?; + let release_id = release.sealed_release_id.clone(); + return self.persist_same_phase(current, |next| { + next.flip = Some(FlipEvidence { + release_id, + previous_release_id, + carry_source: workload.carry.source.clone(), + carried_sha256: None, + flipped_at_unix_millis: None, + }); + next.updated_at_unix_millis = now; + next.last_error = None; + Ok(()) + }); + }; + if !flip.landed() { + // An earlier attempt may have moved the link and failed to record + // it: then nothing is flipped again, and the carry already inside + // the release is what is recorded. + let landed = if self.static_release.observe_current(workload)?.as_deref() + == Some(flip.release_id.as_str()) + { + StaticFlip { + carried_sha256: self + .static_release + .carried_sha256(workload, &flip.release_id)?, + } + } else { + self.static_release + .carry_and_flip(workload, &flip.release_id)? + }; + return self.persist_same_phase(current, |next| { + let flip = next.flip.as_mut().expect("flip intent read above"); + flip.carried_sha256 = Some(landed.carried_sha256); + flip.flipped_at_unix_millis = Some(now); + next.updated_at_unix_millis = now; + Ok(()) + }); + } + ensure!( + self.static_release.observe_current(workload)?.as_deref() == Some(flip.release_id.as_str()), + "the current link does not name the release this transaction flipped to" + ); + let mut complete = current.value.clone(); + complete.enter_phase(DeploymentPhase::Complete, now); + complete.last_error = None; + complete.completion = Some(TransactionCompletion::Admitted { + generation_id: format!("generation-{}", current.value.transaction_id), + }); + complete.post_commit_cleanup = Some(PostCommitCleanup { + incumbent: IncumbentCleanupEvidence::SkippedNoIncumbent, + source: match current.value.command_kind { + CommandKind::Rollback => SourceCleanupEvidence::SkippedContinuity, + _ => SourceCleanupEvidence::Pending, + }, + }); + complete.retime(now); + complete.validate()?; + replace_transaction(&self.options.state_store, current, &complete) + } + fn advance_warming(&self, current: &Stored) -> Result<()> { let expected = required(¤t.value.expected, "Expected projection")?; let activation = required(¤t.value.activation, "activation")?; @@ -7631,6 +8479,14 @@ impl Engine { }); } + // Fencing is the last pre-fencing abort-able point of a deploy: an + // engaged stop halts here, before the incumbent is fenced. A fenced + // deploy runs to its end. Continuity is never gated by this stop. + if current.value.command_kind == CommandKind::Deploy + && let Some(note) = self.stop_note(current)? + { + return self.record_gate_wait(current, ¬e); + } self.transition(current, DeploymentPhase::Fencing) } @@ -8079,18 +8935,21 @@ impl Engine { rollback_allowed, &gate, )?; - let observation = match self.prove_stable_route( + let observation = match self.prove_routed_candidate_through_handoff( &admitted.value, &driver, membership_sha256, + preflight, ) { Ok(observation) => observation, Err(proof_error) if !rollback_allowed => { + self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return Err(proof_error).context( "incumbent authority was fenced; candidate route remains installed for fail-closed retry", ); } Err(proof_error) => { + self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return match driver.rollback( expected, &activation.runtime_instance_id, @@ -8284,7 +9143,7 @@ impl Engine { }, source: match commit_current.value.command_kind { CommandKind::Deploy => SourceCleanupEvidence::Pending, - CommandKind::Continuity => SourceCleanupEvidence::SkippedContinuity, + CommandKind::Continuity | CommandKind::Rollback => SourceCleanupEvidence::SkippedContinuity, }, }; let mut complete = commit_current.value.clone(); @@ -8556,66 +9415,43 @@ impl Engine { .context("no private candidate port remains in the operator range") } - fn deployment_authorization( + /// Why the deployment stop holds this transaction, or None when it admits. + /// Read live at each point that is still reversible: Sealing, the static + /// flip intent and the entry into Fencing. + fn stop_note(&self, current: &Stored) -> Result> { + let notes: Vec = self + .deployment_stops(current)? + .iter() + .filter_map(NamedStop::note) + .collect(); + Ok((!notes.is_empty()).then(|| notes.join("; "))) + } + + /// What the binding's stops say about this transaction's runtime: the + /// operator's store and, when the binding names one, the steward's. Read + /// live; never cached, never consumed. + fn deployment_stops( &self, current: &Stored, - now: u64, - ) -> Result> { - let expected = required(¤t.value.expected, "Expected projection")?; + ) -> Result> { let plan = required(¤t.value.plan, "transaction plan")?; let binding = plan.parsed_inputs()?.1; - let Some((record, canonical_bytes)) = - read_deployment_brake(&binding.brakes.deployment_store)? - else { - return Ok(None); - }; - let anchor = read_trust_anchor::( - &self.options.deployment_brake_operator_anchor, - )?; - if evaluate_idunn_deployment_brake( - IdunnDeploymentBrakeObservation::Present(&record), - &anchor, - &expected.runtime_id, - &expected.sealed_release_id, - ¤t.value.transaction_id, - now, - ) - .is_err() - { - return Ok(None); - } - let authorization_id = record - .authorization_id - .clone() - .context("released brake has no authorization id")?; - let snapshot = ControlSnapshot::read(&self.options.state_store)?; - ensure!( - !snapshot.transactions.iter().any(|stored| { - stored - .value - .deployment_authorization - .as_ref() - .is_some_and(|used| used.authorization_id == authorization_id) - }), - "deployment authorization was already consumed" - ); - Ok(Some(DeploymentAuthorization { - authorization_id, - brake_sha256: sha256_id(&canonical_bytes), - canonical_brake_bytes: canonical_bytes, - authorized_at_unix_millis: now, - })) + Ok(read_stops( + &binding.brakes, + &binding.runtime_identity.runtime_id, + )) } /// Whether a deployment awaiting the handover is free to take it: the - /// deployment brake has released its exact transaction, so the stop a yield - /// makes is followed by the deployment's own landing. A Deploy at its - /// Sealing tail has not passed the brake yet and is read live; one that is - /// still building, or whose brake cannot be read, is held. An operator's + /// deployment stop admits it, so the stop a yield makes is followed by the + /// deployment's own landing. A Deploy at its Sealing tail has not passed + /// the stop yet and is read live; one that is still building, or whose + /// stop is engaged or cannot be read, is held. An operator's /// hold is a reason to leave a serving or retrying body alone. - fn deploy_can_move(&self, deploy: &Stored, now: u64) -> bool { - deploy.value.deployment_authorization.is_some() - || matches!(self.deployment_authorization(deploy, now), Ok(Some(_))) + fn deploy_can_move(&self, deploy: &Stored) -> bool { + deploy.value.deployment_admitted() + || self.deployment_stops(deploy) + .is_ok_and(|stops| stops.iter().all(|named| named.stop.admits())) } /// The deployment a continuity gives way to: one on its target that awaits @@ -8627,13 +9463,12 @@ impl Engine { &self, snapshot: &'a ControlSnapshot, continuity: &DeploymentTransaction, - now: u64, ) -> Option<&'a Stored> { if continuity.serving_proven() { return None; } snapshot.on_target(&continuity.target).find(|other| { - other.value.awaits_handover() && self.deploy_can_move(other, now) + other.value.awaits_handover() && self.deploy_can_move(other) }) } @@ -9029,6 +9864,42 @@ impl Engine { ) } + /// Routing's proof, waiting out nginx's reload handoff. `systemctl reload` + /// only signals the master: the old workers keep answering the stable + /// listener until the new configuration is parsed and they are told to + /// quit, so Idunn's first challenge can reach the incumbent. An answer + /// that names the preflight's incumbent is therefore not-yet for + /// `ROUTE_RELOAD_HANDOFF`; it can delay and never admit (the id is read + /// unauthenticated, and the candidate's proof is checked as before). + /// Anything else fails the attempt at once. One challenge per poll and + /// nothing else: no install, rollback, route gate or persistence. + fn prove_routed_candidate_through_handoff( + &self, + transaction: &DeploymentTransaction, + driver: &NginxRouteDriver, + membership_sha256: String, + preflight: &crate::drivers::RoutePreflightReceipt, + ) -> Result { + let candidate = &required(&transaction.activation, "route activation")?.runtime_instance_id; + let until = handoff_now() + ROUTE_RELOAD_HANDOFF; + loop { + match self.prove_stable_route(transaction, driver, membership_sha256.clone()) { + Err(error) + if preflight.changes_route() + && handoff_now() < until + && error.downcast_ref::().is_some_and(|answer| { + preflight.incumbent_runtime_instance_id.as_deref() + == Some(answer.answered_instance.as_str()) + && answer.answered_instance != *candidate + }) => + { + handoff_sleep(ROUTE_RELOAD_HANDOFF_POLL); + } + result => return result, + } + } + } + fn prove_stable_route_against( &self, expected: &IdunnExpectedIncarnationRecord, @@ -9053,7 +9924,15 @@ impl Engine { challenged_at_unix_millis, received_at_unix_millis, &response.canonical_presence, - )?; + ) + .map_err(|error| match presence_answer_ids(&response.canonical_presence) { + Some((answered_instance, answered_activation_signer)) => error.context(RouteProofAnswer { + answered_instance: route_proof_label(&answered_instance), + answered_activation_signer: route_proof_label(&answered_activation_signer), + candidate_instance: route_proof_label(&activation.runtime_instance_id), + }), + None => error, + })?; let observation = RouteObservation { route_id: driver.binding.route_id.clone(), runtime_instance_id: activation.runtime_instance_id.clone(), @@ -9604,7 +10483,7 @@ impl Engine { fn settle_before_handover(&self, current: &Stored) -> Result { let held = ¤t.value; match held.command_kind { - CommandKind::Deploy => { + CommandKind::Deploy | CommandKind::Rollback => { if self.rebase_onto_admitted(current)? { return Ok(true); } @@ -9665,11 +10544,11 @@ impl Engine { // Lost to a concurrent writer: the next step reads the new world. (Handover::Took | Handover::Lost, _) => Ok(()), // Wait, never abort: a pre-fencing error would end the deployment. - (Handover::OwnedBy(owner), CommandKind::Deploy) => { + (Handover::OwnedBy(owner), CommandKind::Deploy | CommandKind::Rollback) => { self.wait_for_owner(current, &owner) } // The next step rebases onto the generation now admitted. - (Handover::Moved, CommandKind::Deploy) => Ok(()), + (Handover::Moved, CommandKind::Deploy | CommandKind::Rollback) => Ok(()), (Handover::OwnedBy(owner), CommandKind::Continuity) => { bail!("continuity superseded: target {target} is owned by {owner}") } @@ -9799,8 +10678,15 @@ impl Engine { .iter() .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { + // The step's latest failure is the reason the phase never + // finished, and the abort that ends it must not drop it. + let last_attempt = transaction + .last_error + .as_deref() + .map(|error| format!("; its last attempt failed: {error}")) + .unwrap_or_default(); format!( - "the {:?} phase passed its deadline at unix ms {}", + "the {:?} phase passed its deadline at unix ms {}{last_attempt}", transaction.phase, deadline.deadline_at_unix_millis ) } else if requested { @@ -9831,10 +10717,9 @@ impl Engine { } PhaseEnd::RetryForward => { let snapshot = ControlSnapshot::read(&self.options.state_store)?; - let now = now_millis()?; if let Some(deploy) = transaction .yields_to_a_deploy() - .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction, now)) + .then(|| self.deploy_the_continuity_yields_to(&snapshot, transaction)) .flatten() { self.begin_post_fencing_abort( @@ -10305,7 +11190,7 @@ fn reconcile_failed_candidate_projection( ) -> Result<()> { let expected = required(&transaction.expected, "failed Expected projection")?; match transaction.command_kind { - CommandKind::Deploy => topology + CommandKind::Deploy | CommandKind::Rollback => topology .withdraw_incarnation( expected, provider_anchor, @@ -10409,7 +11294,7 @@ where F: FnOnce() -> Result<()>, { match command_kind { - CommandKind::Deploy => validation(), + CommandKind::Deploy | CommandKind::Rollback => validation(), CommandKind::Continuity => Ok(()), } } @@ -10487,24 +11372,6 @@ fn is_semantic_ready(authenticated: &AuthenticatedOdinRuntimeTopologyCorrelation && record.disagreements.is_empty() } -fn read_deployment_brake(path: &Path) -> Result)>> { - let Some(envelope) = read_single_envelope(path)? else { - return Ok(None); - }; - ensure!( - envelope.r#type == IdunnDeploymentBrakeRecord::TYPE - && envelope.schema_id.as_deref() == Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA), - "deployment brake store contains a foreign record" - ); - let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&envelope.payload)?; - record.validate()?; - ensure!( - rmp_serde::to_vec(&record)? == envelope.payload && envelope.key == record.brake_id, - "deployment brake is noncanonical or keyed by another authority" - ); - Ok(Some((record, envelope.payload))) -} - fn read_lifecycle_brake(path: &Path) -> Result> { let Some(envelope) = read_single_envelope(path)? else { return Ok(None); @@ -10553,7 +11420,13 @@ fn required<'a, T>(value: &'a Option, label: &str) -> Result<&'a T> { value.as_ref().with_context(|| format!("{label} is absent")) } -fn now_millis() -> Result { +/// The control store `idunn up` and `idunn serve` use unless told otherwise. +#[cfg(unix)] +pub(crate) fn default_state_store() -> PathBuf { + RuntimeOptions::default().state_store +} + +pub(crate) fn now_millis() -> Result { Ok(SystemTime::now() .duration_since(UNIX_EPOCH) .context("system clock predates Unix epoch")? @@ -10573,12 +11446,12 @@ fn sibling_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(value) } -fn string_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn string_value(args: &mut impl Iterator, name: &str) -> Result { args.next() .ok_or_else(|| anyhow!("{name} requires a value")) } -fn path_value(args: &mut impl Iterator, name: &str) -> Result { +pub(crate) fn path_value(args: &mut impl Iterator, name: &str) -> Result { Ok(PathBuf::from(string_value(args, name)?)) } @@ -10594,7 +11467,7 @@ fn u64_value(args: &mut impl Iterator, name: &str) -> Result .with_context(|| format!("{name} requires a u64")) } -fn require_selector(value: &str) -> Result<()> { +pub(crate) fn require_selector(value: &str) -> Result<()> { let value = value.strip_prefix("profile:").unwrap_or(value); require_id(value, "deployment selector") } @@ -10641,6 +11514,146 @@ const DETAIL_EMPTY: &str = "(no detail)"; /// only judge, and this output always satisfies it. NUL is dropped, empty /// input gets a fixed placeholder, and an over-long value keeps its head and /// its tail (where a build log's error is) cut on char boundaries. +/// The journal line for a failed runner step: the transaction, the subject and +/// status (the failure's `Display`), and where the evidence is. Never output. +fn runner_failure_journal_line( + transaction: &DeploymentTransaction, + failure: &RunnerStepFailure, + store_path: &Path, + recorded: &Result, +) -> String { + let head = format!( + "Idunn transaction {} (command {}, target {}) {failure}", + transaction.transaction_id, transaction.command_id, transaction.target + ); + match recorded { + Ok(kept) => { + let aside = match kept { + Kept::Stored => String::new(), + Kept::SetAside { replaced } => format!( + "; an undecodable evidence store was set aside as {}{}", + unreadable_store_path(store_path).display(), + match replaced { + Some(last_written) => format!( + ", replacing an earlier set-aside store last written at unix millis {last_written}" + ), + None => String::new(), + } + ), + }; + format!( + "{head}; evidence in {}{aside}; read it with idunn status --command {}", + store_path.display(), + transaction.command_id + ) + } + Err(error) => format!("{head}; evidence was not kept: {error:#}"), + } +} + +fn route_proof_report_key(transaction_id: &str) -> String { + format!("route-proof:{transaction_id}") +} + +/// How long Routing waits for nginx's old workers to let go of the stable +/// listener after a reload, derived from two ceilings: one Forward actuation +/// must suffice per attempt (12 an hour), and the 120 s Routing deadline must +/// keep room for at least four more backoff-spaced attempts if the route +/// genuinely never switches. +const ROUTE_RELOAD_HANDOFF: Duration = + if cfg!(test) { Duration::from_millis(400) } else { Duration::from_secs(10) }; +const ROUTE_RELOAD_HANDOFF_POLL: Duration = + if cfg!(test) { Duration::from_millis(20) } else { Duration::from_millis(250) }; + +/// The handoff loop's clock. Production reads the real one; a test may install +/// a virtual one on its own thread, so the window and the poll cadence are +/// measured exactly, whatever the host load. +#[cfg(not(test))] +fn handoff_now() -> std::time::Instant { + std::time::Instant::now() +} +#[cfg(not(test))] +fn handoff_sleep(pause: Duration) { + thread::sleep(pause); +} +#[cfg(test)] +thread_local! { + /// (virtual offset, sleeps taken, clock reads) while a test holds it. + static HANDOFF_CLOCK: std::cell::RefCell, usize)>> = + const { std::cell::RefCell::new(None) }; +} +#[cfg(test)] +fn handoff_now() -> std::time::Instant { + static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); + let base = *BASE.get_or_init(std::time::Instant::now); + HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { + Some((offset, _, reads)) => { + *reads += 1; + assert!(*reads < 10_000, "the handoff loop spins without sleeping"); + base + *offset + } + None => std::time::Instant::now(), + }) +} +#[cfg(test)] +fn handoff_sleep(pause: Duration) { + HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { + Some((offset, sleeps, _)) => { + *offset += pause; + sleeps.push(pause); + } + None => thread::sleep(pause), + }) +} + +/// Who answered a stable route challenge that failed authentication, as the +/// answer claimed itself: ids only, unauthenticated, never signature bytes. +/// A label on a rejection, and the one thing that may say "challenge again". +#[derive(Debug)] +struct RouteProofAnswer { + answered_instance: String, + answered_activation_signer: String, + candidate_instance: String, +} + +impl std::fmt::Display for RouteProofAnswer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "stable route answered as runtime instance {}, activation signer {} (unauthenticated); the candidate is {}", + self.answered_instance, self.answered_activation_signer, self.candidate_instance + ) + } +} + +/// The ids a presence claims, if it decodes at all. No re-encode check and no +/// validation: this labels a rejection and is never evidence. +fn presence_answer_ids(canonical: &[u8]) -> Option<(String, String)> { + let presence = + rmp_serde::from_slice::(canonical).ok()?; + Some((presence.runtime_instance_id, presence.activation_signer_identity_id)) +} + +/// An id cut to 96 bytes at a char boundary, control characters stripped. +fn route_proof_label(id: &str) -> String { + let id: String = id.chars().filter(|c| !c.is_control()).collect(); + id[..id.floor_char_boundary(96)].to_owned() +} + +/// The stable route proof's failure in words: whether the listener said +/// nothing or answered wrongly, then the specific refusal (HTTP status line, +/// presence not active, challenge or authority disagreement). The texts it +/// carries name states and codes, never keys or signature bytes. A rejected +/// answer names the instance and activation signer it claimed, unauthenticated, +/// ids only. +fn route_proof_reason(error: &anyhow::Error) -> String { + match error.downcast_ref::() { + Some(ChallengeFailure::Silent(_)) => format!("silent, nothing answered: {error:#}"), + Some(ChallengeFailure::Refused(_)) => format!("refused: {error:#}"), + None => format!("{error:#}"), + } +} + fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -10679,9 +11692,13 @@ fn usage() -> &'static str { idunn serve [runtime options] [--host-actuator-bind ADDR]\n\ idunn up [--state-store PATH] [--no-wait]\n\ idunn status [--state-store PATH] [--incident-store PATH] [--command ID]\n\ + idunn rollback [--state-store PATH] [--no-wait]\n\ idunn cancel [--state-store PATH]\n\ idunn expire [--state-store PATH]\n\ - idunn validate --recipe PATH [--binding PATH]\n\n\ + idunn validate --recipe PATH [--binding PATH]\n\ + idunn orders serve [--store DIR] [--steward-binding PATH] [--socket PATH]\n\ + idunn order --reason TEXT\n\ + idunn order show ID | idunn order list\n\n\ Recipes describe capability and process requirements. Idunn seals exact\n\ source and artifacts, admits one incarnation, and delegates execution to\n\ systemd and routing mechanics to the configured proxy." @@ -10733,6 +11750,7 @@ mod tests { command_id: match kind { CommandKind::Deploy => "up-test".into(), CommandKind::Continuity => "continuity-test".into(), + CommandKind::Rollback => "rollback-test".into(), }, kind, selector: "ghostlight".into(), @@ -10939,7 +11957,6 @@ mod tests { odin_correlation_store: root.join("odin-correlation.cc"), odin_trust_anchor: odin_anchor, idunn_identity_store: idunn, - deployment_brake_operator_anchor: root.join("brake-anchor.cc"), #[cfg(unix)] route_actuators, ..RuntimeOptions::default() @@ -11163,27 +12180,6 @@ mod tests { Ok(()) } - #[test] - fn an_absent_brake_anchor_does_not_gate_an_empty_store() -> Result<()> { - // `F:\Projects\CLAUDE.md`: a target brake "may never gate Idunn itself - // or unrelated service lifecycle". Reading the operator anchor up front - // meant a missing brake artifact refused startup against a store with - // nothing in it -- the deployment authority held hostage by an artifact - // describing one target's consent. - let world = EngineFixture::new()?; - assert!( - !world - .engine - .options - .deployment_brake_operator_anchor - .exists(), - "fixture must not have written a brake anchor" - ); - let snapshot = ControlSnapshot::read(&world.state_store)?; - world.engine.validate_durable_authority(&snapshot)?; - Ok(()) - } - #[test] fn one_unreadable_record_faults_its_tick_without_touching_the_store() -> Result<()> { let world = EngineFixture::new()?; @@ -11893,9 +12889,45 @@ mod tests { vec!["serve", "--swarm-profile", "yggdrasil-local"], vec!["serve", "--restart-command", "bad"], vec!["validate-runtime-admission"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--command", "sh -c bad"], + vec!["order", "unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["order", "run", "sh", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r", "--deploy-command", "x"], + vec!["order", "unit", "restart", "nginx.service", "extra", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r", "--requested-by", "someone"], + vec!["orders", "serve", "--allow-units", "nginx.service"], + vec!["orders", "run"], ] { assert!(parse(arguments.into_iter().map(str::to_owned)).is_err()); } + // The declarative order forms parse, and only as order commands. + #[cfg(unix)] + for arguments in [ + vec!["order", "unit", "restart", "nginx.service", "--reason", "r"], + vec!["order", "package-install", "git-lfs", "--reason", "r"], + vec!["order", "deploy", "vili", "--reason", "r"], + vec!["order", "list"], + vec!["orders", "serve"], + ] { + let parsed = parse(arguments.into_iter().map(str::to_owned)).unwrap(); + assert!(matches!(parsed, Command::Order(_) | Command::OrdersServe(_))); + } + } + + #[test] + fn deployment_commit_primitive_is_shared_and_never_overwrites() -> Result<()> { + let temp = TempDir::new()?; + let store = temp.path().join("control.cc"); + commit_deployment_command(&store, CommandKind::Deploy, "up-one", "vili", "steward:eureka order ord-1")?; + let snapshot = ControlSnapshot::read(&store)?; + assert_eq!(snapshot.commands.len(), 1); + let command = &snapshot.commands[0].value; + assert_eq!(command.command_id, "up-one"); + assert_eq!(command.selector, "vili"); + assert_eq!(command.requested_by, "steward:eureka order ord-1"); + assert!(commit_deployment_command(&store, CommandKind::Deploy, "up-one", "other", "x").is_err()); + assert_eq!(ControlSnapshot::read(&store)?.commands.len(), 1); + Ok(()) } #[test] @@ -13223,65 +14255,18 @@ mod tests { if incumbent.is_some() { 14 } else { 4 } } - /// A released deployment brake receipt, signed by an operator identity - /// enrolled in the world: what a Deploy's `deployment_authorization` holds. - fn deployment_authorization( - world: &EngineFixture, - expected: &IdunnExpectedIncarnationRecord, - transaction_id: &str, - now: u64, - ) -> Result { - use cultnet_rs::{ - IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, IDUNN_DEPLOYMENT_BRAKE_ID, - IDUNN_DEPLOYMENT_BRAKE_SCOPE, IDUNN_DEPLOYMENT_RELEASE_PURPOSE, - IdunnDeploymentBrakeReleasePurpose, enroll_service_identity_at, - }; - // One operator per world, however many deployments it releases. - let signer_path = world.root.join("identities/brake-operator.cc"); - let signer = if signer_path.exists() { - open_service_identity_at::(&signer_path)? - } else { - let signer = - enroll_service_identity_at::(&signer_path)?; - export_service_identity_trust_anchor(&signer, &world.root.join("brake-anchor.cc"))?; - signer - }; - let authorization_id = format!("authorization-{transaction_id}"); - let mut record = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: expected.runtime_id.clone(), - status: "released".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: "test rollout".into(), - observed_at_unix_millis: now, - expires_at_unix_millis: Some(now + 600_000), - authorization_id: Some(authorization_id.clone()), - authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), - authorized_release_id: Some(expected.sealed_release_id.clone()), - authorized_deployment_id: Some(transaction_id.to_owned()), - authorized_by: Some(signer.trust_anchor()?.identity_id), - authorization_issued_at_unix_millis: Some(now), - authorization_expires_at_unix_millis: Some(now + 600_000), - signature_algorithm: Some("ed25519".into()), - signature: None, - private_state_exposed: false, - updated_by: "operator/test".into(), - }; - record.signature = Some( - signer - .sign::(&rmp_serde::to_vec(&record)?) - .signature, - ); - record.validate()?; - let canonical_brake_bytes = rmp_serde::to_vec(&record)?; - Ok(DeploymentAuthorization { - authorization_id, - brake_sha256: sha256_id(&canonical_brake_bytes), - canonical_brake_bytes, - authorized_at_unix_millis: now, - }) + /// Write the deployment stop of the binding `plan` names, as the verb + /// would: engaged halts that runtime's deployments, clear admits them. + fn set_stop(plan: &CompiledDeploymentPlan, state: StopState, reason: &str) -> Result<()> { + let binding = plan.parsed_inputs()?.1; + crate::brake::replace_stop( + &binding.brakes.deployment_store, + &binding.runtime_identity.runtime_id, + state, + reason, + now_millis()?, + )?; + Ok(()) } /// `binding` with its provider trust anchor and signer pointed at this @@ -13368,6 +14353,14 @@ mod tests { .split_once("[brakes]") .context("binding has no brakes table")? .1 + ) + .replace( + "/var/lib/gamecult/idunn/service-deployment-brake.cc", + &world.root.join("service-deployment-brake.cc").display().to_string(), + ) + .replace( + "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", + &world.root.join("service-lifecycle-brake.cc").display().to_string(), ), )?; let recipe = RECIPE @@ -13419,6 +14412,7 @@ mod tests { match kind { CommandKind::Deploy => "up", CommandKind::Continuity => "continuity", + CommandKind::Rollback => "rollback", }, if incumbent.is_some() { "-again" } else { "" } ); @@ -13457,18 +14451,14 @@ mod tests { CommandKind::Continuity => { transaction.lifecycle_authorized_at_unix_millis = Some(now); } + CommandKind::Rollback => unreachable!("service fixtures have no rollback"), CommandKind::Deploy => { transaction.frozen_source = Some(FrozenSourceReceipt { transaction_id: transaction.transaction_id.clone(), plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - transaction.deployment_authorization = Some(deployment_authorization( - world, - &expected, - &transaction.transaction_id, - now, - )?); + transaction.deployment_admitted_at_unix_millis = Some(now); } } transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); @@ -14274,6 +15264,132 @@ mod tests { odin_reports_ready(world, transaction, sequence) } + /// An Odin-correlated generation, admitted over `workload`, with the + /// sequence of the reading it holds. + fn odin_admitted_world( + workload: Arc, + ) -> Result<(EngineFixture, DeploymentTransaction, u64)> { + let world = EngineFixture::with_workload(workload)?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let first = seeded_sequence(None) + 1; + odin_reports_ready(&world, &seeded, first)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + assert!(world.engine.retire_one_terminal_transaction()?); + Ok((world, seeded, first)) + } + + /// Odin's next reading, about an activation that is not the current one. + fn odin_republishes_another_activation( + world: &EngineFixture, + seeded: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let mut other = seeded.clone(); + other.activation.as_mut().context("no activation")?.runtime_instance_id = + sha256_id(b"another-incarnation"); + odin_republishes(world, &other, sequence) + } + + /// Engage the admitted target's lifecycle brake, at the path its seeded + /// binding names: a dead incarnation mints no continuity, so every later + /// tick reaches the same gates. + fn engage_lifecycle_brake_in(world: &EngineFixture) -> Result<()> { + let generation = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .value + .clone(); + crate::provisioning::replace_lifecycle_brake( + &world.root.join("service-lifecycle-brake.cc"), + IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: generation.expected.runtime_id.clone(), + target: "service".into(), + scope: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "engaged".into(), + reason: "test".into(), + updated_at_unix_millis: now_millis()?, + released_until_unix_millis: None, + }, + ) + } + + #[test] + fn a_dead_generation_does_not_take_odin_correlations() -> Result<()> { + let switch = SwitchWorkload::new(); + let (world, seeded, first) = odin_admitted_world(switch.clone())?; + switch.kill(); + // No continuity is minted, so no transaction ever owns the target and + // every tick reaches the topology gate. + engage_lifecycle_brake_in(&world)?; + let envelope = incumbent_envelope(&world)?; + + // A newer reading that binds the admitted activation, and one that does + // not: neither is taken, nor said, about an incarnation that is not there. + for reading in 0..2 { + if reading == 0 { + odin_republishes(&world, &seeded, first + 4)?; + } else { + odin_republishes_another_activation(&world, &seeded, first + 5)?; + } + for _ in 0..100 { + world.engine.supervise_one_admitted_generation()?; + } + assert!( + ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), + "a transaction owns the target: later ticks never reach the gate" + ); + assert_eq!(incumbent_envelope(&world)?, envelope); + assert!( + !world.engine.fault_reports.lock().unwrap().contains_key("topology:service"), + "a dead generation reported a topology rejection" + ); + } + Ok(()) + } + + fn printed(world: &EngineFixture, key: &str) -> usize { + world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() + } + + #[test] + fn a_running_generation_with_a_non_binding_correlation_is_reported_once() -> Result<()> { + let (world, seeded, first) = odin_admitted_world(Arc::new(StillWorkload))?; + odin_republishes_another_activation(&world, &seeded, first + 4)?; + let envelope = incumbent_envelope(&world)?; + for _ in 0..50 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + assert_eq!(incumbent_envelope(&world)?, envelope); + // The engine said it once over fifty ticks. + assert_eq!(printed(&world, "topology:service"), 1); + assert!(world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); + + // A binding reading is adopted and ends the fault, so the key clears. + odin_republishes(&world, &seeded, first + 6)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); + let adopted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no generation")? + .value + .clone(); + assert_eq!(adopted.odin_publisher_sequence_cursor, first + 6); + + // A recurrence is said again, once; a reading withdrawn ends it too. + odin_republishes_another_activation(&world, &seeded, first + 8)?; + for _ in 0..20 { + world.engine.supervise_one_admitted_generation()?; + } + assert_eq!(printed(&world, "topology:service"), 2); + let store = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); + assert!(store.delete_batch_if_unchanged(&store.pull_all_read_only_snapshot()?)?); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); + Ok(()) + } + fn ready_sequence(transaction: &DeploymentTransaction) -> Option { transaction .ready @@ -16196,6 +17312,60 @@ mod tests { Ok(()) } + #[test] + fn a_route_proof_failure_names_what_the_listener_did() { + let reason = |error: anyhow::Error| route_proof_reason(&error); + let silent = anyhow::Error::new(ChallengeFailure::Silent(anyhow!( + "connecting stable CultNet HTTP route 127.0.0.1:9: refused" + ))); + let text = reason(silent); + assert!(text.contains("silent") && text.contains("127.0.0.1:9"), "{text}"); + + let status = anyhow::Error::new(ChallengeFailure::Refused(anyhow!( + "stable CultNet HTTP route rejected the snapshot challenge: HTTP/1.1 502 Bad Gateway" + ))) + .context("candidate did not answer its stable route challenge"); + let text = reason(status); + assert!(text.contains("refused") && text.contains("502 Bad Gateway"), "{text}"); + + let inactive = anyhow!("route proof runtime is warming, not active"); + assert!(reason(inactive).contains("warming, not active")); + + let disagrees = anyhow::Error::new(PresenceDisagrees { + disagreements: vec![OdinTopologyDisagreement { + code: "runtime-instance-mismatch".into(), + expected: Some("a".into()), + observed: Some("b".into()), + }], + }); + let text = reason(disagrees); + assert!(text.contains("runtime-instance-mismatch (expected a, observed b)"), "{text}"); + + let unbound = anyhow!("route proof response is not bound to the exact challenge"); + assert!(reason(unbound).contains("not bound to the exact challenge")); + } + + #[test] + fn a_route_proof_failure_is_reported_once_per_reason_and_cleared_on_recovery() -> Result<()> { + let world = EngineFixture::new()?; + let key = route_proof_report_key("tx-1"); + let said = |world: &EngineFixture| { + world + .engine + .fault_reports + .lock() + .unwrap() + .get(&key) + .and_then(|report| report.last.lock().unwrap().clone()) + }; + world.engine.report_route_proof_failure("tx-1", &anyhow!("route proof runtime is warming, not active")); + let first = said(&world).context("no report")?; + assert!(first.contains("tx-1") && first.contains("warming, not active"), "{first}"); + world.engine.clear_fault("tx-1"); + assert!(said(&world).is_none(), "recovery left the proof report armed"); + Ok(()) + } + #[test] fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { write_fixture_anchor()?; @@ -17990,11 +19160,12 @@ mod tests { make_stop_first(world, deploy_id)?; let sealing = resident_id(world, deploy_id)?; let mut tail = sealing.value.clone(); - tail.deployment_authorization = None; + tail.deployment_admitted_at_unix_millis = None; replace_transaction(&world.state_store, &sealing, &tail)?; + set_stop(tail.plan.as_ref().context("no plan")?, StopState::Engaged, "test hold")?; let tail = resident_id(world, deploy_id)?.value; assert!(tail.awaits_handover()); - assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?, now_millis()?)); + assert!(!world.engine.deploy_can_move(&resident_id(world, deploy_id)?)); Ok(()) } @@ -18257,13 +19428,14 @@ mod tests { /// for it; once the brake releases, the continuity yields. The yield closes /// the continuity's operator-required incident Recovered. #[test] - fn a_continuity_does_not_yield_to_a_braked_deploy_and_yields_once_the_brake_releases() -> Result<()> { + fn a_continuity_does_not_yield_to_a_stopped_deploy_and_yields_once_the_stop_clears() -> Result<()> { let (world, workload, _, deploy_id, owner_id, _) = waiter_beside_gate_waiting_continuity()?; make_stop_first(&world, &deploy_id).context("A make_stop_first")?; let sealing = resident_id(&world, &deploy_id).context("B sealing")?; let mut tail = sealing.value.clone(); - tail.deployment_authorization = None; + tail.deployment_admitted_at_unix_millis = None; replace_transaction(&world.state_store, &sealing, &tail)?; + set_stop(tail.plan.as_ref().context("no plan")?, StopState::Engaged, "test hold")?; assert!(resident_id(&world, &deploy_id).context("C awaits")?.value.awaits_handover()); pass_deadline_of(&world, &owner_id)?; let stops = stops_of(&workload); @@ -18276,30 +19448,9 @@ mod tests { assert_eq!(operator_required(&world)?, [owner_id.as_str()]); assert_eq!(stops_of(&workload), stops); - // The operator releases the brake for this exact Deploy. + // The steward clears the stop; no per-deploy release exists. let deploy = resident_id(&world, &deploy_id).context("E deploy")?.value; - let authorization = deployment_authorization( - &world, - deploy.expected.as_ref().context("no Expected")?, - &deploy_id, - now_millis()?, - )?; - let store = deploy.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; - let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&authorization.canonical_brake_bytes)?; - assert!(SingleFileMessagePackBackingStore::new(&store).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - key: record.brake_id.clone(), - current: None, - }], - &[CultCacheEnvelope { - key: record.brake_id.clone(), - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - payload: authorization.canonical_brake_bytes.clone(), - stored_at: rfc3339_millis(now_millis()?)?, - schema_id: Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into()), - }], - )?); + set_stop(deploy.plan.as_ref().context("no plan")?, StopState::Clear, "test thaw")?; world.engine.resume_backoff.lock().unwrap().clear(); waiter_then_owner(&world, "", &owner_id)?; let yielded = resident_id(&world, &owner_id).context("F yielded")?.value; @@ -18690,8 +19841,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - next.deployment_authorization = - Some(deployment_authorization(world, &expected, id, now)?); + next.deployment_admitted_at_unix_millis = Some(now); } next.installed_release = Some(crate::drivers::InstalledReleaseObservation { sealed_release_id: release.sealed_release_id.clone(), @@ -19021,8 +20171,9 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide let sealing = resident_id(&world, deploy_id)?; world.engine.advance_transaction(&sealing)?; let after = resident_id(&world, deploy_id)?.value; + // The target has no stop, so Sealing passes it and is not held. assert!( - after.last_error.as_deref().is_some_and(|note| !note.contains(&continuity.transaction_id)), + after.last_error.as_deref().is_none_or(|note| !note.contains(&continuity.transaction_id)), "Sealing was held by the owner: {:?}", after.last_error ); @@ -19278,6 +20429,66 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(deploy) } + /// An engaged stop halts a deploy before it fences, the last pre-fencing + /// abort-able point; clearing lets the same transaction fence. + #[test] + fn an_engaged_stop_halts_entry_into_fencing_and_a_clear_lets_it_through() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let deploy = dead_incumbent_with_a_deployment_ready_to_fence(&workload, &world)?; + let plan = deploy.plan.as_ref().context("no plan")?; + set_stop(plan, StopState::Engaged, "freeze before fencing")?; + for _ in 0..3 { + world.engine.run_scheduler_tick()?; + } + let held = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(held.phase, DeploymentPhase::Warming); + assert!(held.pre_fencing_abort.is_none() && held.completion.is_none()); + assert!( + held.last_error.as_deref().is_some_and(|note| note.contains("deployment brake engaged by") + && note.contains("freeze before fencing")), + "{:?}", + held.last_error + ); + set_stop(plan, StopState::Clear, "thaw")?; + assert!(world.engine.run_scheduler_tick()?); + assert_eq!(resident_id(&world, &deploy.transaction_id)?.value.phase, DeploymentPhase::Fencing); + Ok(()) + } + + /// An Unreadable stop holds a Sealing deploy with a message that names the + /// remedy; it never admits. + #[test] + fn an_unreadable_stop_keeps_a_sealing_deploy_held_with_the_remedy() -> Result<()> { + let world = EngineFixture::new()?; + let deploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, None)?; + make_stop_first(&world, &deploy.transaction_id)?; + let sealing = resident_id(&world, &deploy.transaction_id)?; + let mut tail = sealing.value.clone(); + tail.deployment_admitted_at_unix_millis = None; + tail.deployment_authorization = None; + replace_transaction(&world.state_store, &sealing, &tail)?; + let store = tail.plan.as_ref().context("no plan")?.parsed_inputs()?.1.brakes.deployment_store; + std::fs::create_dir_all(store.parent().context("no parent")?)?; + for bytes in [&b"junk that is no store"[..], &b""[..]] { + std::fs::write(&store, bytes)?; + let held = resident_id(&world, &deploy.transaction_id)?; + rephase(&world, &deploy.transaction_id, DeploymentPhase::Sealing)?; + let held_now = resident_id(&world, &deploy.transaction_id)?; + world.engine.advance_transaction(&held_now)?; + let after = resident_id(&world, &deploy.transaction_id)?.value; + assert_eq!(after.phase, DeploymentPhase::Sealing, "{:?}", held.value.last_error); + assert!(!after.deployment_admitted()); + assert!( + after.last_error.as_deref().is_some_and(|note| note.contains("fails closed") + && note.contains("idunn-provision deployment-brake-clear")), + "{:?}", + after.last_error + ); + } + Ok(()) + } + #[test] fn a_continuity_minted_as_a_deployment_takes_the_target_is_resolved_next_pass() -> Result<()> { // (a) The deployment fences first in the tick, so supervision reads its @@ -20044,7 +21255,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// source resolves `recipe`. One scheduler tick admits its plan or refuses it. fn sealing_world(recipe: &str) -> Result { use crate::deployment_plan::tests::BINDING; - let mut world = EngineFixture::with_workload(Arc::new(StillWorkload))?; let (binding_head, binding_tail) = BINDING .split_once("[route]") .context("binding has no route table")?; @@ -20055,15 +21265,32 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .context("binding has no brakes table")? .1 ); + sealing_world_with(recipe, |_| binding, Arc::new(StillWorkload)) + } + + /// `sealing_world` over any binding (given the world's root, which a + /// binding that names paths needs) and any workload port. + fn sealing_world_with( + recipe: &str, + binding: impl FnOnce(&Path) -> String, + workload: Arc, + ) -> Result { + let mut world = EngineFixture::with_workload(workload)?; std::fs::create_dir_all(world.root.join("bindings"))?; - std::fs::write(world.root.join("bindings/service.toml"), binding)?; + std::fs::write(world.root.join("bindings/service.toml"), binding(&world.root))?; world.engine.source = Arc::new(FixedSource { recipe: recipe.to_owned(), }); + enqueue_deploy(&world, "up-service")?; + Ok(world) + } + + /// A Deploy command for target `service` and its transaction, still Sealing. + fn enqueue_deploy(world: &EngineFixture, command_id: &str) -> Result<()> { let now = now_millis()?; let command = DeploymentCommand { schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), - command_id: "up-service".into(), + command_id: command_id.into(), kind: CommandKind::Deploy, selector: "service".into(), requested_by: "test".into(), @@ -20090,7 +21317,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide ], )? ); - Ok(world) + Ok(()) } fn unrouted_recipe() -> String { @@ -20751,6 +21978,25 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + #[test] + fn status_says_how_overdue_a_route_challenge_is_and_claims_no_cause() -> Result<()> { + let now = 10_000_000_u64; + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.route_supervision = Some(RouteSupervisionState { + last_challenge_at_unix_millis: Some(now - 5_000), + next_challenge_at_unix_millis: Some(now - 4_000), + ..RouteSupervisionState::default() + }); + let lines = render_supervision("service", Some(&generation), None, now).join("\n"); + assert!(lines.contains("last-challenge-at 9995000 (5000 ms ago)"), "{lines}"); + assert!(lines.contains("next-challenge-at 9996000 (past due 4000 ms)"), "{lines}"); + assert!(!lines.contains("waits on"), "{lines}"); + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(now); + let lines = render_supervision("service", Some(&generation), None, now).join("\n"); + assert!(!lines.contains("past due"), "{lines}"); + Ok(()) + } + #[test] fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { let now = 10_000_000_u64; @@ -20850,6 +22096,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation: IdunnRuntimeActivationRecord, provider: cultnet_rs::ServiceIdentitySigner, activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, + /// A second key the transaction's activation record never named. + other_signer: cultnet_rs::IdunnRuntimeActivationSigner, + /// The instance the stable listener answered as before the reload. + incumbent_instance: Mutex>, + stable_answers: AtomicUsize, state: Mutex<&'static str>, capacity: AtomicU32, sequence: AtomicU64, @@ -20879,11 +22130,36 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// A first Odin candidate answering its Warming challenge: the /// detail names the challenge as `idunn-warming:`. FirstOdinWarming, + /// The stable listener's first n answers come from the incumbent + /// (its instance id, a key the record never named), then Honest. + IncumbentFirst(usize), + /// The candidate's own instance id, signed with the wrong key. + ForeignActivation, + /// Neither the candidate nor the incumbent, wrong key. + StrangerInstance, } impl RuntimeStub { fn answer(&self, message_id: &str) -> Result> { + self.answer_on(message_id, false) + } + + fn answer_on(&self, message_id: &str, stable: bool) -> Result> { let expected = &self.expected; + let own = self.activation.runtime_instance_id.clone(); + let (instance, signer) = match *self.reply.lock().unwrap() { + Reply::IncumbentFirst(n) + if stable && self.stable_answers.fetch_add(1, Ordering::SeqCst) < n => + { + let incumbent = self.incumbent_instance.lock().unwrap().clone(); + (incumbent.context("no incumbent instance set")?, &self.other_signer) + } + Reply::ForeignActivation if stable => (own, &self.other_signer), + Reply::StrangerInstance if stable => { + (sha256_id(b"stranger instance"), &self.other_signer) + } + _ => (own, &self.activation_signer), + }; let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), target: expected.target.clone(), @@ -20895,7 +22171,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide state_schema_generation: expected.state_schema_generation.clone(), state_contract_sha256: expected.state_contract_sha256.clone(), runtime_id: expected.runtime_id.clone(), - runtime_instance_id: self.activation.runtime_instance_id.clone(), + runtime_instance_id: instance, bound_endpoint: expected .route .as_ref() @@ -20922,7 +22198,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide observed_at_unix_millis: now_millis()?, signature_algorithm: "ed25519".into(), signature: Vec::new(), - activation_signer_identity_id: self.activation_signer.identity_id(), + activation_signer_identity_id: signer.identity_id(), activation_signature: Vec::new(), }; let proof = presence.canonical_proof_payload()?; @@ -20930,8 +22206,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .provider .sign::(&proof) .signature; - presence.activation_signature = - self.activation_signer.sign_presence_proof(&presence)?; + presence.activation_signature = signer.sign_presence_proof(&presence)?; Ok(rmp_serde::to_vec(&presence)?) } @@ -20939,7 +22214,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide *self.state.lock().unwrap() = state; } - fn serve_one(&self, mut stream: TcpStream) -> Result<()> { + fn serve_one(&self, mut stream: TcpStream, stable: bool) -> Result<()> { stream.set_read_timeout(Some(Duration::from_secs(3)))?; let mut request = Vec::new(); let mut chunk = [0_u8; 4096]; @@ -20986,7 +22261,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide record_key: self.expected.target.clone(), stored_at: rfc3339_millis(now_millis()?)?, payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, - payload: self.answer(&message_id)?, + payload: self.answer_on(&message_id, stable)?, source_runtime_id: None, source_agent_id: None, source_role: None, @@ -21028,7 +22303,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide hook(); } if !stub.hang_up.load(Ordering::SeqCst) { - let _ = stub.serve_one(stream); + let _ = stub.serve_one(stream, !candidate); } } Err(_) => std::thread::sleep(Duration::from_millis(5)), @@ -21140,6 +22415,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .replace( "/etc/nginx/idunn-stream-routes/service.conf", &config.display().to_string().replace('\\', "/"), + ) + .replace( + "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", + &world.root.join("service-lifecycle-brake.cc").display().to_string(), ); let (binding, recipe_source) = if stateful { std::fs::create_dir_all(world.root.join("lease"))?; @@ -21212,6 +22491,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide command_id: match kind { CommandKind::Deploy => "up-service", CommandKind::Continuity => "continuity-service", + CommandKind::Rollback => "rollback-service", } .into(), kind, @@ -21225,18 +22505,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide CommandKind::Continuity => { transaction.lifecycle_authorized_at_unix_millis = Some(now); } + CommandKind::Rollback => unreachable!("service fixtures have no rollback"), CommandKind::Deploy => { transaction.frozen_source = Some(FrozenSourceReceipt { transaction_id: transaction.transaction_id.clone(), plan_id: plan.plan_id.clone(), snapshot_sha256: sha256_id(b"frozen source snapshot"), }); - transaction.deployment_authorization = Some(deployment_authorization( - &world, - &expected, - &transaction.transaction_id, - now, - )?); + transaction.deployment_admitted_at_unix_millis = Some(now); } } let launch = IdunnRuntimeActivationLaunch::issue( @@ -21247,6 +22523,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide )?; let mut seed = Vec::new(); let activation = launch.write_credential(&mut seed)?; + let mut other_seed = Vec::new(); + IdunnRuntimeActivationLaunch::issue( + &expected, + sha256_id(b"unrecorded activation"), + now, + &world.engine.idunn_signer, + )? + .write_credential(&mut other_seed)?; let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; let mut workload = recorded.workload.clone().context("no recorded workload")?; match &mut workload { @@ -21278,6 +22562,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( &seed[..], )?, + other_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( + &other_seed[..], + )?, + incumbent_instance: Mutex::new(None), + stable_answers: AtomicUsize::new(0), state: Mutex::new("warming"), capacity: AtomicU32::new(minimum_capacity), sequence: AtomicU64::new(0), @@ -23257,30 +24546,325 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[cfg(unix)] #[test] - fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { + fn a_dead_unit_is_restarted_without_a_route_challenge() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.admit()?; routed.workload.kill(); routed.stub.hang_up.store(true, Ordering::SeqCst); routed.drift()?; + let before = admitted(&routed)?.route_supervision; + let reloads = routed.reloads(); + let survivals = meters_of(&routed.world, "service")?.route_survivals; - // Tick 1: the repair charges the target's meters and the proof - // fails. Both are writes, so they end the pass: nothing else is - // decided from the snapshot they made stale. - assert!(routed.tick()?); - let state = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(state.consecutive_failures, 1); - assert!(state.degraded_since_unix_millis.is_some()); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let meters = meters_of(&routed.world, "service")?; - assert_eq!(meters.route_survivals.len(), 2); - assert!(meters.route_actuations.is_empty()); - - // Tick 2: the route waits, and the dead unit is restarted from a - // fresh read, counted in the same write that mints it. + // Tick 1: restart, not route repair, is the response to a dead + // incarnation. The continuity is minted and no route survival is + // charged for a proof nobody could have passed. assert!(routed.tick()?); assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); + assert_eq!(meters_of(&routed.world, "service")?.route_survivals, survivals); + assert_eq!(admitted(&routed)?.route_supervision, before); + assert_eq!(routed.reloads(), reloads, "the dead unit's route was repaired"); + Ok(()) + } + + /// A target with a process write lease, admitted and running. + #[cfg(unix)] + fn admitted_stateful_world() -> Result { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let granted_lease = routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(granted_lease); + routed.step()?; + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + routed.promote()?; + routed.drive_until(|transaction| transaction.is_terminal())?; + assert!(routed.world.engine.retire_one_terminal_transaction()?); + Ok(routed) + } + + /// Engage the target's lifecycle brake: the dead path may not mint. + fn engage_lifecycle_brake(routed: &RoutedWorld) -> Result<()> { + engage_lifecycle_brake_in(&routed.world) + } + #[cfg(unix)] + #[test] + fn a_parked_routed_target_is_not_route_supervised() -> Result<()> { + let routed = admitted_stateful_world()?; + let generation = admitted(&routed)?; + let lease = generation.leasing.lease().context("no lease")?.clone(); + CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ) + .revoke_exact(Some(&lease))?; + routed.workload.kill(); + engage_lifecycle_brake(&routed)?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + let programs = routed.world.route_stubs.count(""); + + for _ in 0..100 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "a parked target's route was reloaded"); + assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let reports = routed.world.engine.fault_reports.lock().unwrap(); + assert!( + !reports.keys().any(|key| key.starts_with("route-supervision:") || key.starts_with("route:")), + "{:?}", + reports.keys().collect::>() + ); + Ok(()) + } + + fn fault_keys(routed: &RoutedWorld) -> Vec { + routed.world.engine.fault_reports.lock().unwrap().keys().cloned().collect() + } + + /// Soul's probe: a route record that cannot be written leaves the + /// challenge past due on a running incarnation. Status shows the age + /// and does not name a cause it cannot see. + #[cfg(unix)] + #[test] + fn status_of_a_past_due_route_on_a_running_incarnation_claims_no_wait() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + assert!(routed.tick()?); + assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); + routed.stub.hang_up.store(false, Ordering::SeqCst); + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); + })?; + let fragment = routed.fragment_path(); + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..20 { + routed.tick()?; + } + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + assert!(fault_keys(&routed).contains(&"route-supervision:service".to_owned())); + let generation = admitted(&routed)?; + let lines = render_supervision("service", Some(&generation), None, now_millis()?).join("\n"); + assert!(lines.contains("(past due "), "{lines}"); + assert!(!lines.contains("waits on"), "{lines}"); + Ok(()) + } + + /// A target whose lease store is empty while its process runs is a + /// dead-path case: the route is not challenged for it. + #[cfg(unix)] + #[test] + fn a_running_target_with_an_empty_lease_is_not_route_challenged() -> Result<()> { + let routed = admitted_stateful_world()?; + let lease = admitted(&routed)?.leasing.lease().context("no lease")?.clone(); + CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ) + .revoke_exact(Some(&lease))?; + engage_lifecycle_brake(&routed)?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + let programs = routed.world.route_stubs.count(""); + for _ in 0..50 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "an empty-lease target's route was reloaded"); + assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + assert!( + !fault_keys(&routed).iter().any(|key| key.starts_with("route")), + "{:?}", + fault_keys(&routed) + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_host_that_answers_with_a_death_ends_the_silence_fault() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + engage_lifecycle_brake(&routed)?; + routed.workload.silence(true); + for _ in 0..5 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "observe:service"), 1); + routed.workload.silence(false); + routed.workload.kill(); + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"observe:service".to_owned())); + Ok(()) + } + + /// Replace the topology store with a directory: every read of it fails. + #[cfg(unix)] + fn break_topology(routed: &RoutedWorld) -> Result<()> { + let store = routed.world.root.join("topology.cc"); + if store.exists() { + std::fs::rename(&store, routed.world.root.join("topology.aside"))?; + } + std::fs::create_dir(&store)?; + Ok(()) + } + + #[cfg(unix)] + fn mend_topology(routed: &RoutedWorld) -> Result<()> { + let store = routed.world.root.join("topology.cc"); + std::fs::remove_dir(&store)?; + let aside = routed.world.root.join("topology.aside"); + if aside.exists() { + std::fs::rename(aside, &store)?; + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_stale_incarnation_and_a_projection_repair_that_fail_are_each_said_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let keys = ["stale-incarnation:service", "projection-repair:service"]; + for round in 1..=2 { + break_topology(&routed)?; + for _ in 0..30 { + routed.tick()?; + } + for key in keys { + assert_eq!(printed_in(&routed, key), round, "{key}"); + assert!(fault_keys(&routed).contains(&key.to_owned()), "{key}"); + } + mend_topology(&routed)?; + routed.tick()?; + for key in keys { + assert!(!fault_keys(&routed).contains(&key.to_owned()), "{key}"); + } + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_refused_demotion_is_said_once_and_cleared_when_it_succeeds() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.workload.kill(); + break_topology(&routed)?; + // Each attempt is let through by forgetting the deferral it wrote. + for _ in 0..6 { + set_meters(&routed.world, &TargetSupervision::new("service"))?; + routed.tick()?; + } + assert_eq!(printed_in(&routed, "demotion:service"), 1); + assert!(fault_keys(&routed).contains(&"demotion:service".to_owned())); + mend_topology(&routed)?; + set_meters(&routed.world, &TargetSupervision::new("service"))?; + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"demotion:service".to_owned())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unsafe_write_lease_store_is_said_once_and_cleared_when_it_is_exact() -> Result<()> { + let routed = admitted_stateful_world()?; + let store = routed.world.root.join("lease/process-write-lease.cc"); + let exact = std::fs::read(&store)?; + for round in 1..=2 { + std::fs::write(&store, b"not a lease store")?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "write-lease:service"), round); + assert!(fault_keys(&routed).contains(&"write-lease:service".to_owned())); + std::fs::write(&store, &exact)?; + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"write-lease:service".to_owned())); + } + Ok(()) + } + + fn printed_in(routed: &RoutedWorld, key: &str) -> usize { + routed.world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() + } + + #[cfg(unix)] + #[test] + fn a_running_route_whose_fragment_cannot_be_read_is_reported_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let key = "route-supervision:service"; + let fragment = routed.fragment_path(); + let bytes = std::fs::read(&fragment)?; + // The fragment is a directory: reading it fails, and nothing is + // written, so every tick meets the same error. + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, key), 1); + assert!(routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); + + // The fragment is back: the fault ends and the key clears. + std::fs::remove_dir(&fragment)?; + std::fs::write(&fragment, &bytes)?; + routed.tick()?; + assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); + + // A recurrence is said again, once. + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, key), 2); + Ok(()) + } + + /// Skipping route and topology on a tick the host could not be asked is + /// the steward's ruling (no activation can be bound without an + /// observation; no routed target runs on a host actuator today). + #[cfg(unix)] + #[test] + fn a_host_that_cannot_be_asked_skips_route_supervision_and_is_said_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + routed.workload.silence(true); + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "route supervision ran without an observation"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert_eq!(printed_in(&routed, "observe:service"), 1); + + routed.workload.silence(false); + routed.tick()?; + assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key("observe:service")); + routed.workload.silence(true); + for _ in 0..5 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "observe:service"), 2); Ok(()) } @@ -23445,9 +25029,39 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.run_to_routing()?; let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; routed.stub.hang_up.store(true, Ordering::SeqCst); - for _ in 0..3 { + // The Routing tick itself reports why the proof failed, once per + // distinct reason, and recovery clears it. + let proof_key = route_proof_report_key(&routed.transaction()?.transaction_id); + let said = |routed: &RoutedWorld| -> Option { + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .get(&proof_key) + .and_then(|report| report.last.lock().unwrap().clone()) + }; + assert!(said(&routed).is_none(), "a proof report existed before any failure"); + routed.resume()?; + let first = said(&routed).context("the failing Routing tick reported no proof failure")?; + assert!(first.contains("route proof") && first.contains("silent, nothing answered"), "{first}"); + // The hung-up stub fails the same way each attempt, but the OS + // words the tail either as an early EOF or a broken pipe, so a + // retry may honestly report a second distinct reason. What may + // not change is everything up to that tail, and a repeat must not + // mint a new line each attempt. + let head = |text: &str| text.split("answered:").next().unwrap().to_owned(); + let mut seen = vec![first.clone()]; + for _ in 0..2 { routed.resume()?; + let again = said(&routed).context("a retry cleared the proof report")?; + assert_eq!(head(&again), head(&first), "a retry reworded the report"); + if !seen.contains(&again) { + seen.push(again); + } } + assert!(seen.len() <= 2, "the same failure was reported afresh on every retry: {seen:#?}"); // Before the deadline each attempt installs and rolls back, and no // resume moves the deadline. assert_eq!(routed.reloads(), 6); @@ -23490,6 +25104,200 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(records.len(), 1); assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(said(&routed).is_none(), "recovery left the proof report armed"); + Ok(()) + } + + // ----------------------------------------------------------------- + // The reload handoff: nginx's old workers may answer the stable + // listener for a moment after `systemctl reload` returns. + // ----------------------------------------------------------------- + + /// A deploy world at Routing whose preflight found an incumbent route + /// (another instance, another membership), so the install changes it. + #[cfg(unix)] + fn handoff_world(reply: Reply) -> Result<(RoutedWorld, String)> { + handoff_world_with(reply, false) + } + + /// `same_membership`: the incumbent's membership equals the + /// candidate's, so the install changes no route. + #[cfg(unix)] + fn handoff_world_with(reply: Reply, same_membership: bool) -> Result<(RoutedWorld, String)> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let incumbent = sha256_id(b"incumbent instance"); + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + let mut on_disk = b"incumbent route\n".to_vec(); + if same_membership { + on_disk = NginxRouteDriver::new( + next.plan.as_ref().unwrap().parsed_inputs()?.1.route.context("no route binding")?, + ) + .render(next.expected.as_ref().context("no Expected")?)?; + } + let preflight = next.route_preflight.as_mut().context("no preflight")?; + preflight.incumbent_runtime_instance_id = Some(incumbent.clone()); + preflight.incumbent_membership_sha256 = Some(sha256_id(&on_disk)); + preflight.incumbent_configuration = Some(on_disk.clone()); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + std::fs::write(routed.fragment_path(), &on_disk)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + *routed.stub.incumbent_instance.lock().unwrap() = Some(incumbent.clone()); + *routed.stub.reply.lock().unwrap() = reply; + Ok((routed, incumbent)) + } + + fn forwards(routed: &RoutedWorld) -> Result { + Ok(meters_of(&routed.world, "service")?.route_actuations.len()) + } + + fn proof_report(routed: &RoutedWorld) -> Result> { + let key = route_proof_report_key(&routed.transaction()?.transaction_id); + Ok(routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .get(&key) + .and_then(|report| report.last.lock().unwrap().clone())) + } + + /// Holds a virtual handoff clock on this thread: the wait is measured + /// in the loop's own sleeps, so no host load can move it. + struct VirtualHandoffClock; + + impl VirtualHandoffClock { + fn hold() -> Self { + HANDOFF_CLOCK.with(|c| *c.borrow_mut() = Some((Duration::ZERO, Vec::new(), 0))); + Self + } + + /// The pauses the loop took so far. + fn sleeps(&self) -> Vec { + HANDOFF_CLOCK.with(|c| c.borrow().as_ref().unwrap().1.clone()) + } + } + + impl Drop for VirtualHandoffClock { + fn drop(&mut self) { + HANDOFF_CLOCK.with(|c| *c.borrow_mut() = None); + } + } + + #[cfg(unix)] + #[test] + fn routing_waits_out_the_incumbent_answering_after_reload() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world(Reply::IncumbentFirst(3))?; + let before = routed.reloads(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + routed.promote()?; + assert_eq!(routed.reloads() - before, 1, "an install and no rollback"); + assert_eq!(forwards(&routed)?, 1); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 4); + assert_eq!(clock.sleeps(), vec![ROUTE_RELOAD_HANDOFF_POLL; 3]); + assert!(proof_report(&routed)?.is_none(), "a route proof report remains"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_once_when_the_incumbent_never_lets_go() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, incumbent) = handoff_world(Reply::IncumbentFirst(usize::MAX))?; + let before = routed.reloads(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + let error = routed.step().unwrap_err(); + // The window is exactly ROUTE_RELOAD_HANDOFF of polls at exactly + // the poll cadence: one challenge before each pause and one after + // the last. + let sleeps = clock.sleeps(); + let polls = (ROUTE_RELOAD_HANDOFF.as_millis() / ROUTE_RELOAD_HANDOFF_POLL.as_millis()) as usize; + assert_eq!(sleeps, vec![ROUTE_RELOAD_HANDOFF_POLL; polls]); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, polls + 1); + assert_eq!(routed.reloads() - before, 2, "install and one rollback"); + assert_eq!(forwards(&routed)?, 1); + let said = proof_report(&routed)?.context("no route proof report")?; + let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; + let signer = routed.stub.other_signer.identity_id(); + for text in [format!("{error:#}"), said] { + assert!(text.contains(&format!("runtime instance {incumbent}")), "{text}"); + assert!(text.contains(&format!("activation signer {signer}")), "{text}"); + assert!(text.contains(&format!("the candidate is {candidate}")), "{text}"); + } + routed.step().unwrap_err(); + assert_eq!(forwards(&routed)?, 2, "the second attempt charges one more Forward"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_at_once_when_the_candidate_answers_with_a_foreign_activation() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world(Reply::ForeignActivation)?; + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + let error = routed.step().unwrap_err(); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); + let text = format!("{error:#}"); + let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; + assert!(text.contains("runtime activation proof verification failed"), "{text}"); + assert!(text.contains(&format!("runtime instance {candidate}")), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_at_once_for_a_stranger_instance() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world(Reply::StrangerInstance)?; + let before = routed.reloads(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + routed.step().unwrap_err(); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); + assert_eq!(routed.reloads() - before, 2); + Ok(()) + } + + /// An incumbent whose membership equals the candidate's has no route + /// to hand off, so its answer is a plain failure, never waited on. + #[cfg(unix)] + #[test] + fn routing_does_not_wait_when_the_route_is_unchanged() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world_with(Reply::IncumbentFirst(1), true)?; + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + routed.step().unwrap_err(); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); + Ok(()) + } + + #[test] + fn route_proof_answer_names_ids_only() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let mut presence = rmp_serde::from_slice::( + &routed.stub.answer("probe")?, + )?; + presence.runtime_instance_id = format!("a\u{7}{}", "é".repeat(5_000)); + presence.activation_signer_identity_id = "b".repeat(10_240); + presence.signature = vec![0xAB; 64]; + presence.activation_signature = vec![0xCD; 64]; + let (instance, signer) = presence_answer_ids(&rmp_serde::to_vec(&presence)?) + .context("a decodable presence gave no ids")?; + let text = RouteProofAnswer { + answered_instance: route_proof_label(&instance), + answered_activation_signer: route_proof_label(&signer), + candidate_instance: route_proof_label(&"c".repeat(10_240)), + } + .to_string(); + assert!(text.len() <= 400, "{} bytes", text.len()); + assert!(!text.contains('\u{7}')); + assert!(!text.contains("abab") && !text.contains("cdcd")); + assert!(presence_answer_ids(b"not msgpack presence").is_none()); Ok(()) } @@ -24098,6 +25906,16 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .post_fencing_abort .context("the passed deadline did not end a deployment")?; assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!( + abort.error.contains("its last attempt failed"), + "the deadline abort dropped the proof's failure: {}", + abort.error + ); + assert!( + abort.error.contains("stable CultNet route stopped before its headers ended"), + "the deadline abort dropped the proof's own failure text: {}", + abort.error + ); assert!( routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), "the boot lift took a deployment's abort" @@ -24657,4 +26475,1595 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } } + + // BEGIN runner-failure-evidence tests + // A failed runner step is named, its output tail is kept in + // runner-evidence.cc, and no output reaches an error, control.cc or the + // journal line. The runner is the real DockerRunnerDriver driving a fake + // docker program, through the engine's Sealing path. + + const RUNNER_CANARY: &str = "CANARY-7c41e0d9"; + + /// A source that freezes a fixed tree, so a deployment reaches the runner. + struct TreeSource { + recipe: String, + root: PathBuf, + } + + impl SourcePort for TreeSource { + fn resolve( + &self, + _: &OperatorBinding, + _: &str, + _: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, transaction_id: &str, plan: &CompiledDeploymentPlan) -> Result { + Ok(FrozenSourceReceipt { + transaction_id: transaction_id.to_owned(), + plan_id: plan.plan_id.clone(), + snapshot_sha256: format!("sha256-{}", "a".repeat(64)), + }) + } + + fn observe_frozen( + &self, + plan: &CompiledDeploymentPlan, + receipt: &FrozenSourceReceipt, + ) -> Result { + Ok(crate::drivers::FrozenSource::fixture( + receipt.clone(), + plan.source.clone(), + plan.recipe_blob.clone(), + self.root.clone(), + )) + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// The unrouted recipe with two steps ahead of its build, and no external + /// input: the second step, `test-daemon`, is the one the fake docker fails. + fn two_step_recipe() -> String { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let (head, rest) = recipe + .split_once("[[external_inputs]]") + .expect("the recipe declares an external input"); + let (_, tail) = rest.split_once("[[artifacts]]").expect("the recipe declares artifacts"); + let head = head.replacen( + "[[steps]]\nid = \"build\"", + "[[steps]]\nid = \"install\"\nphase = \"prepare\"\nrunner = \"rust\"\nargv = [\"cargo\", \"fetch\"]\n\n\ + [[steps]]\nid = \"test-daemon\"\nphase = \"test\"\nrunner = \"rust\"\nargv = [\"cargo\", \"test\"]\n\n\ + [[steps]]\nid = \"build\"", + 1, + ); + format!("{head}[[artifacts]]{tail}") + } + + /// A Sealing world whose runner is the real driver over a fake docker: the + /// first call succeeds, the second prints the canary and an escape + /// sequence on both streams and exits 3. + #[cfg(unix)] + fn failing_runner_world() -> Result<(EngineFixture, TempDir)> { + use std::os::unix::fs::PermissionsExt; + let tools = TempDir::new()?; + let docker = tools.path().join("docker"); + std::fs::write( + &docker, + format!( + "#!/bin/sh\nn=$(cat \"$0.count\" 2>/dev/null || echo 0)\nn=$((n+1))\necho $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + printf 'out {RUNNER_CANARY} \\033[2J\\n'\nprintf 'err {RUNNER_CANARY}\\n' >&2\nexit 3\n" + ), + )?; + std::fs::set_permissions(&docker, std::fs::Permissions::from_mode(0o755))?; + let recipe = two_step_recipe(); + let mut world = sealing_world(&recipe)?; + let tree = tools.path().join("tree"); + std::fs::create_dir_all(&tree)?; + std::fs::write(tree.join("README"), b"source")?; + world.engine.source = Arc::new(TreeSource { recipe, root: tree }); + world.engine.docker_runner = DockerRunnerDriver { docker_program: docker }; + Ok((world, tools)) + } + + /// Scheduler ticks until the deployment ends (or the budget is spent). + fn tick_until_ended(world: &EngineFixture) -> Result { + for _ in 0..24 { + world.engine.resume_backoff.lock().unwrap().clear(); + let _ = world.engine.run_scheduler_tick(); + if latest(world)?.completion.is_some() { + break; + } + } + latest(world) + } + + fn printed_status(world: &EngineFixture) -> Result { + let mut printed = Vec::new(); + status(&mut printed, &world.state_store, &world.engine.options.incident_store, Some("up-service"))?; + Ok(String::from_utf8(printed)?) + } + + fn canary_failure(stdout: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject: crate::drivers::RunnerSubject::Step("test-daemon".into()), + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stdout.to_vec(), + stderr_total_bytes: stdout.len() as u64, + } + } + + #[cfg(unix)] + #[test] + fn a_failed_runner_step_is_named_kept_and_never_echoed() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the deployment did not fail before fencing: {:?} {:?}", ended.phase, ended.last_error); + }; + // Named: the second step, on its runner, with the exit status. + assert!( + error.contains("runner step test-daemon on runner rust exited with exit status 3"), + "{error}" + ); + // Never echoed: no error text, no control.cc byte, carries the output. + let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; + for text in [error.as_str(), abort.error.as_str(), &format!("{ended:?}")] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached a record: {text}"); + } + // The terminal transaction has moved to history.cc: both stores. + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + let bytes = std::fs::read(&store)?; + assert!(!bytes.is_empty(), "{} is empty", store.display()); + assert!( + !bytes.windows(RUNNER_CANARY.len()).any(|window| window == RUNNER_CANARY.as_bytes()), + "runner output reached {}", + store.display() + ); + } + // Kept: in the private store, tails of both streams, and on status. + let evidence = runner_evidence_store_path(&world.state_store); + assert_eq!(std::fs::metadata(&evidence)?.permissions().mode() & 0o777, 0o600); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("no evidence for the failed transaction")?; + assert_eq!(record.subject_id, "test-daemon"); + assert_eq!(record.exit_code, Some(3)); + assert!(String::from_utf8_lossy(&record.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&record.stderr_tail).contains(RUNNER_CANARY)); + let printed = printed_status(&world)?; + assert!(printed.contains("failed step test-daemon"), "{printed}"); + assert!(printed.contains(RUNNER_CANARY), "{printed}"); + assert!(!printed.contains('\u{1b}'), "status printed a raw escape byte"); + Ok(()) + } + + #[test] + fn the_journal_line_names_the_store_and_never_the_output() -> Result<()> { + let world = EngineFixture::new()?; + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let line = world + .engine + .keep_runner_failure(&transaction, &anyhow::Error::new(failure.clone()).context("sealing")) + .context("a runner failure produced no journal line")?; + assert!(!line.contains(RUNNER_CANARY), "{line}"); + for needle in [ + transaction.transaction_id.as_str(), + "up-ghostlight", + "ghostlight", + "runner step test-daemon on runner rust exited with exit status 3", + "runner-evidence.cc", + "idunn status --command up-ghostlight", + ] { + assert!(line.contains(needle), "{needle} missing from {line}"); + } + // Not a runner failure: nothing is kept and nothing is said. + assert!( + world + .engine + .keep_runner_failure(&transaction, &anyhow::anyhow!("a different failure")) + .is_none() + ); + // The line a failed write produces says so, with no output. + let unwritable = runner_failure_journal_line( + &transaction, + &failure, + Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"), + &Err(anyhow::anyhow!("disk full")), + ); + assert!(unwritable.contains("evidence was not kept: disk full"), "{unwritable}"); + assert!(!unwritable.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_evidence_write_does_not_stop_the_abort() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + // A directory where the store should be: every write to it fails. + std::fs::create_dir(runner_evidence_store_path(&world.state_store))?; + let ended = tick_until_ended(&world)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the unwritable store stopped the abort: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b""))) + .context("no journal line")?; + assert!(line.contains("evidence was not kept"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_undecodable_evidence_store_does_not_stop_the_next_failure_keeping_its_evidence() -> Result<()> { + let damage = format!("\u{c1} not a store {RUNNER_CANARY}-in-the-damage"); + let (world, _tools) = failing_runner_world()?; + let evidence = runner_evidence_store_path(&world.state_store); + std::fs::write(&evidence, damage.as_bytes())?; + let ended = tick_until_ended(&world)?; + // The transaction's own outcome is the runner failure, not the store. + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the damaged store changed the outcome: {:?} {:?}", ended.phase, ended.last_error); + }; + assert!(error.contains("runner step test-daemon on runner rust"), "{error}"); + assert!(!error.contains("damage"), "{error}"); + // The damage is kept aside; the failure's evidence is in a fresh store. + assert_eq!(std::fs::read(unreadable_store_path(&evidence))?, damage.as_bytes()); + let record = runner_evidence::for_transaction(&evidence, &ended.transaction_id)? + .context("the failure kept no evidence beside a damaged store")?; + assert_eq!(record.subject_id, "test-daemon"); + assert!(printed_status(&world)?.contains("failed step test-daemon")); + // The journal says it, naming the file and none of its content. + std::fs::write(&evidence, damage.as_bytes())?; + let line = world + .engine + .keep_runner_failure(&ended, &anyhow::Error::new(canary_failure(b"x"))) + .context("no journal line")?; + assert!(line.contains("an undecodable evidence store was set aside as"), "{line}"); + assert!(line.contains("runner-evidence.unreadable.cc"), "{line}"); + assert!(!line.contains("damage") && !line.contains("not a store"), "{line}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_evidence_store_costs_only_its_lines() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + std::fs::write(runner_evidence_store_path(&world.state_store), b"\xc1not a store")?; + let printed = printed_status(&world)?; + assert!(printed.contains(&format!("transaction {}", ended.transaction_id)), "{printed}"); + assert!(printed.contains("target service phase"), "{printed}"); + assert!(printed.contains("pre-fencing abort"), "{printed}"); + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + Ok(()) + } + + /// Everything status prints from the evidence store goes through one + /// allowlist, so a hostile build log reaches the terminal as visible + /// escapes whatever its class. + #[cfg(unix)] + #[test] + fn status_prints_hostile_runner_output_as_visible_escapes_only() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + let ended = tick_until_ended(&world)?; + let hostile: Vec<(&str, &str)> = vec![ + ("\u{1b}[2J", "\\u{1b}[2J"), + ("\u{7}", "\\u{7}"), + ("\r", "\\u{d}"), + ("\u{202e}", "\\u{202e}"), + ("\u{2066}", "\\u{2066}"), + ("\u{85}", "\\u{85}"), + ("\u{e0041}", "\\u{e0041}"), + ("\u{200b}", "\\u{200b}"), + ("\u{feff}", "\\u{feff}"), + ]; + let mut tail: Vec = Vec::new(); + for &(raw, _) in &hostile { + tail.extend_from_slice(b"a"); + tail.extend_from_slice(raw.as_bytes()); + tail.extend_from_slice(b"b\n"); + } + tail.extend_from_slice(b"bad \xff\xc1 bytes\n"); + runner_evidence::record( + &runner_evidence_store_path(&world.state_store), + &RunnerFailureRecord::from_failure( + &ended.transaction_id, + &ended.command_id, + &ended.target, + &canary_failure(&tail), + 9, + ), + )?; + let printed = printed_status(&world)?; + for &(raw, escaped) in &hostile { + assert!(!printed.contains(raw), "{escaped} reached the terminal raw"); + assert!(printed.contains(&format!("a{escaped}b")), "{escaped} missing: {printed}"); + } + assert!(printed.contains("bad \\xff\\xc1 bytes"), "{printed}"); + let section = printed.find(" failed step").context("no evidence section")?; + assert!(printed[section..].is_ascii(), "non-ASCII reached the terminal: {printed}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn status_prints_the_unreadable_store_line_escaped() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + let evidence = runner_evidence_store_path(&world.state_store); + // The decode error of this store quotes the stored subject kind. + let mut bytes = std::fs::read(&evidence)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker in the evidence store")?; + bytes[at..at + 4].copy_from_slice(b"\x1b[2J"); + std::fs::write(&evidence, bytes)?; + let printed = printed_status(&world)?; + assert_eq!(printed.matches("runner evidence unreadable").count(), 1, "{printed}"); + assert!(!printed.contains('\u{1b}'), "an escape byte reached the terminal"); + assert!(printed.contains("\\u{1b}[2J"), "{printed}"); + Ok(()) + } + + #[test] + fn the_journal_line_names_a_displaced_set_aside_store_by_time_only() -> Result<()> { + let (transaction, _) = terminal_transaction_with_command("ghostlight")?; + let failure = canary_failure(RUNNER_CANARY.as_bytes()); + let path = Path::new("/var/lib/gamecult/idunn/runner-evidence.cc"); + let first = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: None }), + ); + assert!(first.contains("was set aside as"), "{first}"); + assert!(!first.contains("replacing"), "{first}"); + let second = runner_failure_journal_line( + &transaction, + &failure, + path, + &Ok(Kept::SetAside { replaced: Some(1234) }), + ); + assert!( + second.contains("replacing an earlier set-aside store last written at unix millis 1234"), + "{second}" + ); + assert!(!second.contains(RUNNER_CANARY)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn control_store_types_are_unchanged_by_a_runner_failure() -> Result<()> { + let (world, _tools) = failing_runner_world()?; + tick_until_ended(&world)?; + // The store a rolled-back Idunn reads: it refuses any foreign type. + ControlSnapshot::read(&world.state_store)?; + let mut transactions = 0; + for store in [world.state_store.clone(), history_store_path(&world.state_store)] { + for envelope in SingleFileMessagePackBackingStore::new(&store).pull_all_read_only_snapshot()? { + assert!(!envelope.r#type.contains("runner_failure"), "{}", envelope.r#type); + if envelope.r#type == DeploymentTransaction::TYPE { + transactions += 1; + assert_eq!(envelope.schema_id.as_deref(), Some("idunn.deployment_transaction.v6")); + } + } + } + assert_eq!(transactions, 1, "the failed transaction is in neither store"); + Ok(()) + } + + /// The records control.cc holds re-encode byte for byte at v5. A runner + /// failure adds no field, no type and no schema to the control store. + #[test] + fn control_records_encode_exactly_as_pinned() -> Result<()> { + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, 100)?; + transaction.transaction_id = "tx-pinned".into(); + transaction.last_error = Some("runner step test-daemon on runner rust exited with exit status 3".into()); + transaction.pre_fencing_abort = Some(PreFencingAbort { + error: "runner step test-daemon on runner rust exited with exit status 3".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02x}")).collect::(); + let command_hex = hex(&command_envelope(&command, 100)?.payload); + let transaction_hex = hex(&transaction_envelope(&transaction, 100)?.payload); + assert_eq!(command_hex, PINNED_COMMAND_HEX, "command bytes changed"); + assert_eq!(transaction_hex, PINNED_TRANSACTION_HEX, "transaction bytes changed"); + Ok(()) + } + + const PINNED_COMMAND_HEX: &str = "96bb6964756e6e2e6465706c6f796d656e745f636f6d6d616e642e7632aa75702d73657276696365a66465706c6f79a773657276696365a47465737464"; + /// The v4 bytes of the same record, captured at base 804ed75: a legacy + /// fixture. v5 added key 37 (`flip`) and v6 key 38, so v4 is two elements shorter. + const PINNED_TRANSACTION_HEX_V4: &str = "dc0025bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7634a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0"; + const PINNED_TRANSACTION_HEX_V5: &str = "dc0026bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7635a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0c0"; + /// The v6 bytes: v5 with key 38 (`deployment_admitted_at_unix_millis`) appended, so + /// one element longer and naming v6. + const PINNED_TRANSACTION_HEX: &str = "dc0027bf6964756e6e2e6465706c6f796d656e745f7472616e73616374696f6e2e7636a974782d70696e6e6564aa75702d73657276696365a66465706c6f79a77365727669636500a77365616c696e676464c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c000d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033c094d94072756e6e6572207374657020746573742d6461656d6f6e206f6e2072756e6e657220727573742065786974656420776974682065786974207374617475732033a7736b6970706564a7736b6970706564a8636f6d706c657465c0c0c0c0c0c0"; + + fn pinned_v4_envelope() -> Result { + let payload = (0..PINNED_TRANSACTION_HEX_V4.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&PINNED_TRANSACTION_HEX_V4[i..i + 2], 16)) + .collect::, _>>()?; + Ok(CultCacheEnvelope { + key: "tx-pinned".into(), + r#type: DeploymentTransaction::TYPE.into(), + payload, + stored_at: rfc3339_millis(100)?, + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA_V4.into()), + }) + } + + #[test] + fn a_pinned_v4_transaction_lifts_and_reencodes_as_the_pinned_v6() -> Result<()> { + let lifted = read_transaction_record(&pinned_v4_envelope()?)?; + assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + assert_eq!(lifted.flip, None); + assert_eq!(lifted.deployment_admitted_at_unix_millis, None); + let bytes = encode_transaction(&lifted, 100)?.payload; + let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + assert_eq!(hex, PINNED_TRANSACTION_HEX); + // Read as v6 the v4 bytes are refused, never silently accepted. + let mut mislabeled = pinned_v4_envelope()?; + mislabeled.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()); + assert!(read_transaction_record(&mislabeled).is_err()); + Ok(()) + } + + /// Always a copy in a temp dir: never a live control.cc. + #[test] + fn a_v4_control_store_boots_into_v6_and_a_v6_store_is_left_alone() -> Result<()> { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + let v4 = pinned_v4_envelope()?; + assert!(store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: v4.r#type.clone(), + key: v4.key.clone(), + current: None, + }], + std::slice::from_ref(&v4), + )?); + assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); + let after = store.pull_all_read_only_snapshot()?; + assert_eq!(after.len(), 1); + assert_eq!(after[0].schema_id.as_deref(), Some(DEPLOYMENT_TRANSACTION_SCHEMA)); + assert_eq!(after[0].stored_at, v4.stored_at); + let migrated: String = after[0].payload.iter().map(|byte| format!("{byte:02x}")).collect(); + assert_eq!(migrated, PINNED_TRANSACTION_HEX); + // Negative: an already-v6 store is untouched, byte for byte. + let file_before = std::fs::read(&path)?; + assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); + assert_eq!(std::fs::read(&path)?, file_before); + read_transaction_record(&after[0])?; + Ok(()) + } + fn pinned_v5_envelope() -> Result { + let payload = (0..PINNED_TRANSACTION_HEX_V5.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&PINNED_TRANSACTION_HEX_V5[i..i + 2], 16)) + .collect::, _>>()?; + Ok(CultCacheEnvelope { + key: "tx-pinned".into(), + r#type: DeploymentTransaction::TYPE.into(), + payload, + stored_at: rfc3339_millis(100)?, + schema_id: Some(DEPLOYMENT_TRANSACTION_SCHEMA_V5.into()), + }) + } + + #[test] + fn a_pinned_v5_transaction_lifts_and_reencodes_as_the_pinned_v6() -> Result<()> { + let lifted = read_transaction_record(&pinned_v5_envelope()?)?; + assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + assert_eq!(lifted.deployment_admitted_at_unix_millis, None); + let bytes = encode_transaction(&lifted, 100)?.payload; + let hex: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + assert_eq!(hex, PINNED_TRANSACTION_HEX); + // Read as v6 the v5 bytes are refused, never silently accepted. + let mut mislabeled = pinned_v5_envelope()?; + mislabeled.schema_id = Some(DEPLOYMENT_TRANSACTION_SCHEMA.into()); + assert!(read_transaction_record(&mislabeled).is_err()); + // History is never migrated: it lifts a v5 record the same way. + let (history, undecodable) = decode_history_transactions(vec![pinned_v5_envelope()?]); + assert!(undecodable.is_empty(), "{undecodable:?}"); + assert_eq!(history, vec![lifted]); + Ok(()) + } + + /// Always a copy in a temp dir: never a live control.cc. + #[test] + fn a_v5_control_store_copy_boots_into_v6_and_a_v6_store_is_left_alone() -> Result<()> { + let temporary = tempfile::tempdir()?; + let path = temporary.path().join("control.cc"); + let store = SingleFileMessagePackBackingStore::new(&path); + let v5 = pinned_v5_envelope()?; + assert!(store.compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: v5.r#type.clone(), + key: v5.key.clone(), + current: None, + }], + std::slice::from_ref(&v5), + )?); + assert_eq!(migrate_control_store_to_current_schema(&path)?, 1); + let after = store.pull_all_read_only_snapshot()?; + assert_eq!(after.len(), 1); + assert_eq!(after[0].schema_id.as_deref(), Some(DEPLOYMENT_TRANSACTION_SCHEMA)); + assert_eq!(after[0].stored_at, v5.stored_at); + let migrated: String = after[0].payload.iter().map(|byte| format!("{byte:02x}")).collect(); + assert_eq!(migrated, PINNED_TRANSACTION_HEX); + let file_before = std::fs::read(&path)?; + assert_eq!(migrate_control_store_to_current_schema(&path)?, 0); + assert_eq!(std::fs::read(&path)?, file_before); + read_transaction_record(&after[0])?; + Ok(()) + } + // END runner-failure-evidence tests + + // BEGIN static-release tests + // A static release over a stand-in site tree: the real engine, the real + // Sealing path over a fake docker, the real filesystem flip, and an install + // that copies instead of hardening (hardening wants root). Nothing here + // is a real site. + + /// Installs the sealed release by copying it where the binding says. + struct InstallingWorkload; + + impl WorkloadPort for InstallingWorkload { + fn install( + &self, + plan: &CompiledDeploymentPlan, + release: &crate::drivers::MaterializedRelease, + ) -> Result { + let (_, binding) = plan.parsed_inputs()?; + let release_root = binding.workload.release_root()?; + std::fs::create_dir_all(release_root)?; + let root = release_root.join(&release.release.sealed_release_id); + crate::drivers::copy_tree(&release.root, &root)?; + Ok(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.release.sealed_release_id.clone(), + root, + }) + } + fn prepare_activation( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: IdunnRuntimeActivationLaunch, + ) -> Result { + bail!("a static release prepares no activation") + } + fn start_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &SealedRelease, + _: &crate::drivers::InstalledReleaseObservation, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result { + bail!("a static release starts nothing") + } + fn discard_prepared( + &self, + _: &CompiledDeploymentPlan, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + ) -> Result<()> { + bail!("a static release discards nothing") + } + fn observe( + &self, + _: &IdunnExpectedIncarnationRecord, + _: &IdunnRuntimeActivationRecord, + _: &WorkloadObservation, + ) -> Result { + bail!("a static release observes no workload") + } + fn stop(&self, _: &WorkloadObservation) -> Result<()> { + bail!("a static release stops nothing") + } + fn is_permanently_stopped(&self, _: &WorkloadObservation) -> Result { + bail!("a static release has no workload to be dead") + } + fn lost_to_host_reboot(&self, _: &WorkloadObservation, _: u64) -> Result { + bail!("a static release has no workload to lose") + } + } + + /// The real artifact sealing over the frozen tree, without the Docker + /// step and the root-only workspace ownership that go with it. + struct SealingRunner { + tree: PathBuf, + } + + impl RunnerPort for SealingRunner { + fn materialize( + &self, + _: &crate::drivers::FrozenSource, + plan: &CompiledDeploymentPlan, + staging_parent: &Path, + sealed_at_unix_millis: u64, + ) -> Result { + let (declaration, _) = plan.parsed_inputs()?; + let staging_root = staging_parent.join(format!("stand-in-{}", Uuid::new_v4())); + std::fs::create_dir_all(&staging_root)?; + let mut artifacts = Vec::new(); + for artifact in &declaration.artifacts { + artifacts.push(crate::drivers::seal_artifact( + artifact, + &self.tree, + &self.tree.join(&artifact.source), + &staging_root, + )?); + } + Ok(crate::drivers::MaterializedRelease { + release: SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?, + root: staging_root, + }) + } + } + + const SITE_CONFIG_ONE: &[u8] = br#"{"api":"https://one.test"}"#; + const SITE_CONFIG_TWO: &[u8] = br#"{"api":"https://two.test"}"#; + + struct StaticWorld { + world: EngineFixture, + _tools: TempDir, + site: PathBuf, + } + + impl StaticWorld { + fn site_config(&self) -> PathBuf { + self.site.join("config/site.json") + } + + fn current_link(&self) -> PathBuf { + self.site.join("www/current") + } + + fn current_release(&self) -> Option { + std::fs::read_link(self.current_link()) + .ok() + .and_then(|target| target.file_name().map(|name| name.to_string_lossy().into_owned())) + } + + /// Scheduler ticks until `done`, or fails when the budget is spent. + fn tick_until( + &self, + id: &str, + what: &str, + done: impl Fn(&DeploymentTransaction) -> bool, + ) -> Result<()> { + for _ in 0..24 { + self.world.engine.resume_backoff.lock().unwrap().clear(); + let _ = self.world.engine.run_scheduler_tick(); + if done(&latest_id(&self.world, id)?) { + return Ok(()); + } + } + let last = latest_id(&self.world, id)?; + bail!("never reached {what}: phase {:?} completion {:?} error {:?}", last.phase, last.completion, last.last_error) + } + + /// The Deploy command's transaction, by command id. + fn transaction_of(&self, command_id: &str) -> Result { + let live = ControlSnapshot::read(&self.world.state_store)? + .transactions + .into_iter() + .map(|stored| stored.value); + live.chain(read_history_transactions(&self.world.state_store)) + .find(|transaction| transaction.command_id == command_id) + .map(|transaction| transaction.transaction_id) + .context("no transaction for the command") + } + + /// A full deploy through the brake, on the stand-in site. + fn deploy(&self, command_id: &str) -> Result { + if command_id != "up-service" { + enqueue_deploy(&self.world, command_id)?; + } + let id = self.transaction_of(command_id)?; + let waiting = self.run_to_brake(&id)?; + self.release_brake(&waiting)?; + self.tick_until(&id, "the deploy end", |t| t.is_terminal())?; + latest_id(&self.world, &id) + } + + /// Engage the stop for this world's runtime, as a steward would. + fn hold(&self) -> Result<()> { + crate::brake::replace_stop( + &self.site.join("authority/deployment-brake.cc"), + "site-yggdrasil", + StopState::Engaged, + "test hold", + now_millis()?, + )?; + Ok(()) + } + + /// Submit a rollback command and tick until the daemon freezes it. + /// A rollback is born at Sealing with its evidence, so a test that + /// wants it held at the stop engages the stop first. + fn submit_rollback(&self, command_id: &str) -> Result { + self.hold()?; + commit_deployment_command( + &self.world.state_store, + CommandKind::Rollback, + command_id, + "service", + "operator", + )?; + for _ in 0..24 { + self.world.engine.resume_backoff.lock().unwrap().clear(); + let _ = self.world.engine.run_scheduler_tick(); + if let Ok(id) = self.transaction_of(command_id) { + return Ok(id); + } + } + bail!("the rollback command was never frozen") + } + + /// Roll back through the brake to the end. + fn rollback(&self, command_id: &str) -> Result { + let id = self.submit_rollback(command_id)?; + let waiting = self.run_to_brake(&id)?; + self.release_brake(&waiting)?; + self.tick_until(&id, "the rollback end", |t| t.is_terminal())?; + latest_id(&self.world, &id) + } + + fn release_dir(&self, release_id: &str) -> PathBuf { + self.site.join("releases").join(release_id) + } + + /// Two deploys (config one, then two): current is the second release. + fn two_deploys(&self) -> Result<(String, String)> { + let first = self.deploy("up-service")?.flip.context("no first flip")?.release_id; + std::fs::write(self.site_config(), SITE_CONFIG_TWO)?; + let second = self.deploy("up-service-2")?.flip.context("no second flip")?.release_id; + assert_eq!(self.current_release().as_deref(), Some(second.as_str())); + Ok((first, second)) + } + + /// The terminal rejection a refused rollback is recorded as. + fn refusal_of(&self, command_id: &str) -> Result { + let id = self.submit_rollback(command_id)?; + self.tick_until(&id, "the refusal", |t| t.is_terminal())?; + let refused = latest_id(&self.world, &id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = refused.completion else { + bail!("the rollback was not refused: {refused:?}"); + }; + assert!(refused.plan.is_none() && refused.flip.is_none()); + Ok(error) + } + + /// The steward clears the stop. It is not bound to this transaction and + /// not consumed by it. + fn release_brake(&self, transaction: &DeploymentTransaction) -> Result<()> { + set_stop( + transaction.plan.as_ref().context("no plan")?, + StopState::Clear, + "test thaw", + ) + } + + /// Run a Deploy or a Rollback up to the stop and say where it waits. + fn run_to_brake(&self, id: &str) -> Result { + self.hold()?; + self.tick_until(id, "the sealed release", |t| t.installed_release.is_some())?; + for _ in 0..2 { + self.world.engine.resume_backoff.lock().unwrap().clear(); + let _ = self.world.engine.run_scheduler_tick(); + } + let waiting = latest_id(&self.world, id)?; + assert_eq!(waiting.phase, DeploymentPhase::Sealing, "{waiting:?}"); + assert!(!waiting.deployment_admitted()); + assert!( + waiting.last_error.as_deref().is_some_and(|why| why.contains("deployment brake engaged by")), + "{:?}", + waiting.last_error + ); + Ok(waiting) + } + } + + /// A target world: the static recipe over a fixed vite-like tree, a runner + /// that seals it, and a stand-in site beside it. + #[cfg(unix)] + fn static_world() -> Result { + static_world_with_steward(None) + } + + /// The static world, its binding naming `steward` (a login) as a second + /// stop whose store is `steward/brake.cc`. + #[cfg(unix)] + fn static_world_with_steward(steward: Option<&str>) -> Result { + use crate::deployment::tests::{STATIC_BINDING, STATIC_RECIPE}; + let tools = TempDir::new()?; + let tree = tools.path().join("tree"); + std::fs::create_dir_all(tree.join("dist/assets"))?; + std::fs::write(tree.join("dist/index.html"), b"site")?; + std::fs::write(tree.join("dist/index.html.br"), b"br")?; + std::fs::write(tree.join("dist/index.html.gz"), b"gz")?; + std::fs::write(tree.join("dist/assets/app.js"), b"app")?; + let mut site = PathBuf::new(); + let mut world = sealing_world_with( + STATIC_RECIPE, + |root| { + site = root.join("site"); + let text = STATIC_BINDING.replace("@SITE@", &site.display().to_string()); + match steward { + None => text, + Some(user) => text.replace( + "[rollout]", + &format!( + "[brakes.steward]\nuser = \"{user}\"\nstore = \"{}/steward/brake.cc\"\n\n[rollout]", + site.display() + ), + ), + } + }, + Arc::new(InstallingWorkload), + )?; + std::fs::create_dir_all(site.join("www"))?; + std::fs::create_dir_all(site.join("config"))?; + std::fs::create_dir_all(site.join("authority"))?; + std::fs::create_dir_all(site.join("steward"))?; + std::fs::write(site.join("config/site.json"), SITE_CONFIG_ONE)?; + *world.engine.runner_stand_in.lock().unwrap() = + Some(Arc::new(SealingRunner { tree: tree.clone() })); + world.engine.source = Arc::new(TreeSource { recipe: STATIC_RECIPE.into(), root: tree }); + Ok(StaticWorld { world, _tools: tools, site }) + } + + fn sha256_of(bytes: &[u8]) -> String { + sha256_id(bytes) + } + + #[cfg(unix)] + #[test] + fn static_release_waits_at_the_brake_then_flips_and_completes_without_a_workload() -> Result<()> { + let w = static_world()?; + let id = w.transaction_of("up-service")?; + let waiting = w.run_to_brake(&id)?; + let sealed = waiting.sealed_release.clone().context("no sealed release")?; + let release_dir = w.site.join("releases").join(&sealed.sealed_release_id); + assert!(release_dir.join("dist/index.html.br").is_file(), "the release is installed"); + // At the brake nothing external has changed: no link, no carry. + assert_eq!(w.current_release(), None); + assert!(!release_dir.join("config.json").exists()); + assert!(waiting.expected.is_none() && waiting.flip.is_none()); + // Status names what a brake release must name, though no Expected exists. + let printed = printed_status(&w.world)?; + assert!(printed.contains("runtime site-yggdrasil"), "{printed}"); + assert!(printed.contains(&format!("release {}", sealed.sealed_release_id)), "{printed}"); + + // The site file changes while the deployment waits: the sealed release + // does not, because the carry lives outside every artifact digest. + std::fs::write(w.site_config(), SITE_CONFIG_TWO)?; + w.release_brake(&waiting)?; + w.tick_until(&id, "completion", |t| t.is_terminal())?; + + let done = latest_id(&w.world, &id)?; + assert_eq!( + done.sealed_release.as_ref().map(|release| &release.sealed_release_id), + Some(&sealed.sealed_release_id), + "a changed site file changed the sealed release" + ); + let (installed_sha, _) = crate::drivers::digest_artifact(&release_dir.join("dist"))?; + assert_eq!(format!("sha256-{installed_sha}"), sealed.artifacts[0].sha256); + assert_eq!( + done.completion, + Some(TransactionCompletion::Admitted { + generation_id: format!("generation-{}", done.transaction_id) + }) + ); + let flip = done.flip.as_ref().context("no flip evidence")?; + assert_eq!(flip.release_id, sealed.sealed_release_id); + assert_eq!(flip.previous_release_id, None); + assert_eq!(flip.carried_sha256.as_deref(), Some(sha256_of(SITE_CONFIG_TWO).as_str())); + assert_eq!(flip.carry_source, w.site_config()); + assert!(done.deployment_admitted_at_unix_millis.is_some(), "the stop did not stamp its admission"); + assert_eq!(w.current_release().as_deref(), Some(sealed.sealed_release_id.as_str())); + let carried = w.current_link().join("config.json"); + assert_eq!(std::fs::read(&carried)?, SITE_CONFIG_TWO); + assert!(std::fs::symlink_metadata(&carried)?.is_file(), "the carry must be a copy, not a link to the site file"); + + // A static release runs nothing: no workload evidence, no admitted + // generation, no route program was touched. + assert!( + done.expected.is_none() + && done.activation.is_none() + && done.workload.is_none() + && done.warming.is_none() + && done.isolation.is_none() + && done.fencing.is_none() + && done.leasing.is_none() + && done.ready.is_none() + && done.routing.is_none() + && done.expected_publication_sha256.is_none() + ); + assert!(ControlSnapshot::read(&w.world.state_store)?.admitted_for("service").is_none()); + assert_eq!(w.world.route_stubs.count(""), 0, "the static path ran a host program"); + assert!(done.is_terminal(), "the post-commit cleanup did not finish: {done:?}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_release_second_deploy_records_the_release_it_replaced() -> Result<()> { + let w = static_world()?; + let first = w.run_to_brake(&w.transaction_of("up-service")?)?; + w.release_brake(&first)?; + w.tick_until(&first.transaction_id, "the first completion", |t| t.is_terminal())?; + let first_done = latest_id(&w.world, &first.transaction_id)?; + let first_release = first_done.flip.context("no first flip")?.release_id; + assert_eq!(w.current_release().as_deref(), Some(first_release.as_str())); + + // The same tree is deployed again with a changed site config. Release + // ids are per transaction (the plan names its own incarnation), so the + // redeploy is a new release that carries the new file; the first keeps + // the bytes it was flipped with. No redeploy can name its own release. + std::fs::write(w.site_config(), SITE_CONFIG_TWO)?; + enqueue_deploy(&w.world, "up-service-2")?; + let second_id = w.transaction_of("up-service-2")?; + let second = w.run_to_brake(&second_id)?; + // The first deployment's brake release names the first transaction + // and cannot release the second. + assert_eq!(w.current_release().as_deref(), Some(first_release.as_str())); + w.release_brake(&second)?; + w.tick_until(&second_id, "the second completion", |t| t.is_terminal())?; + let second_done = latest_id(&w.world, &second_id)?; + let flip = second_done.flip.context("no second flip")?; + assert_eq!(flip.previous_release_id.as_deref(), Some(first_release.as_str())); + assert_ne!(flip.release_id, first_release); + assert_eq!(w.current_release().as_deref(), Some(flip.release_id.as_str())); + assert_eq!(std::fs::read(w.current_link().join("config.json"))?, SITE_CONFIG_TWO); + assert_eq!( + std::fs::read(w.site.join("releases").join(&first_release).join("config.json"))?, + SITE_CONFIG_ONE + ); + // The first release is still on disk: rollback is a flip back to it. + assert!(w.site.join("releases").join(&first_release).join("dist/index.html").is_file()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn absent_store_admits_and_stamps_admission() -> Result<()> { + let w = static_world()?; + let id = w.transaction_of("up-service")?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + assert!(!w.site.join("authority/deployment-brake.cc").exists()); + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert!(done.deployment_admitted_at_unix_millis.is_some()); + assert!(done.deployment_authorization.is_none()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_steward_stop_holds_a_deploy_beside_the_operators_and_names_who() -> Result<()> { + let me = crate::brake::process_uid(); + if me == 0 { + eprintln!("SKIPPED: a steward is never root, and this run is root"); + return Ok(()); + } + let user = crate::brake::login_name(me)?; + let w = static_world_with_steward(Some(&user))?; + let steward_store = w.site.join("steward/brake.cc"); + let op_store = w.site.join("authority/deployment-brake.cc"); + let stop = |store: &Path, state, reason: &str| -> Result<()> { + crate::brake::replace_stop(store, "site-yggdrasil", state, reason, now_millis()?)?; + Ok(()) + }; + stop(&steward_store, StopState::Engaged, "steward hold")?; + let id = w.transaction_of("up-service")?; + w.tick_until(&id, "the sealed release", |t| t.installed_release.is_some())?; + let settle = |w: &StaticWorld| -> Result { + for _ in 0..2 { + w.world.engine.resume_backoff.lock().unwrap().clear(); + let _ = w.world.engine.run_scheduler_tick(); + } + latest_id(&w.world, &id) + }; + let held = settle(&w)?; + assert_eq!(held.phase, DeploymentPhase::Sealing); + assert_eq!( + held.last_error.as_deref(), + Some(format!("steward {user} deployment brake engaged: steward hold").as_str()) + ); + // The operator engages too; the steward clears its own. The operator's holds. + stop(&op_store, StopState::Engaged, "operator hold")?; + stop(&steward_store, StopState::Clear, "steward thaw")?; + let held = settle(&w)?; + assert!(!held.deployment_admitted()); + assert_eq!( + held.last_error.as_deref(), + Some(format!("deployment brake engaged by {user}: operator hold").as_str()) + ); + // Both engaged: both named. + stop(&steward_store, StopState::Engaged, "steward again")?; + let held = settle(&w)?; + assert!(held.last_error.as_deref().is_some_and(|note| note.contains("operator hold") + && note.contains("steward again"))); + // The operator clears, the steward still holds; then both clear. + stop(&op_store, StopState::Clear, "operator thaw")?; + assert!(!settle(&w)?.deployment_admitted()); + stop(&steward_store, StopState::Clear, "steward thaw")?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + assert!(latest_id(&w.world, &id)?.deployment_admitted()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn one_clear_admits_many_deploys_and_an_engaged_stop_names_who() -> Result<()> { + let w = static_world()?; + let first_id = w.transaction_of("up-service")?; + let first = w.run_to_brake(&first_id)?; + let held = first.last_error.clone().context("no wait note")?; + let me = crate::brake::process_uid(); + let store = w.site.join("authority/deployment-brake.cc"); + let crate::brake::Stop::Engaged(engaged) = crate::brake::read_stop(&store, "site-yggdrasil", me) else { + bail!("the stop is not engaged") + }; + assert_eq!(held, format!("deployment brake engaged by {}: {}", engaged.by, engaged.reason)); + w.release_brake(&first)?; + w.tick_until(&first_id, "the first end", |t| t.is_terminal())?; + // No engage in between: one Clear admits the second deploy too, with + // no release id or transaction id in sight. + enqueue_deploy(&w.world, "up-service-2")?; + let second_id = w.transaction_of("up-service-2")?; + w.tick_until(&second_id, "the second end", |t| t.is_terminal())?; + let crate::brake::Stop::Clear(record) = crate::brake::read_stop(&store, "site-yggdrasil", me) else { + bail!("the stop did not stay clear") + }; + assert_eq!(record.sequence, 2); + for id in [&first_id, &second_id] { + let done = latest_id(&w.world, id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert!(done.deployment_admitted_at_unix_millis.is_some()); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_stop_engaged_after_admission_halts_the_static_flip_until_it_clears() -> Result<()> { + let w = static_world()?; + let first = w.deploy("up-service")?; + let incumbent = w.current_release().context("no incumbent")?; + assert!(first.is_terminal()); + enqueue_deploy(&w.world, "up-service-2")?; + let id = w.transaction_of("up-service-2")?; + // Admitted by the absent-or-clear stop, flip intent not yet recorded. + w.tick_until(&id, "admission", |t| t.phase == DeploymentPhase::Starting)?; + assert!(latest_id(&w.world, &id)?.flip.is_none()); + w.hold()?; + for _ in 0..4 { + w.world.engine.resume_backoff.lock().unwrap().clear(); + let _ = w.world.engine.run_scheduler_tick(); + } + let held = latest_id(&w.world, &id)?; + assert_eq!(held.phase, DeploymentPhase::Starting); + assert!(held.flip.is_none(), "a halted flip recorded an intent"); + assert!(held.pre_fencing_abort.is_none() && held.completion.is_none()); + assert!(held.last_error.as_deref().is_some_and(|n| n.contains("deployment brake engaged by")), "{:?}", held.last_error); + assert_eq!(w.current_release().as_deref(), Some(incumbent.as_str()), "the link moved under a stop"); + w.release_brake(&held)?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert_ne!(w.current_release().as_deref(), Some(incumbent.as_str())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_stop_engaged_after_the_flip_intent_does_not_halt_the_flip() -> Result<()> { + let w = static_world()?; + let id = w.transaction_of("up-service")?; + w.tick_until(&id, "the flip intent", |t| t.flip.is_some())?; + w.hold()?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert!(done.flip.as_ref().is_some_and(FlipEvidence::landed)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_release_failed_flip_leaves_current_and_ends_before_fencing() -> Result<()> { + let w = static_world()?; + let first = w.run_to_brake(&w.transaction_of("up-service")?)?; + w.release_brake(&first)?; + w.tick_until(&first.transaction_id, "the first completion", |t| t.is_terminal())?; + let incumbent = w.current_release().context("no incumbent release")?; + + enqueue_deploy(&w.world, "up-service-2")?; + let second_id = w.transaction_of("up-service-2")?; + let second = w.run_to_brake(&second_id)?; + // The operator's site file vanishes before the flip. + std::fs::remove_file(w.site_config())?; + w.release_brake(&second)?; + w.tick_until(&second_id, "the second end", |t| t.is_terminal())?; + + let ended = latest_id(&w.world, &second_id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the flip did not end before fencing: {ended:?}"); + }; + assert!(error.contains("carry source"), "{error}"); + assert_eq!(w.current_release().as_deref(), Some(incumbent.as_str()), "a failed flip moved current"); + assert!(ended.is_terminal(), "the abort cleanup did not finish"); + let abort = ended.pre_fencing_abort.as_ref().context("no pre-fencing abort")?; + assert!(abort.is_complete()); + let flip = ended.flip.as_ref().context("the intent was not recorded before the act")?; + assert_eq!(flip.previous_release_id.as_deref(), Some(incumbent.as_str())); + assert!(!flip.landed(), "a failed flip claims it landed"); + assert!( + !w.site.join("releases").join(&flip.release_id).join("config.json").exists(), + "a failed flip left a carry" + ); + Ok(()) + } + + + /// The filesystem port, with faults injected where the real one could only + /// fail after the link had moved: `flip_fault_after_rename` fails the flip + /// call after the rename landed, `observe_fault_at` fails the nth observe. + struct FaultyPort { + inner: FilesystemStaticRelease, + flip_fault_after_rename: std::sync::atomic::AtomicBool, + observe_calls: std::sync::atomic::AtomicUsize, + observe_fault_at: usize, + } + + impl FaultyPort { + fn new(flip_fault_after_rename: bool, observe_fault_at: usize) -> Arc { + Arc::new(Self { + inner: FilesystemStaticRelease, + flip_fault_after_rename: flip_fault_after_rename.into(), + observe_calls: 0.into(), + observe_fault_at, + }) + } + } + + impl crate::drivers::StaticReleasePort for FaultyPort { + fn observe_current(&self, binding: &crate::deployment::StaticReleaseBinding) -> Result> { + use std::sync::atomic::Ordering; + let call = self.observe_calls.fetch_add(1, Ordering::SeqCst) + 1; + ensure!(call != self.observe_fault_at, "injected observe fault (call {call})"); + self.inner.observe_current(binding) + } + + fn carry_and_flip( + &self, + binding: &crate::deployment::StaticReleaseBinding, + release_id: &str, + ) -> Result { + let flip = self.inner.carry_and_flip(binding, release_id)?; + if self.flip_fault_after_rename.swap(false, std::sync::atomic::Ordering::SeqCst) { + bail!("injected fault after the rename"); + } + Ok(flip) + } + + fn carried_sha256( + &self, + binding: &crate::deployment::StaticReleaseBinding, + release_id: &str, + ) -> Result { + self.inner.carried_sha256(binding, release_id) + } + + fn verify_installed( + &self, + binding: &crate::deployment::StaticReleaseBinding, + release: &SealedRelease, + ) -> Result<()> { + self.inner.verify_installed(binding, release) + } + } + + /// Drive the first Deploy through the brake to its end. + fn drive_static_to_end(w: &StaticWorld) -> Result { + let id = w.transaction_of("up-service")?; + let waiting = w.run_to_brake(&id)?; + w.release_brake(&waiting)?; + w.tick_until(&id, "the end", |t| t.is_terminal())?; + latest_id(&w.world, &id) + } + + #[cfg(unix)] + #[test] + fn static_release_error_after_the_rename_never_ends_before_fencing() -> Result<()> { + for (flip_fault, observe_fault_at) in [(true, 0), (false, 4)] { + let mut w = static_world()?; + w.world.engine.static_release = FaultyPort::new(flip_fault, observe_fault_at); + let done = drive_static_to_end(&w)?; + assert!( + matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), + "a post-rename fault ended the transaction: {:?} {:?}", + done.completion, + done.last_error + ); + assert!(done.pre_fencing_abort.is_none()); + let flip = done.flip.as_ref().context("no flip")?; + assert!(flip.landed()); + assert_eq!(w.current_release().as_deref(), Some(flip.release_id.as_str())); + assert_eq!( + flip.carried_sha256.as_deref(), + Some(sha256_of(SITE_CONFIG_ONE).as_str()) + ); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_release_validation_refuses_a_landed_flip_that_failed_before_fencing() -> Result<()> { + let w = static_world()?; + let mut done = drive_static_to_end(&w)?; + assert!(done.flip.as_ref().is_some_and(FlipEvidence::landed)); + done.completion = Some(TransactionCompletion::FailedBeforeFencing { error: "x".into() }); + done.pre_fencing_abort = Some(PreFencingAbort { + error: "x".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + done.post_commit_cleanup = None; + done.phase_deadline = None; + let error = done.validate().unwrap_err(); + assert!(format!("{error:#}").contains("flip landed"), "{error:#}"); + Ok(()) + } + + /// A populated record an older Idunn could have written: a static deploy + /// that consumed a signed brake release (key 16, now legacy) and ended + /// before fencing. `drop_keys` is how many trailing nil keys the older + /// layout lacks: v5 lacks key 38, v4 also lacks key 37. + #[cfg(unix)] + fn populated_legacy_lifts_to_the_exact_current_field_by_field( + schema: &str, + drop_keys: usize, + ) -> Result<()> { + let w = static_world()?; + let done = drive_static_to_end(&w)?; + let mut expected = done.clone(); + expected.flip = None; + expected.deployment_admitted_at_unix_millis = None; + expected.deployment_authorization = Some(DeploymentAuthorization { + authorization_id: "authorization-legacy".into(), + brake_sha256: sha256_id(b"legacy brake bytes"), + canonical_brake_bytes: b"legacy brake bytes".to_vec(), + authorized_at_unix_millis: 99, + }); + expected.completion = Some(TransactionCompletion::FailedBeforeFencing { error: "refused".into() }); + expected.pre_fencing_abort = Some(PreFencingAbort { + error: "refused".into(), + candidate_cleanup: CleanupEvidence::Skipped, + topology_reconciliation: CleanupEvidence::Skipped, + source_cleanup: CleanupEvidence::Complete, + }); + expected.post_commit_cleanup = None; + expected.phase_deadline = None; + expected.last_error = Some("refused".into()); + // Starting accepts the legacy receipt as the admission. + expected.validate()?; + assert_ne!(expected.created_at_unix_millis, expected.updated_at_unix_millis); + // The older bytes: the current encoding with the old schema name and + // `drop_keys` fewer elements, every one a nil. + let mut older = expected.clone(); + older.schema_version = schema.into(); + let mut payload = rmp_serde::to_vec(&older)?; + assert_eq!(&payload[..3], &[0xdc, 0x00, 0x27]); + payload[2] -= u8::try_from(drop_keys)?; + for _ in 0..drop_keys { + assert_eq!(payload.pop(), Some(0xc0)); + } + let envelope = CultCacheEnvelope { + key: expected.transaction_id.clone(), + r#type: DeploymentTransaction::TYPE.into(), + payload, + stored_at: rfc3339_millis(100)?, + schema_id: Some(schema.into()), + }; + let lifted = read_transaction_record(&envelope)?; + assert_eq!(lifted.schema_version, DEPLOYMENT_TRANSACTION_SCHEMA); + assert_eq!(lifted.deployment_authorization, expected.deployment_authorization); + assert_eq!(lifted.deployment_admitted_at_unix_millis, None); + assert_eq!(lifted.flip, None); + assert_eq!(lifted, expected, "some field changed in the lift"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_populated_v4_transaction_lifts_to_the_exact_v6_field_by_field() -> Result<()> { + populated_legacy_lifts_to_the_exact_current_field_by_field(DEPLOYMENT_TRANSACTION_SCHEMA_V4, 2) + } + + #[cfg(unix)] + #[test] + fn a_populated_v5_transaction_with_a_consumed_release_lifts_to_the_exact_v6_field_by_field() -> Result<()> { + populated_legacy_lifts_to_the_exact_current_field_by_field(DEPLOYMENT_TRANSACTION_SCHEMA_V5, 1) + } + + #[cfg(unix)] + #[test] + fn static_release_with_a_foreign_current_retries_visibly_and_cannot_be_cancelled() -> Result<()> { + let w = static_world()?; + let id = w.transaction_of("up-service")?; + let waiting = w.run_to_brake(&id)?; + w.release_brake(&waiting)?; + w.tick_until(&id, "the flip intent", |t| t.flip.is_some())?; + // The operator replaces the link's place with a directory. + std::fs::create_dir(w.current_link())?; + for _ in 0..4 { + w.world.engine.resume_backoff.lock().unwrap().clear(); + let _ = w.world.engine.run_scheduler_tick(); + } + let stuck = latest_id(&w.world, &id)?; + assert!(!stuck.is_terminal() && stuck.phase == DeploymentPhase::Starting, "{stuck:?}"); + assert!(stuck.pre_fencing_abort.is_none(), "an error after the intent ended the flip"); + let reason = stuck.last_error.clone().context("the retry kept no error")?; + assert!(reason.contains("not a symlink"), "{reason}"); + let printed = printed_status(&w.world)?; + assert!(printed.contains(&format!("waiting on {reason}")), "{printed}"); + let refused = cancel(&w.world.state_store, "up-service", "operator").unwrap_err(); + assert!(format!("{refused:#}").contains("can only be cancelled"), "{refused:#}"); + Ok(()) + } + + const SITE_CONFIG_THREE: &[u8] = br#"{"api":"https://three.test"}"#; + + #[cfg(unix)] + #[test] + fn static_rollback_flips_back_as_a_new_brake_gated_transaction() -> Result<()> { + let w = static_world()?; + let (first, second) = w.two_deploys()?; + let second_carry = std::fs::read(w.release_dir(&second).join("config.json"))?; + std::fs::write(w.site_config(), SITE_CONFIG_THREE)?; + + let id = w.submit_rollback("rollback-1")?; + let waiting = w.run_to_brake(&id)?; + // At the brake: a new transaction of its own, nothing moved, nothing frozen. + assert_eq!(waiting.command_kind, CommandKind::Rollback); + assert!(waiting.frozen_source.is_none() && waiting.expected.is_none() && waiting.flip.is_none()); + assert_eq!(waiting.sealed_release.as_ref().map(|r| r.sealed_release_id.as_str()), Some(first.as_str())); + assert_eq!(w.current_release().as_deref(), Some(second.as_str()), "a braked rollback moved current"); + assert_eq!(std::fs::read(w.release_dir(&second).join("config.json"))?, second_carry); + let printed = { + let mut out = Vec::new(); + status(&mut out, &w.world.state_store, &w.world.engine.options.incident_store, Some("rollback-1"))?; + String::from_utf8(out)? + }; + assert!(printed.contains(&format!("release {first}")), "{printed}"); + + w.release_brake(&waiting)?; + w.tick_until(&id, "the rollback end", |t| t.is_terminal())?; + let done = latest_id(&w.world, &id)?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + let flip = done.flip.as_ref().context("no flip")?; + assert_eq!((flip.release_id.as_str(), flip.previous_release_id.as_deref()), (first.as_str(), Some(second.as_str()))); + // The carry is re-applied from the site file as it is now. + assert_eq!(flip.carried_sha256.as_deref(), Some(sha256_of(SITE_CONFIG_THREE).as_str())); + assert_eq!(w.current_release().as_deref(), Some(first.as_str())); + assert_eq!(std::fs::read(w.current_link().join("config.json"))?, SITE_CONFIG_THREE); + assert!(done.deployment_admitted_at_unix_millis.is_some(), "the stop did not stamp its admission"); + // Negatives: no source, no source cleanup, no generation, no pruning. + assert!(done.frozen_source.is_none()); + assert_eq!( + done.post_commit_cleanup.as_ref().map(|c| c.source), + Some(SourceCleanupEvidence::SkippedContinuity) + ); + assert!(ControlSnapshot::read(&w.world.state_store)?.admitted_for("service").is_none()); + assert!(w.release_dir(&first).is_dir() && w.release_dir(&second).is_dir()); + assert_eq!(w.world.route_stubs.count(""), 0); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_rollback_of_a_rollback_flips_back() -> Result<()> { + let w = static_world()?; + let (first, second) = w.two_deploys()?; + let one = w.rollback("rollback-1")?; + assert_eq!(w.current_release().as_deref(), Some(first.as_str())); + assert!(matches!(one.completion, Some(TransactionCompletion::Admitted { .. }))); + let two = w.rollback("rollback-2")?; + let flip = two.flip.as_ref().context("no flip")?; + assert_eq!((flip.release_id.as_str(), flip.previous_release_id.as_deref()), (second.as_str(), Some(first.as_str()))); + assert_eq!(w.current_release().as_deref(), Some(second.as_str())); + // A third goes back again: the latest flip always names what it displaced. + w.rollback("rollback-3")?; + assert_eq!(w.current_release().as_deref(), Some(first.as_str())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_rollback_with_no_previous_release_is_a_typed_refusal() -> Result<()> { + let w = static_world()?; + let first = w.deploy("up-service")?.flip.context("no flip")?.release_id; + let error = w.refusal_of("rollback-1")?; + assert!(error.contains("no previous release"), "{error}"); + assert_eq!(w.current_release().as_deref(), Some(first.as_str())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_rollback_refuses_when_current_is_not_the_newest_flip() -> Result<()> { + let w = static_world()?; + let (first, second) = w.two_deploys()?; + // The operator pointed current at the older release by hand. + std::fs::remove_file(w.current_link())?; + std::os::unix::fs::symlink(w.release_dir(&first), w.current_link())?; + let error = w.refusal_of("rollback-1")?; + assert!(error.contains("rollback does not guess") && error.contains(&second), "{error}"); + assert_eq!(w.current_release().as_deref(), Some(first.as_str())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_rollback_refuses_a_previous_release_that_is_gone_or_drifted() -> Result<()> { + let w = static_world()?; + let (first, second) = w.two_deploys()?; + // Drift: a file in the previous release changed after it was sealed. + let page = w.release_dir(&first).join("dist/index.html"); + std::fs::write(&page, b"tampered")?; + let error = w.refusal_of("rollback-1")?; + assert!(error.contains("differs from its sealed receipt"), "{error}"); + // Gone: the directory was removed. + std::fs::remove_dir_all(w.release_dir(&first))?; + let error = w.refusal_of("rollback-2")?; + assert!(error.contains("no longer installed"), "{error}"); + assert_eq!(w.current_release().as_deref(), Some(second.as_str())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_rollback_with_a_missing_site_file_fails_before_the_link_moves() -> Result<()> { + let w = static_world()?; + let (first, second) = w.two_deploys()?; + let id = w.submit_rollback("rollback-1")?; + let waiting = w.run_to_brake(&id)?; + // The operator moved the site file (the plan's carry source is the old one). + std::fs::remove_file(w.site_config())?; + w.release_brake(&waiting)?; + w.tick_until(&id, "the rollback end", |t| t.is_terminal())?; + let ended = latest_id(&w.world, &id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the rollback did not fail before fencing: {ended:?}"); + }; + assert!(error.contains("carry source"), "{error}"); + assert_eq!(w.current_release().as_deref(), Some(second.as_str()), "a failed rollback moved current"); + assert!(!ended.flip.as_ref().is_some_and(FlipEvidence::landed)); + assert!(w.release_dir(&first).is_dir()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_rollback_error_after_the_rename_still_completes() -> Result<()> { + let mut w = static_world()?; + let port = FaultyPort::new(false, 0); + w.world.engine.static_release = port.clone(); + let (first, _) = w.two_deploys()?; + port.flip_fault_after_rename.store(true, std::sync::atomic::Ordering::SeqCst); + let done = w.rollback("rollback-1")?; + assert!(matches!(done.completion, Some(TransactionCompletion::Admitted { .. })), "{done:?}"); + assert!(done.pre_fencing_abort.is_none()); + assert_eq!(w.current_release().as_deref(), Some(first.as_str())); + assert!(done.flip.as_ref().is_some_and(FlipEvidence::landed)); + Ok(()) + } + + #[test] + fn parse_rollback_reads_one_target_and_refuses_a_profile() -> Result<()> { + let parsed = parse(["rollback", "site", "--no-wait", "--requested-by", "me"].into_iter().map(str::to_owned))?; + let Command::Up { kind, selector, requested_by, wait, .. } = parsed else { + bail!("expected a rollback command"); + }; + assert_eq!((kind, selector.as_str(), requested_by.as_str(), wait), (CommandKind::Rollback, "site", "me", false)); + assert!(parse(["rollback", "profile:all"].into_iter().map(str::to_owned)).is_err()); + assert!(parse(["rollback"].into_iter().map(str::to_owned)).is_err()); + assert!(usage().contains("idunn rollback ")); + let command = |id: &str, kind| DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: id.into(), + kind, + selector: "site".into(), + requested_by: "me".into(), + requested_at_unix_millis: 100, + }; + command("rollback-1", CommandKind::Rollback).validate()?; + assert!(command("up-1", CommandKind::Rollback).validate().is_err()); + assert!(command("rollback-1", CommandKind::Deploy).validate().is_err()); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_rollback_drift_after_freezing_ends_before_fencing_with_current_untouched() -> Result<()> { + let w = static_world()?; + let (first, second) = w.two_deploys()?; + let id = w.submit_rollback("rollback-1")?; + let waiting = w.run_to_brake(&id)?; + // Frozen and intact; now the previous release drifts before Starting. + std::fs::write(w.release_dir(&first).join("dist/index.html"), b"tampered")?; + w.release_brake(&waiting)?; + w.tick_until(&id, "the rollback end", |t| t.is_terminal())?; + let ended = latest_id(&w.world, &id)?; + let Some(TransactionCompletion::FailedBeforeFencing { error }) = &ended.completion else { + bail!("the drifted rollback did not fail before fencing: {ended:?}"); + }; + assert!(error.contains("differs from its sealed receipt"), "{error}"); + assert!(ended.flip.is_none()); + assert_eq!(w.current_release().as_deref(), Some(second.as_str())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_rollback_refuses_a_previous_release_deployed_under_another_binding() -> Result<()> { + let w = static_world()?; + w.two_deploys()?; + let snapshot = ControlSnapshot::read(&w.world.state_store)?; + let mut history = read_history_transactions(&w.world.state_store); + // The older transaction's plan names another release root. + let older = history + .iter_mut() + .min_by_key(|t| t.created_at_unix_millis) + .context("no history")?; + let plan = older.plan.as_mut().context("no plan")?; + let text = String::from_utf8(plan.binding_blob.clone())?; + assert!(text.contains("/releases\"")); + plan.binding_blob = text.replace("/releases\"", "/releases-old\"").into_bytes(); + let error = w.world.engine.rollback_source(&snapshot, &history, "service").unwrap_err(); + assert!(format!("{error:#}").contains("does not cross bindings"), "{error:#}"); + Ok(()) + } + // END static-release tests } diff --git a/src/deployment.rs b/src/deployment.rs index 56cd8d8..265e59f 100644 --- a/src/deployment.rs +++ b/src/deployment.rs @@ -45,7 +45,12 @@ pub struct TargetDeclaration { #[serde(default)] pub external_inputs: Vec, pub artifacts: Vec, - pub service: ServiceDeclaration, + /// A running workload. Exactly one of `service` and `static_release`. + #[serde(default)] + pub service: Option, + /// A sealed tree that is published by repointing a link, with no process. + #[serde(default)] + pub static_release: Option, #[serde(default)] pub state: Option, #[serde(default)] @@ -113,6 +118,10 @@ pub struct ArtifactOutput { pub expected_sha256: Option, #[serde(default)] pub executable: bool, + /// Regular files inside a directory artifact that are sealed executable + /// (0555); every other file in the tree is sealed 0444. + #[serde(default)] + pub executable_members: BTreeSet, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -122,6 +131,15 @@ pub enum ArtifactSource { WorktreeTree, } +/// A target that is a sealed tree and nothing else: Idunn installs the release +/// and repoints the binding's current link at it. `artifact` names the tree a +/// web server reads through that link. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StaticReleaseDeclaration { + pub artifact: String, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ServiceDeclaration { @@ -471,20 +489,42 @@ pub struct DockerRunnerBinding { pub enum WorkloadBinding { SystemdTransient(SystemdWorkloadBinding), HostActuator(HostWorkloadBinding), + /// No process: a release directory and the link that names the current one. + StaticRelease(StaticReleaseBinding), } +static NO_ENVIRONMENT: BTreeMap = BTreeMap::new(); + impl WorkloadBinding { pub fn systemd(&self) -> Result<&SystemdWorkloadBinding> { match self { Self::SystemdTransient(binding) => Ok(binding), Self::HostActuator(_) => bail!("workload is host-actuator, not systemd-transient"), + Self::StaticRelease(_) => bail!("workload is a static release, not systemd-transient"), + } + } + + pub fn static_release(&self) -> Result<&StaticReleaseBinding> { + match self { + Self::StaticRelease(binding) => Ok(binding), + _ => bail!("workload is not a static release"), + } + } + + /// Where Idunn installs sealed releases on its own host: the one thing a + /// systemd workload and a static release share. + pub fn release_root(&self) -> Result<&Path> { + match self { + Self::SystemdTransient(binding) => Ok(&binding.release_root), + Self::StaticRelease(binding) => Ok(&binding.release_root), + Self::HostActuator(_) => bail!("a host-actuator workload installs on its own host"), } } pub fn host(&self) -> Result<&HostWorkloadBinding> { match self { Self::HostActuator(binding) => Ok(binding), - Self::SystemdTransient(_) => bail!("workload is systemd-transient, not host-actuator"), + _ => bail!("workload is not host-actuator"), } } @@ -492,6 +532,7 @@ impl WorkloadBinding { match self { Self::SystemdTransient(binding) => binding.state_root.is_some(), Self::HostActuator(binding) => binding.state_root.is_some(), + Self::StaticRelease(_) => false, } } @@ -499,6 +540,7 @@ impl WorkloadBinding { match self { Self::SystemdTransient(binding) => &binding.environment, Self::HostActuator(binding) => &binding.environment, + Self::StaticRelease(_) => &NO_ENVIRONMENT, } } @@ -506,6 +548,7 @@ impl WorkloadBinding { match self { Self::SystemdTransient(binding) => binding.secret_files.keys().collect(), Self::HostActuator(binding) => binding.secret_files.keys().collect(), + Self::StaticRelease(_) => Vec::new(), } } @@ -513,6 +556,7 @@ impl WorkloadBinding { match self { Self::SystemdTransient(binding) => &binding.argument_bindings, Self::HostActuator(binding) => &binding.argument_bindings, + Self::StaticRelease(_) => &NO_ENVIRONMENT, } } @@ -547,6 +591,49 @@ impl WorkloadBinding { } } +/// A static release on the Idunn host. `current_link` is the symlink a web +/// server's root reads through; Idunn repoints it at one sealed release. +/// `carry` names one operator-owned file (a per-site config) that Idunn copies +/// into each release as `destination` when it flips: the copy sits beside the +/// artifacts, outside every sealed digest. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StaticReleaseBinding { + pub release_root: PathBuf, + pub current_link: PathBuf, + pub carry: StaticCarryBinding, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StaticCarryBinding { + pub source: PathBuf, + pub destination: PathBuf, +} + +impl StaticReleaseBinding { + fn validate(&self) -> Result<()> { + require_absolute_path(&self.release_root, "static release root")?; + require_absolute_path(&self.current_link, "static current link")?; + require_absolute_path(&self.carry.source, "static carry source")?; + require_leaf_path(&self.carry.destination, "static carry destination")?; + ensure!( + !paths_overlap(&self.current_link, &self.release_root), + "static current link and release root overlap" + ); + ensure!( + !self.carry.source.starts_with(&self.release_root), + "static carry source is inside the release root" + ); + ensure!( + self.carry.source != self.current_link + && !self.carry.source.starts_with(&self.current_link), + "static carry source is the current link" + ); + Ok(()) + } +} + /// A workload on a managed host. Every path is the host's and is validated by /// the actuator that owns the host; here they are opaque non-empty values. /// The host's actuator is authenticated against `host_trust_anchor_store`, @@ -666,6 +753,20 @@ impl ProcessWriteLeaseBinding { pub struct BrakeBinding { pub deployment_store: PathBuf, pub lifecycle_store: PathBuf, + /// A second deployment stop, written by the target's steward in a + /// directory only the steward can write. Absent means the operator's store + /// is the only stop. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub steward: Option, +} + +/// The steward's own stop: the file's owner uid proves who wrote it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StewardBrakeBinding { + /// Login name; must resolve to a non-root uid when the store is read. + pub user: String, + pub store: PathBuf, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -912,6 +1013,14 @@ impl TargetDeclaration { if let Some(expected_sha256) = &artifact.expected_sha256 { require_sha256(expected_sha256, "expected artifact sha256")?; } + ensure!( + !(artifact.executable && !artifact.executable_members.is_empty()), + "artifact {} declares both executable and executable_members", + artifact.id + ); + for member in &artifact.executable_members { + require_relative_path(member, "executable member", false)?; + } ensure!( destinations.insert(artifact.destination.as_path()), "artifact destination {} is declared twice", @@ -965,57 +1074,76 @@ impl TargetDeclaration { "target declaration uses no runner classes" ); - ensure!( - artifact_ids.contains(self.service.executable_artifact.as_str()), - "service executable artifact is not declared" - ); - for artifact in &self.service.required_adjacent_artifacts { + match (&self.service, &self.static_release) { + (Some(service), None) => { + ensure!( + artifact_ids.contains(service.executable_artifact.as_str()), + "service executable artifact is not declared" + ); + for artifact in &service.required_adjacent_artifacts { + ensure!( + artifact_ids.contains(artifact.as_str()), + "adjacent service artifact {artifact} is not declared" + ); + } + for argument in &service.arguments { + validate_launch_argument(argument)?; + } + require_capability(&service.health.contract, "health contract")?; + for name in &service.required_environment { + require_environment_name(name)?; + } ensure!( - artifact_ids.contains(artifact.as_str()), - "adjacent service artifact {artifact} is not declared" + service + .required_environment + .contains(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT), + "service does not require the standard Idunn runtime bundle" ); - } - for argument in &self.service.arguments { - validate_launch_argument(argument)?; - } - require_capability(&self.service.health.contract, "health contract")?; - for name in &self.service.required_environment { - require_environment_name(name)?; - } - ensure!( - self.service - .required_environment - .contains(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT), - "service does not require the standard Idunn runtime bundle" - ); - ensure!( - self.service - .required_environment - .contains(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) - == self.service.route_required, - "routed services must require exactly one Idunn-owned candidate bind" - ); - ensure!( - !self - .service - .required_environment - .contains(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) - || self.service.route_required, - "only routed services may require the Idunn-owned member bind" - ); - ensure!( - self.service - .required_environment - .contains(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT) - == self.write_lease_required(), - "state-writing services must require exactly one Idunn-owned process write lease" - ); - for name in &self.service.optional_environment { - require_environment_name(name)?; ensure!( - !self.service.required_environment.contains(name), - "service environment {name} is both required and optional" + service + .required_environment + .contains(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT) + == service.route_required, + "routed services must require exactly one Idunn-owned candidate bind" ); + ensure!( + !service + .required_environment + .contains(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT) + || service.route_required, + "only routed services may require the Idunn-owned member bind" + ); + ensure!( + service + .required_environment + .contains(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT) + == self.write_lease_required(), + "state-writing services must require exactly one Idunn-owned process write lease" + ); + for name in &service.optional_environment { + require_environment_name(name)?; + ensure!( + !service.required_environment.contains(name), + "service environment {name} is both required and optional" + ); + } + + } + (None, Some(static_release)) => { + ensure!( + artifact_ids.contains(static_release.artifact.as_str()), + "static release artifact is not declared" + ); + ensure!( + self.state.is_none() + && self.provides.is_empty() + && self.dependencies.is_empty() + && self.conflicts.is_empty(), + "a static release declares no state, capability, dependency or conflict" + ); + } + (Some(_), Some(_)) => bail!("target declares both a service and a static release"), + (None, None) => bail!("target declares neither a service nor a static release"), } if let Some(state) = &self.state { @@ -1132,6 +1260,14 @@ impl TargetDeclaration { Ok(()) } + /// The service of a service target. A static release has none, and a code + /// path that asks for one has been reached by the wrong kind of target. + pub fn service(&self) -> Result<&ServiceDeclaration> { + self.service + .as_ref() + .context("target declares no service") + } + pub fn write_lease_required(&self) -> bool { self.state.as_ref().is_some_and(|state| { state @@ -1308,6 +1444,39 @@ impl OperatorBinding { self.brakes.deployment_store != self.brakes.lifecycle_store, "deployment and lifecycle brakes are identical" ); + if let Some(steward) = &self.brakes.steward { + require_absolute_path(&steward.store, "steward deployment brake")?; + ensure!( + crate::steward_orders::login_grammar_ok(&steward.user) && steward.user != "root", + "steward brake user is invalid or root" + ); + ensure!( + steward.store != self.brakes.deployment_store + && steward.store != self.brakes.lifecycle_store, + "steward brake store is identical to another brake store" + ); + // The steward owns its store's directory, so it could unlink any + // file there. Every other authority store must therefore live + // outside that directory, and the steward's directory must be + // clear of every directory an operator store lives in. + let steward_dir = steward.store.parent().unwrap_or(&steward.store); + let others = [ + Some(&self.brakes.deployment_store), + Some(&self.brakes.lifecycle_store), + Some(&self.runtime_identity.trust_anchor_store), + self.process_write_lease.as_ref().map(|lease| &lease.record_path), + self.repository.release_authority_store.as_ref(), + ]; + for other in others.into_iter().flatten() { + let other_dir = other.parent().unwrap_or(other); + ensure!( + !paths_overlap(steward_dir, other_dir), + "steward brake directory {} is shared with or nested in the directory of Idunn authority {}: the steward owns its directory and could remove that file", + steward_dir.display(), + other.display() + ); + } + } self.workload.validate_launch_environment()?; let workload = match &self.workload { WorkloadBinding::HostActuator(workload) => { @@ -1326,6 +1495,32 @@ impl OperatorBinding { ); return self.validate_shared_tail(); } + WorkloadBinding::StaticRelease(workload) => { + workload.validate()?; + ensure!( + self.process_write_lease.is_none() && self.route.is_none(), + "a static release has no write lease or route" + ); + ensure!( + self.rollout.strategy == RolloutStrategy::CandidateThenPromote, + "a static release rolls out candidate-then-promote: it never stops an incumbent" + ); + for authority in [ + &self.brakes.deployment_store, + &self.brakes.lifecycle_store, + &self.runtime_identity.trust_anchor_store, + ] + .into_iter() + .chain(self.brakes.steward.as_ref().map(|steward| &steward.store)) + { + ensure!( + !paths_overlap(authority, &workload.release_root) + && !paths_overlap(authority, &workload.current_link), + "static release path overlaps an Idunn authority path" + ); + } + return self.validate_shared_tail(); + } WorkloadBinding::SystemdTransient(workload) => { ensure!( self.rollout.strategy == RolloutStrategy::CandidateThenPromote, @@ -1386,6 +1581,7 @@ impl OperatorBinding { self.brakes.deployment_store.clone(), self.brakes.lifecycle_store.clone(), ]; + authority_paths.extend(self.brakes.steward.as_ref().map(|steward| steward.store.clone())); if let Some(write_lease) = &self.process_write_lease { authority_paths.push(write_lease.record_path.clone()); authority_paths.push(write_lease.lock_path()); @@ -1553,6 +1749,11 @@ impl OperatorBinding { self.target == declaration.target, "binding target does not match declaration" ); + ensure!( + declaration.static_release.is_some() + == matches!(self.workload, WorkloadBinding::StaticRelease(_)), + "a static release declaration needs a static-release workload, and only it" + ); let bound_gitlinks: BTreeSet<_> = self.repository.gitlinks.keys().cloned().collect(); ensure!( bound_gitlinks == declaration.required_gitlinks, @@ -1664,6 +1865,19 @@ impl OperatorBinding { input.id ); } + let Some(service) = declaration.service.as_ref() else { + let WorkloadBinding::StaticRelease(workload) = &self.workload else { + bail!("a static release declaration needs a static-release workload"); + }; + ensure!( + declaration + .artifacts + .iter() + .all(|artifact| artifact.destination != workload.carry.destination), + "static carry destination collides with an artifact destination" + ); + return Ok(()); + }; let workload_names: BTreeSet<&String> = self .workload .environment() @@ -1690,7 +1904,7 @@ impl OperatorBinding { if host_workload { available_environment.insert(IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT.into()); } - if declaration.service.route_required { + if service.route_required { available_environment.insert(IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT.into()); if self.route.as_ref().is_some_and(|route| route.member.is_some()) { available_environment.insert(IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT.into()); @@ -1700,16 +1914,14 @@ impl OperatorBinding { available_environment.insert(IDUNN_PROCESS_WRITE_LEASE_ENVIRONMENT.into()); } ensure!( - declaration - .service + service .required_environment .is_subset(&available_environment), "operator service environment omits a required launch input" ); - let declared_environment: BTreeSet<_> = declaration - .service + let declared_environment: BTreeSet<_> = service .required_environment - .union(&declaration.service.optional_environment) + .union(&service.optional_environment) .cloned() .collect(); // On the Idunn host the presence identity reaches the workload as a @@ -1729,8 +1941,7 @@ impl OperatorBinding { available_environment.is_subset(&declared_environment), "operator service environment contains an undeclared launch input" ); - let required_arguments: BTreeSet<_> = declaration - .service + let required_arguments: BTreeSet<_> = service .arguments .iter() .filter_map(|argument| match argument { @@ -1746,6 +1957,7 @@ impl OperatorBinding { .as_ref() .map(|path| path.display().to_string()), WorkloadBinding::HostActuator(workload) => workload.state_root.clone(), + WorkloadBinding::StaticRelease(_) => None, }; match ( state_root.as_deref(), @@ -1768,8 +1980,8 @@ impl OperatorBinding { "operator argument bindings do not exactly match the launch contract" ); match ( - declaration.service.route_required, - declaration.service.transport, + service.route_required, + service.transport, self.route.as_ref(), ) { (true, ServiceTransport::Http, Some(route)) @@ -2211,6 +2423,76 @@ nodes = ["yggdrasil"] assert!(TargetDeclaration::parse(&input).is_err()); } + const MEMBER_CANARY: &str = "zzcanary"; + + fn recipe_with_members(members: &str) -> String { + let input = RECIPE.replace("executable = true", &format!("executable_members = [{members}]")); + assert_ne!(input, RECIPE, "the fixture lost its executable line"); + input + } + + /// A refused member list is refused by name of the field, never by echoing + /// the member. + fn assert_members_refused(members: &[&str]) { + let control = TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).unwrap(); + assert!(control.artifacts[0].executable_members.contains(Path::new("bin/daemon"))); + for member in members { + let error = TargetDeclaration::parse(&recipe_with_members(&format!("\"{member}\""))) + .expect_err(member); + assert!( + !format!("{error:#}").contains(MEMBER_CANARY), + "the refusal echoed a member path" + ); + } + } + + #[test] + fn recipe_executable_member_refuses_parent_traversal() { + assert_members_refused(&[ + "zzcanary/../b", + "../zzcanary", + "bin/zzcanary/..", + "bin/../../zzcanary", + ]); + } + + #[test] + fn recipe_executable_member_refuses_absolute_path() { + assert_members_refused(&["/zzcanary/sh", "/zzcanary"]); + } + + #[test] + fn recipe_executable_member_refuses_empty_and_current_dir() { + assert_members_refused(&["", ".", "./", "zzcanary/.."]); + } + + #[test] + fn recipe_refuses_executable_flag_with_members() { + let input = RECIPE.replace( + "executable = true", + "executable = true\nexecutable_members = [\"bin/daemon\"]", + ); + let error = TargetDeclaration::parse(&input).unwrap_err(); + assert!(format!("{error:#}").contains("executable_members")); + assert!(!format!("{error:#}").contains("bin/daemon")); + // executable = true alone, and members alone, both stay admitted. + assert!(TargetDeclaration::parse(RECIPE).is_ok()); + assert!(TargetDeclaration::parse(&recipe_with_members("\"bin/daemon\"")).is_ok()); + } + + #[test] + fn recipe_admits_members_on_both_source_kinds() { + let runner_output = TargetDeclaration::parse(&recipe_with_members("\"a/b\", \"c\"")).unwrap(); + assert_eq!(runner_output.artifacts[0].source_kind, ArtifactSource::RunnerOutput); + assert_eq!(runner_output.artifacts[0].executable_members.len(), 2); + let worktree = recipe_with_members("\"a/b\", \"c\"") + .replace("source_kind = \"runner-output\"\nrunner = \"rust\"\n", "source_kind = \"worktree-tree\"\n") + .replace("expected_sha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"\n", ""); + let worktree = TargetDeclaration::parse(&worktree).unwrap(); + assert_eq!(worktree.artifacts[0].source_kind, ArtifactSource::WorktreeTree); + assert_eq!(worktree.artifacts[0].executable_members.len(), 2); + } + #[test] fn operator_cannot_omit_an_affordance_derived_from_the_recipe() { let recipe = TargetDeclaration::parse(RECIPE).unwrap(); @@ -2262,7 +2544,7 @@ nodes = ["yggdrasil"] ) .is_err() ); - assert!(recipe.service.route_required); + assert!(recipe.service().unwrap().route_required); } #[test] @@ -2750,4 +3032,230 @@ max_connections = {value}"), "v2", )); } + + /// A static release: a vite-style tree published by repointing a link. + pub(crate) const STATIC_RECIPE: &str = r#" +schema = "gamecult.idunn.target_declaration.v1" +target = "service" +source_stamp_environment = "SERVICE_BUILD_COMMIT" + +[[steps]] +id = "build" +phase = "build" +runner = "rust" +argv = ["true"] + +[[artifacts]] +id = "dist" +source_kind = "worktree-tree" +source = "dist" +destination = "dist" + +[static_release] +artifact = "dist" +"#; + + /// `@SITE@` stands for the stand-in site tree a test lays out. + pub(crate) const STATIC_BINDING: &str = r#" +schema = "gamecult.idunn.operator_binding.v2" +target = "service" + +[repository] +origin = "https://github.com/GameCult/Service.git" +admitted_ref = "refs/heads/main" +minimum_revision = "1111111111111111111111111111111111111111" +selection = "ref-head" +checkout = "/srv/build/Service" +recipe_path = "deployment/idunn/recipe.toml" + +[runners.rust] +driver = "docker" +image = "rust@sha256:4c2fd73ef19c5ef9d54bee03b06b2839a392604fbfcd578ed948b71b37c1d7fb" +user = "1000:1000" +affordances = ["source-read", "artifact-write"] +allowed_programs = ["true"] +memory_mebibytes = 1024 +cpu_quota_percent = 100 +pids_limit = 128 +tmpfs_mebibytes = 64 + +[workload] +driver = "static-release" +release_root = "@SITE@/releases" +current_link = "@SITE@/www/current" + +[workload.carry] +source = "@SITE@/config/site.json" +destination = "config.json" + +[runtime_identity] +runtime_id = "site-yggdrasil" +expected_signer_identity_id = "site-runtime-signer" +trust_anchor_store = "@SITE@/trust/site.cc" + +[brakes] +deployment_store = "@SITE@/authority/deployment-brake.cc" +lifecycle_store = "@SITE@/authority/lifecycle-brake.cc" + +[rollout] +strategy = "candidate-then-promote" +drain_seconds = 1 +retain_releases = 2 + +[placement] +desired_replicas = 1 +nodes = ["yggdrasil"] +"#; + + fn static_binding(replace: &[(&str, &str)]) -> String { + let mut text = STATIC_BINDING.replace("@SITE@", "/srv/site"); + for (from, to) in replace { + assert!(text.contains(from), "the fixture lost {from}"); + text = text.replace(from, to); + } + text + } + + const STEWARD: &str = "\n[brakes.steward]\nuser = \"mind-streampixels\"\nstore = \"/var/lib/gamecult/idunn-steward/streampixels/deployment-brake.cc\"\n"; + + #[test] + fn binding_without_steward_parses_and_admits_on_operator_store_only() { + for text in [BINDING.to_string(), static_binding(&[])] { + assert_eq!(OperatorBinding::parse(&text).unwrap().brakes.steward, None); + } + let steward = OperatorBinding::parse(&BINDING.replace("\n[rollout]", &format!("{STEWARD}\n[rollout]"))) + .unwrap() + .brakes + .steward + .unwrap(); + assert_eq!(steward.user, "mind-streampixels"); + } + + #[test] + fn a_steward_brake_must_be_a_distinct_absolute_non_root_login_and_clear_of_the_workload() { + let with = |steward: &str| BINDING.replace("\n[rollout]", &format!("{steward}\n[rollout]")); + let store = "/var/lib/gamecult/idunn-steward/x/deployment-brake.cc"; + for bad in [ + format!("\n[brakes.steward]\nuser = \"root\"\nstore = \"{store}\"\n"), + format!("\n[brakes.steward]\nuser = \"Bad User\"\nstore = \"{store}\"\n"), + "\n[brakes.steward]\nuser = \"mind\"\nstore = \"relative.cc\"\n".into(), + "\n[brakes.steward]\nuser = \"mind\"\nstore = \"/var/lib/gamecult/idunn-authority/test-service-deployment-brake.cc\"\n".into(), + "\n[brakes.steward]\nuser = \"mind\"\nstore = \"/var/lib/gamecult/idunn-authority/test-service-lifecycle-brake.cc\"\n".into(), + format!("\n[brakes.steward]\nuser = \"mind\"\nstore = \"{store}\"\nextra = 1\n"), + ] { + assert!(OperatorBinding::parse(&with(&bad)).is_err(), "{bad}"); + } + // A steward directory shared with, or enclosing, an operator store + // lets the steward unlink a root-owned file: refused. + for (deployment, steward_store) in [ + ("/var/lib/gamecult/steward-x/op.cc", "/var/lib/gamecult/steward-x/s.cc"), + ("/var/lib/gamecult/steward-x/sub/op.cc", "/var/lib/gamecult/steward-x/s.cc"), + ("/var/lib/gamecult/op.cc", "/var/lib/gamecult/steward-x/s.cc"), + ] { + let shared = BINDING + .replace("/var/lib/gamecult/idunn-authority/test-service-deployment-brake.cc", deployment) + .replace("\n[rollout]", &format!("\n[brakes.steward]\nuser = \"mind\"\nstore = \"{steward_store}\"\n\n[rollout]")); + assert!(OperatorBinding::parse(&shared).is_err(), "{deployment} {steward_store}"); + } + // Disjoint directories are the one admitted shape. + let disjoint = BINDING.replace( + "\n[rollout]", + "\n[brakes.steward]\nuser = \"mind\"\nstore = \"/var/lib/gamecult/steward-x/s.cc\"\n\n[rollout]", + ); + OperatorBinding::parse(&disjoint).unwrap(); + // steward_store_overlap_with_workload_path_refused + let overlapping = BINDING + .replace("\n[rollout]", &format!("\n[brakes.steward]\nuser = \"mind\"\nstore = \"{store}\"\n\n[rollout]")) + .replace( + "cpu_quota_percent = 200\n\n[workload.argument_bindings]", + "cpu_quota_percent = 200\nread_write_paths = [\"/var/lib/gamecult/idunn-steward/x\"]\n\n[workload.argument_bindings]", + ); + assert!(OperatorBinding::parse(&overlapping).is_err()); + let static_overlap = static_binding(&[("[rollout]", "[brakes.steward]\nuser = \"mind\"\nstore = \"/srv/site/releases/x/brake.cc\"\n\n[rollout]")]); + assert!(OperatorBinding::parse(&static_overlap).is_err()); + } + + #[test] + fn static_recipe_and_binding_admit() { + let recipe = TargetDeclaration::parse(STATIC_RECIPE).unwrap(); + let binding = OperatorBinding::parse(&static_binding(&[])).unwrap(); + binding.admit(&recipe).unwrap(); + assert!(recipe.service().is_err()); + assert_eq!(binding.workload.release_root().unwrap(), Path::new("/srv/site/releases")); + } + + #[test] + fn static_recipe_declares_exactly_one_of_service_and_static_release() { + let both = format!("{RECIPE}\n[static_release]\nartifact = \"daemon\"\n"); + let error = TargetDeclaration::parse(&both).unwrap_err(); + assert!(format!("{error:#}").contains("both a service and a static release")); + let neither = STATIC_RECIPE.replace("[static_release]\nartifact = \"dist\"\n", ""); + let error = TargetDeclaration::parse(&neither).unwrap_err(); + assert!(format!("{error:#}").contains("neither a service nor a static release")); + let unknown = STATIC_RECIPE.replace("artifact = \"dist\"", "artifact = \"nope\""); + assert!(TargetDeclaration::parse(&unknown).is_err()); + } + + #[test] + fn static_release_refuses_a_workload_contract() { + for extra in [ + "\n[[provides]]\ncapability = \"site.runtime\"\nschema = \"site.v1\"\ncompatibility = \"v1\"\n", + "\n[[conflicts]]\ncapability = \"site.old\"\nreason = \"replaced\"\n", + "\n[state]\nschema_generation = \"v1\"\n\n[[state.slots]]\nid = \"world\"\nrelative_path = \"world.cc\"\nkind = \"cultcache-file\"\nschema = \"site.v1\"\nwriter = \"none\"\nrecovery = \"preserve\"\nstartup = \"open-at-start\"\n", + ] { + let error = TargetDeclaration::parse(&format!("{STATIC_RECIPE}{extra}")).unwrap_err(); + assert!( + format!("{error:#}").contains("a static release declares no state"), + "{extra}: {error:#}" + ); + } + for (from, to, refusal) in [ + ("[rollout]", "[route]\ndriver = \"nginx-stream-tcp\"\nroute_id = \"r\"\nstable_endpoint = \"http://127.0.0.1:18830\"\nprivate_host = \"127.0.0.1\"\nprivate_port_start = 18831\nprivate_port_end = 18839\nconfig_path = \"/etc/nginx/r.conf\"\nreload_unit = \"nginx.service\"\n\n[rollout]", "no write lease or route"), + ("[rollout]", "[process_write_lease]\nrecord_path = \"/srv/site/lease.cc\"\n\n[rollout]", "no write lease or route"), + ("retain_releases = 2", "retain_releases = 1", "retain at least current and prior"), + ("candidate-then-promote", "stop-then-start", "never stops an incumbent"), + ] { + let error = OperatorBinding::parse(&static_binding(&[(from, to)])).unwrap_err(); + assert!(format!("{error:#}").contains(refusal), "{to}: {error:#}"); + } + } + + #[test] + fn static_release_and_service_recipes_need_their_own_binding_kind() { + let static_recipe = TargetDeclaration::parse(STATIC_RECIPE).unwrap(); + let service_recipe = TargetDeclaration::parse(RECIPE).unwrap(); + let static_binding = OperatorBinding::parse(&static_binding(&[])).unwrap(); + let service_binding = OperatorBinding::parse(BINDING).unwrap(); + assert!(static_binding.admit(&service_recipe).is_err()); + assert!(service_binding.admit(&static_recipe).is_err()); + } + + #[test] + fn static_carry_destination_must_not_be_an_artifact_destination() { + let recipe = TargetDeclaration::parse(STATIC_RECIPE).unwrap(); + let binding = + OperatorBinding::parse(&static_binding(&[("destination = \"config.json\"", "destination = \"dist\"")])) + .unwrap(); + let error = binding.admit(&recipe).unwrap_err(); + assert!(format!("{error:#}").contains("collides with an artifact destination")); + for destination in ["a/b", "../b", "/b", ""] { + let line = format!("destination = \"{destination}\""); + assert!( + OperatorBinding::parse(&static_binding(&[("destination = \"config.json\"", &line)])).is_err(), + "{destination}" + ); + } + } + + #[test] + fn static_carry_source_and_link_stay_out_of_the_release_root() { + for (from, to) in [ + ("source = \"/srv/site/config/site.json\"", "source = \"/srv/site/releases/site.json\""), + ("current_link = \"/srv/site/www/current\"", "current_link = \"/srv/site/releases/current\""), + ("source = \"/srv/site/config/site.json\"", "source = \"site.json\""), + ("release_root = \"/srv/site/releases\"", "release_root = \"/srv/site/www\""), + ] { + assert!(OperatorBinding::parse(&static_binding(&[(from, to)])).is_err(), "{to}"); + } + } } diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 1363c48..da790c7 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -734,6 +734,11 @@ pub struct ArtifactReceipt { pub sha256: String, pub size_bytes: u64, pub executable: bool, + /// Files sealed executable inside a directory artifact. Skipped when empty + /// so a receipt with no members keeps its original encoding, and so its + /// sealed release id. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub executable_members: BTreeSet, } impl ArtifactReceipt { @@ -747,7 +752,9 @@ impl ArtifactReceipt { .find(|artifact| artifact.id == self.artifact_id) .with_context(|| format!("plan declares no artifact {}", self.artifact_id))?; ensure!( - self.destination == declared.destination && self.executable == declared.executable, + self.destination == declared.destination + && self.executable == declared.executable + && self.executable_members == declared.executable_members, "artifact receipt differs from declared output" ); if let Some(expected_sha256) = &declared.expected_sha256 { @@ -847,14 +854,15 @@ impl SealedRelease { ) -> Result { self.validate_against(plan)?; let (declaration, binding) = plan.parsed_inputs()?; + let service = declaration.service()?; let route = match (&binding.route, plan.candidate_port) { (Some(bound), Some(port)) => Some(IdunnExpectedRoute { route_id: bound.route_id.clone(), - transport: endpoint_scheme(declaration.service.transport)?.into(), + transport: endpoint_scheme(service.transport)?.into(), stable_endpoint: bound.stable_endpoint.clone(), candidate_endpoint: format!( "{}://{}:{port}", - endpoint_scheme(declaration.service.transport)?, + endpoint_scheme(service.transport)?, bound.private_host ), }), @@ -864,7 +872,7 @@ impl SealedRelease { let artifact_sha256 = self .artifacts .iter() - .find(|artifact| artifact.artifact_id == declaration.service.executable_artifact) + .find(|artifact| artifact.artifact_id == service.executable_artifact) .context("sealed release has no executable service artifact")? .sha256 .clone(); @@ -911,7 +919,7 @@ impl SealedRelease { .runtime_identity .expected_signer_identity_id .clone(), - health_contract: declaration.service.health.contract.clone(), + health_contract: service.health.contract.clone(), artifact_sha256, state_schema_generation: declaration .state @@ -1425,6 +1433,7 @@ nodes = ["yggdrasil"] sha256: digest(b'c'), size_bytes: 42, executable: true, + executable_members: BTreeSet::new(), } } @@ -1791,6 +1800,113 @@ nodes = ["yggdrasil"] assert!(changed.validate_against(&plan).is_err()); } + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + } + + #[test] + fn receipt_without_members_encodes_as_before() { + // Both values were computed outside this code, from the five-field + // msgpack array the receipt encoded to at base d74ac8d. + const BASE_RECEIPT: &str = "95a66461656d6f6ea773657276696365d9477368613235362d636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363632ac3"; + const BASE_RELEASE_ID: &str = + "sha256-b9143d8f546ad71c493c70037813547aaec10fd87994a6f3bdc77ac3a8b3495e"; + let receipt = artifact_receipt(); + let encoded = rmp_serde::to_vec(&receipt).unwrap(); + assert_eq!(hex(&encoded), BASE_RECEIPT); + let bytes = (0..BASE_RECEIPT.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&BASE_RECEIPT[at..at + 2], 16).unwrap()) + .collect::>(); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), receipt); + let release = SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![receipt], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + assert_eq!(release.recomputed_release_id().unwrap(), BASE_RELEASE_ID); + } + + #[test] + fn sealed_release_address_covers_executable_members() { + let release_with = |members: &[&str]| SealedRelease { + schema: SEALED_RELEASE_SCHEMA.into(), + sealed_release_id: String::new(), + plan_id: digest(b'a'), + artifacts: vec![ArtifactReceipt { + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + ..artifact_receipt() + }], + external_inputs: vec![], + sealed_at_unix_millis: 0, + }; + let none = release_with(&[]).recomputed_release_id().unwrap(); + let one = release_with(&["bin/daemon"]).recomputed_release_id().unwrap(); + let other = release_with(&["bin/other"]).recomputed_release_id().unwrap(); + let two = release_with(&["bin/daemon", "bin/other"]) + .recomputed_release_id() + .unwrap(); + let all = [&none, &one, &other, &two]; + for (index, left) in all.iter().enumerate() { + for right in &all[index + 1..] { + assert_ne!(left, right); + } + } + + // The declaration is the other half: a receipt must carry exactly the + // declared list, in a plan that declares one and in one that does not. + let plain = plan(); + let mut with_members = artifact_receipt(); + with_members.executable_members = BTreeSet::from([PathBuf::from("bin/daemon")]); + assert!( + SealedRelease::new(&plain, vec![with_members], vec![external_input_receipt()], 120) + .is_err() + ); + let recipe = RECIPE.replace("executable = true", "executable_members = [\"bin/daemon\", \"lib/tool\"]"); + let declaring = compile_deployment_plan( + recipe.as_bytes(), + BINDING.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(18001), + 110, + &[ready_odin_provider("odin", "odin-yggdrasil", 1)], + ) + .unwrap(); + let honest = ArtifactReceipt { + executable: false, + executable_members: BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]), + ..artifact_receipt() + }; + let release = SealedRelease::new( + &declaring, + vec![honest.clone()], + vec![external_input_receipt()], + 120, + ) + .unwrap(); + release.validate_against(&declaring).unwrap(); + for drifted in [ + BTreeSet::from([PathBuf::from("bin/daemon")]), + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/other")]), + BTreeSet::new(), + ] { + let mut receipt = honest.clone(); + receipt.executable_members = drifted; + assert!( + SealedRelease::new(&declaring, vec![receipt], vec![external_input_receipt()], 120) + .is_err() + ); + } + let mut changed = release.clone(); + changed.artifacts[0].executable_members.insert(PathBuf::from("lib/extra")); + assert!(changed.validate_against(&declaring).is_err()); + } + fn binding_with_deadlines(table: &str) -> String { BINDING.replace("[placement]", &format!("{table}\n[placement]")) } diff --git a/src/drivers.rs b/src/drivers.rs index 44d9976..7dfeaee 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; @@ -25,7 +25,8 @@ use crate::deployment::{ IDUNN_RUNTIME_CANDIDATE_BIND_ENVIRONMENT, IDUNN_RUNTIME_MEMBER_BIND_ENVIRONMENT, LaunchArgument, OperatorBinding, RUNTIME_PRESENCE_IDENTITY_BINDING, RUNTIME_PRESENCE_IDENTITY_FD_NAME, RouteBinding, - RouteDriver, SourceSelectionPolicy, TargetDeclaration, WorkloadNetwork, + RouteDriver, SourceSelectionPolicy, StaticReleaseBinding, TargetDeclaration, + WorkloadNetwork, }; use crate::deployment_plan::{ ArtifactReceipt, CompiledDeploymentPlan, ExternalInputMaterializationReceipt, GitlinkTreeFact, @@ -157,6 +158,24 @@ pub struct FrozenSourceReceipt { pub snapshot_sha256: String, } +#[cfg(test)] +impl FrozenSource { + /// A frozen source over an existing tree, for tests that drive a runner. + pub(crate) fn fixture( + receipt: FrozenSourceReceipt, + facts: SourceSelectionFacts, + recipe_bytes: Vec, + root: PathBuf, + ) -> Self { + Self { + receipt, + facts, + recipe_bytes, + root, + } + } +} + impl FrozenSourceReceipt { pub fn validate_against(&self, plan: &CompiledDeploymentPlan) -> Result<()> { plan.validate()?; @@ -470,6 +489,9 @@ pub struct SystemdWorkloadObservation { pub no_new_privileges: bool, pub umask: String, pub inaccessible_paths: String, + /// Retired wire slot. systemd 257 displays `LoadCredential` as `[unprintable]` for + /// every unit, so it was never evidence: written empty, never compared. Kept so + /// compact (positional) MessagePack records of earlier Idunns still decode. pub load_credential: String, pub main_pid: u32, pub process_start_time: u64, @@ -584,7 +606,9 @@ pub trait WorkloadPort { /// /// Idunn launches candidates as transient units with `Restart=no`, so a /// unit that has failed, or that systemd no longer knows, will not come - /// back on its own. Past the fence every error is otherwise treated as + /// back on its own. Those units are collected once they fail + /// (`INCARNATION_UNIT_LIFECYCLE`), so a failed candidate usually reads as + /// forgotten rather than failed. Past the fence every error is otherwise treated as /// resumable and retried forever, which is right while a candidate can /// still recover and a trap once it cannot. fn is_permanently_stopped(&self, observation: &WorkloadObservation) -> Result; @@ -601,6 +625,273 @@ pub trait WorkloadPort { ) -> Result; } +/// The largest per-site file a static release will carry. +const STATIC_CARRY_LIMIT_BYTES: u64 = 1024 * 1024; + +/// What the flip landed: the digest of the exact bytes now inside the release. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StaticFlip { + pub carried_sha256: String, +} + +/// A flip that was refused before the link moved. It is the only error that may +/// end a static transaction before fencing: once the rename has happened, +/// nothing may say the flip did not. +#[derive(Debug)] +pub(crate) struct FlipRefused(pub(crate) String); + +impl std::fmt::Display for FlipRefused { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for FlipRefused {} + +/// The one external change a static release makes: copy the operator's site +/// file into an installed release and repoint the current link at it. The port +/// decides nothing; the engine decides when, and records what it reports. +pub trait StaticReleasePort { + /// The sealed release id the current link names: `None` when the link does + /// not exist yet, an error when it names anything but a release under the + /// binding's release root. + fn observe_current(&self, binding: &StaticReleaseBinding) -> Result>; + + /// Copy the carry into `release_id` and repoint the link at it. Nothing + /// moves unless the carry source was read, so a missing site file leaves + /// the link where it was. Repeating it is harmless. + /// + /// An error is `FlipRefused` exactly when the link did not move; the rename + /// is the last fallible step, and what follows it cannot fail the flip. + fn carry_and_flip(&self, binding: &StaticReleaseBinding, release_id: &str) + -> Result; + + /// The release is installed where the binding says and every artifact in it + /// still digests to its sealed receipt. A rollback reuses a release sealed + /// long ago, so this is the check that it is still what was released. + fn verify_installed(&self, binding: &StaticReleaseBinding, release: &SealedRelease) -> Result<()>; + + /// The digest of the carry already inside an installed release: what a + /// resumed flip records when an earlier attempt landed. + fn carried_sha256(&self, binding: &StaticReleaseBinding, release_id: &str) -> Result; +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct FilesystemStaticRelease; + +#[cfg(unix)] +impl StaticReleasePort for FilesystemStaticRelease { + fn observe_current(&self, binding: &StaticReleaseBinding) -> Result> { + let metadata = match fs::symlink_metadata(&binding.current_link) { + Ok(metadata) => metadata, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| { + format!("reading current link {}", binding.current_link.display()) + }); + } + }; + ensure!( + metadata.file_type().is_symlink(), + "current {} is not a symlink", + binding.current_link.display() + ); + let target = fs::read_link(&binding.current_link)?; + let target = if target.is_absolute() { + target + } else { + binding + .current_link + .parent() + .context("current link has no parent")? + .join(target) + }; + ensure!( + target.parent() == Some(binding.release_root.as_path()), + "current link names {}, which is not a release under {}", + target.display(), + binding.release_root.display() + ); + let id = target + .file_name() + .and_then(OsStr::to_str) + .context("current link names no release")?; + require_sha256_id(id, "release named by the current link")?; + Ok(Some(id.to_owned())) + } + + fn verify_installed(&self, binding: &StaticReleaseBinding, release: &SealedRelease) -> Result<()> { + let root = binding.release_root.join(&release.sealed_release_id); + ensure!( + fs::symlink_metadata(&root).is_ok_and(|metadata| metadata.is_dir()), + "release {} is no longer installed under {}", + release.sealed_release_id, + binding.release_root.display() + ); + for artifact in &release.artifacts { + let (sha256, size_bytes) = digest_artifact(&root.join(&artifact.destination)) + .with_context(|| format!("digesting installed artifact {}", artifact.artifact_id))?; + ensure!( + format!("sha256-{sha256}") == artifact.sha256 && size_bytes == artifact.size_bytes, + "installed artifact {} of release {} differs from its sealed receipt", + artifact.artifact_id, + release.sealed_release_id + ); + } + Ok(()) + } + + fn carried_sha256(&self, binding: &StaticReleaseBinding, release_id: &str) -> Result { + require_sha256_id(release_id, "release carrying the site file")?; + let carried = binding + .release_root + .join(release_id) + .join(&binding.carry.destination); + ensure!( + fs::symlink_metadata(&carried)?.is_file(), + "carried {} is not a regular file", + carried.display() + ); + let mut hasher = Sha256::new(); + hash_file_exact(&carried, fs::metadata(&carried)?.len(), &mut hasher)?; + Ok(format!("sha256-{:x}", hasher.finalize())) + } + + fn carry_and_flip( + &self, + binding: &StaticReleaseBinding, + release_id: &str, + ) -> Result { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt, symlink}; + + // Everything up to and including the rename can refuse, and only that + // refuses: the link has not moved when `FlipRefused` is returned. + let refuse = |error: anyhow::Error| anyhow::Error::new(FlipRefused(format!("{error:#}"))); + let staged = (|| -> Result<(PathBuf, PathBuf, String)> { + require_sha256_id(release_id, "release to flip to")?; + let release = binding.release_root.join(release_id); + let release_metadata = fs::symlink_metadata(&release) + .with_context(|| format!("installed release {}", release.display()))?; + ensure!( + release_metadata.is_dir(), + "installed release {} is not a directory", + release.display() + ); + let link_parent = binding + .current_link + .parent() + .context("current link has no parent")?; + match fs::symlink_metadata(&binding.current_link) { + Ok(metadata) => ensure!( + metadata.file_type().is_symlink(), + "current {} is not a symlink and will not be replaced", + binding.current_link.display() + ), + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => return Err(error).context("reading current link"), + } + + let source = &binding.carry.source; + let source_metadata = fs::symlink_metadata(source) + .with_context(|| format!("reading carry source {}", source.display()))?; + ensure!( + source_metadata.is_file(), + "carry source {} is not a regular file", + source.display() + ); + ensure!( + source_metadata.len() <= STATIC_CARRY_LIMIT_BYTES, + "carry source {} is larger than {STATIC_CARRY_LIMIT_BYTES} bytes", + source.display() + ); + let mut bytes = Vec::new(); + fs::File::open(source)? + .take(STATIC_CARRY_LIMIT_BYTES + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() as u64 <= STATIC_CARRY_LIMIT_BYTES, + "carry source {} grew past {STATIC_CARRY_LIMIT_BYTES} bytes", + source.display() + ); + let carried_sha256 = format!("sha256-{:x}", Sha256::digest(&bytes)); + + let carried = release.join(&binding.carry.destination); + let temporary_carry = release.join(format!(".carry-{}", Uuid::new_v4())); + let written = (|| -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o444) + .open(&temporary_carry)?; + // The creation mode is masked by the service's umask + // (UMask=0027 yields 0440), and the web server reads this file + // as "other"; say 0444 on the open handle. + file.set_permissions(fs::Permissions::from_mode(0o444))?; + file.write_all(&bytes)?; + file.sync_all()?; + fs::rename(&temporary_carry, &carried)?; + fs::File::open(&release)?.sync_all()?; + Ok(()) + })(); + if let Err(error) = written { + let _ = fs::remove_file(&temporary_carry); + return Err(error) + .with_context(|| format!("carrying the site file into {}", release.display())); + } + + // A link is renamed over, never edited: a reader sees the old + // release or the new one. A temporary link a crashed flip left is + // replaced. + let temporary_link = link_parent.join(format!( + ".{}.flip", + binding + .current_link + .file_name() + .and_then(OsStr::to_str) + .context("current link has no name")? + )); + if fs::symlink_metadata(&temporary_link).is_ok() { + fs::remove_file(&temporary_link).context("removing a stale flip link")?; + } + symlink(&release, &temporary_link).context("creating the flip link")?; + Ok((temporary_link, release, carried_sha256)) + })() + .map_err(refuse)?; + let (temporary_link, _, carried_sha256) = staged; + if let Err(error) = fs::rename(&temporary_link, &binding.current_link) { + let _ = fs::remove_file(&temporary_link); + return Err(refuse( + anyhow::Error::new(error).context("repointing the current link"), + )); + } + // The link has moved and nothing below can say it did not. The parent + // directory sync is durability only, so it is best-effort. + if let Some(parent) = binding.current_link.parent() { + let _ = fs::File::open(parent).and_then(|directory| directory.sync_all()); + } + Ok(StaticFlip { carried_sha256 }) + } +} + +#[cfg(not(unix))] +impl StaticReleasePort for FilesystemStaticRelease { + fn observe_current(&self, _: &StaticReleaseBinding) -> Result> { + bail!("a static release flip requires Unix symlinks") + } + + fn carry_and_flip(&self, _: &StaticReleaseBinding, _: &str) -> Result { + bail!("a static release flip requires Unix symlinks") + } + + fn carried_sha256(&self, _: &StaticReleaseBinding, _: &str) -> Result { + bail!("a static release flip requires Unix symlinks") + } + + fn verify_installed(&self, _: &StaticReleaseBinding, _: &SealedRelease) -> Result<()> { + bail!("a static release flip requires Unix symlinks") + } +} + pub trait TopologyPort { fn publish_expected( &self, @@ -2149,6 +2440,115 @@ pub(crate) fn docker_run_args( Ok(args) } +/// Bytes of each output stream that a failed runner step keeps. The daemon +/// never holds more than this per stream while the step runs. +pub(crate) const RUNNER_OUTPUT_TAIL_BYTES: usize = 8192; + +/// What a runner container was running: a recipe step or an external input, +/// by the id the recipe gave it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RunnerSubject { + Step(String), + ExternalInput(String), +} + +impl RunnerSubject { + pub(crate) fn id(&self) -> &str { + match self { + Self::Step(id) | Self::ExternalInput(id) => id, + } + } + + fn noun(&self) -> &'static str { + match self { + Self::Step(_) => "step", + Self::ExternalInput(_) => "external input", + } + } +} + +/// A runner container that did not exit successfully: what failed, how, and +/// the last `RUNNER_OUTPUT_TAIL_BYTES` of each stream with its total size. +/// The tails are evidence for `runner-evidence.cc`. `Display` and `Debug` +/// carry none of them, so no error chain, record or log line renders output. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RunnerStepFailure { + pub(crate) subject: RunnerSubject, + pub(crate) runner: String, + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stdout_total_bytes: u64, + pub(crate) stderr_tail: Vec, + pub(crate) stderr_total_bytes: u64, +} + +impl RunnerStepFailure { + /// The status as an operator reads it. + pub(crate) fn status(&self) -> String { + runner_status(self.exit_code, self.signal) + } +} + +/// How a container ended, as an operator reads it. +pub(crate) fn runner_status(exit_code: Option, signal: Option) -> String { + match (exit_code, signal) { + (Some(code), _) => format!("exit status {code}"), + (None, Some(signal)) => format!("signal {signal}"), + (None, None) => "an unknown status".to_owned(), + } +} + +impl std::fmt::Display for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "runner {} {} on runner {} exited with {}; its output tail is kept in runner-evidence.cc", + self.subject.noun(), + self.subject.id(), + self.runner, + self.status() + ) + } +} + +impl std::fmt::Debug for RunnerStepFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerStepFailure") + .field("subject", &self.subject) + .field("runner", &self.runner) + .field("exit_code", &self.exit_code) + .field("signal", &self.signal) + .field("stdout_total_bytes", &self.stdout_total_bytes) + .field("stderr_total_bytes", &self.stderr_total_bytes) + .finish_non_exhaustive() + } +} + +impl std::error::Error for RunnerStepFailure {} + +/// Read `reader` to its end keeping only the last `cap` bytes and counting +/// every byte, so a verbose step never buffers whole in the daemon. +fn drain_tail(mut reader: impl Read, cap: usize) -> std::io::Result<(Vec, u64)> { + let mut tail = Vec::new(); + let mut total = 0u64; + let mut chunk = [0u8; 8192]; + loop { + let read = match reader.read(&mut chunk) { + Ok(0) => return Ok((tail, total)), + Ok(read) => read, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + total += read as u64; + tail.extend_from_slice(&chunk[..read]); + if tail.len() > cap { + tail.drain(..tail.len() - cap); + } + } +} + /// Docker is only a runner substrate. Recipe argv stays an argv vector, the /// operator binding selects the exact image/network/mount affordances, and the /// driver returns complete materialization receipts rather than build truth by @@ -2167,7 +2567,7 @@ impl Default for DockerRunnerDriver { } impl DockerRunnerDriver { - fn docker(&self, args: I) -> Result + fn docker(&self, subject: RunnerSubject, runner_id: &str, args: I) -> Result<()> where I: IntoIterator, S: AsRef, @@ -2176,23 +2576,50 @@ impl DockerRunnerDriver { self.docker_program.is_absolute(), "Docker runner program is not absolute" ); - let output = Command::new(&self.docker_program) + let mut child = Command::new(&self.docker_program) .args(args) .stdin(Stdio::null()) - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .with_context(|| format!("starting Docker runner {}", self.docker_program.display()))?; - if !output.status.success() { - bail!( - "Docker runner exited with {}: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); + let stdout = child.stdout.take().context("Docker runner has no stdout pipe")?; + let stderr = child.stderr.take().context("Docker runner has no stderr pipe")?; + let stdout = thread::spawn(move || drain_tail(stdout, RUNNER_OUTPUT_TAIL_BYTES)); + let stderr = thread::spawn(move || drain_tail(stderr, RUNNER_OUTPUT_TAIL_BYTES)); + let status = child.wait().context("waiting for the Docker runner")?; + let (stdout_tail, stdout_total_bytes) = stdout + .join() + .map_err(|_| anyhow!("the runner stdout reader panicked"))? + .context("reading the runner stdout")?; + let (stderr_tail, stderr_total_bytes) = stderr + .join() + .map_err(|_| anyhow!("the runner stderr reader panicked"))? + .context("reading the runner stderr")?; + if status.success() { + return Ok(()); } - Ok(output) + #[cfg(unix)] + let signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let signal = None; + Err(RunnerStepFailure { + subject, + runner: runner_id.to_owned(), + exit_code: status.code(), + signal, + stdout_tail, + stdout_total_bytes, + stderr_tail, + stderr_total_bytes, + } + .into()) } fn run_in_workspace( &self, + step_id: &str, + runner_id: &str, runner: &DockerRunnerBinding, workspace: &Path, working_directory: &Path, @@ -2209,8 +2636,7 @@ impl DockerRunnerDriver { build_machine_id_file(workspace)?; let spec = ContainerSpec::for_step(runner, required_environment, source_stamp)?; let args = docker_run_args(&spec, workspace, working_directory, argv)?; - self.docker(args)?; - Ok(()) + self.docker(RunnerSubject::Step(step_id.to_owned()), runner_id, args) } fn materialize_external_input( @@ -2252,7 +2678,11 @@ impl DockerRunnerDriver { input.url.clone(), ]; let args = docker_run_args(&spec, workspace, Path::new("."), &curl_argv)?; - self.docker(args)?; + self.docker( + RunnerSubject::ExternalInput(input.id.clone()), + &input.runner, + args, + )?; let bytes = fs::read(&destination).with_context(|| { format!( "reading materialized external input {}", @@ -2302,27 +2732,7 @@ impl DockerRunnerDriver { } ArtifactSource::WorktreeTree => (source.root.clone(), source.root.join(&artifact.source)), }; - ensure!(source_path.exists(), "declared artifact output is absent"); - let destination = staging_root.join(&artifact.destination); - ensure!( - destination.starts_with(staging_root), - "artifact destination escaped its release staging root" - ); - copy_artifact(&containment_root, &source_path, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact output differs from its recipe-pinned digest" - ); - } - Ok(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - }) + seal_artifact(artifact, &containment_root, &source_path, staging_root) } } @@ -2394,6 +2804,8 @@ impl RunnerPort for DockerRunnerDriver { ); } self.run_in_workspace( + &step.id, + &step.runner, runner, workspace, &step.working_directory, @@ -2432,6 +2844,13 @@ impl RunnerPort for DockerRunnerDriver { } } +/// How every incarnation unit lives and dies. It runs once (`Restart=no`) +/// and systemd forgets it once it is inactive or failed (`--collect`): the +/// outcome is Idunn's to record in its own store, and a failed transient unit +/// left loaded is only clutter that marks the host degraded. +const INCARNATION_UNIT_LIFECYCLE: [&str; 3] = + ["--collect", "--property=Type=exec", "--property=Restart=no"]; + /// systemd owns process execution. This driver lowers one validated launch /// contract into a transient unit and then proves the native process and /// executable that systemd actually started. It does not decide admission, @@ -2479,7 +2898,6 @@ fn changed_systemd_workload_identity_fields( no_new_privileges, umask, inaccessible_paths, - load_credential, main_pid, process_start_time, process_uids, @@ -2753,7 +3171,7 @@ impl SystemdTransientWorkloadDriver { ) -> Result { release.release.validate_against(plan)?; let (_, binding) = plan.parsed_inputs()?; - let release_root = &binding.workload.systemd()?.release_root; + let release_root = binding.workload.release_root()?; fs::create_dir_all(release_root) .with_context(|| format!("creating release root {}", release_root.display()))?; let installed = release_root.join(&release.release.sealed_release_id); @@ -2805,8 +3223,7 @@ impl SystemdTransientWorkloadDriver { && installed.root == binding .workload - .systemd()? - .release_root + .release_root()? .join(&release.sealed_release_id), "installed release observation belongs to another sealed release" ); @@ -2879,7 +3296,6 @@ impl SystemdTransientWorkloadDriver { OsString::from("--property=NoNewPrivileges"), OsString::from("--property=UMask"), OsString::from("--property=InaccessiblePaths"), - OsString::from("--property=LoadCredential"), OsString::from("--property=OpenFile"), OsString::from("--property=WorkingDirectory"), OsString::from("--property=ControlGroup"), @@ -2991,38 +3407,7 @@ impl SystemdTransientWorkloadDriver { let umask = required_systemd_property(&values, "UMask")?.to_owned(); let inaccessible_paths = required_systemd_property(&values, "InaccessiblePaths")?.to_owned(); - let load_credential = required_systemd_property(&values, "LoadCredential")?.to_owned(); - ensure!( - service_type == "exec" - && restart_policy == "no" - && kill_mode == "mixed" - // `User` is deliberately not asserted here. Idunn never emits a - // User property, and systemd reports the identity it *allocated* - // for a DynamicUser unit -- the unit name, or a generated `_du…` - // when that is too long -- so the property is never empty for a - // running workload and cannot tell an allocated identity from a - // pinned one. The invariant it was reaching for, that the workload - // does not run as a persistent named account, is carried by - // `dynamic_user` below and by Idunn never passing User at all. - && dynamic_user - && supplementary_groups.is_empty() - && capability_bounding_set.is_empty() - && ambient_capabilities.is_empty() - && private_mounts - && private_pids - && protect_proc == "invisible" - && proc_subset == "all" - && no_new_privileges - && umask == "0007" - && inaccessible_paths == self.credential_root.display().to_string() - && load_credential - == if service_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }, - "systemd workload isolation properties differ from the Idunn contract" - ); + require_systemd_isolation_contract(&values, &self.credential_root)?; let working_directory = PathBuf::from(required_systemd_property(&values, "WorkingDirectory")?); ensure!( @@ -3164,6 +3549,11 @@ impl SystemdTransientWorkloadDriver { ); Some(path) }; + require_delivered_credentials( + &process_root.join("root"), + unit, + service_credential_names, + )?; let activation_descriptor = &parent_only_file_descriptors[0]; ensure!( activation_descriptor.fd_name == IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME @@ -3271,7 +3661,7 @@ impl SystemdTransientWorkloadDriver { no_new_privileges, umask, inaccessible_paths, - load_credential, + load_credential: String::new(), main_pid, process_start_time, process_uids: process_security.uids, @@ -3312,11 +3702,11 @@ impl SystemdTransientWorkloadDriver { installed: &Path, ) -> Result> { let executable_artifact = - release_artifact(declaration, &declaration.service.executable_artifact)?; + release_artifact(declaration, &declaration.service()?.executable_artifact)?; let executable = installed.join(&executable_artifact.destination); ensure!(executable.is_file(), "sealed service executable is absent"); let mut command = vec![executable.into_os_string()]; - for argument in &declaration.service.arguments { + for argument in &declaration.service()?.arguments { command.push(match argument { LaunchArgument::Literal { value } => value.into(), LaunchArgument::Binding { name } => binding.workload.systemd()?.argument_bindings @@ -3485,12 +3875,7 @@ impl SystemdTransientWorkloadDriver { // workload. The same applies to `Group` when no fixed state group is // bound. In that case the numeric primary GID must match the allocated // UID; a fixed state group still requires both its exact name and GID. - let expected_load_credential = if regular_credential_names.is_empty() { - "" - } else { - "[unprintable]" - }; - let launch_contract: [(bool, &str); 27] = [ + let launch_contract: [(bool, &str); 26] = [ ( observation.unit_description == expected_description, "unit description", @@ -3530,10 +3915,6 @@ impl SystemdTransientWorkloadDriver { observation.inaccessible_paths == self.credential_root.display().to_string(), "inaccessible paths", ), - ( - observation.load_credential == expected_load_credential, - "load credential", - ), ( observation.working_directory == installed, "working directory", @@ -3620,7 +4001,7 @@ impl SystemdTransientWorkloadDriver { ) -> Result<()> { ensure!( declaration - .service + .service()? .required_environment .contains(IDUNN_RUNTIME_BUNDLE_ENVIRONMENT), "target does not declare the standard Idunn runtime bundle" @@ -3643,7 +4024,7 @@ impl SystemdTransientWorkloadDriver { parent_only_open_file_properties(parent_only_file_descriptors)?; let environment = self.launch_environment(binding, bundle, expected, unit)?; let executable_artifact = - release_artifact(declaration, &declaration.service.executable_artifact)?; + release_artifact(declaration, &declaration.service()?.executable_artifact)?; let executable = installed.join(&executable_artifact.destination); ensure!(executable.is_file(), "sealed service executable is absent"); let unit_description = format!( @@ -3659,8 +4040,9 @@ impl SystemdTransientWorkloadDriver { OsString::from("--no-ask-password"), OsString::from("--expand-environment=no"), OsString::from(format!("--unit={unit}")), - OsString::from("--property=Type=exec"), - OsString::from("--property=Restart=no"), + ]; + args.extend(INCARNATION_UNIT_LIFECYCLE.map(OsString::from)); + args.extend([ OsString::from("--property=KillMode=mixed"), OsString::from("--property=DynamicUser=yes"), OsString::from("--property=SupplementaryGroups="), @@ -3696,7 +4078,7 @@ impl SystemdTransientWorkloadDriver { OsString::from(format!("--working-directory={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", installed.display())), OsString::from(format!("--property=ReadOnlyPaths={}", bundle.display())), - ]; + ]); for open_file in parent_only_open_files { args.push(OsString::from(format!("--property=OpenFile={open_file}"))); } @@ -3852,7 +4234,7 @@ impl WorkloadPort for SystemdTransientWorkloadDriver { let (declaration, binding) = plan.parsed_inputs()?; self.validate_writable_bindings(&binding)?; let executable_artifact = - release_artifact(&declaration, &declaration.service.executable_artifact)?; + release_artifact(&declaration, &declaration.service()?.executable_artifact)?; let executable = installed.root.join(&executable_artifact.destination); let unit = self.unit_name( &binding.workload.systemd()?.unit_prefix, @@ -6810,21 +7192,7 @@ fn hash_frozen_source_tree(root: &Path, current: &Path, hasher: &mut Sha256) -> ); hasher.update(b"\0"); hasher.update(metadata.len().to_le_bytes()); - // S9: streamed in fixed-size chunks rather than `fs::read`ing the - // whole file, so peak RSS is bounded by the buffer, not by the - // largest file in the tree. - let mut file = fs::File::open(&path) - .with_context(|| format!("opening {} to hash it", path.display()))?; - let mut buffer = [0u8; 1 << 16]; - loop { - let read = file - .read(&mut buffer) - .with_context(|| format!("reading {} to hash it", path.display()))?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - } + hash_file_exact(&path, metadata.len(), hasher)?; } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; hasher.update(b"link\0"); @@ -6846,20 +7214,28 @@ fn frozen_source_sha256(_root: &Path) -> Result { #[cfg(unix)] fn harden_installed_release(root: &Path, artifacts: &[ArtifactReceipt]) -> Result<()> { - let executable_paths = artifacts - .iter() - .filter(|artifact| artifact.executable) - .map(|artifact| root.join(&artifact.destination)) - .collect::>(); - harden_root_tree(root, root, &executable_paths) + let mut executable_paths = BTreeSet::new(); + for artifact in artifacts { + let destination = root.join(&artifact.destination); + for member in &artifact.executable_members { + executable_paths.insert(destination.join(member)); + } + if artifact.executable { + executable_paths.insert(destination); + } + } + harden_root_tree(root, &mut executable_paths)?; + ensure!( + executable_paths.is_empty(), + "declared executable is missing or not a regular file" + ); + Ok(()) } +/// Each regular file the walk seals 0555 is removed from `executable_paths`; +/// whatever remains was declared executable and never sealed. #[cfg(unix)] -fn harden_root_tree( - root: &Path, - current: &Path, - executable_paths: &std::collections::BTreeSet, -) -> Result<()> { +fn harden_root_tree(current: &Path, executable_paths: &mut BTreeSet) -> Result<()> { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let metadata = fs::symlink_metadata(current)?; @@ -6871,48 +7247,28 @@ fn harden_root_tree( let mut entries = fs::read_dir(current)?.collect::>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { - harden_root_tree(root, &entry.path(), executable_paths)?; + harden_root_tree(&entry.path(), &mut *executable_paths)?; } fs::set_permissions(current, fs::Permissions::from_mode(0o555))?; } else if metadata.is_file() { fs::set_permissions( current, - fs::Permissions::from_mode(if executable_paths.contains(current) { + fs::Permissions::from_mode(if executable_paths.remove(current) { 0o555 } else { 0o444 }), )?; } else if metadata.file_type().is_symlink() { - validate_release_symlink(root, current)?; + // Left untouched: containment was decided by check_release_link in + // digest_tree, which install_release and validate_installed_release + // run over every receipt before this walk (map T16). } else { bail!("sealed release contains a special filesystem entry") } Ok(()) } -#[cfg(unix)] -fn validate_release_symlink(root: &Path, path: &Path) -> Result<()> { - let target = fs::read_link(path)?; - ensure!(target.is_relative(), "sealed release symlink is absolute"); - ensure!( - target.components().all(|component| matches!( - component, - std::path::Component::CurDir | std::path::Component::Normal(_) - )), - "sealed release symlink has a parent traversal" - ); - let destination = path - .parent() - .context("sealed release symlink has no parent")? - .join(target); - ensure!( - destination.starts_with(root), - "sealed release symlink escaped its release" - ); - Ok(()) -} - #[cfg(not(unix))] fn harden_installed_release(_root: &Path, _artifacts: &[ArtifactReceipt]) -> Result<()> { bail!("systemd workload installation requires Unix permissions") @@ -7315,6 +7671,44 @@ fn path_inside_process_root(process_root: &Path, absolute_path: &Path) -> Result Ok(process_root.join(absolute_path.strip_prefix(Path::new("/"))?)) } +/// The workload holds exactly the credentials Idunn delivered: the entries of +/// `/run/credentials/` in the MainPID's root (`process_root` is that root) +/// equal `expected`, and an absent directory is the empty set. Only entry names +/// are read; a credential is never opened here. The message names no credential. +fn require_delivered_credentials( + process_root: &Path, + unit: &str, + expected: &[String], +) -> Result<()> { + let directory = path_inside_process_root( + process_root, + &Path::new("/run/credentials").join(unit), + )?; + let mut delivered = Vec::new(); + match fs::read_dir(&directory) { + Ok(entries) => { + for entry in entries { + let name = entry + .context("reading a delivered credential entry")? + .file_name() + .into_string() + .map_err(|_| anyhow!("delivered credential name is not UTF-8"))?; + delivered.push(name); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).context("reading the delivered credential directory"); + } + } + delivered.sort(); + ensure!( + delivered == expected, + "systemd delivered credentials that differ from the Idunn launch" + ); + Ok(()) +} + fn linux_process_security( status_path: &Path, expected_host_pid: u32, @@ -7530,6 +7924,56 @@ fn parse_systemd_boolean(values: &BTreeMap, name: &str) -> Resul } } +/// The isolation properties of a transient workload unit, judged from `systemctl +/// show`. `LoadCredential` is absent on purpose: systemd 257 displays it as +/// `[unprintable]` whatever the unit holds, so what the workload was given is +/// judged by `require_delivered_credentials` from the delivered directory. +fn require_systemd_isolation_contract( + values: &BTreeMap, + credential_root: &Path, +) -> Result<()> { + let service_type = required_systemd_property(values, "Type")?; + let restart_policy = required_systemd_property(values, "Restart")?; + let kill_mode = required_systemd_property(values, "KillMode")?; + let dynamic_user = parse_systemd_boolean(values, "DynamicUser")?; + let supplementary_groups = systemd_property(values, "SupplementaryGroups")?; + let capability_bounding_set = systemd_property(values, "CapabilityBoundingSet")?; + let ambient_capabilities = systemd_property(values, "AmbientCapabilities")?; + let private_mounts = parse_systemd_boolean(values, "PrivateMounts")?; + let private_pids = parse_systemd_boolean(values, "PrivatePIDs")?; + let protect_proc = required_systemd_property(values, "ProtectProc")?; + let proc_subset = required_systemd_property(values, "ProcSubset")?; + let no_new_privileges = parse_systemd_boolean(values, "NoNewPrivileges")?; + let umask = required_systemd_property(values, "UMask")?; + let inaccessible_paths = required_systemd_property(values, "InaccessiblePaths")?; + ensure!( + service_type == "exec" + && restart_policy == "no" + && kill_mode == "mixed" + // `User` is deliberately not asserted here. Idunn never emits a + // User property, and systemd reports the identity it *allocated* + // for a DynamicUser unit -- the unit name, or a generated `_du…` + // when that is too long -- so the property is never empty for a + // running workload and cannot tell an allocated identity from a + // pinned one. The invariant it was reaching for, that the workload + // does not run as a persistent named account, is carried by + // `dynamic_user` below and by Idunn never passing User at all. + && dynamic_user + && supplementary_groups.is_empty() + && capability_bounding_set.is_empty() + && ambient_capabilities.is_empty() + && private_mounts + && private_pids + && protect_proc == "invisible" + && proc_subset == "all" + && no_new_privileges + && umask == "0007" + && inaccessible_paths == credential_root.display().to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + Ok(()) +} + fn read_process_environment(path: &Path) -> Result>> { let bytes = fs::read(path) .with_context(|| format!("reading process environment {}", path.display()))?; @@ -8102,7 +8546,7 @@ fn copy_symlink(source: &Path, destination: &Path) -> Result<()> { /// checked to contain the fully resolved `source` with the same /// whole-chain-aware resolver `validate_frozen_source_symlink` uses, before /// anything is read from it. -pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { +fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> Result<()> { let canonical_root = root.canonicalize().context("resolving artifact root")?; let canonical_source = source .canonicalize() @@ -8125,11 +8569,124 @@ pub(crate) fn copy_artifact(root: &Path, source: &Path, destination: &Path) -> R } } +/// The one seal-time owner of mode declarations for a copied artifact: a +/// directory output is never `executable` (it declares `executable_members`), +/// a file output never declares members, and every member is a regular file +/// reached through real directories, never through a symlink. Errors name the +/// rule, never the member path. +fn check_artifact_modes(artifact: &ArtifactOutput, destination: &Path) -> Result<()> { + let is_dir = fs::symlink_metadata(destination) + .context("inspecting a copied artifact")? + .is_dir(); + ensure!( + !(is_dir && artifact.executable), + "directory artifact {} cannot be executable; declare executable_members", + artifact.id + ); + if !is_dir { + ensure!( + artifact.executable_members.is_empty(), + "file artifact {} cannot declare executable_members", + artifact.id + ); + return Ok(()); + } + for member in &artifact.executable_members { + ensure!(!member.as_os_str().is_empty(), "executable member is empty"); + let mut current = destination.to_path_buf(); + let mut components = member.components().peekable(); + while let Some(component) = components.next() { + let std::path::Component::Normal(name) = component else { + bail!("executable member has a forbidden path component"); + }; + current.push(name); + let metadata = match fs::symlink_metadata(¤t) { + Ok(metadata) => metadata, + Err(error) if error.kind() == ErrorKind::NotFound => { + bail!("executable member is missing") + } + Err(error) => return Err(error).context("inspecting an executable member"), + }; + if components.peek().is_some() { + ensure!( + metadata.is_dir(), + "executable member passes through a symlink or non-directory" + ); + } else { + ensure!(metadata.is_file(), "executable member is not a regular file"); + } + } + } + Ok(()) +} + +/// The one seal step for a declared artifact, for systemd and host targets +/// alike: copy it into the staging root, check its mode declarations, digest it +/// and check the recipe pin. Modes are applied by harden on systemd targets; +/// host targets (Windows, map T15) carry `executable` and `executable_members` +/// in the receipt and do not apply them. +pub(crate) fn seal_artifact( + artifact: &ArtifactOutput, + containment_root: &Path, + source_path: &Path, + staging_root: &Path, +) -> Result { + ensure!( + source_path.exists(), + "declared artifact output {} is absent", + artifact.id + ); + let destination = staging_root.join(&artifact.destination); + ensure!( + destination.starts_with(staging_root), + "artifact {} destination escaped its staging root", + artifact.id + ); + copy_artifact(containment_root, source_path, &destination)?; + check_artifact_modes(artifact, &destination)?; + let (sha256, size_bytes) = digest_artifact(&destination)?; + if let Some(expected) = &artifact.expected_sha256 { + ensure!( + &sha256 == expected, + "artifact {} differs from its recipe-pinned digest", + artifact.id + ); + } + Ok(ArtifactReceipt { + artifact_id: artifact.id.clone(), + destination: artifact.destination.clone(), + sha256: format!("sha256-{sha256}"), + size_bytes, + executable: artifact.executable, + executable_members: artifact.executable_members.clone(), + }) +} + +/// Streams `path` into `hasher` in 64 KiB chunks, so peak memory is the buffer +/// and not the largest file in the tree, and refuses if the bytes read differ +/// from `expected_len`. The only owner of the length check. +pub(crate) fn hash_file_exact(path: &Path, expected_len: u64, hasher: &mut Sha256) -> Result<()> { + let mut file = fs::File::open(path).context("opening a file to hash it")?; + let mut buffer = [0u8; 1 << 16]; + let mut total = 0_u64; + loop { + let read = file.read(&mut buffer).context("reading a file to hash it")?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + total += read as u64; + } + ensure!(total == expected_len, "file changed while hashing"); + Ok(()) +} + pub(crate) fn digest_artifact(path: &Path) -> Result<(String, u64)> { let metadata = fs::symlink_metadata(path)?; if metadata.is_file() { - let bytes = fs::read(path)?; - return Ok((raw_sha256(&bytes), bytes.len().try_into()?)); + let mut hasher = Sha256::new(); + hash_file_exact(path, metadata.len(), &mut hasher)?; + return Ok((format!("{:x}", hasher.finalize()), metadata.len())); } ensure!(metadata.is_dir(), "artifact is not a file or directory"); let mut hasher = Sha256::new(); @@ -8153,15 +8710,16 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) hasher.update(b"\0"); digest_tree(root, &path, hasher, size)?; } else if metadata.is_file() { - let bytes = fs::read(&path)?; + let length = metadata.len(); hasher.update(b"file\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); - hasher.update((bytes.len() as u64).to_le_bytes()); - hasher.update(&bytes); - *size = size.saturating_add(bytes.len().try_into()?); + hasher.update(length.to_le_bytes()); + hash_file_exact(&path, length, hasher)?; + *size = size.saturating_add(length); } else if metadata.file_type().is_symlink() { let target = fs::read_link(&path)?; + check_release_link(path.strip_prefix(root)?, &target)?; hasher.update(b"link\0"); hasher.update(relative.as_bytes()); hasher.update(b"\0"); @@ -8174,6 +8732,35 @@ fn digest_tree(root: &Path, current: &Path, hasher: &mut Sha256, size: &mut u64) Ok(()) } +/// The one judge of link containment in a release tree, pure and static: +/// `link` is the link's path relative to its artifact's top and `target` what +/// it reads. A target is relative, its `..` steps all come first, and they +/// climb no higher than the link's own directory depth, so the link can only +/// land inside the artifact. Lexical normalisation of parent and target is not +/// enough: with `s -> ..` it admits `s/../x` and escapes (map T14). +fn check_release_link(link: &Path, target: &Path) -> Result<()> { + use std::path::Component; + + let depth = link.components().count().saturating_sub(1); + let mut climbs = 0_usize; + let mut named = false; + for component in target.components() { + match component { + Component::CurDir => {} + Component::RootDir | Component::Prefix(_) => { + bail!("sealed release symlink is absolute") + } + Component::ParentDir => { + ensure!(!named, "sealed release symlink climbs after a name"); + climbs += 1; + } + Component::Normal(_) => named = true, + } + } + ensure!(climbs <= depth, "sealed release symlink escapes its artifact"); + Ok(()) +} + pub(crate) fn raw_sha256(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } @@ -8204,7 +8791,7 @@ fn require_git_sha(value: &str, label: &str) -> Result<()> { Ok(()) } -fn require_sha256_id(value: &str, label: &str) -> Result<()> { +pub(crate) fn require_sha256_id(value: &str, label: &str) -> Result<()> { let digest = value .strip_prefix("sha256-") .with_context(|| format!("{label} has no sha256 prefix"))?; @@ -9632,6 +10219,16 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// An incarnation unit runs once and is collected when it ends, so a + /// failed candidate does not stay loaded and mark the host degraded. + /// Collection is safe because a forgotten unit already reads as + /// permanently stopped (the test below). + #[test] + fn incarnation_units_run_once_and_are_collected() { + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--collect")); + assert!(INCARNATION_UNIT_LIFECYCLE.contains(&"--property=Restart=no")); + } + /// Admission audit, claim 3: `is_permanently_stopped` now lets the /// Isolation gate skip `prove_isolation` against the incumbent. Enumerate /// what it answers `true` for, using a fake `systemctl show` that replays @@ -9871,6 +10468,168 @@ processes 9 ); } + #[cfg(unix)] + fn fixture_show_properties(text: &str) -> Result> { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir()?; + let shown = temp.path().join("shown"); + std::fs::write(&shown, text)?; + let systemctl = temp.path().join("systemctl"); + std::fs::write( + &systemctl, + format!("#!/bin/sh\ncat '{}'\nexit 0\n", shown.display()), + )?; + std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o755))?; + let driver = SystemdTransientWorkloadDriver { + systemctl_program: systemctl, + ..Default::default() + }; + Ok(driver + .show_unit("idunn-test.service")? + .context("fixture unit was not found")? + .properties) + } + + /// systemd 257 prints `LoadCredential=[unprintable]` for every unit, so the + /// isolation contract must admit the real output of a unit with no service + /// credential as well as one with one. The inputs are real captures, not a + /// re-spelling. + #[cfg(unix)] + #[test] + fn systemd_257_isolation_contract_admits_a_presence_only_target_and_one_with_service_credentials( + ) -> Result<()> { + let root = Path::new("/run/idunn/activation-credentials"); + let vili = include_str!("../tests/fixtures/systemd-257/vili-presence-identity-only.show"); + let odin = include_str!( + "../tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show" + ); + for text in [vili, odin] { + assert!(text.contains("LoadCredential=[unprintable]")); + require_systemd_isolation_contract(&fixture_show_properties(text)?, root)?; + } + let default_proc = vili.replace("ProtectProc=invisible", "ProtectProc=default"); + assert_ne!(default_proc, vili); + let other_path = vili.replace( + "/run/idunn/activation-credentials", + "/run/idunn/elsewhere", + ); + assert_ne!(other_path, vili); + for text in [default_proc, other_path] { + let error = + require_systemd_isolation_contract(&fixture_show_properties(&text)?, root) + .expect_err("a drifted isolation property must be refused"); + assert_eq!( + error.to_string(), + "systemd workload isolation properties differ from the Idunn contract" + ); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn delivered_credentials_are_exactly_the_launch_names() -> Result<()> { + let unit = "idunn-test.service"; + let names = |list: &[&str]| list.iter().map(|name| name.to_string()).collect::>(); + // `Some(entries)` creates the unit's directory in a fresh process root. + let check = |delivered: Option<&[&str]>, expected: &[&str]| -> Result<()> { + let temp = tempfile::tempdir()?; + if let Some(entries) = delivered { + let directory = temp.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + for entry in entries { + std::fs::write(directory.join(entry), "canary-credential-contents")?; + } + } + let result = require_delivered_credentials(temp.path(), unit, &names(expected)); + if let Err(error) = &result { + let text = format!("{error:#}"); + for canary in ["STOWAWAY", "ODIN_TOPOLOGY_IDENTITY", "canary-credential"] { + assert!(!text.contains(canary), "error echoed {canary}"); + } + assert!(!text.contains(&*temp.path().to_string_lossy())); + } + result + }; + check(None, &[])?; + check(Some(&[]), &[])?; + check(Some(&["ODIN_TOPOLOGY_IDENTITY"]), &["ODIN_TOPOLOGY_IDENTITY"])?; + check(Some(&["B", "A"]), &["A", "B"])?; + for (delivered, expected) in [ + (None, &["ODIN_TOPOLOGY_IDENTITY"][..]), + (Some(&[][..]), &["ODIN_TOPOLOGY_IDENTITY"][..]), + ( + Some(&["ODIN_TOPOLOGY_IDENTITY", "STOWAWAY"][..]), + &["ODIN_TOPOLOGY_IDENTITY"][..], + ), + (Some(&["STOWAWAY"][..]), &[][..]), + (Some(&["A"][..]), &["B"][..]), + (Some(&["A"][..]), &["A", "B"][..]), + ] { + let error = check(delivered, expected) + .expect_err("delivered credentials differing from the launch must be refused"); + assert_eq!( + error.to_string(), + "systemd delivered credentials that differ from the Idunn launch" + ); + } + // A directory that cannot be listed is a refusal, never the empty set. + let temp = tempfile::tempdir()?; + let parent = temp.path().join("run/credentials"); + std::fs::create_dir_all(&parent)?; + std::fs::write(parent.join(unit), "")?; + let error = require_delivered_credentials(temp.path(), unit, &[]) + .expect_err("an unreadable credential directory must be refused"); + assert!(!format!("{error:#}").contains(&*temp.path().to_string_lossy())); + // The rule reads the process root it is given, never the host's /run. + let temp = tempfile::tempdir()?; + let outside = tempfile::tempdir()?; + let directory = outside.path().join("run/credentials").join(unit); + std::fs::create_dir_all(&directory)?; + std::fs::write(directory.join("A"), "")?; + require_delivered_credentials(temp.path(), unit, &[])?; + assert!(require_delivered_credentials(temp.path(), unit, &names(&["A"])).is_err()); + Ok(()) + } + + /// Live Odin and StreamPixels hold records persisted by the Idunn that + /// compared `LoadCredential`: `[unprintable]` in the retired slot. The bytes + /// are that Idunn's own output (tests/fixtures/pre-cut-*); they must decode + /// under both encodings and read as no drift against a new observation, or + /// `stop` refuses to stop the incumbents. + #[test] + fn an_incumbent_recorded_with_the_display_form_is_not_identity_drift() -> Result<()> { + let decode_hex = |text: &str| -> Vec { + (0..text.trim().len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text.trim()[at..at + 2], 16).unwrap()) + .collect() + }; + let mut recorded = systemd_audit_observation("no", 61_000); + recorded.load_credential = "[unprintable]".into(); + let observed = systemd_audit_observation("no", 61_000); + assert!(observed.load_credential.is_empty()); + for hex in [ + include_str!("../tests/fixtures/pre-cut-systemd-observation.compact.hex"), + include_str!("../tests/fixtures/pre-cut-systemd-observation.named.hex"), + ] { + let decoded: WorkloadObservation = rmp_serde::from_slice(&decode_hex(hex))?; + assert_eq!(decoded, WorkloadObservation::Systemd(recorded.clone())); + let WorkloadObservation::Systemd(prior) = decoded else { + unreachable!() + }; + assert!(changed_systemd_workload_identity_fields(&observed, &prior).is_empty()); + } + let mut drifted = observed.clone(); + drifted.main_pid += 1; + assert_eq!( + changed_systemd_workload_identity_fields(&drifted, &recorded), + vec!["main_pid"] + ); + Ok(()) + } + #[test] fn host_isolation_is_two_live_distinct_processes_on_one_host() { let live = |pid: u32, created: u64| { @@ -13862,6 +14621,7 @@ processes 9 sha256: sha256_id(b"sealed"), size_bytes: 6, executable: true, + executable_members: BTreeSet::new(), }], )?; let root_metadata = fs::symlink_metadata(&root)?; @@ -13873,6 +14633,554 @@ processes 9 Ok(()) } + #[cfg(unix)] + const MEMBER_CANARY: &str = "zzcanary"; + + #[cfg(unix)] + fn tree_output(executable: bool, members: &[&str]) -> ArtifactOutput { + ArtifactOutput { + id: "upstream".into(), + source_kind: ArtifactSource::WorktreeTree, + runner: None, + source: PathBuf::from("upstream"), + destination: PathBuf::from("upstream"), + expected_sha256: None, + executable, + executable_members: members.iter().map(PathBuf::from).collect(), + } + } + + #[cfg(unix)] + fn write_file(path: &Path, bytes: &[u8]) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + fs::create_dir_all(path.parent().context("file has no parent")?)?; + fs::write(path, bytes)?; + // Builds and git leave arbitrary exec bits; the seal must not inherit them. + fs::set_permissions(path, fs::Permissions::from_mode(0o777))?; + Ok(()) + } + + #[cfg(unix)] + fn sealed_mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + + fs::symlink_metadata(path).unwrap().permissions().mode() & 0o777 + } + + /// `/src/upstream` is the worktree tree; `/staging` the release. + #[cfg(unix)] + fn sample_tree(root: &Path) -> Result<(PathBuf, PathBuf)> { + let source = root.join("src"); + let staging = root.join("staging"); + fs::create_dir_all(&staging)?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/daemon"), b"daemon")?; + write_file(&tree.join("bin/helper"), b"helper")?; + write_file(&tree.join("lib/tool"), b"tool")?; + write_file(&tree.join("lib/data.txt"), b"data")?; + write_file(&tree.join("NOTICE"), b"notice")?; + Ok((source, staging)) + } + + #[cfg(unix)] + fn collect_tree( + source: &Path, + staging: &Path, + output: ArtifactOutput, + ) -> Result { + let mut declaration = + TargetDeclaration::parse(crate::deployment::tests::RECIPE)?; + let id = output.id.clone(); + declaration.artifacts = vec![output]; + let frozen = FrozenSource { + receipt: FrozenSourceReceipt { + transaction_id: "tx".into(), + plan_id: "plan".into(), + snapshot_sha256: "snapshot".into(), + }, + facts: crate::deployment_plan::tests::source(crate::deployment::tests::RECIPE), + recipe_bytes: Vec::new(), + root: source.to_path_buf(), + }; + DockerRunnerDriver::default().collect_artifact( + &declaration, + &frozen, + &BTreeMap::new(), + staging, + &id, + ) + } + + #[cfg(unix)] + fn refusal_text(result: Result) -> String { + let Err(error) = result else { + panic!("the release was not refused"); + }; + let text = format!("{error:#}"); + assert!( + !text.contains(MEMBER_CANARY), + "the refusal echoed a member path: {text}" + ); + text + } + + #[cfg(unix)] + #[test] + fn tree_member_sealed_0555_and_siblings_0444() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let receipt = collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib/tool"]), + )?; + assert_eq!( + receipt.executable_members, + BTreeSet::from([PathBuf::from("bin/daemon"), PathBuf::from("lib/tool")]) + ); + harden_installed_release(&staging, &[receipt])?; + let tree = staging.join("upstream"); + assert_eq!(sealed_mode(&tree.join("bin/daemon")), 0o555); + assert_eq!(sealed_mode(&tree.join("lib/tool")), 0o555); + assert_eq!(sealed_mode(&tree.join("bin/helper")), 0o444); + assert_eq!(sealed_mode(&tree.join("lib/data.txt")), 0o444); + assert_eq!(sealed_mode(&tree.join("NOTICE")), 0o444); + for directory in [&staging, &tree, &tree.join("bin"), &tree.join("lib")] { + assert_eq!(sealed_mode(directory), 0o555); + } + Ok(()) + } + + #[test] + fn release_link_rule_is_containment() { + for (link, target) in [ + ("a/b/l", "../t/x"), + ("a/l", ".."), + ("a/l", "./../b"), + ("a/l", "b/./c"), + ("l", "python3.12"), + ("a/b/c/l", "../../x"), + ] { + check_release_link(Path::new(link), Path::new(target)) + .unwrap_or_else(|error| panic!("({link}, {target}) refused: {error:#}")); + } + for (link, target, rule) in [ + ("l", "/x", "absolute"), + ("a/l", "s/../x", "climbs after a name"), + ("a/l", "D/D/../../../etc/passwd", "climbs after a name"), + ("a/l", "../../x", "escapes its artifact"), + ("l", "..", "escapes its artifact"), + ("a/b/l", "../../../x", "escapes its artifact"), + ("a/l", "./s/../x", "climbs after a name"), + ] { + let text = format!( + "{:#}", + check_release_link(Path::new(link), Path::new(target)) + .expect_err(&format!("({link}, {target}) was admitted")) + ); + assert!(text.contains(rule), "({link}, {target}): {text}"); + } + } + + #[cfg(unix)] + #[test] + fn missing_member_refuses_the_release() -> Result<()> { + // The bad member sorts after the good one in one case and before it in + // the other, so a check that stops at the first (or last) member fails. + for members in [ + &["bin/daemon", "lib/zzcanary-missing"][..], + &["aaa-zzcanary-missing", "bin/daemon"][..], + &["zzcanary-dir/tool"][..], + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, members))); + assert!(text.contains("executable member is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn symlink_member_refuses_the_release() -> Result<()> { + for link_target in ["daemon", "../zzcanary-nowhere"] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink( + link_target, + source.join("upstream/bin/zzcanary-link"), + )?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "bin/zzcanary-link"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn member_through_symlinked_directory_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("real-bin/zzcanary-tool"), b"tool")?; + std::os::unix::fs::symlink("real-bin", tree.join("linked-bin"))?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["linked-bin/zzcanary-tool"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + // A member below a regular file is refused the same way. + let second_staging = temp.path().join("second-staging"); + fs::create_dir_all(&second_staging)?; + let text = refusal_text(collect_tree( + &source, + &second_staging, + tree_output(false, &["bin/daemon/zzcanary-below-file"]), + )); + assert!(text.contains("symlink or non-directory"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn directory_member_refuses_the_release() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree( + &source, + &staging, + tree_output(false, &["bin/daemon", "lib"]), + )); + assert!(text.contains("not a regular file"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn executable_flag_on_tree_output_refuses() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(true, &[]))); + assert!(text.contains("cannot be executable"), "{text}"); + // The same flag on a file output stays admitted. + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + let mut output = tree_output(true, &[]); + output.id = "file".into(); + check_artifact_modes(&output, &file)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn members_on_file_output_refuse() -> Result<()> { + let temp = tempfile::tempdir()?; + let file = temp.path().join("file-artifact"); + fs::write(&file, b"x")?; + check_artifact_modes(&tree_output(false, &[]), &file)?; + let text = refusal_text(check_artifact_modes( + &tree_output(false, &["zzcanary"]), + &file, + )); + assert!(text.contains("cannot declare executable_members"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn check_artifact_modes_refuses_a_member_that_is_not_a_clean_relative_path() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&temp.path().join("zzcanary-outside"), b"x")?; + for member in ["../../zzcanary-outside", "bin/../bin/daemon", "/zzcanary", ""] { + refusal_text(check_artifact_modes(&tree_output(false, &[member]), &tree)); + } + check_artifact_modes(&tree_output(false, &["bin/daemon"]), &tree)?; + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unreadable_member_is_not_reported_as_missing() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, _) = sample_tree(temp.path())?; + // A component past NAME_MAX fails with an io error that is not NotFound. + let long = "z".repeat(300); + let text = refusal_text(check_artifact_modes( + &tree_output(false, &[long.as_str()]), + &source.join("upstream"), + )); + assert!(!text.contains("missing"), "{text}"); + assert!(text.contains("inspecting an executable member"), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn harden_fails_when_a_declared_member_is_absent() -> Result<()> { + let receipt = |members: &[&str]| ArtifactReceipt { + artifact_id: "upstream".into(), + destination: PathBuf::from("upstream"), + sha256: sha256_id(b"x"), + size_bytes: 1, + executable: false, + executable_members: members.iter().map(PathBuf::from).collect(), + }; + let installed = |root: &Path| -> Result<()> { + write_file(&root.join("upstream/bin/daemon"), b"daemon")?; + write_file(&root.join("upstream/lib/tool"), b"tool")?; + fs::create_dir_all(root.join("upstream/zzcanary-dir"))?; + std::os::unix::fs::symlink("daemon", root.join("upstream/bin/zzcanary-link"))?; + Ok(()) + }; + let temp = tempfile::tempdir()?; + installed(temp.path())?; + harden_installed_release(temp.path(), &[receipt(&["bin/daemon", "lib/tool"])])?; + for members in [ + &["bin/daemon", "lib/zzcanary-gone"][..], + &["zzcanary-gone", "bin/daemon"][..], + &["zzcanary-dir"][..], + &["bin/zzcanary-link"][..], + &["bin/daemon/zzcanary-below-file"][..], + ] { + let temp = tempfile::tempdir()?; + installed(temp.path())?; + let text = refusal_text(harden_installed_release(temp.path(), &[receipt(members)])); + assert!(text.contains("declared executable is missing"), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn relative_in_tree_symlink_survives_the_seal() -> Result<()> { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let tree = source.join("upstream"); + write_file(&tree.join("bin/python3.12"), b"python")?; + std::os::unix::fs::symlink("python3.12", tree.join("bin/python3"))?; + write_file(&tree.join("lib/terminfo/b/real"), b"terminfo")?; + fs::create_dir_all(tree.join("lib/terminfo/a"))?; + std::os::unix::fs::symlink("../b/real", tree.join("lib/terminfo/a/alias"))?; + let receipt = collect_tree(&source, &staging, tree_output(false, &["bin/python3.12"]))?; + let (before, _) = digest_artifact(&staging.join("upstream"))?; + harden_installed_release(&staging, &[receipt])?; + let link = staging.join("upstream/bin/python3"); + assert!(fs::symlink_metadata(&link)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&link)?, PathBuf::from("python3.12")); + assert_eq!(sealed_mode(&staging.join("upstream/bin/python3.12")), 0o555); + let alias = staging.join("upstream/lib/terminfo/a/alias"); + assert!(fs::symlink_metadata(&alias)?.file_type().is_symlink()); + assert_eq!(fs::read_link(&alias)?, PathBuf::from("../b/real")); + assert_eq!(digest_artifact(&staging.join("upstream"))?.0, before); + + // A link that escapes, is absolute or climbs after a name is refused at + // the seal, before any receipt exists. + for (target, rule) in [ + ("../../zzcanary-up", "escapes its artifact"), + ("/zzcanary/absolute", "absolute"), + ("zzcanary-s/../x", "climbs after a name"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + std::os::unix::fs::symlink(target, source.join("upstream/lib/zzcanary-link"))?; + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); + assert!(text.contains(rule), "{text}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn links_to_existing_targets_are_refused_at_the_seal() -> Result<()> { + // Every refused shape, aimed at a target that EXISTS (file and + // directory): a judge guarded by `exists()`, `is_dir()` or a file-type + // test passes the dangling fixtures and fails these. + for kind in ["file", "dir"] { + for (shape, rule) in [ + ("absolute", "absolute"), + ("escape", "escapes its artifact"), + ("after-name", "climbs after a name"), + ] { + let temp = tempfile::tempdir()?; + let (source, staging) = sample_tree(temp.path())?; + let target = match (shape, kind) { + ("absolute", "file") => source.join("upstream/bin/daemon"), + ("absolute", _) => source.join("upstream/lib"), + ("escape", "file") => { + write_file(&source.join("outside-file"), b"outside")?; + PathBuf::from("../../outside-file") + } + ("escape", _) => { + fs::create_dir_all(source.join("outside-dir"))?; + PathBuf::from("../../outside-dir") + } + (_, "file") => PathBuf::from("../bin/../NOTICE"), + _ => PathBuf::from("../bin/../lib"), + }; + let link = source.join("upstream/lib/zzcanary-link"); + std::os::unix::fs::symlink(&target, &link)?; + assert!(fs::metadata(&link).is_ok(), "{shape} {kind} link must resolve"); + let text = refusal_text(collect_tree(&source, &staging, tree_output(false, &["bin/daemon"]))); + assert!(text.contains(rule), "{shape} {kind}: {text}"); + } + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn streamed_digest_is_byte_compatible() -> Result<()> { + // Not a multiple of the 64 KiB read buffer, so a short last chunk + // matters; the aligned and empty files are the other boundaries. + let big = (0..3 * 65536 + 17).map(|i| (i % 251) as u8).collect::>(); + let aligned = vec![7_u8; 65536]; + let temp = tempfile::tempdir()?; + let tree = temp.path().join("tree"); + write_file(&tree.join("a.txt"), b"alpha")?; + write_file(&tree.join("empty"), b"")?; + write_file(&tree.join("sub/aligned.bin"), &aligned)?; + write_file(&tree.join("sub/big.bin"), &big)?; + std::os::unix::fs::symlink("a.txt", tree.join("zlink"))?; + + // The encoding digest_tree had before it streamed, written out by hand. + fn add_file(hasher: &mut Sha256, name: &str, bytes: &[u8]) { + hasher.update(b"file\0"); + hasher.update(name.as_bytes()); + hasher.update(b"\0"); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + let mut expected = Sha256::new(); + add_file(&mut expected, "a.txt", b"alpha"); + add_file(&mut expected, "empty", b""); + let mut expected_size = 5_u64; + expected.update(b"dir\0sub\0"); + add_file(&mut expected, "sub/aligned.bin", &aligned); + add_file(&mut expected, "sub/big.bin", &big); + expected_size += (aligned.len() + big.len()) as u64; + expected.update(b"link\0zlink\0a.txt"); + expected_size += "a.txt".len() as u64; + assert_eq!( + digest_artifact(&tree)?, + (format!("{:x}", expected.finalize()), expected_size) + ); + + let single = tree.join("sub/big.bin"); + assert_eq!( + digest_artifact(&single)?, + (raw_sha256(&big), big.len() as u64) + ); + let mut hasher = Sha256::new(); + hash_file_exact(&single, big.len() as u64, &mut hasher)?; + assert_eq!(format!("{:x}", hasher.finalize()), raw_sha256(&big)); + for wrong in [big.len() as u64 + 1, big.len() as u64 - 1] { + let text = refusal_text(hash_file_exact(&single, wrong, &mut Sha256::new())); + assert!(text.contains("changed while hashing"), "{text}"); + } + Ok(()) + } + + /// Acceptance on the real trees: `IDUNN_BIG_TREE` holds `python/` and + /// `blender/`. Run on Yggdrasil as its own job (see the cut spec). + #[cfg(unix)] + #[test] + #[ignore = "needs IDUNN_BIG_TREE: a CPython+torch tree and a Blender tree"] + fn big_tree_seals_from_env() -> Result<()> { + use std::time::Instant; + + /// (links, links with a `..` step, regular files, files at 0555), never + /// following a link; every regular file must be 0444 or a member's 0555. + fn census( + root: &Path, + path: &Path, + members: &BTreeSet, + modes: bool, + totals: &mut [usize; 4], + ) -> Result<()> { + for entry in fs::read_dir(path)? { + let entry = entry?; + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.is_dir() { + census(root, &entry.path(), members, modes, totals)?; + } else if metadata.file_type().is_symlink() { + totals[0] += 1; + let target = fs::read_link(entry.path())?; + if target + .components() + .any(|part| matches!(part, std::path::Component::ParentDir)) + { + totals[1] += 1; + } + } else { + totals[2] += 1; + let member = members.contains(entry.path().strip_prefix(root)?); + let mode = sealed_mode(&entry.path()); + if modes { + assert_eq!(mode, if member { 0o555 } else { 0o444 }); + totals[3] += usize::from(mode == 0o555); + } + } + } + Ok(()) + } + fn peak_rss() -> String { + fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| { + status + .lines() + .find(|line| line.starts_with("VmHWM")) + .map(str::to_owned) + }) + .unwrap_or_default() + } + + let tree = PathBuf::from( + std::env::var_os("IDUNN_BIG_TREE").context("IDUNN_BIG_TREE is not set")?, + ); + let members = ["python/bin/python3.12", "blender/blender"]; + let temp = tempfile::tempdir()?; + let staging = temp.path().join("staging"); + fs::create_dir_all(&staging)?; + let destination = staging.join("tree"); + let output = ArtifactOutput { + destination: PathBuf::from("tree"), + ..tree_output(false, &members) + }; + + let started = Instant::now(); + let receipt = seal_artifact(&output, &tree, &tree, &staging)?; + println!("seal {:?} {}", started.elapsed(), peak_rss()); + let started = Instant::now(); + let again = digest_artifact(&destination)?; + assert_eq!(format!("sha256-{}", again.0), receipt.sha256); + assert_eq!(again.1, receipt.size_bytes); + println!("digest {:?} {}", started.elapsed(), peak_rss()); + let mut before = [0; 4]; + census(&destination, &destination, &output.executable_members, false, &mut before)?; + let started = Instant::now(); + harden_installed_release(&staging, &[receipt.clone()])?; + println!("harden {:?} {}", started.elapsed(), peak_rss()); + let mut totals = [0; 4]; + census(&destination, &destination, &output.executable_members, true, &mut totals)?; + assert_eq!(totals[0], before[0], "links changed across harden"); + assert_eq!(totals[3], members.len()); + println!( + "dotdot links {} links before {} after {} files {} exec {}", + totals[1], before[0], totals[0], totals[2], totals[3] + ); + println!("bytes {}", receipt.size_bytes); + Ok(()) + } + #[cfg(unix)] #[test] fn runner_cache_is_created_once_for_one_exact_identity() -> Result<()> { @@ -15369,4 +16677,492 @@ processes 9 ); Ok(()) } + + // BEGIN runner-failure-evidence tests + // The runner decides what failed and what it kept: these drive the real + // DockerRunnerDriver over a fake docker program, never the capture alone. + + const RUNNER_CANARY: &str = "CANARY-5d2e91b7"; + + #[cfg(unix)] + fn fake_docker(dir: &Path, body: &str) -> Result { + use std::os::unix::fs::PermissionsExt; + let program = dir.join("docker"); + fs::write(&program, format!("#!/bin/sh\n{body}\n"))?; + fs::set_permissions(&program, fs::Permissions::from_mode(0o755))?; + Ok(DockerRunnerDriver { + docker_program: program, + }) + } + + /// One recipe step through the real `run_in_workspace` path. + #[cfg(unix)] + fn run_step(driver: &DockerRunnerDriver, dir: &Path, step_id: &str) -> Result<()> { + let workspace = dir.join("workspace"); + fs::create_dir_all(&workspace)?; + driver.run_in_workspace( + step_id, + "rust", + &fixture_docker_runner_binding(), + &workspace, + Path::new("."), + &["cargo".to_owned(), "test".to_owned()], + &std::collections::BTreeSet::new(), + ("IDUNN_STAMP", "rev"), + ) + } + + #[cfg(unix)] + fn failure_of(result: Result<()>) -> Result { + let error = result.expect_err("the step did not fail"); + error + .downcast_ref::() + .cloned() + .with_context(|| format!("the failure is not typed: {error:#}")) + } + + #[cfg(unix)] + #[test] + fn a_failed_step_is_named_with_both_tails() -> Result<()> { + let temp = tempfile::tempdir()?; + // The first step passes, the second fails: the count is the fake's. + let driver = fake_docker( + temp.path(), + &format!( + "n=$(cat \"$0.count\" 2>/dev/null || echo 0); n=$((n+1)); echo $n > \"$0.count\"\n\ + [ \"$n\" -lt 2 ] && exit 0\n\ + echo 'stdout line one'; echo 'stdout report {RUNNER_CANARY}'\n\ + echo 'stderr report' >&2\nexit 3" + ), + )?; + run_step(&driver, temp.path(), "first")?; + let failure = failure_of(run_step(&driver, temp.path(), "second"))?; + assert_eq!(failure.subject, RunnerSubject::Step("second".into())); + assert_eq!(failure.runner, "rust"); + assert_eq!(failure.exit_code, Some(3)); + assert_eq!(failure.signal, None); + assert!( + failure + .stdout_tail + .ends_with(format!("stdout report {RUNNER_CANARY}\n").as_bytes()), + "stdout was dropped: {:?}", + String::from_utf8_lossy(&failure.stdout_tail) + ); + assert_eq!(failure.stderr_tail, b"stderr report\n"); + assert_eq!(failure.stdout_total_bytes, failure.stdout_tail.len() as u64); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_step_killed_by_a_signal_is_named_by_its_signal() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "kill -9 $$")?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert_eq!((failure.exit_code, failure.signal), (None, Some(9))); + assert!(failure.to_string().contains("exited with signal 9"), "{failure}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_external_input_is_named_as_one() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker(temp.path(), "echo curl failed >&2\nexit 22")?; + let declaration = TargetDeclaration::parse(crate::deployment_plan::tests::RECIPE)?; + let binding = OperatorBinding::parse(crate::deployment_plan::tests::BINDING)?; + let input_id = declaration.external_inputs[0].id.clone(); + let workspaces = BTreeMap::from([("rust".to_owned(), temp.path().join("workspace"))]); + fs::create_dir_all(&workspaces["rust"])?; + let error = driver + .materialize_external_input(&declaration, &binding, &workspaces, &input_id) + .expect_err("the download did not fail"); + let failure = error + .downcast_ref::() + .with_context(|| format!("the failure is not typed: {error:#}"))?; + assert_eq!(failure.subject, RunnerSubject::ExternalInput(input_id.clone())); + assert_eq!(failure.exit_code, Some(22)); + assert!( + failure.to_string().contains(&format!("runner external input {input_id} on runner rust")), + "{failure}" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn runner_output_never_reaches_the_error_text() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'out {RUNNER_CANARY}'\necho 'err {RUNNER_CANARY}' >&2\nexit 1"), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + assert!(String::from_utf8_lossy(&failure.stdout_tail).contains(RUNNER_CANARY)); + assert!(String::from_utf8_lossy(&failure.stderr_tail).contains(RUNNER_CANARY)); + let error = anyhow::Error::new(failure.clone()).context("sealing"); + for text in [ + format!("{error:#}"), + format!("{error:?}"), + format!("{failure}"), + format!("{failure:?}"), + ] { + assert!(!text.contains(RUNNER_CANARY), "runner output reached text: {text}"); + } + Ok(()) + } + + /// A reader that answers from a script, so a read error the OS rarely + /// produces (an interrupted call) is produced on demand. + struct ScriptedReader(Vec>>); + + impl Read for ScriptedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.0.is_empty() { + return Ok(0); + } + let bytes = self.0.remove(0)?; + buffer[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } + } + + struct BrokenReader; + + impl Read for BrokenReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("persistent read failure")) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_loses_no_byte() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + Ok(b"cd".to_vec()), + Err(std::io::Error::from(ErrorKind::Interrupted)), + ]); + let drained = drain_tail(reader, 3).expect("an interrupted read ended the drain"); + assert_eq!(drained, (b"bcd".to_vec(), 4)); + } + + #[test] + fn a_persistent_read_error_ends_the_drain_instead_of_spinning() { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = sender.send(drain_tail(BrokenReader, 8).is_err()); + }); + assert_eq!( + receiver.recv_timeout(Duration::from_secs(5)), + Ok(true), + "a persistent read error spun or was swallowed" + ); + } + + #[test] + fn a_non_interrupted_read_error_after_data_is_returned_not_retried() { + let reader = ScriptedReader(vec![ + Ok(b"ab".to_vec()), + Err(std::io::Error::from(ErrorKind::BrokenPipe)), + Ok(b"cd".to_vec()), + ]); + let error = drain_tail(reader, 8).expect_err("a broken pipe was retried past"); + assert_eq!(error.kind(), ErrorKind::BrokenPipe); + } + + #[cfg(unix)] + #[test] + fn each_tail_is_the_last_bound_bytes_with_the_total() -> Result<()> { + let temp = tempfile::tempdir()?; + let data = temp.path().join("data"); + for size in [ + 0, + 1, + RUNNER_OUTPUT_TAIL_BYTES - 1, + RUNNER_OUTPUT_TAIL_BYTES, + RUNNER_OUTPUT_TAIL_BYTES + 1, + 1024 * 1024, + ] { + // Distinct neighbours, so a shifted or head-keeping tail differs. + let bytes: Vec = (0..size).map(|index| (index % 251) as u8).collect(); + fs::write(&data, &bytes)?; + let driver = fake_docker( + temp.path(), + &format!("cat '{}'\ncat '{}' >&2\nexit 1", data.display(), data.display()), + )?; + let failure = failure_of(run_step(&driver, temp.path(), "build"))?; + let expected = &bytes[size.saturating_sub(RUNNER_OUTPUT_TAIL_BYTES)..]; + assert_eq!(failure.stdout_tail, expected, "stdout tail at {size} bytes"); + assert_eq!(failure.stderr_tail, expected, "stderr tail at {size} bytes"); + assert_eq!(failure.stdout_total_bytes, size as u64); + assert_eq!(failure.stderr_total_bytes, size as u64); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_successful_step_and_a_missing_program_keep_their_outcomes() -> Result<()> { + let temp = tempfile::tempdir()?; + run_step(&fake_docker(temp.path(), "echo fine")?, temp.path(), "build")?; + let missing = DockerRunnerDriver { + docker_program: temp.path().join("no-such-docker"), + }; + let error = run_step(&missing, temp.path(), "build").expect_err("a missing program ran"); + assert!(error.downcast_ref::().is_none()); + assert!(format!("{error:#}").contains("starting Docker runner"), "{error:#}"); + Ok(()) + } + + /// The 2026-10-25 vili failure: `node --test` reported on stdout alone, + /// stderr stayed empty (the runner's /tmp is a noexec tmpfs) and the step + /// exited 1. The old capture dropped stdout and printed an empty message. + #[cfg(unix)] + #[test] + fn a_step_that_reports_only_on_stdout_keeps_that_report_and_its_name() -> Result<()> { + let temp = tempfile::tempdir()?; + let driver = fake_docker( + temp.path(), + &format!("echo 'not ok 1 - daemon {RUNNER_CANARY}'\necho '# fail 1'\nexit 1"), + )?; + let error = run_step(&driver, temp.path(), "test-daemon").expect_err("the step passed"); + // The error names the step and carries no output. + let text = format!("{error:#}"); + assert!(text.contains("runner step test-daemon on runner rust exited with exit status 1"), "{text}"); + assert!(!text.contains(RUNNER_CANARY) && !text.contains("not ok"), "{text}"); + // The evidence keeps the stdout tail; the empty stderr is a recorded 0. + let failure = error + .downcast_ref::() + .context("the failure is not typed")?; + let store = temp.path().join("runner-evidence.cc"); + let record = crate::runner_evidence::RunnerFailureRecord::from_failure( + "tx-vili", + "up-vili", + "vili", + failure, + 1, + ); + crate::runner_evidence::record(&store, &record)?; + let kept = crate::runner_evidence::for_transaction(&store, "tx-vili")?.context("no evidence kept")?; + assert_eq!(kept.subject_id, "test-daemon"); + assert_eq!(kept.exit_code, Some(1)); + assert_eq!( + String::from_utf8_lossy(&kept.stdout_tail), + format!("not ok 1 - daemon {RUNNER_CANARY}\n# fail 1\n") + ); + assert_eq!(kept.stdout_total_bytes, kept.stdout_tail.len() as u64); + assert!(kept.stderr_tail.is_empty()); + assert_eq!(kept.stderr_total_bytes, 0); + let mut shown = Vec::new(); + crate::runner_evidence::show_status(&mut shown, &store, "tx-vili")?; + let rendered = String::from_utf8(shown)?; + assert!(rendered.contains("stderr (last 0 of 0 bytes)"), "{rendered}"); + assert!(rendered.contains(&format!("| not ok 1 - daemon {RUNNER_CANARY}")), "{rendered}"); + // Debug names the failure and prints no tail. + let debug = format!("{failure:?}"); + assert!(debug.starts_with("RunnerStepFailure") && !debug.contains("not ok"), "{debug}"); + Ok(()) + } + // END runner-failure-evidence tests + + // --------------------------------------------------------------------- + // A static release flips on a stand-in site tree: releases/, a www/ that + // holds the current link, and the operator's per-site config. Nothing here + // is a real site. + // --------------------------------------------------------------------- + + #[cfg(unix)] + struct StandInSite { + _temp: tempfile::TempDir, + binding: StaticReleaseBinding, + } + + #[cfg(unix)] + const RELEASE_A: &str = "sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + #[cfg(unix)] + const RELEASE_B: &str = "sha256-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + + /// Two installed releases, each a fake vite build with compressed twins, + /// and a site config the operator owns. + #[cfg(unix)] + fn stand_in_site() -> Result { + let temp = tempfile::tempdir()?; + let root = temp.path(); + for release in [RELEASE_A, RELEASE_B] { + let dist = root.join("releases").join(release).join("dist"); + fs::create_dir_all(dist.join("assets"))?; + fs::write(dist.join("index.html"), format!("{release}"))?; + fs::write(dist.join("index.html.br"), b"br")?; + fs::write(dist.join("index.html.gz"), b"gz")?; + fs::write(dist.join("assets/app.js"), b"app")?; + } + fs::create_dir_all(root.join("www"))?; + fs::create_dir_all(root.join("config"))?; + fs::write(root.join("config/site.json"), br#"{"api":"https://example.test"}"#)?; + Ok(StandInSite { + binding: StaticReleaseBinding { + release_root: root.join("releases"), + current_link: root.join("www/current"), + carry: crate::deployment::StaticCarryBinding { + source: root.join("config/site.json"), + destination: "config.json".into(), + }, + }, + _temp: temp, + }) + } + + #[cfg(unix)] + #[test] + fn static_release_flip_carries_the_site_file_and_names_the_release() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let site = stand_in_site()?; + let port = FilesystemStaticRelease; + assert_eq!(port.observe_current(&site.binding)?, None, "no link before the first flip"); + + let flip = port.carry_and_flip(&site.binding, RELEASE_A)?; + assert_eq!(port.observe_current(&site.binding)?.as_deref(), Some(RELEASE_A)); + let carried = site.binding.current_link.join("config.json"); + let bytes = fs::read(&carried)?; + assert_eq!(bytes, fs::read(&site.binding.carry.source)?); + assert_eq!(flip.carried_sha256, format!("sha256-{:x}", Sha256::digest(&bytes))); + assert_eq!(fs::metadata(&carried)?.permissions().mode() & 0o777, 0o444); + assert!(fs::symlink_metadata(&carried)?.is_file(), "the carry is a copy, never a link"); + // The web root reads the release's own tree through the link. + assert_eq!(fs::read(site.binding.current_link.join("dist/index.html.br"))?, b"br"); + + // The site file changes between releases: the second release carries + // the new bytes and the first keeps the ones it was flipped with. + fs::write(&site.binding.carry.source, br#"{"api":"https://other.test"}"#)?; + let second = port.carry_and_flip(&site.binding, RELEASE_B)?; + assert_ne!(second.carried_sha256, flip.carried_sha256); + assert_eq!(port.observe_current(&site.binding)?.as_deref(), Some(RELEASE_B)); + assert_eq!(fs::read(&carried)?, br#"{"api":"https://other.test"}"#); + assert_eq!( + fs::read(site.binding.release_root.join(RELEASE_A).join("config.json"))?, + br#"{"api":"https://example.test"}"# + ); + // Rolling back is the same flip to the release the link named before. + port.carry_and_flip(&site.binding, RELEASE_A)?; + assert_eq!(port.observe_current(&site.binding)?.as_deref(), Some(RELEASE_A)); + Ok(()) + } + + /// The service runs with UMask=0027, which masks the create mode to 0440 + /// and hides the carry from the web user. The umask is process-global, so + /// the proof runs in a child copy of this test binary and no sibling test + /// ever sees it. + #[cfg(unix)] + #[test] + fn static_release_carry_is_0444_under_the_service_umask() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + const CHILD: &str = "IDUNN_TEST_CARRY_UMASK_CHILD"; + const NAME: &str = "static_release_carry_is_0444_under_the_service_umask"; + if std::env::var_os(CHILD).is_none() { + let path = format!("{}::{NAME}", module_path!().split_once("::").map_or("", |(_, rest)| rest)); + let output = std::process::Command::new(std::env::current_exe()?) + .args(["--exact", path.as_str(), "--nocapture", "--test-threads=1"]) + .env(CHILD, "1") + .output()?; + assert!( + output.status.success(), + "child under umask 0027 failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "the child filter matched no test:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + return Ok(()); + } + unsafe { libc::umask(0o027) }; + let site = stand_in_site()?; + FilesystemStaticRelease.carry_and_flip(&site.binding, RELEASE_A)?; + let carried = site.binding.current_link.join("config.json"); + assert_eq!(fs::metadata(&carried)?.permissions().mode() & 0o777, 0o444); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_release_missing_site_config_fails_before_the_link_moves() -> Result<()> { + let site = stand_in_site()?; + let port = FilesystemStaticRelease; + port.carry_and_flip(&site.binding, RELEASE_A)?; + fs::remove_file(&site.binding.carry.source)?; + let error = port.carry_and_flip(&site.binding, RELEASE_B).unwrap_err(); + assert!(format!("{error:#}").contains("carry source"), "{error:#}"); + assert!(error.downcast_ref::().is_some(), "a pre-rename refusal is typed"); + assert_eq!(port.observe_current(&site.binding)?.as_deref(), Some(RELEASE_A)); + assert!( + !site.binding.release_root.join(RELEASE_B).join("config.json").exists(), + "a failed flip wrote its carry" + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_release_refuses_a_site_file_that_is_not_a_plain_small_file() -> Result<()> { + let site = stand_in_site()?; + let port = FilesystemStaticRelease; + let source = site.binding.carry.source.clone(); + fs::remove_file(&source)?; + fs::write(source.with_extension("real"), b"{}")?; + std::os::unix::fs::symlink(source.with_extension("real"), &source)?; + assert!(port.carry_and_flip(&site.binding, RELEASE_A).is_err(), "a linked site file was carried"); + fs::remove_file(&source)?; + fs::write(&source, vec![b'x'; STATIC_CARRY_LIMIT_BYTES as usize + 1])?; + assert!(port.carry_and_flip(&site.binding, RELEASE_A).is_err(), "an oversize site file was carried"); + assert_eq!(port.observe_current(&site.binding)?, None); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_release_replaces_a_stale_flip_link_a_crash_left() -> Result<()> { + let site = stand_in_site()?; + let port = FilesystemStaticRelease; + let stale = site.binding.current_link.with_file_name(".current.flip"); + std::os::unix::fs::symlink("/nowhere", &stale)?; + port.carry_and_flip(&site.binding, RELEASE_A)?; + assert_eq!(port.observe_current(&site.binding)?.as_deref(), Some(RELEASE_A)); + assert!(fs::symlink_metadata(&stale).is_err(), "the stale flip link survived"); + // Repeating the flip, as a resumed transaction does, changes nothing. + port.carry_and_flip(&site.binding, RELEASE_A)?; + assert_eq!(port.observe_current(&site.binding)?.as_deref(), Some(RELEASE_A)); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn static_release_will_not_replace_or_trust_what_is_not_its_link() -> Result<()> { + let site = stand_in_site()?; + let port = FilesystemStaticRelease; + // A real directory where the link belongs is left alone. + fs::create_dir(&site.binding.current_link)?; + let error = port.carry_and_flip(&site.binding, RELEASE_A).unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + assert!(site.binding.current_link.is_dir()); + assert!( + !site.binding.release_root.join(RELEASE_A).join("config.json").exists(), + "a refused flip wrote its carry" + ); + assert!(port.observe_current(&site.binding).is_err()); + fs::remove_dir(&site.binding.current_link)?; + // A link that names something outside the release root has no release + // to roll back to, and Idunn says so instead of guessing. + std::os::unix::fs::symlink("/usr", &site.binding.current_link)?; + let error = port.observe_current(&site.binding).unwrap_err(); + assert!(format!("{error:#}").contains("not a release under"), "{error:#}"); + // A release that was never installed is not flipped to. + fs::remove_file(&site.binding.current_link)?; + let missing = format!("sha256-{}", "c".repeat(64)); + assert!(port.carry_and_flip(&site.binding, &missing).is_err()); + assert_eq!(port.observe_current(&site.binding)?, None); + Ok(()) + } } diff --git a/src/host.rs b/src/host.rs index 5221f12..5605a8d 100644 --- a/src/host.rs +++ b/src/host.rs @@ -30,13 +30,13 @@ use serde::Deserialize; use crate::control_plane::read_trust_anchor; use crate::deployment::{ - ArtifactSource, HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, + HostWorkloadBinding, IDUNN_ACTIVATION_CREDENTIAL_ENVIRONMENT, IDUNN_RUNTIME_BUNDLE_ENVIRONMENT, LaunchArgument, OperatorBinding, TargetDeclaration, }; -use crate::deployment_plan::{ArtifactReceipt, CompiledDeploymentPlan, SealedRelease}; +use crate::deployment_plan::{CompiledDeploymentPlan, SealedRelease}; use crate::drivers::{ - HostWorkloadObservation, InstalledReleaseObservation, copy_artifact, copy_tree, - digest_artifact, release_artifact, remove_tree_inside, sha256_id, write_runtime_bundle_records, + HostWorkloadObservation, InstalledReleaseObservation, copy_tree, digest_artifact, + release_artifact, remove_tree_inside, seal_artifact, sha256_id, write_runtime_bundle_records, }; use crate::host_actuator::{ HOST_ACTUATOR_CHANNEL, HOST_ACTUATOR_CONNECTION_ID, HOST_ACTUATOR_PING_MS, @@ -631,11 +631,11 @@ impl Actuator { workload: &HostWorkloadBinding, installed: &Path, ) -> Result<(PathBuf, Vec)> { - let artifact = release_artifact(declaration, &declaration.service.executable_artifact)?; + let artifact = release_artifact(declaration, &declaration.service()?.executable_artifact)?; let executable = installed.join(&artifact.destination); ensure!(executable.is_file(), "sealed service executable is absent"); let mut arguments = Vec::new(); - for argument in &declaration.service.arguments { + for argument in &declaration.service()?.arguments { arguments.push(match argument { LaunchArgument::Literal { value } => value.clone(), LaunchArgument::Binding { name } => workload @@ -1048,37 +1048,12 @@ fn materialize( fs::create_dir_all(&staging)?; let mut artifacts = Vec::new(); for artifact in &declaration.artifacts { - let source_path = match artifact.source_kind { - ArtifactSource::RunnerOutput | ArtifactSource::WorktreeTree => { - source.join(&artifact.source) - } - }; - ensure!( - source_path.exists(), - "declared artifact output {} is absent", - artifact.id - ); - let destination = staging.join(&artifact.destination); - ensure!( - destination.starts_with(&staging), - "artifact destination escaped its staging root" - ); - copy_artifact(&source, &source_path, &destination)?; - let (sha256, size_bytes) = digest_artifact(&destination)?; - if let Some(expected) = &artifact.expected_sha256 { - ensure!( - &sha256 == expected, - "artifact {} differs from its recipe-pinned digest", - artifact.id - ); - } - artifacts.push(ArtifactReceipt { - artifact_id: artifact.id.clone(), - destination: artifact.destination.clone(), - sha256: format!("sha256-{sha256}"), - size_bytes, - executable: artifact.executable, - }); + artifacts.push(seal_artifact( + artifact, + &source, + &source.join(&artifact.source), + &staging, + )?); } let release = SealedRelease::new(plan, artifacts, Vec::new(), sealed_at_unix_millis)?; Ok((release, staging)) diff --git a/src/host_actuator.rs b/src/host_actuator.rs index f7bf4c7..8fd776c 100644 --- a/src/host_actuator.rs +++ b/src/host_actuator.rs @@ -1151,7 +1151,7 @@ mod tests { let answerer = thread::spawn(move || -> Result<()> { for _ in 0..500 { client.poll_resends()?; - while let Some(frame) = client.receive_once()? { + if let Some(frame) = client.receive_once()? { let envelope = HostActuatorEnvelope::decode(&frame.payload)?; let request = envelope.open_request("raven", &idunn_anchor)?; assert_eq!(envelope.session_nonce, nonce); diff --git a/src/lib.rs b/src/lib.rs index e71a17f..6b034d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +mod brake; pub mod control_plane; pub mod deployment; pub mod deployment_plan; @@ -7,4 +8,7 @@ pub mod host_actuator; #[cfg(unix)] pub mod launcher; mod incident; +mod runner_evidence; pub mod provisioning; +#[cfg(unix)] +mod steward_orders; diff --git a/src/provisioning.rs b/src/provisioning.rs index 1c74a83..324785e 100644 --- a/src/provisioning.rs +++ b/src/provisioning.rs @@ -1,4 +1,5 @@ -use anyhow::{Context, Result, anyhow, bail}; +use crate::brake::{Stop, StopState, process_uid, read_stop, replace_stop}; +use anyhow::{Context, Result, anyhow, bail, ensure}; use cultcache_rs::{ CultCacheEnvelope, CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, }; @@ -6,15 +7,13 @@ use cultnet_rs::{ GAMECULT_SERVICE_TRUST_ANCHOR_SCHEMA, GameCultProviderHealthIdentity, GameCultServiceTrustAnchorRecord, IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SCHEMA, IDUNN_AUTHENTICATED_PROVIDER_HEALTH_PROJECTION_SIGNING_PURPOSE, - IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_AUTHORITY, - IDUNN_DEPLOYMENT_BRAKE_ID, IDUNN_DEPLOYMENT_BRAKE_SCHEMA, IDUNN_DEPLOYMENT_BRAKE_SCOPE, - IDUNN_DEPLOYMENT_RELEASE_PURPOSE, IDUNN_LIFECYCLE_BRAKE_AUTHORITY, + IDUNN_DAEMON_HEALTH_TRUST_BINDING_SCHEMA, + IDUNN_LIFECYCLE_BRAKE_AUTHORITY, IDUNN_LIFECYCLE_BRAKE_SCHEMA, IDUNN_LIFECYCLE_BRAKE_SCOPE, IdunnDaemonHealthTrustBindingRecord, - IdunnDeploymentBrakeObservation, IdunnDeploymentBrakeOperatorIdentity, - IdunnDeploymentBrakeRecord, IdunnDeploymentBrakeReleasePurpose, IdunnLifecycleBrakeObservation, + IdunnLifecycleBrakeObservation, IdunnLifecycleBrakeRecord, IdunnServiceIdentity, OdinTopologyIdentity, ServiceIdentityProfile, ServiceIdentityTrustAnchor, derive_service_identity_id, enroll_service_identity_at, - evaluate_idunn_continuity_restart, evaluate_idunn_deployment_brake, + evaluate_idunn_continuity_restart, export_service_identity_trust_anchor, open_service_identity_at, }; use std::collections::{BTreeMap, BTreeSet}; @@ -80,17 +79,8 @@ pub fn run(args: impl IntoIterator) -> Result<()> { "export-odin-topology-public-anchor" => { export_identity_anchor::(&options, "Odin topology public anchor")? } - "enroll-deployment-brake-operator" => { - enroll_identity::(&options)? - } - "export-deployment-brake-operator-anchor" => { - export_identity_anchor::( - &options, - "deployment brake operator anchor", - )? - } - "deployment-brake-engage" => deployment_brake_engage(&options)?, - "deployment-brake-release" => deployment_brake_release(&options)?, + "deployment-brake-engage" => deployment_brake_write(&options, StopState::Engaged)?, + "deployment-brake-clear" => deployment_brake_write(&options, StopState::Clear)?, "deployment-brake-status" => deployment_brake_status(&options)?, "lifecycle-brake-engage" => lifecycle_brake_engage(&options)?, "lifecycle-brake-release" => lifecycle_brake_release(&options)?, @@ -157,153 +147,60 @@ fn provider_health_public_key(options: &BTreeMap) -> Result) -> Result<()> { - require_only( - options, - &[ - "store", - "runtime-id", - "owner", - "reason", - "observed-at-unix-millis", - ], +/// Engage or clear the deployment stop. The author is the process uid's login +/// name and the time is this verb's clock: neither is an argument. +fn deployment_brake_write(options: &BTreeMap, state: StopState) -> Result<()> { + require_only(options, &["store", "runtime-id", "reason"])?; + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), )?; - let observed = parse_u64(options, "observed-at-unix-millis")?; - let record = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: required(options, "runtime-id")?.into(), - status: "engaged".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: required(options, "reason")?.into(), - observed_at_unix_millis: observed, - expires_at_unix_millis: None, - authorization_id: None, - authorization_purpose: None, - authorized_release_id: None, - authorized_deployment_id: None, - authorized_by: None, - authorization_issued_at_unix_millis: None, - authorization_expires_at_unix_millis: None, - signature_algorithm: None, - signature: None, - private_state_exposed: false, - updated_by: required(options, "owner")?.into(), - }; - record.validate()?; - replace_brake(&path(options, "store")?, record, observed) -} - -fn deployment_brake_release(options: &BTreeMap) -> Result<()> { - require_only( - options, - &[ - "store", - "private-store", - "runtime-id", - "owner", - "reason", - "authorization-id", - "release-id", - "deployment-id", - "issued-at-unix-millis", - "expires-at-unix-millis", - ], - )?; - let issued = parse_u64(options, "issued-at-unix-millis")?; - let expires = parse_u64(options, "expires-at-unix-millis")?; - let signer = open_service_identity_at::(&path( - options, - "private-store", - )?)?; - let mut record = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: required(options, "runtime-id")?.into(), - status: "released".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: required(options, "reason")?.into(), - observed_at_unix_millis: issued, - expires_at_unix_millis: Some(expires), - authorization_id: Some(required(options, "authorization-id")?.into()), - authorization_purpose: Some(IDUNN_DEPLOYMENT_RELEASE_PURPOSE.into()), - authorized_release_id: Some(required(options, "release-id")?.into()), - authorized_deployment_id: Some(required(options, "deployment-id")?.into()), - authorized_by: Some(signer.trust_anchor()?.identity_id), - authorization_issued_at_unix_millis: Some(issued), - authorization_expires_at_unix_millis: Some(expires), - signature_algorithm: Some("ed25519".into()), - signature: None, - private_state_exposed: false, - updated_by: required(options, "owner")?.into(), - }; - record.signature = Some( - signer - .sign::(&rmp_serde::to_vec(&record)?) - .signature, - ); - record.validate()?; - replace_brake(&path(options, "store")?, record, issued) -} - -fn deployment_brake_status(options: &BTreeMap) -> Result<()> { - allow_only( - options, - &[ - "store", - "operator-anchor", - "runtime-id", - "release-id", - "deployment-id", - "now-unix-millis", - ], - )?; - let exact_actuation_fields = ["release-id", "deployment-id", "now-unix-millis"]; - let exact_actuation_field_count = exact_actuation_fields - .into_iter() - .filter(|field| options.contains_key(*field)) - .count(); - if exact_actuation_field_count != 0 - && exact_actuation_field_count != exact_actuation_fields.len() - { - bail!("exact deployment-brake actuation validation requires release, deployment, and time"); - } - let record = read_brake(&path(options, "store")?)?; - if record.runtime_id != required(options, "runtime-id")? { - bail!("deployment brake belongs to another runtime") - } - if record.status == "engaged" { - if exact_actuation_field_count != 0 { - bail!("engaged deployment brake denies exact actuation validation"); - } - println!( - "engaged runtime={} scope={} owner={} reason={}", - record.runtime_id, record.scope, record.updated_by, record.reason - ); - return Ok(()); - } - let anchor = read_operator_anchor(&path(options, "operator-anchor")?)?; - evaluate_idunn_deployment_brake( - IdunnDeploymentBrakeObservation::Present(&record), - &anchor, + let record = replace_stop( + &path(options, "store")?, required(options, "runtime-id")?, - required(options, "release-id")?, - required(options, "deployment-id")?, - parse_u64(options, "now-unix-millis")?, - ) - .map_err(|d| anyhow!("deployment brake denied actuation: {d:?}"))?; + state, + required(options, "reason")?, + now, + )?; println!( - "released runtime={} release={} deployment={} owner={}", + "{} runtime={} by={} sequence={}", + record.state.name(), record.runtime_id, - record.authorized_release_id.unwrap(), - record.authorized_deployment_id.unwrap(), - record.updated_by + record.by, + record.sequence ); Ok(()) } +/// Print the stop and exit nonzero unless deployments admit, so a script can +/// gate on it as it could on the retired exact-actuation validation. +fn deployment_brake_status(options: &BTreeMap) -> Result<()> { + allow_only(options, &["store", "runtime-id", "owner-user"])?; + // A steward's store is owned by the steward: the operator names that user + // to read it, and the owner check is the same one Idunn makes. + let owner = match options.get("owner-user") { + Some(user) => crate::steward_orders::user_uid(user)?, + None => process_uid(), + }; + let stop = read_stop(&path(options, "store")?, required(options, "runtime-id")?, owner); + match &stop { + Stop::Absent => println!("absent: deployments admit"), + Stop::Unreadable(why) => println!("unreadable: deployments wait ({why})"), + Stop::Clear(record) | Stop::Engaged(record) => println!( + "{} runtime={} by={} at={} sequence={} reason={}", + record.state.name(), + record.runtime_id, + record.by, + record.at_unix_millis, + record.sequence, + record.reason + ), + } + ensure!(stop.admits(), "the deployment brake does not admit deployments"); + Ok(()) +} + fn lifecycle_brake_engage(options: &BTreeMap) -> Result<()> { require_only( options, @@ -407,7 +304,7 @@ fn lifecycle_brake_status(options: &BTreeMap) -> Result<()> { Ok(()) } -fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { +pub(crate) fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { record.validate()?; let store = SingleFileMessagePackBackingStore::new(path); let entries = store.pull_all_read_only_snapshot()?; @@ -482,70 +379,6 @@ fn lifecycle_identifier<'a>(options: &'a BTreeMap, name: &str) - Ok(value) } -fn replace_brake(path: &Path, record: IdunnDeploymentBrakeRecord, millis: u64) -> Result<()> { - let store = SingleFileMessagePackBackingStore::new(path); - let entries = store.pull_all_read_only_snapshot()?; - let current = match entries.as_slice() { - [] => None, - [entry] - if entry.r#type == IdunnDeploymentBrakeRecord::TYPE - && entry.key == IDUNN_DEPLOYMENT_BRAKE_ID => - { - Some(entry.clone()) - } - _ => bail!("deployment brake store is corrupt or ambiguous"), - }; - let next = typed_envelope( - IDUNN_DEPLOYMENT_BRAKE_ID, - &record, - IDUNN_DEPLOYMENT_BRAKE_SCHEMA, - millis, - )?; - if !store.compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - key: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - current, - }], - &[next], - )? { - bail!("deployment brake changed during transition"); - } - Ok(()) -} - -fn read_brake(path: &Path) -> Result { - let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - let [entry] = entries.as_slice() else { - bail!("deployment brake is missing or ambiguous") - }; - if entry.r#type != IdunnDeploymentBrakeRecord::TYPE - || entry.key != IDUNN_DEPLOYMENT_BRAKE_ID - || entry.schema_id.as_deref() != Some(IDUNN_DEPLOYMENT_BRAKE_SCHEMA) - { - bail!("deployment brake is foreign") - } - let record: IdunnDeploymentBrakeRecord = rmp_serde::from_slice(&entry.payload)?; - if rmp_serde::to_vec(&record)? != entry.payload { - bail!("deployment brake is noncanonical") - } - record.validate()?; - Ok(record) -} - -fn read_operator_anchor(path: &Path) -> Result { - let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; - let [entry] = entries.as_slice() else { - bail!("operator anchor is missing or ambiguous") - }; - if entry.r#type != IdunnDeploymentBrakeOperatorIdentity::TRUST_ANCHOR_TYPE - || entry.key != IdunnDeploymentBrakeOperatorIdentity::TRUST_ANCHOR_KEY - { - bail!("operator anchor is foreign") - } - Ok(rmp_serde::from_slice(&entry.payload)?) -} - fn validate_health_binding_store(path: &Path) -> Result<()> { let entries = SingleFileMessagePackBackingStore::new(path).pull_all_read_only_snapshot()?; validated_health_bindings(&entries)?; @@ -1127,7 +960,7 @@ fn normalized(path: &Path) -> Result { } fn usage() -> &'static str { - "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision export-provider-health-public-anchor --private-store --public-anchor \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-odin-topology-identity --private-store \n idunn-provision export-odin-topology-public-anchor --private-store --public-anchor \n idunn-provision enroll-deployment-brake-operator --private-store \n idunn-provision export-deployment-brake-operator-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --owner --reason --observed-at-unix-millis \n idunn-provision deployment-brake-release --store --private-store --runtime-id --owner --reason --authorization-id --release-id --deployment-id --issued-at-unix-millis --expires-at-unix-millis \n idunn-provision deployment-brake-status --store --operator-anchor --runtime-id [--release-id --deployment-id --now-unix-millis ]\n idunn-provision lifecycle-brake-engage --store --runtime-id --target --reason --updated-at-unix-millis \n idunn-provision lifecycle-brake-release --store --runtime-id --target --reason --updated-at-unix-millis [--released-until-unix-millis ]\n idunn-provision lifecycle-brake-status --store --runtime-id --target --now-unix-millis \n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " + "Usage: idunn-provision enroll-provider-health-identity --private-store \n idunn-provision provider-health-public-key --private-store \n idunn-provision export-provider-health-public-anchor --private-store --public-anchor \n idunn-provision enroll-idunn-identity --private-store \n idunn-provision export-idunn-public-anchor --private-store --public-anchor \n idunn-provision enroll-odin-topology-identity --private-store \n idunn-provision export-odin-topology-public-anchor --private-store --public-anchor \n idunn-provision deployment-brake-engage --store --runtime-id --reason \n idunn-provision deployment-brake-clear --store --runtime-id --reason \n idunn-provision deployment-brake-status --store --runtime-id [--owner-user ]\n idunn-provision lifecycle-brake-engage --store --runtime-id --target --reason --updated-at-unix-millis \n idunn-provision lifecycle-brake-release --store --runtime-id --target --reason --updated-at-unix-millis [--released-until-unix-millis ]\n idunn-provision lifecycle-brake-status --store --runtime-id --target --now-unix-millis \n idunn-provision create-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision add-daemon-health-trust-binding --output --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex --bound-at-unix-millis --release-binding-required \n idunn-provision require-daemon-health-release-binding --store --binding-id --daemon --health-contract --source-runtime --signer-public-key-hex \n idunn-provision rotate-daemon-health-trust-signer --output --binding-id --signer-public-key-hex --bound-at-unix-millis \n idunn-provision validate-daemon-health-trust-binding --input \n idunn-provision create-provider-projection-trust-anchor --output --trust-anchor-id --runtime-id --idunn-public-anchor --bound-at-unix-millis --expires-at-unix-millis \n idunn-provision validate-provider-projection-trust-anchor --input --idunn-public-anchor " } #[cfg(test)] @@ -2157,120 +1990,74 @@ mod tests { } #[test] - fn brake_operator_can_issue_one_exact_bounded_rollout_grant() -> Result<()> { + fn provision_engage_clear_sequence() -> Result<()> { let temp = TempDir::new()?; - let private = temp.path().join("operator-private.cc"); - let anchor = temp.path().join("operator-public.cc"); let store = temp.path().join("brake.cc"); - invoke(&[ - "enroll-deployment-brake-operator", - "--private-store", - private.to_str().unwrap(), - ])?; - invoke(&[ - "export-deployment-brake-operator-anchor", - "--private-store", - private.to_str().unwrap(), - "--public-anchor", - anchor.to_str().unwrap(), - ])?; - invoke(&[ - "deployment-brake-engage", - "--store", - store.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--owner", - "operator/metacrat", - "--reason", - "sleep mode", - "--observed-at-unix-millis", - "100", - ])?; - assert_eq!(read_brake(&store)?.status, "engaged"); + let me = process_uid(); + let engage = |verb: &str, reason: &str| { + invoke(&[ + verb, + "--store", + store.to_str().unwrap(), + "--runtime-id", + "yggdrasil", + "--reason", + reason, + ]) + }; + assert_eq!(read_stop(&store, "yggdrasil", me), Stop::Absent); invoke(&[ "deployment-brake-status", "--store", store.to_str().unwrap(), - "--operator-anchor", - anchor.to_str().unwrap(), "--runtime-id", "yggdrasil", ])?; - assert!( + engage("deployment-brake-engage", "freeze")?; + let status = || { invoke(&[ "deployment-brake-status", "--store", store.to_str().unwrap(), - "--operator-anchor", - anchor.to_str().unwrap(), "--runtime-id", "yggdrasil", - "--release-id", - "commit-4", - "--deployment-id", - "request-4", - "--now-unix-millis", - "150", ]) - .is_err() - ); - invoke(&[ - "deployment-brake-release", - "--store", - store.to_str().unwrap(), - "--private-store", - private.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--owner", - "operator/metacrat", - "--reason", - "one rollout", - "--authorization-id", - "auth/r4", - "--release-id", - "commit-4", - "--deployment-id", - "request-4", - "--issued-at-unix-millis", - "200", - "--expires-at-unix-millis", - "800", - ])?; - invoke(&[ - "deployment-brake-status", - "--store", - store.to_str().unwrap(), - "--operator-anchor", - anchor.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--release-id", - "commit-4", - "--deployment-id", - "request-4", - "--now-unix-millis", - "500", - ])?; - assert!( - invoke(&[ - "deployment-brake-status", - "--store", - store.to_str().unwrap(), - "--operator-anchor", - anchor.to_str().unwrap(), - "--runtime-id", - "yggdrasil", - "--release-id", - "substituted", - "--deployment-id", - "request-4", - "--now-unix-millis", - "500" - ]) - .is_err() - ); + }; + assert!(status().is_err(), "an engaged stop must not pass status"); + let Stop::Engaged(first) = read_stop(&store, "yggdrasil", me) else { + bail!("engage did not engage") + }; + assert_eq!((first.sequence, first.reason.as_str()), (1, "freeze")); + assert!(!first.by.is_empty() && first.at_unix_millis > 0); + engage("deployment-brake-clear", "thawed")?; + status()?; + let Stop::Clear(second) = read_stop(&store, "yggdrasil", me) else { + bail!("clear did not clear") + }; + assert_eq!((second.sequence, second.by), (2, first.by)); + assert!(second.at_unix_millis >= first.at_unix_millis); + // The author and the time are not arguments, and the retired + // signing arguments are refused. + for stray in ["--owner", "--observed-at-unix-millis", "--private-store"] { + assert!( + invoke(&[ + "deployment-brake-engage", + "--store", + store.to_str().unwrap(), + "--runtime-id", + "yggdrasil", + "--reason", + "x", + stray, + "y", + ]) + .is_err(), + "{stray}" + ); + } + for gone in ["deployment-brake-release", "enroll-deployment-brake-operator"] { + assert!(invoke(&[gone]).is_err(), "{gone}"); + } Ok(()) } @@ -2309,7 +2096,6 @@ mod tests { assert_eq!(engaged.status, "engaged"); assert_eq!(engaged.runtime_id, "idunn-yggdrasil"); assert_eq!(engaged.target, "ghostlight"); - assert!(read_brake(&store).is_err()); assert!( invoke(&[ "lifecycle-brake-status", @@ -2426,57 +2212,4 @@ mod tests { assert_eq!(std::fs::read(&store)?, released_bytes); Ok(()) } - - #[test] - fn brake_compare_exchange_rejects_stale_transition_snapshot() -> Result<()> { - let temp = TempDir::new()?; - let store_path = temp.path().join("brake.cc"); - let base = IdunnDeploymentBrakeRecord { - schema_version: IDUNN_DEPLOYMENT_BRAKE_SCHEMA.into(), - brake_id: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - authority: IDUNN_DEPLOYMENT_BRAKE_AUTHORITY.into(), - runtime_id: "yggdrasil".into(), - status: "engaged".into(), - scope: IDUNN_DEPLOYMENT_BRAKE_SCOPE.into(), - reason: "first".into(), - observed_at_unix_millis: 100, - expires_at_unix_millis: None, - authorization_id: None, - authorization_purpose: None, - authorized_release_id: None, - authorized_deployment_id: None, - authorized_by: None, - authorization_issued_at_unix_millis: None, - authorization_expires_at_unix_millis: None, - signature_algorithm: None, - signature: None, - private_state_exposed: false, - updated_by: "operator/a".into(), - }; - replace_brake(&store_path, base.clone(), 100)?; - let stale = SingleFileMessagePackBackingStore::new(&store_path) - .pull_all_read_only_snapshot()? - .remove(0); - let mut changed = base.clone(); - changed.reason = "racing writer".into(); - changed.observed_at_unix_millis = 101; - replace_brake(&store_path, changed, 101)?; - let candidate = typed_envelope( - IDUNN_DEPLOYMENT_BRAKE_ID, - &base, - IDUNN_DEPLOYMENT_BRAKE_SCHEMA, - 102, - )?; - assert!( - !SingleFileMessagePackBackingStore::new(&store_path).compare_exchange( - &[CultCacheExpectedEnvelope { - r#type: IdunnDeploymentBrakeRecord::TYPE.into(), - key: IDUNN_DEPLOYMENT_BRAKE_ID.into(), - current: Some(stale) - }], - &[candidate] - )? - ); - Ok(()) - } } diff --git a/src/runner_evidence.rs b/src/runner_evidence.rs new file mode 100644 index 0000000..30dd1d2 --- /dev/null +++ b/src/runner_evidence.rs @@ -0,0 +1,805 @@ +//! The output tail of a failed runner step. +//! +//! One file, `runner-evidence.cc`, beside `control.cc`, holds one type: the +//! last bytes of a failed runner container's stdout and stderr, one record per +//! transaction. The engine's pre-fence failure path is its only writer, and +//! `idunn status --command` its only reader. It is never `control.cc`, which +//! refuses any type it does not know, and never `incidents.cc`, which is +//! published and carries no free text. +//! +//! The tail is raw runner output. It is kept unredacted and readable by root +//! alone (mode 0600): root can already read every secret the runner was given. +//! It never enters an error, a transaction record or the journal. The status +//! reader is a terminal, so everything it prints from this store goes through +//! one allowlist sink (`emit`): printable ASCII passes, all else is escaped. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail, ensure}; +use cultcache_rs::{ + CultCacheEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, + TryCompareExchangeSnapshotOutcome, +}; +use serde::{Deserialize, Serialize}; + +use crate::control_plane::{decode_record, require_id, typed_envelope}; +use crate::drivers::{ + LockContended, RUNNER_OUTPUT_TAIL_BYTES, RunnerStepFailure, RunnerSubject, + exchange_behind_private_lock, runner_status, +}; + +const RUNNER_FAILURE_SCHEMA: &str = "idunn.runner_failure.v1"; + +/// Records kept; the writer drops the oldest past this, so the store needs no +/// other owner. +pub(crate) const RUNNER_EVIDENCE_RECORDS: usize = 32; + +/// `runner-evidence.cc`, beside the control store. +pub(crate) fn runner_evidence_store_path(state_store: &Path) -> PathBuf { + state_store.with_file_name("runner-evidence.cc") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum SubjectKind { + Step, + ExternalInput, +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.runner_failure", schema = "idunn.runner_failure.v1")] +pub(crate) struct RunnerFailureRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) transaction_id: String, + #[cultcache(key = 2)] + pub(crate) command_id: String, + #[cultcache(key = 3)] + pub(crate) target: String, + #[cultcache(key = 4)] + pub(crate) subject_kind: SubjectKind, + #[cultcache(key = 5)] + pub(crate) subject_id: String, + #[cultcache(key = 6)] + pub(crate) runner: String, + #[cultcache(key = 7)] + pub(crate) exit_code: Option, + #[cultcache(key = 8)] + pub(crate) signal: Option, + #[cultcache(key = 9)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 10)] + pub(crate) stdout_total_bytes: u64, + #[cultcache(key = 11)] + pub(crate) stderr_tail: Vec, + #[cultcache(key = 12)] + pub(crate) stderr_total_bytes: u64, + #[cultcache(key = 13)] + pub(crate) recorded_at_unix_millis: u64, +} + +/// Tails stay out of `Debug`, so a failed assertion or a logged record never +/// prints runner output. +impl std::fmt::Debug for RunnerFailureRecord { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RunnerFailureRecord") + .field("transaction_id", &self.transaction_id) + .field("subject_id", &self.subject_id) + .finish_non_exhaustive() + } +} + +impl RunnerFailureRecord { + pub(crate) fn from_failure( + transaction_id: &str, + command_id: &str, + target: &str, + failure: &RunnerStepFailure, + now: u64, + ) -> Self { + let (subject_kind, subject_id) = match &failure.subject { + RunnerSubject::Step(id) => (SubjectKind::Step, id), + RunnerSubject::ExternalInput(id) => (SubjectKind::ExternalInput, id), + }; + Self { + schema_version: RUNNER_FAILURE_SCHEMA.to_owned(), + transaction_id: transaction_id.to_owned(), + command_id: command_id.to_owned(), + target: target.to_owned(), + subject_kind, + subject_id: subject_id.clone(), + runner: failure.runner.clone(), + exit_code: failure.exit_code, + signal: failure.signal, + stdout_tail: failure.stdout_tail.clone(), + stdout_total_bytes: failure.stdout_total_bytes, + stderr_tail: failure.stderr_tail.clone(), + stderr_total_bytes: failure.stderr_total_bytes, + recorded_at_unix_millis: now, + } + } + + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + self.schema_version == RUNNER_FAILURE_SCHEMA, + "runner failure schema is unsupported" + ); + require_id(&self.transaction_id, "runner failure transaction")?; + require_id(&self.command_id, "runner failure command")?; + require_id(&self.target, "runner failure target")?; + require_id(&self.subject_id, "runner failure subject")?; + require_id(&self.runner, "runner failure runner")?; + for (tail, total, stream) in [ + (&self.stdout_tail, self.stdout_total_bytes, "stdout"), + (&self.stderr_tail, self.stderr_total_bytes, "stderr"), + ] { + ensure!( + tail.len() <= RUNNER_OUTPUT_TAIL_BYTES, + "runner failure {stream} tail exceeds its bound" + ); + ensure!( + tail.len() as u64 <= total, + "runner failure {stream} tail exceeds its stream" + ); + } + Ok(()) + } + + fn envelope(&self) -> Result { + self.validate()?; + typed_envelope( + &self.transaction_id, + Self::TYPE, + RUNNER_FAILURE_SCHEMA, + self, + self.recorded_at_unix_millis, + ) + } +} + +/// One read of the store: the envelopes exactly as stored, and the records +/// decoded from them in the same order. +fn snapshot(path: &Path) -> Result<(Vec, Vec)> { + let envelopes = SingleFileMessagePackBackingStore::new(path) + .pull_all_read_only_snapshot() + .context("reading the runner evidence store")?; + let mut records = Vec::new(); + for envelope in &envelopes { + ensure!( + envelope.r#type == RunnerFailureRecord::TYPE + && envelope.schema_id.as_deref() == Some(RUNNER_FAILURE_SCHEMA), + "the runner evidence store holds a document that is not a runner failure" + ); + let record: RunnerFailureRecord = decode_record(envelope)?; + record.validate()?; + ensure!( + envelope.key == record.transaction_id, + "runner evidence store key differs from the record's transaction" + ); + records.push(record); + } + Ok((envelopes, records)) +} + +/// How a write left the store. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kept { + Stored, + /// The store held bytes that do not decode; it was moved to + /// `unreadable_store_path` and the record opened a fresh one. `replaced` + /// is the last-written time (unix milliseconds) of an earlier set-aside + /// store this one displaced, which is gone: one slot, and it says so. + SetAside { replaced: Option }, +} + +/// Where an undecodable store is kept. One slot: the next one replaces it, so +/// the damage a bad store can cost is one extra file. +pub(crate) fn unreadable_store_path(path: &Path) -> PathBuf { + path.with_file_name("runner-evidence.unreadable.cc") +} + +/// Moves the store over the previous set-aside one, root-only, and reports +/// what it displaced. Nothing of the store's content is named: a decode error +/// can quote stored values, and none is carried here. +fn set_aside_unreadable(path: &Path) -> Result> { + let aside = unreadable_store_path(path); + let replaced = std::fs::symlink_metadata(&aside).ok().map(|earlier| { + earlier + .modified() + .ok() + .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) + .map_or(0, |since| since.as_millis() as u64) + }); + std::fs::rename(path, &aside) + .context("the runner evidence store is undecodable and could not be set aside")?; + keep_private(&aside)?; + Ok(replaced) +} + +/// How the writer reads the store's bytes; tests inject an I/O error here. +type ReadStore<'a> = &'a dyn Fn(&Path) -> std::io::Result>; + +fn unreadable_fault(error: &std::io::Error) -> anyhow::Error { + anyhow::anyhow!("the runner evidence store could not be read: {}", error.kind()) +} + +/// The one writer. Upsert by transaction id; past `RUNNER_EVIDENCE_RECORDS` the +/// oldest records go in the same exchange. The file ends at mode 0600. +/// +/// A store is set aside only when bytes that were read do not decode. An I/O +/// error leaves the store in place, keeps no evidence for that failure, and +/// the error names its kind for the journal. +pub(crate) fn record(path: &Path, record: &RunnerFailureRecord) -> Result { + record_reading(path, record, &|path| std::fs::read(path)) +} + +fn record_reading(path: &Path, record: &RunnerFailureRecord, read: ReadStore) -> Result { + let first = match read(path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(unreadable_fault(&error)), + }; + let (envelopes, records, how) = match snapshot(path) { + Ok((envelopes, records)) => (envelopes, records, Kept::Stored), + Err(_) => { + let Some(first) = first else { + bail!("the runner evidence store could not be decoded"); + }; + // Undecodable only if the same bytes are still there to read: a + // second read that fails, or finds other bytes, is not a verdict. + match read(path) { + Ok(second) if second == first => {} + Ok(_) => bail!("the runner evidence store changed while it was being read"), + Err(error) => return Err(unreadable_fault(&error)), + } + let replaced = set_aside_unreadable(path)?; + (Vec::new(), Vec::new(), Kept::SetAside { replaced }) + } + }; + let mut kept: Vec<(u64, CultCacheEnvelope)> = envelopes + .iter() + .zip(&records) + .filter(|(_, stored)| stored.transaction_id != record.transaction_id) + .map(|(envelope, stored)| (stored.recorded_at_unix_millis, envelope.clone())) + .collect(); + kept.push((record.recorded_at_unix_millis, record.envelope()?)); + // Stable: records of one instant keep their stored order, so the record + // being written survives a tie. + kept.sort_by_key(|(recorded_at, _)| *recorded_at); + let excess = kept.len().saturating_sub(RUNNER_EVIDENCE_RECORDS); + let replacements: Vec<_> = kept + .into_iter() + .skip(excess) + .map(|(_, envelope)| envelope) + .collect(); + match exchange_behind_private_lock(path, &envelopes, &replacements)? { + TryCompareExchangeSnapshotOutcome::Exchanged => {} + TryCompareExchangeSnapshotOutcome::Mismatch => { + bail!("the runner evidence store changed while it was being written") + } + TryCompareExchangeSnapshotOutcome::LockContended => { + return Err(LockContended { + store: "runner evidence store", + } + .into()); + } + } + keep_private(path)?; + Ok(how) +} + +#[cfg(unix)] +fn keep_private(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("making the runner evidence store private") +} + +#[cfg(not(unix))] +fn keep_private(_path: &Path) -> Result<()> { + Ok(()) +} + +/// The one reader. An absent store holds nothing. +pub(crate) fn for_transaction( + path: &Path, + transaction_id: &str, +) -> Result> { + Ok(snapshot(path)? + .1 + .into_iter() + .find(|stored| stored.transaction_id == transaction_id)) +} + +/// The record as `idunn status --command` shows it, as raw lines. They are +/// not text yet: `show_status` is the only caller, and `emit` the only +/// writer. +fn render(record: &RunnerFailureRecord) -> Vec> { + let noun = match record.subject_kind { + SubjectKind::Step => "step", + SubjectKind::ExternalInput => "external input", + }; + let mut lines: Vec> = vec![ + format!(" failed {noun} {}", record.subject_id).into_bytes(), + format!(" runner {}", record.runner).into_bytes(), + format!(" status {}", runner_status(record.exit_code, record.signal)).into_bytes(), + ]; + for (stream, tail, total) in [ + ("stdout", &record.stdout_tail, record.stdout_total_bytes), + ("stderr", &record.stderr_tail, record.stderr_total_bytes), + ] { + lines.push(format!(" {stream} (last {} of {total} bytes)", tail.len()).into_bytes()); + let text = tail.strip_suffix(b"\n").unwrap_or(tail); + if !text.is_empty() { + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); + } + } + lines +} + +/// What `idunn status --command` prints for a transaction's runner evidence: +/// its record, or one line naming why the store is unreadable. Nothing is +/// written when the transaction has no record. +pub(crate) fn show_status( + out: &mut impl std::io::Write, + store: &Path, + transaction_id: &str, +) -> std::io::Result<()> { + let lines = match for_transaction(store, transaction_id) { + Ok(Some(record)) => render(&record), + Ok(None) => return Ok(()), + Err(error) => { + vec![format!(" runner evidence unreadable: {}", error.root_cause()).into_bytes()] + } + }; + emit(out, &lines) +} + +/// The one sink: every byte the status read prints from the evidence store, +/// or from an error about it, is written here and nowhere else. An allowlist: +/// printable ASCII and tab pass; every other code point is a visible +/// `\u{..}` escape and every invalid UTF-8 byte a `\x..` escape. A newline +/// inside a line is escaped too: this function writes the only ones. +pub(crate) fn emit(out: &mut impl std::io::Write, lines: &[Vec]) -> std::io::Result<()> { + for line in lines { + out.write_all(escape(line).as_bytes())?; + out.write_all(b"\n")?; + } + Ok(()) +} + +pub(crate) fn escape(bytes: &[u8]) -> String { + let mut shown = String::with_capacity(bytes.len()); + for chunk in bytes.utf8_chunks() { + for character in chunk.valid().chars() { + match character { + ' '..='~' | '\t' => shown.push(character), + _ => shown.push_str(&format!("\\u{{{:x}}}", u32::from(character))), + } + } + for byte in chunk.invalid() { + shown.push_str(&format!("\\x{byte:02x}")); + } + } + shown +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn failure(subject: RunnerSubject, stdout: &[u8], stderr: &[u8]) -> RunnerStepFailure { + RunnerStepFailure { + subject, + runner: "rust".into(), + exit_code: Some(3), + signal: None, + stdout_tail: stdout.to_vec(), + stdout_total_bytes: stdout.len() as u64, + stderr_tail: stderr.to_vec(), + stderr_total_bytes: stderr.len() as u64, + } + } + + /// What status writes for a record, through the one sink. + fn shown(record: &RunnerFailureRecord) -> String { + let mut written = Vec::new(); + emit(&mut written, &render(record)).expect("a Vec takes any write"); + String::from_utf8(written).expect("the sink writes ASCII only") + } + + fn shown_lines(record: &RunnerFailureRecord) -> Vec { + shown(record).lines().map(str::to_owned).collect() + } + + fn record_at(transaction: &str, now: u64) -> RunnerFailureRecord { + RunnerFailureRecord::from_failure( + transaction, + "up-service", + "service", + &failure(RunnerSubject::Step("build".into()), b"out\n", b"err\n"), + now, + ) + } + + #[test] + fn the_evidence_store_keeps_the_newest_32() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // The literal, not the constant: the spec's bound is 32. + for index in 0..=32 { + record(&path, &record_at(&format!("tx-{index:02}"), 1000 + index as u64))?; + } + let kept: Vec<_> = snapshot(&path)?.1.into_iter().map(|stored| stored.transaction_id).collect(); + assert_eq!(kept.len(), 32); + assert!(!kept.contains(&"tx-00".to_owned()), "the oldest survived"); + assert!(kept.contains(&"tx-32".to_owned()), "the newest was evicted"); + assert!(kept.contains(&"tx-01".to_owned()), "the second oldest was evicted early"); + // Rewriting a resident transaction replaces it and drops nothing. + let mut again = record_at("tx-05", 5000); + again.stdout_tail = b"rewritten\n".to_vec(); + again.stdout_total_bytes = 10; + record(&path, &again)?; + let kept = snapshot(&path)?.1; + assert_eq!(kept.len(), 32); + assert!(kept.iter().any(|stored| stored.transaction_id == "tx-01")); + let stored = for_transaction(&path, "tx-05")?.context("rewritten record is gone")?; + assert_eq!(stored.stdout_tail, b"rewritten\n"); + assert_eq!(for_transaction(&path, "tx-00")?, None); + Ok(()) + } + + #[test] + fn a_record_round_trips_byte_for_byte() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let written = RunnerFailureRecord::from_failure( + "tx-a", + "up-service", + "service", + &failure(RunnerSubject::ExternalInput("toolchain".into()), &[0, 1, 255, 10], b""), + 77, + ); + record(&path, &written)?; + assert_eq!(for_transaction(&path, "tx-a")?, Some(written)); + assert_eq!(for_transaction(&path, "tx-b")?, None); + Ok(()) + } + + #[test] + fn the_evidence_store_is_private() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // Whatever the umask, the mode is the store's own rule, on every write. + for index in 0..2 { + record(&path, &record_at(&format!("tx-{index}"), 10 + index))?; + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + } + Ok(()) + } + + #[test] + fn a_tail_over_its_bound_or_its_stream_is_refused() { + let mut oversize = record_at("tx-a", 1); + oversize.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES + 1]; + oversize.stdout_total_bytes = u64::MAX; + assert!(oversize.validate().is_err()); + let mut impossible = record_at("tx-a", 1); + impossible.stderr_total_bytes = 1; + assert!(impossible.validate().is_err()); + let mut exact = record_at("tx-a", 1); + exact.stdout_tail = vec![b'x'; RUNNER_OUTPUT_TAIL_BYTES]; + exact.stdout_total_bytes = RUNNER_OUTPUT_TAIL_BYTES as u64; + assert!(exact.validate().is_ok()); + } + + #[test] + fn status_renders_the_tail_with_control_bytes_escaped() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"first\n\x1b[2Jsecond\x00end\tTab\nlast\n".to_vec(); + stored.stdout_total_bytes = 5000; + stored.stderr_tail = b"bad\r\x07".to_vec(); + stored.stderr_total_bytes = 6; + let lines = shown_lines(&stored); + let text = lines.join("\n"); + assert!(!text.contains('\u{1b}') && !text.contains('\0') && !text.contains('\r')); + assert!(!text.contains('\u{7}')); + assert_eq!( + lines, + vec![ + " failed step build", + " runner rust", + " status exit status 3", + " stdout (last 30 of 5000 bytes)", + " | first", + " | \\u{1b}[2Jsecond\\u{0}end\tTab", + " | last", + " stderr (last 5 of 6 bytes)", + " | bad\\u{d}\\u{7}", + ] + ); + } + + #[test] + fn debug_names_the_record_and_prints_no_tail() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = b"SECRET-OUTPUT".to_vec(); + stored.stdout_total_bytes = 13; + let debug = format!("{stored:?}"); + assert!(debug.contains("tx-a") && !debug.contains("SECRET"), "{debug}"); + } + + #[test] + fn an_empty_tail_renders_only_its_header() { + let mut stored = record_at("tx-a", 1); + stored.stdout_tail = Vec::new(); + stored.stdout_total_bytes = 0; + let lines = shown_lines(&stored); + assert!(lines.contains(&" stdout (last 0 of 0 bytes)".to_owned())); + assert!(!lines.iter().any(|line| line == " | ")); + } + + #[test] + fn the_reader_names_an_unreadable_store_as_an_error() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 not a store")?; + assert!(for_transaction(&path, "tx-a").is_err()); + Ok(()) + } + + fn files_beside(path: &Path) -> Result> { + let mut names: Vec = std::fs::read_dir(path.parent().context("no parent")?)? + .map(|entry| Ok(entry?.file_name().to_string_lossy().into_owned())) + .collect::>()?; + names.retain(|name| name.starts_with("runner-evidence") && !name.contains("lock")); + names.sort(); + Ok(names) + } + + /// A store a newer Idunn wrote: well-formed, but a schema this one does not know. + fn write_future_store(path: &Path) -> Result<()> { + let mut future = record_at("tx-future", 1); + future.schema_version = "idunn.runner_failure.v2".into(); + let envelope = typed_envelope( + &future.transaction_id, + RunnerFailureRecord::TYPE, + "idunn.runner_failure.v2", + &future, + 1, + )?; + let outcome = exchange_behind_private_lock(path, &[], &[envelope])?; + ensure!( + matches!(outcome, TryCompareExchangeSnapshotOutcome::Exchanged), + "the future store was not written" + ); + Ok(()) + } + + #[test] + fn a_corrupt_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damage = b"\xc1 not a store \x00 damage"; + std::fs::write(&path, damage)?; + let kept = record(&path, &record_at("tx-a", 5))?; + assert_eq!(kept, Kept::SetAside { replaced: None }); + // Kept, not deleted: the damage is where the journal says, root-only. + let aside = unreadable_store_path(&path); + assert_eq!(std::fs::read(&aside)?, damage); + assert_eq!(std::fs::metadata(&aside)?.permissions().mode() & 0o777, 0o600); + // The failure's evidence is in a fresh, private store. + assert_eq!( + for_transaction(&path, "tx-a")?.map(|stored| stored.transaction_id), + Some("tx-a".to_owned()) + ); + assert_eq!(std::fs::metadata(&path)?.permissions().mode() & 0o777, 0o600); + // A healthy store is not set aside again. + assert_eq!(record(&path, &record_at("tx-b", 6))?, Kept::Stored); + assert_eq!(std::fs::read(&aside)?, damage); + Ok(()) + } + + #[test] + fn a_future_version_store_is_set_aside_and_the_next_failure_keeps_its_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + write_future_store(&path)?; + let before = std::fs::read(&path)?; + assert!(for_transaction(&path, "tx-future").is_err()); + assert_eq!( + record(&path, &record_at("tx-a", 5))?, + Kept::SetAside { replaced: None } + ); + assert_eq!(std::fs::read(unreadable_store_path(&path))?, before); + assert!(for_transaction(&path, "tx-a")?.is_some()); + assert_eq!(for_transaction(&path, "tx-future")?, None); + Ok(()) + } + + #[test] + fn at_most_one_unreadable_store_is_kept() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + let damages: [&[u8]; 3] = [b"\xc1 first", b"\xc1 second", b"\xc1 third"]; + for (index, damage) in damages.into_iter().enumerate() { + std::fs::write(&path, damage)?; + record(&path, &record_at(&format!("tx-{index}"), 5 + index as u64))?; + assert_eq!( + std::fs::read(unreadable_store_path(&path))?, + damage, + "the newest damage is the one kept" + ); + } + assert_eq!( + files_beside(&path)?, + vec![ + "runner-evidence.cc".to_owned(), + "runner-evidence.unreadable.cc".to_owned() + ] + ); + Ok(()) + } + + #[test] + fn a_store_that_cannot_be_read_is_left_in_place_and_the_write_fails() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + // A directory where the store should be: unreadable, not undecodable. + std::fs::create_dir(&path)?; + assert!(record(&path, &record_at("tx-a", 1)).is_err()); + assert!(path.is_dir()); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn every_code_point_but_printable_ascii_is_a_visible_escape() { + // The allowlist, exhaustively: no class list decides anything. + for code in 0..=0x10ffffu32 { + let Some(character) = char::from_u32(code) else { + continue; + }; + let text = character.to_string(); + let shown = escape(text.as_bytes()); + if matches!(character, ' '..='~' | '\t') { + assert_eq!(shown, text, "U+{code:04X} should pass"); + } else { + assert_eq!(shown, format!("\\u{{{code:x}}}"), "U+{code:04X}"); + } + } + // Invalid UTF-8 is escaped byte by byte; valid text round it passes. + assert_eq!(escape(b"a\xc1\xffb\xe2\x82"), "a\\xc1\\xffb\\xe2\\x82"); + assert_eq!(escape(b"ok \xe2\x82\xac"), "ok \\u{20ac}"); + } + + #[test] + fn a_newline_inside_a_line_cannot_forge_a_line() { + let mut stored = record_at("tx-a", 1); + stored.subject_id = "build\n status exit status 0".into(); + let lines = shown_lines(&stored); + assert_eq!(lines[0], " failed step build\\u{a} status exit status 0"); + assert_eq!( + lines.iter().filter(|line| line.starts_with(" status ")).count(), + 1 + ); + } + + /// A store whose first record is well-formed except that its subject kind + /// holds `marker`: the decode error quotes it. + fn store_with_hostile_subject_kind(path: &Path, marker: &[u8; 4]) -> Result<()> { + record(path, &record_at("tx-a", 1))?; + let mut bytes = std::fs::read(path)?; + let at = bytes + .windows(4) + .position(|window| window == b"step") + .context("no step marker")?; + bytes[at..at + 4].copy_from_slice(marker); + std::fs::write(path, bytes)?; + Ok(()) + } + + #[test] + fn the_unreadable_store_line_is_escaped_with_the_rest() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"\x1b[2J")?; + let mut written = Vec::new(); + show_status(&mut written, &path, "tx-a")?; + assert!(!written.contains(&0x1b), "an escape byte reached the terminal"); + assert!(written.iter().all(|byte| matches!(byte, b' '..=b'~' | b'\t' | b'\n'))); + let text = String::from_utf8(written)?; + assert!(text.starts_with(" runner evidence unreadable: "), "{text}"); + assert!(text.contains("\\u{1b}[2J"), "the quoted value was not shown escaped: {text}"); + Ok(()) + } + + #[test] + fn a_failed_set_aside_names_itself_keeps_the_store_and_echoes_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + store_with_hostile_subject_kind(&path, b"CNRY")?; + let damage = std::fs::read(&path)?; + // Sanity: this damage decodes to an error that quotes the canary. + assert!(format!("{:#}", snapshot(&path).unwrap_err()).contains("CNRY")); + // A non-empty directory where the aside goes: rename fails even as root. + let aside = unreadable_store_path(&path); + std::fs::create_dir(&aside)?; + std::fs::write(aside.join("occupied"), b"x")?; + let error = record(&path, &record_at("tx-b", 2)).unwrap_err(); + for text in [format!("{error:#}"), format!("{error:?}")] { + assert!(text.contains("could not be set aside"), "{text}"); + assert!(!text.contains("CNRY"), "stored content reached the error: {text}"); + } + // The failure cost nothing: the store is as it was, with no evidence + // written over it. + assert_eq!(std::fs::read(&path)?, damage); + assert!(aside.join("occupied").exists()); + Ok(()) + } + + #[test] + fn an_io_error_leaves_the_store_in_place_and_keeps_no_evidence() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + record(&path, &record_at("tx-a", 1))?; + let before = std::fs::read(&path)?; + let denied = || std::io::Error::from(std::io::ErrorKind::PermissionDenied); + // The first read fails on a healthy store. + let error = record_reading(&path, &record_at("tx-b", 2), &|_| Err(denied())).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, before, "the store changed"); + assert!(!unreadable_store_path(&path).exists()); + assert!(for_transaction(&path, "tx-b")?.is_none()); + // The second read, the one that confirms the bytes do not decode, fails. + std::fs::write(&path, b"\xc1 not a store")?; + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| { + calls.set(calls.get() + 1); + if calls.get() == 2 { + return Err(denied()); + } + std::fs::read(path) + }; + let error = record_reading(&path, &record_at("tx-b", 2), &read).unwrap_err(); + assert!(format!("{error:#}").contains("permission denied"), "{error:#}"); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + // Bytes that changed between the reads are no verdict either. + let calls = std::cell::Cell::new(0usize); + let read = |path: &Path| -> std::io::Result> { + calls.set(calls.get() + 1); + let mut bytes = std::fs::read(path)?; + bytes.push(calls.get() as u8); + Ok(bytes) + }; + assert!(record_reading(&path, &record_at("tx-b", 2), &read).is_err()); + assert_eq!(std::fs::read(&path)?, b"\xc1 not a store"); + assert!(!unreadable_store_path(&path).exists()); + Ok(()) + } + + #[test] + fn a_second_set_aside_says_it_replaced_the_first() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = runner_evidence_store_path(&temp.path().join("control.cc")); + std::fs::write(&path, b"\xc1 first")?; + assert_eq!( + record(&path, &record_at("tx-a", 1))?, + Kept::SetAside { replaced: None } + ); + std::fs::write(&path, b"\xc1 second")?; + let Kept::SetAside { replaced: Some(_) } = record(&path, &record_at("tx-b", 2))? else { + bail!("a displaced set-aside store went unreported"); + }; + assert_eq!(std::fs::read(unreadable_store_path(&path))?, b"\xc1 second"); + Ok(()) + } +} diff --git a/src/steward_orders.rs b/src/steward_orders.rs new file mode 100644 index 0000000..379e09c --- /dev/null +++ b/src/steward_orders.rs @@ -0,0 +1,4086 @@ +//! Typed orders from Idunn's steward. +//! +//! `idunn orders serve` is a second root unit from the same binary as +//! `idunn serve`, with its own store (`orders.cc`) and socket. It alone mints +//! order ids, decides acceptance and refusal, actuates, and records outcomes. +//! The survival daemon takes no part: `control.cc` never learns an order type. +//! +//! An order is a closed enum, never a command string: install packages, act on +//! a systemd unit, or deploy a target through the same commit primitive +//! `idunn up` uses. Every unit is allowed except the always-refused set, which +//! is compiled in here and read from nowhere: no binding field, record or flag +//! holds it. The `#operator` post is a retried projection of the store. +//! +//! Lifecycle, one row per step: ISSUE a frame on the socket; AUTHENTICATE the +//! peer's uid; VALIDATE the grammar (a malformed order is a typed refusal and +//! no record); ACCEPT a `steward_order`; SCOPE the always-refused check; BRAKE +//! `mind-control check`; START record before actuation, at most once; RUN; +//! RESULT outcome; POST accepted and result. +//! +//! No refusal or error here carries a value the issuer supplied: it names the +//! field and a code. A refused unit that a record names was resolved by +//! systemd and passed the unit grammar. + +use std::fmt; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail, ensure}; +use cultcache_rs::{ + CultCacheExpectedEnvelope, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::control_plane::{ + command_state, commit_deployment_command, decode_record, default_state_store, now_millis, + path_value, require_id, require_selector, string_value, typed_envelope, +}; +use crate::runner_evidence::{emit, escape}; + +const ORDER_SCHEMA: &str = "idunn.steward_order.v1"; +const START_SCHEMA: &str = "idunn.steward_order_start.v1"; +const OUTCOME_SCHEMA: &str = "idunn.steward_order_outcome.v1"; +const POST_SCHEMA: &str = "idunn.steward_order_post.v1"; + +const REQUEST_FRAME_LIMIT: usize = 16 * 1024; +const RESPONSE_FRAME_LIMIT: usize = 256 * 1024; +const TAIL_BYTES: usize = 8 * 1024; +const REASON_LIMIT: usize = 300; +const PACKAGE_LIMIT: usize = 16; +const UNIT_LIMIT: usize = 200; +const LIST_LIMIT: usize = 50; +const UNIT_SUFFIXES: [&str; 5] = [".service", ".socket", ".timer", ".path", ".mount"]; + +/// The always-refused set: Idunn's own units and the minds' own. Matched on +/// every name a unit answers to, ASCII case-insensitively (systemd units are +/// case-sensitive, so the fold only widens the refusal). +fn is_refused_name(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + ["idunn-", "mind-self@", "mind-persona@"] + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +/// Units whose binaries suspend, hibernate or open recovery: no property +/// reveals it, so they are refused by name, on every name they answer to. +const HOST_NAMES: [&str; 6] = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", +]; + +fn is_host_name(name: &str) -> bool { + HOST_NAMES.iter().any(|host| name.eq_ignore_ascii_case(host)) +} + +// ---------------------------------------------------------------- grammar -- + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Code { + Empty, + TooLong, + TooMany, + BadCharacter, + BadShape, + Malformed, + Unauthorized, + Unknown, + Unavailable, +} + +/// A refusal names the field and a code. It never carries the offending value. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Refusal { + pub(crate) field: String, + pub(crate) code: Code, +} + +impl Refusal { + fn new(field: &'static str, code: Code) -> Self { + Self { + field: field.to_owned(), + code, + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{} refused: {:?}", self.field, self.code) + } +} + +impl std::error::Error for Refusal {} + +fn validate_package(spec: &str) -> Result<(), Refusal> { + const FIELD: &str = "package"; + if spec.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if spec.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let (name, version) = match spec.split_once('=') { + Some((name, version)) => (name, Some(version)), + None => (spec, None), + }; + let bytes = name.as_bytes(); + if bytes.len() < 2 { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + let head = bytes[0].is_ascii_lowercase() || bytes[0].is_ascii_digit(); + let rest = bytes[1..].iter().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'.' | b'-') + }); + if !head || !rest { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + // apt reads `name-` as a removal and `name.deb` as a local file. + if name.ends_with('-') || name.ends_with(".deb") { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if let Some(version) = version { + if version.is_empty() { + return Err(Refusal::new(FIELD, Code::BadShape)); + } + if !version.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'~' | b':' | b'-') + }) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + } + Ok(()) +} + +fn validate_unit(unit: &str) -> Result<(), Refusal> { + const FIELD: &str = "unit"; + if unit.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if unit.len() > UNIT_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + let allowed = unit.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b':' | b'_' | b'.' | b'@' | b'-') + }); + if !allowed || unit.starts_with('-') { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + match UNIT_SUFFIXES + .iter() + .find_map(|suffix| unit.strip_suffix(suffix)) + { + Some(stem) if !stem.is_empty() => Ok(()), + _ => Err(Refusal::new(FIELD, Code::BadShape)), + } +} + +fn validate_reason(reason: &str) -> Result<(), Refusal> { + const FIELD: &str = "reason"; + if reason.is_empty() { + return Err(Refusal::new(FIELD, Code::Empty)); + } + if reason.len() > REASON_LIMIT { + return Err(Refusal::new(FIELD, Code::TooLong)); + } + if !reason.bytes().all(|byte| (0x20..=0x7e).contains(&byte)) { + return Err(Refusal::new(FIELD, Code::BadCharacter)); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Verb { + Start, + Stop, + Restart, + Reload, +} + +impl Verb { + fn as_str(self) -> &'static str { + match self { + Verb::Start => "start", + Verb::Stop => "stop", + Verb::Restart => "restart", + Verb::Reload => "reload", + } + } + + fn parse(word: &str) -> Option { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + .into_iter() + .find(|verb| verb.as_str() == word) + } +} + +/// The closed set of things an order can ask for. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Operation { + PackageInstall { packages: Vec }, + UnitAction { verb: Verb, unit: String }, + Deploy { selector: String }, +} + +/// One fixed argv for `systemd-run`, built from the typed operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Step { + pub(crate) argv: Vec, + pub(crate) environment: Vec<(String, String)>, +} + +impl Operation { + fn validate(&self) -> Result<(), Refusal> { + match self { + Operation::PackageInstall { packages } => { + if packages.is_empty() { + return Err(Refusal::new("package", Code::Empty)); + } + if packages.len() > PACKAGE_LIMIT { + return Err(Refusal::new("package", Code::TooMany)); + } + packages.iter().try_for_each(|package| validate_package(package)) + } + Operation::UnitAction { unit, .. } => validate_unit(unit), + Operation::Deploy { selector } => require_selector(selector) + .map_err(|_| Refusal::new("selector", Code::BadShape)), + } + } + + fn steps(&self) -> Vec { + let words = |words: &[&str]| words.iter().map(|word| (*word).to_owned()).collect(); + let apt_environment = vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())]; + match self { + Operation::PackageInstall { packages } => { + // Never a removal, never a pattern: each name selects exactly + // itself, and apt aborts rather than displace an installed package. + let mut install: Vec = + words(&["apt-get", "install", "-y", "-q", "--no-remove", "--"]); + let mut present: Vec = words(&["dpkg", "-s", "--"]); + for package in packages { + let (name, version) = package.split_once('=').unwrap_or((package, "")); + install.push(match version { + "" => format!("?exact-name({name})"), + version => format!("?exact-name({name})={version}"), + }); + present.push(name.to_owned()); + } + // An exact name that matches nothing installs nothing and exits 0, + // so the last step fails unless every named package is installed. + vec![ + Step { + argv: words(&["apt-get", "update", "-q"]), + environment: apt_environment.clone(), + }, + Step { + argv: install, + environment: apt_environment, + }, + Step { + argv: present, + environment: Vec::new(), + }, + ] + } + Operation::UnitAction { verb, unit } => vec![Step { + argv: vec![ + "systemctl".into(), + verb.as_str().into(), + "--job-mode=ignore-requirements".into(), + "--".into(), + unit.clone(), + ], + environment: Vec::new(), + }], + Operation::Deploy { .. } => Vec::new(), + } + } + + fn describe(&self) -> String { + match self { + Operation::PackageInstall { packages } => { + format!("package-install {}", packages.join(" ")) + } + Operation::UnitAction { verb, unit } => format!("unit {} {unit}", verb.as_str()), + Operation::Deploy { selector } => format!("deploy {selector}"), + } + } +} + +// ---------------------------------------------------------------- records -- + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "idunn.steward_order", schema = "idunn.steward_order.v1")] +pub(crate) struct OrderRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) principal: String, + #[cultcache(key = 3)] + pub(crate) issuer_uid: u32, + #[cultcache(key = 4)] + pub(crate) issuer_unit: String, + #[cultcache(key = 5)] + pub(crate) issuer_invocation: String, + #[cultcache(key = 6)] + pub(crate) issued_at_unix_millis: u64, + #[cultcache(key = 7)] + pub(crate) operation: Operation, + #[cultcache(key = 8)] + pub(crate) reason: String, +} + +impl fmt::Debug for OrderRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OrderRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_start", + schema = "idunn.steward_order_start.v1" +)] +pub(crate) struct StartRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) started_at_unix_millis: u64, + /// The transient unit name, or the deployment command id. + #[cultcache(key = 3)] + pub(crate) actuation: String, +} + +impl fmt::Debug for StartRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StartRecord") + .field("order_id", &self.order_id) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Via { + Name, + Alias, + Host, + OnEnd, + Bound, + Upholds, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Refused { + AlwaysRefused { unit: String, via: Via }, + /// The unit only wraps `member` (`ConsistsOf=`): a stop, restart or reload + /// of the wrapper exits 0 and leaves the member alone. + Wrapper { member: String }, + Braked, + /// The order's principal or issuing uid is not the steward bound now. + NotSteward, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Outcome { + Ran, + Failed { + exit_code: Option, + signal: Option, + }, + Refused(Refused), + Lost, + Submitted { command_id: String }, +} + +impl Outcome { + fn describe(&self) -> String { + match self { + Outcome::Ran => "ran (exit 0)".into(), + Outcome::Failed { + exit_code: Some(code), + .. + } => format!("failed (exit {code})"), + Outcome::Failed { + signal: Some(signal), + .. + } => format!("failed (signal {signal})"), + Outcome::Failed { .. } => "failed (not run to an exit)".into(), + Outcome::Refused(Refused::AlwaysRefused { unit, via }) => format!( + "refused: always-refused unit {} reached by {}", + escape(unit.as_bytes()), + match via { + Via::Name => "name", + Via::Alias => "alias", + Via::Host => "host", + Via::OnEnd => "on-end", + Via::Bound => "bound", + Via::Upholds => "upholds", + } + ), + Outcome::Refused(Refused::Wrapper { member }) => format!( + "refused: wrapper unit, it only wraps {}", + escape(member.as_bytes()) + ), + Outcome::Refused(Refused::Braked) => "refused: braked".into(), + Outcome::Refused(Refused::NotSteward) => { + "refused: the issuer is not the current steward".into() + } + Outcome::Lost => "lost: the orders service restarted mid-run".into(), + Outcome::Submitted { command_id } => { + format!("submitted deployment {}", escape(command_id.as_bytes())) + } + } + } +} + +#[derive(Clone, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_outcome", + schema = "idunn.steward_order_outcome.v1" +)] +pub(crate) struct OutcomeRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + #[cultcache(key = 1)] + pub(crate) order_id: String, + #[cultcache(key = 2)] + pub(crate) ended_at_unix_millis: u64, + #[cultcache(key = 3)] + pub(crate) outcome: Outcome, + #[cultcache(key = 4)] + pub(crate) stdout_tail: Vec, + #[cultcache(key = 5)] + pub(crate) stderr_tail: Vec, +} + +/// Tails stay out of `Debug`: they are raw program output. +impl fmt::Debug for OutcomeRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OutcomeRecord") + .field("order_id", &self.order_id) + .field("outcome", &self.outcome) + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum PostKind { + Accepted, + Result, +} + +impl PostKind { + fn as_str(self) -> &'static str { + match self { + PostKind::Accepted => "accepted", + PostKind::Result => "result", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache( + type = "idunn.steward_order_post", + schema = "idunn.steward_order_post.v1" +)] +pub(crate) struct PostRecord { + #[cultcache(key = 0)] + pub(crate) schema_version: String, + /// `:`. + #[cultcache(key = 1)] + pub(crate) post_key: String, + #[cultcache(key = 2)] + pub(crate) event_id: String, + #[cultcache(key = 3)] + pub(crate) posted_at_unix_millis: u64, +} + +// ------------------------------------------------------------------ store -- + +/// `orders.cc`: root-only, written by `idunn orders serve` alone. +pub(crate) struct OrderStore { + directory: PathBuf, +} + +#[derive(Default)] +struct Ledger { + orders: Vec, + starts: Vec, + outcomes: Vec, + posts: Vec, +} + +impl Ledger { + fn start_of(&self, order_id: &str) -> Option<&StartRecord> { + self.starts.iter().find(|start| start.order_id == order_id) + } + + fn outcome_of(&self, order_id: &str) -> Option<&OutcomeRecord> { + self.outcomes + .iter() + .find(|outcome| outcome.order_id == order_id) + } + + fn posted(&self, order_id: &str, kind: PostKind) -> bool { + let key = post_key(order_id, kind); + self.posts.iter().any(|post| post.post_key == key) + } +} + +fn post_key(order_id: &str, kind: PostKind) -> String { + format!("{order_id}:{}", kind.as_str()) +} + +impl OrderStore { + pub(crate) fn new(directory: &Path) -> Self { + Self { + directory: directory.to_owned(), + } + } + + fn path(&self) -> PathBuf { + self.directory.join("orders.cc") + } + + fn prepare(&self) -> Result<()> { + std::fs::create_dir_all(&self.directory).context("creating the orders directory")?; + std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700)) + .context("making the orders directory private") + } + + /// Insert a record whose (type, key) is absent. `false` means it was there. + fn put(&self, key: &str, schema: &str, value: &T) -> Result { + self.prepare()?; + let path = self.path(); + let now = now_millis()?; + let inserted = SingleFileMessagePackBackingStore::new(&path).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: T::TYPE.into(), + key: key.into(), + current: None, + }], + &[typed_envelope(key, T::TYPE, schema, value, now)?], + )?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + .context("making the orders store private")?; + Ok(inserted) + } + + fn ledger(&self) -> Result { + let path = self.path(); + let mut ledger = Ledger::default(); + if !path.exists() { + return Ok(ledger); + } + let envelopes = SingleFileMessagePackBackingStore::new(&path) + .pull_all_read_only_snapshot() + .context("reading the orders store")?; + for envelope in &envelopes { + let schema = envelope.schema_id.as_deref(); + let key = envelope.key.as_str(); + match envelope.r#type.as_str() { + OrderRecord::TYPE if schema == Some(ORDER_SCHEMA) => { + let record: OrderRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an order is stored under another key"); + ledger.orders.push(record); + } + StartRecord::TYPE if schema == Some(START_SCHEMA) => { + let record: StartRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "a start is stored under another key"); + ledger.starts.push(record); + } + OutcomeRecord::TYPE if schema == Some(OUTCOME_SCHEMA) => { + let record: OutcomeRecord = decode_record(envelope)?; + ensure!(record.order_id == key, "an outcome is stored under another key"); + ledger.outcomes.push(record); + } + PostRecord::TYPE if schema == Some(POST_SCHEMA) => { + let record: PostRecord = decode_record(envelope)?; + ensure!(record.post_key == key, "a post is stored under another key"); + ledger.posts.push(record); + } + _ => bail!("the orders store holds a document that is not an order record"), + } + } + ledger + .orders + .sort_by_key(|order| (order.issued_at_unix_millis, order.order_id.clone())); + Ok(ledger) + } +} + +// ------------------------------------------------------------------ ports -- + +/// The properties naming what systemd does when a unit ends or its job times +/// out: each holds an emergency action (`reboot`, `poweroff-force`, ...). +const ACTIONS: [&str; 4] = ["SuccessAction", "FailureAction", "StartLimitAction", "JobTimeoutAction"]; + +/// The properties naming units systemd starts when a unit ends. +const ON_END: [&str; 2] = ["OnFailure", "OnSuccess"]; + +/// What `systemctl show` says about one unit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct UnitFacts { + pub(crate) id: String, + pub(crate) names: Vec, + pub(crate) not_found: bool, + /// The properties that make the unit act when it ends: each emergency + /// action other than `none`, and each `OnFailure=` and `OnSuccess=` unit. + pub(crate) on_end: Vec, + /// The units bound to this one at runtime (`BoundBy=`): systemd stops + /// them when this unit stops, whatever the job mode. + pub(crate) bound_by: Vec, + /// The units this one upholds (`Upholds=`): systemd starts them in a full + /// transaction whenever this unit is active, whatever the job mode. + pub(crate) upholds: Vec, + /// The units this one groups (`ConsistsOf=`): the verbs reach them only + /// through propagation, which the job mode removes. + pub(crate) consists_of: Vec, +} + +impl UnitFacts { + fn identities(&self) -> impl Iterator { + std::iter::once(&self.id).chain(self.names.iter()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RunReport { + pub(crate) exit_code: Option, + pub(crate) signal: Option, + pub(crate) stdout_tail: Vec, + pub(crate) stderr_tail: Vec, +} + +pub(crate) trait Systemd: Send + Sync { + fn show(&self, unit: &str) -> Result; + fn invocation(&self, unit: &str) -> Option; + /// Run one step as the transient unit `unit`, to its end. + fn run(&self, unit: &str, step: &Step) -> Result; +} + +pub(crate) struct Peer { + pub(crate) uid: u32, + pub(crate) issuer_unit: String, +} + +pub(crate) trait PeerCredentials: Send + Sync { + fn peer(&self, stream: &UnixStream) -> Result; +} + +pub(crate) trait BrakeCheck: Send + Sync { + /// `true` only when the instance's brake check exits 0. + fn clear(&self, principal: &str) -> bool; +} + +pub(crate) trait Homeserver: Send + Sync { + /// Idempotent by `txn_id`; returns the event id. + fn post(&self, txn_id: &str, text: &str) -> Result; +} + +pub(crate) struct Ctx { + pub(crate) store: OrderStore, + pub(crate) state_store: PathBuf, + pub(crate) steward_uid: u32, + pub(crate) principal: String, + pub(crate) systemd: Arc, + pub(crate) peers: Arc, + pub(crate) brake: Arc, + pub(crate) homeserver: Arc, +} + +// ------------------------------------------------------------------ scope -- + +fn refused(unit: &str, via: Via) -> Refused { + Refused::AlwaysRefused { + unit: unit.to_owned(), + via, + } +} + +/// Would `verb` on `unit` be refused. systemd builds the transaction and the +/// order tells it to build one job (`--job-mode=ignore-requirements`), so this +/// reads only what that job mode cannot remove from the one named unit: the +/// name it answers to, what fires when it ends, who is bound to it, what it upholds and +/// what it only wraps. +fn refusal_for(systemd: &dyn Systemd, unit: &str, verb: Verb) -> Result> { + if is_refused_name(unit) { + return Ok(Some(refused(unit, Via::Name))); + } + let facts = systemd.show(unit)?; + ensure!(!facts.not_found, "systemd has no such unit to check"); + for name in facts.identities() { + if is_refused_name(name) { + return Ok(Some(refused(name, Via::Alias))); + } + if is_host_name(name) { + return Ok(Some(refused(name, Via::Host))); + } + } + if verb != Verb::Reload && !facts.on_end.is_empty() { + return Ok(Some(refused(unit, Via::OnEnd))); + } + if matches!(verb, Verb::Stop | Verb::Restart) + && let Some(bound) = facts.bound_by.first() + { + return Ok(Some(refused(bound, Via::Bound))); + } + if matches!(verb, Verb::Start | Verb::Restart) && !facts.upholds.is_empty() { + return Ok(Some(refused(unit, Via::Upholds))); + } + if verb != Verb::Start + && let Some(member) = facts.consists_of.first() + { + return Ok(Some(Refused::Wrapper { + member: member.clone(), + })); + } + Ok(None) +} + +// -------------------------------------------------------------- execution -- + +fn tail(bytes: &[u8]) -> Vec { + bytes[bytes.len().saturating_sub(TAIL_BYTES)..].to_vec() +} + +fn conclude( + ctx: &Ctx, + order_id: &str, + outcome: Outcome, + stdout_tail: &[u8], + stderr_tail: &[u8], +) -> Result<()> { + ctx.store.put( + order_id, + OUTCOME_SCHEMA, + &OutcomeRecord { + schema_version: OUTCOME_SCHEMA.into(), + order_id: order_id.into(), + ended_at_unix_millis: now_millis()?, + outcome, + stdout_tail: tail(stdout_tail), + stderr_tail: tail(stderr_tail), + }, + )?; + Ok(()) +} + +fn order_uuid(order_id: &str) -> &str { + order_id.strip_prefix("ord-").unwrap_or(order_id) +} + +pub(crate) fn requester(principal: &str, order_id: &str) -> String { + format!("steward:{principal} order {order_id}") +} + +/// Run one accepted order. A start record is written before anything is +/// actuated and an order that has one is never run again. +pub(crate) fn execute(ctx: &Ctx, order_id: &str) -> Result<()> { + let ledger = ctx.store.ledger()?; + let order = ledger + .orders + .iter() + .find(|order| order.order_id == order_id) + .context("the order is not in the store")?; + if ledger.start_of(order_id).is_some() || ledger.outcome_of(order_id).is_some() { + return Ok(()); + } + // One gate for a fresh order and a recovered one: who may order is what the + // binding says now, not what it said when the order was accepted. + if order.principal != ctx.principal || order.issuer_uid != ctx.steward_uid { + return conclude(ctx, order_id, Outcome::Refused(Refused::NotSteward), b"", b""); + } + if order.operation.validate().is_err() { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the stored order no longer satisfies the grammar", + ); + } + if let Operation::UnitAction { verb, unit } = &order.operation { + match refusal_for(ctx.systemd.as_ref(), unit, *verb) { + Ok(None) => {} + Ok(Some(hit)) => return conclude(ctx, order_id, Outcome::Refused(hit), b"", b""), + Err(_) => { + return conclude( + ctx, + order_id, + Outcome::Failed { + exit_code: None, + signal: None, + }, + b"", + b"the unit could not be resolved, so it was not acted on", + ); + } + } + } + if !ctx.brake.clear(&ctx.principal) { + return conclude(ctx, order_id, Outcome::Refused(Refused::Braked), b"", b""); + } + let uuid = order_uuid(order_id); + let command_id = format!("up-{uuid}"); + let steps = order.operation.steps(); + let actuation = match &order.operation { + Operation::Deploy { .. } => command_id.clone(), + _ => format!("idunn-order-{uuid}"), + }; + let started = StartRecord { + schema_version: START_SCHEMA.into(), + order_id: order_id.into(), + started_at_unix_millis: now_millis()?, + actuation, + }; + if !ctx.store.put(order_id, START_SCHEMA, &started)? { + return Ok(()); + } + if let Operation::Deploy { selector } = &order.operation { + let outcome = match commit_deployment_command( + &ctx.state_store, + crate::control_plane::CommandKind::Deploy, + &command_id, + selector, + &requester(&order.principal, order_id), + ) { + Ok(_) => Outcome::Submitted { command_id }, + Err(_) => Outcome::Failed { + exit_code: None, + signal: None, + }, + }; + return conclude(ctx, order_id, outcome, b"", b""); + } + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let mut outcome = Outcome::Ran; + for (index, step) in steps.iter().enumerate() { + let unit = if steps.len() == 1 { + format!("idunn-order-{uuid}") + } else { + format!("idunn-order-{uuid}-{}", index + 1) + }; + match ctx.systemd.run(&unit, step) { + Ok(report) => { + stdout = report.stdout_tail; + stderr = report.stderr_tail; + if report.exit_code != Some(0) { + outcome = Outcome::Failed { + exit_code: report.exit_code, + signal: report.signal, + }; + break; + } + } + Err(_) => { + outcome = Outcome::Failed { + exit_code: None, + signal: None, + }; + stderr = b"the transient unit could not be started".to_vec(); + break; + } + } + } + conclude(ctx, order_id, outcome, &stdout, &stderr) +} + +/// On service start: an order with a start record and no outcome ran, or began +/// to, in a life that ended. It is Lost and never run again. Returns the +/// orders that never started, oldest first. +pub(crate) fn recover(ctx: &Ctx) -> Result> { + let ledger = ctx.store.ledger()?; + for start in &ledger.starts { + if ledger.outcome_of(&start.order_id).is_none() { + conclude(ctx, &start.order_id, Outcome::Lost, b"", b"")?; + } + } + Ok(ledger + .orders + .iter() + .filter(|order| { + ledger.start_of(&order.order_id).is_none() && ledger.outcome_of(&order.order_id).is_none() + }) + .map(|order| order.order_id.clone()) + .collect()) +} + +// ---------------------------------------------------------------- posting -- + +struct Due { + order_id: String, + kind: PostKind, + text: String, +} + +fn quoted(reason: &str) -> String { + format!("\"{}\"", escape(reason.as_bytes())) +} + +fn accepted_text(order: &OrderRecord) -> String { + format!( + "Steward order {} accepted: {}\nissuer {} (invocation {})\nreason: {}", + order.order_id, + escape(order.operation.describe().as_bytes()), + escape(order.issuer_unit.as_bytes()), + escape(order.issuer_invocation.as_bytes()), + quoted(&order.reason) + ) +} + +/// A deploy's command has finished once its state is neither queued nor running. +fn deploy_finished(state: &str) -> bool { + !matches!(state, "queued" | "running") +} + +/// What is owed to the room. `deploy_state` is a deployment command's derived +/// state and detail, read from the control store by the caller. +fn due_posts( + ledger: &Ledger, + deploy_state: &dyn Fn(&str) -> Result<(&'static str, String)>, +) -> Vec { + let mut due = Vec::new(); + for order in &ledger.orders { + let id = &order.order_id; + if !ledger.posted(id, PostKind::Accepted) { + due.push(Due { + order_id: id.clone(), + kind: PostKind::Accepted, + text: accepted_text(order), + }); + } + if ledger.posted(id, PostKind::Result) { + continue; + } + let Some(record) = ledger.outcome_of(id) else { + continue; + }; + let mut line = record.outcome.describe(); + if let Outcome::Submitted { command_id } = &record.outcome { + match deploy_state(command_id) { + Ok((state, detail)) if deploy_finished(state) => { + line = format!("{line}: {state} {}", escape(detail.as_bytes())); + } + _ => continue, + } + } + due.push(Due { + order_id: id.clone(), + kind: PostKind::Result, + text: format!( + "Steward order {id}: {line}\n{}", + escape(order.operation.describe().as_bytes()) + ), + }); + } + due +} + +/// Post what is due, in order, stopping at the first failure. Returns how many +/// went out. +pub(crate) fn post_pass(ctx: &Ctx) -> Result { + let ledger = ctx.store.ledger()?; + let mut sent = 0; + let deploy_state = |command_id: &str| command_state(&ctx.state_store, command_id); + for due in due_posts(&ledger, &deploy_state) { + let transaction = format!("idunn-{}-{}", due.order_id, due.kind.as_str()); + let event_id = ctx.homeserver.post(&transaction, &due.text)?; + ctx.store.put( + &post_key(&due.order_id, due.kind), + POST_SCHEMA, + &PostRecord { + schema_version: POST_SCHEMA.into(), + post_key: post_key(&due.order_id, due.kind), + event_id, + posted_at_unix_millis: now_millis()?, + }, + )?; + sent += 1; + } + Ok(sent) +} + +// ----------------------------------------------------------------- frames -- + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct OrderRequest { + pub(crate) operation: Operation, + pub(crate) reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ShowRequest { + pub(crate) id: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Request { + Order(OrderRequest), + Show(ShowRequest), + List, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderView { + pub(crate) order: OrderRecord, + pub(crate) start: Option, + pub(crate) outcome: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OrderLine { + pub(crate) order_id: String, + pub(crate) status: String, + pub(crate) summary: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum Response { + Accepted { order_id: String }, + Refused(Refusal), + Show(Box), + List(Vec), +} + +fn write_frame(stream: &mut impl Write, value: &impl Serialize) -> Result<()> { + let bytes = rmp_serde::to_vec_named(value)?; + stream.write_all(&u32::try_from(bytes.len())?.to_be_bytes())?; + stream.write_all(&bytes)?; + Ok(stream.flush()?) +} + +fn read_frame(stream: &mut impl Read, limit: usize) -> Result> { + let mut length = [0u8; 4]; + stream.read_exact(&mut length)?; + let length = u32::from_be_bytes(length) as usize; + ensure!(length <= limit, "a frame exceeds its limit"); + let mut bytes = vec![0u8; length]; + stream.read_exact(&mut bytes)?; + Ok(bytes) +} + +fn status_word(view: &OrderView) -> &'static str { + match (&view.outcome, &view.start) { + (Some(record), _) => match record.outcome { + Outcome::Ran => "ran", + Outcome::Failed { .. } => "failed", + Outcome::Refused(_) => "refused", + Outcome::Lost => "lost", + Outcome::Submitted { .. } => "submitted", + }, + (None, Some(_)) => "running", + (None, None) => "accepted", + } +} + +fn view_of(ledger: &Ledger, order: &OrderRecord) -> OrderView { + OrderView { + order: order.clone(), + start: ledger.start_of(&order.order_id).cloned(), + outcome: ledger.outcome_of(&order.order_id).cloned(), + } +} + +fn answer(stream: &mut UnixStream, response: Response) -> Result<()> { + write_frame(stream, &response) +} + +fn refuse(stream: &mut UnixStream, field: &'static str, code: Code) -> Result<()> { + answer(stream, Response::Refused(Refusal::new(field, code))) +} + +/// Serve one connection. The peer is authenticated before its frame is read. +pub(crate) fn handle( + ctx: &Ctx, + stream: &mut UnixStream, + queue: &mpsc::Sender, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + let peer = ctx.peers.peer(stream)?; + // The steward may order and read; root, the operator, may only read. + let steward = peer.uid == ctx.steward_uid; + if !steward && peer.uid != 0 { + eprintln!("steward socket: refused a peer that is not the steward"); + return refuse(stream, "peer", Code::Unauthorized); + } + let request = read_frame(stream, REQUEST_FRAME_LIMIT) + .ok() + .and_then(|bytes| rmp_serde::from_slice::(&bytes).ok()); + let Some(request) = request else { + eprintln!("steward socket: refused a malformed frame"); + return refuse(stream, "frame", Code::Malformed); + }; + match request { + Request::Order(_) if !steward => { + eprintln!("steward socket: refused an order from a peer that is not the steward"); + refuse(stream, "peer", Code::Unauthorized) + } + Request::Order(request) => { + let checked = + validate_reason(&request.reason).and_then(|()| request.operation.validate()); + if let Err(refusal) = checked { + eprintln!("steward socket: refused an order at the grammar"); + return answer(stream, Response::Refused(refusal)); + } + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: format!("ord-{}", Uuid::new_v4()), + principal: ctx.principal.clone(), + issuer_uid: peer.uid, + issuer_invocation: ctx.systemd.invocation(&peer.issuer_unit).unwrap_or_default(), + issuer_unit: peer.issuer_unit, + issued_at_unix_millis: now_millis()?, + operation: request.operation, + reason: request.reason, + }; + match ctx.store.put(&order.order_id, ORDER_SCHEMA, &order) { + Ok(true) => {} + _ => return refuse(stream, "store", Code::Unavailable), + } + queue + .send(order.order_id.clone()) + .map_err(|_| anyhow!("the executor has stopped"))?; + answer( + stream, + Response::Accepted { + order_id: order.order_id, + }, + ) + } + Request::Show(request) => { + if request.id.strip_prefix("ord-").map(Uuid::parse_str).is_none_or(|parsed| parsed.is_err()) + { + return refuse(stream, "id", Code::BadShape); + } + let ledger = ctx.store.ledger()?; + match ledger.orders.iter().find(|order| order.order_id == request.id) { + Some(order) => answer(stream, Response::Show(Box::new(view_of(&ledger, order)))), + None => refuse(stream, "id", Code::Unknown), + } + } + Request::List => { + let ledger = ctx.store.ledger()?; + let skip = ledger.orders.len().saturating_sub(LIST_LIMIT); + let lines = ledger + .orders + .iter() + .skip(skip) + .map(|order| { + let view = view_of(&ledger, order); + OrderLine { + order_id: order.order_id.clone(), + status: status_word(&view).into(), + summary: order.operation.describe(), + } + }) + .collect(); + answer(stream, Response::List(lines)) + } + } +} + +// ------------------------------------------------------------- real ports -- + +/// `systemctl` and `systemd-run`, found by name. +pub(crate) struct HostSystemd; + +/// The names in one `systemctl show` list value. systemd quotes a name that +/// holds a backslash and doubles the backslash inside the quotes. +fn parse_names(value: &str) -> Vec { + let mut names = Vec::new(); + let mut current = String::new(); + let (mut quoted, mut open, mut escaped) = (false, false, false); + for character in value.chars() { + match character { + _ if escaped => { + current.push(character); + escaped = false; + } + '\\' if quoted => escaped = true, + '"' => { + quoted = !quoted; + open = true; + } + _ if character.is_whitespace() && !quoted => { + if open { + names.push(std::mem::take(&mut current)); + open = false; + } + } + _ => { + current.push(character); + open = true; + } + } + } + if open { + names.push(current); + } + names +} + +/// The properties one `systemctl show` is asked for: identity, what fires at +/// the unit's end, who is bound to it, and what it upholds or groups. +fn show_properties() -> String { + ["Id", "Names", "LoadState"] + .into_iter() + .chain(ACTIONS) + .chain(ON_END) + .chain(["BoundBy", "Upholds", "ConsistsOf"]) + .collect::>() + .join(",") +} + +fn parse_show(output: &str) -> Result { + let mut facts = UnitFacts::default(); + for line in output.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + match key { + "Id" => facts.id = value.to_owned(), + "Names" => facts.names = parse_names(value), + "LoadState" => facts.not_found = value == "not-found", + "BoundBy" => facts.bound_by = parse_names(value), + "Upholds" => facts.upholds = parse_names(value), + "ConsistsOf" => facts.consists_of = parse_names(value), + _ if ACTIONS.contains(&key) => { + if !matches!(value, "" | "none") { + facts.on_end.push(key.to_owned()); + } + } + _ if ON_END.contains(&key) => { + facts.on_end.extend(parse_names(value).into_iter().map(|_| key.to_owned())); + } + _ => {} + } + } + ensure!(!facts.id.is_empty(), "systemctl named no unit"); + Ok(facts) +} + +impl Systemd for HostSystemd { + fn show(&self, unit: &str) -> Result { + let output = Command::new("systemctl") + .args(["show", "--no-pager"]) + .arg(format!("--property={}", show_properties())) + .args(["--", unit]) + .stdin(Stdio::null()) + .output() + .context("running systemctl show")?; + ensure!(output.status.success(), "systemctl show failed"); + parse_show(&String::from_utf8_lossy(&output.stdout)) + } + + fn invocation(&self, unit: &str) -> Option { + let output = Command::new("systemctl") + .args(["show", "--no-pager", "--property=InvocationID", "--value", "--"]) + .arg(unit) + .stdin(Stdio::null()) + .output() + .ok()?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + (output.status.success() && !value.is_empty()).then_some(value) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + let mut command = Command::new("systemd-run"); + command.args(["--unit", unit, "--wait", "--pipe", "--collect"]); + command.args(["-p", "RuntimeMaxSec=15min"]); + for (name, value) in &step.environment { + command.arg(format!("--setenv={name}={value}")); + } + let output = command + .arg("--") + .args(&step.argv) + .stdin(Stdio::null()) + .output() + .context("running systemd-run")?; + Ok(RunReport { + exit_code: output.status.code(), + signal: output.status.signal(), + stdout_tail: tail(&output.stdout), + stderr_tail: tail(&output.stderr), + }) + } +} + +/// A system service or scope names the issuer: the first unit under +/// `system.slice` in the peer's cgroup path. A peer cannot choose it. Slices +/// above it are the manager's, and what a delegated subtree adds below it is +/// ignored. Any other path (a user manager's, a bare directory) is `unknown`. +fn unit_from_cgroup(text: &str) -> String { + let line = text + .lines() + .find(|line| line.starts_with("0::") || line.contains(":name=systemd:")) + .unwrap_or_default(); + let path = line.rsplit_once(':').map_or("", |(_, path)| path); + let unit = path.strip_prefix("/system.slice/").and_then(|under| { + for segment in under.split('/') { + if segment.ends_with(".slice") { + continue; + } + let unit = segment.ends_with(".service") || segment.ends_with(".scope"); + return (unit && segment.len() <= UNIT_LIMIT).then_some(segment); + } + None + }); + unit.map_or_else(|| "unknown".to_owned(), str::to_owned) +} + +/// Kernel peer credentials (`SO_PEERCRED`) and the peer's cgroup. +pub(crate) struct KernelPeers; + +impl PeerCredentials for KernelPeers { + fn peer(&self, stream: &UnixStream) -> Result { + use std::os::fd::AsRawFd; + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = std::mem::size_of::() as libc::socklen_t; + // SAFETY: the descriptor is a live socket and the pointer and length + // describe `credentials`, which outlives the call. + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&mut credentials as *mut libc::ucred).cast(), + &mut length, + ) + }; + ensure!( + status == 0 && length as usize == std::mem::size_of::(), + "peer credentials are unavailable" + ); + let cgroup = + std::fs::read_to_string(format!("/proc/{}/cgroup", credentials.pid)).unwrap_or_default(); + Ok(Peer { + uid: credentials.uid, + issuer_unit: unit_from_cgroup(&cgroup), + }) + } +} + +/// `mind-control --instance check`; a missing binary is a held brake. +pub(crate) struct MindControl { + pub(crate) program: PathBuf, +} + +impl BrakeCheck for MindControl { + fn clear(&self, principal: &str) -> bool { + Command::new(&self.program) + .args(["--instance", principal, "check"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } +} + +/// The Synapse client API on loopback, over plain HTTP/1.0 so the reply is +/// never chunked. The token is read from its credential file for each post. +pub(crate) struct Synapse { + pub(crate) address: std::net::SocketAddr, + pub(crate) room: String, + pub(crate) token_file: PathBuf, + /// The whole wait for a reply, however slowly it drips. + pub(crate) reply_within: Duration, +} + +fn percent_encode(text: &str) -> String { + let mut encoded = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +impl Homeserver for Synapse { + fn post(&self, txn_id: &str, text: &str) -> Result { + let token = std::fs::read_to_string(&self.token_file) + .context("the homeserver token is unreadable")?; + let token = token.trim(); + ensure!( + !token.is_empty() && token.bytes().all(|byte| byte.is_ascii_graphic()), + "the homeserver token is not a bearer token" + ); + let body = serde_json::to_vec(&serde_json::json!({"msgtype": "m.text", "body": text}))?; + let head = format!( + "PUT /_matrix/client/v3/rooms/{}/send/m.room.message/{txn_id} HTTP/1.0\r\n\ + Host: {}\r\nAuthorization: Bearer {token}\r\n\ + Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", + percent_encode(&self.room), + self.address, + body.len() + ); + let mut socket = std::net::TcpStream::connect_timeout(&self.address, Duration::from_secs(5)) + .context("the homeserver is unreachable")?; + socket.set_write_timeout(Some(self.reply_within))?; + socket.write_all(head.as_bytes())?; + socket.write_all(&body)?; + let deadline = std::time::Instant::now() + self.reply_within; + let mut reply = Vec::new(); + let mut chunk = [0u8; 4096]; + while reply.len() < RESPONSE_FRAME_LIMIT { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|left| !left.is_zero()) + .context("the homeserver reply was too slow")?; + socket.set_read_timeout(Some(remaining))?; + let count = socket + .read(&mut chunk) + .context("the homeserver reply was cut short")?; + if count == 0 { + break; + } + reply.extend_from_slice(&chunk[..count]); + } + let split = reply + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("the homeserver reply has no body")?; + let status = String::from_utf8_lossy(&reply[..split]); + ensure!( + status.split_whitespace().nth(1) == Some("200"), + "the homeserver refused the post" + ); + #[derive(Deserialize)] + struct Sent { + event_id: String, + } + let sent: Sent = serde_json::from_slice(&reply[split + 4..]) + .map_err(|_| anyhow!("the homeserver reply has no event id"))?; + Ok(sent.event_id) + } +} + +// ---------------------------------------------------------------- binding -- + +/// `/etc/gamecult/idunn/steward.toml`: who the steward is and where to post. +/// It lists no unit of any kind, and any field outside this set is refused. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct StewardBinding { + pub(crate) principal: String, + pub(crate) user: String, + pub(crate) mind_control: PathBuf, + pub(crate) room: String, + pub(crate) homeserver: String, + pub(crate) token_credential: String, +} + +impl StewardBinding { + pub(crate) fn parse(text: &str) -> Result { + let binding: Self = toml::from_str(text) + .map_err(|error| anyhow!("the steward binding does not parse: {}", error.message()))?; + binding.validate()?; + Ok(binding) + } + + fn validate(&self) -> Result<()> { + require_id(&self.principal, "steward principal")?; + ensure!( + !self.principal.starts_with('-') && !self.principal.contains('/'), + "steward principal is invalid" + ); + let user_ok = login_grammar_ok(&self.user); + ensure!(user_ok && self.user != "root", "steward user is invalid or root"); + ensure!( + self.mind_control.is_absolute(), + "mind_control must be an absolute path" + ); + let room_ok = self.room.starts_with('!') + && self.room.len() <= 255 + && self.room.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'!' | b'.' | b'_' | b':' | b'=' | b'-') + }); + ensure!(room_ok, "room must be a Matrix room id"); + self.address()?; + let credential_ok = !self.token_credential.is_empty() + && !self.token_credential.starts_with('.') + && self.token_credential.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-') + }); + ensure!(credential_ok, "token_credential is not a credential name"); + Ok(()) + } + + fn address(&self) -> Result { + let address: std::net::SocketAddr = self + .homeserver + .strip_prefix("http://") + .and_then(|rest| rest.parse().ok()) + .context("homeserver must be http://:")?; + ensure!(address.ip().is_loopback(), "homeserver must be loopback"); + Ok(address) + } +} + +/// Root-admitted: owned by root and writable by no one else. +fn admitted(owner: u32, mode: u32) -> bool { + owner == 0 && mode & 0o022 == 0 +} + +fn load_binding(path: &Path) -> Result { + let metadata = std::fs::metadata(path).context("reading the steward binding")?; + ensure!( + metadata.is_file() && admitted(metadata.uid(), metadata.mode()), + "the steward binding is not a root-owned file closed to writes" + ); + let text = std::fs::read_to_string(path).context("reading the steward binding")?; + StewardBinding::parse(&text) +} + +/// A login name Idunn will resolve: the one grammar for every binding that names +/// a user. +pub(crate) fn login_grammar_ok(user: &str) -> bool { + !user.is_empty() + && user.len() <= 32 + && user.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) + && !user.starts_with(|first: char| first.is_ascii_digit() || first == '-') +} + +pub(crate) fn user_uid(name: &str) -> Result { + let name = std::ffi::CString::new(name)?; + // SAFETY: `passwd` is plain data and getpwnam_r fills it, with strings in + // `buffer`, which outlives every read of them below (none are read). + let (status, found, uid) = unsafe { + let mut passwd: libc::passwd = std::mem::zeroed(); + let mut buffer = vec![0 as libc::c_char; 4096]; + let mut result: *mut libc::passwd = std::ptr::null_mut(); + let status = libc::getpwnam_r( + name.as_ptr(), + &mut passwd, + buffer.as_mut_ptr(), + buffer.len(), + &mut result, + ); + (status, !result.is_null(), passwd.pw_uid) + }; + ensure!(status == 0 && found, "the steward user is unknown"); + ensure!(uid != 0, "the steward user resolves to root"); + Ok(uid) +} + +// ------------------------------------------------------------------ serve -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ServeOptions { + pub(crate) state_store: PathBuf, + pub(crate) store: PathBuf, + pub(crate) steward_binding: PathBuf, + pub(crate) socket: PathBuf, +} + +const DEFAULT_STORE: &str = "/var/lib/gamecult/idunn-orders"; +const DEFAULT_BINDING: &str = "/etc/gamecult/idunn/steward.toml"; +const DEFAULT_SOCKET: &str = "/run/idunn-orders/steward.sock"; + +pub(crate) fn parse_serve(mut args: impl Iterator) -> Result { + ensure!( + args.next().as_deref() == Some("serve"), + "idunn orders takes the subcommand serve" + ); + let mut options = ServeOptions { + state_store: default_state_store(), + store: DEFAULT_STORE.into(), + steward_binding: DEFAULT_BINDING.into(), + socket: DEFAULT_SOCKET.into(), + }; + while let Some(argument) = args.next() { + match argument.as_str() { + "--state-store" => options.state_store = path_value(&mut args, "--state-store")?, + "--store" => options.store = path_value(&mut args, "--store")?, + "--steward-binding" => { + options.steward_binding = path_value(&mut args, "--steward-binding")? + } + "--socket" => options.socket = path_value(&mut args, "--socket")?, + _ => bail!("an orders serve option is not part of the grammar"), + } + } + Ok(options) +} + +/// Seconds between post passes: two when the last pass went through, and +/// doubling for each failed pass in a row up to a minute. +fn post_pause(failures: u32) -> u64 { + if failures == 0 { + 2 + } else { + (2u64 << failures.min(5)).min(60) + } +} + +/// Run a thread whose end ends the service: systemd restarts it, and the +/// start recovery settles what it left. +fn vital(name: &'static str, work: impl FnOnce() + Send + 'static) { + struct ExitOnDrop(&'static str); + impl Drop for ExitOnDrop { + fn drop(&mut self) { + eprintln!("steward orders: the {} thread ended", self.0); + std::process::exit(1); + } + } + thread::Builder::new() + .name(name.into()) + .spawn(move || { + let _guard = ExitOnDrop(name); + work(); + }) + .expect("a thread can be spawned"); +} + +/// The socket belongs to the steward and no one else may open it; the peer +/// credential check on every connection stays the lock. +fn bind_socket(path: &Path, steward_uid: u32) -> Result { + if let Ok(metadata) = std::fs::symlink_metadata(path) { + use std::os::unix::fs::FileTypeExt; + ensure!( + metadata.file_type().is_socket(), + "the socket path holds something that is not a socket" + ); + std::fs::remove_file(path).context("removing a stale socket")?; + } + let listener = UnixListener::bind(path).context("binding the steward socket")?; + std::os::unix::fs::chown(path, Some(steward_uid), None) + .context("giving the steward socket to the steward")?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .context("setting the steward socket mode")?; + Ok(listener) +} + +/// Files this service creates in Idunn's own stores (a deploy order writes +/// `control.cc`) take Idunn's umask, whatever the unit that runs it sets. +const IDUNN_UMASK: libc::mode_t = 0o027; + +pub(crate) fn serve(options: ServeOptions) -> Result<()> { + // SAFETY: umask only sets the process file-mode mask; it cannot fail. + unsafe { libc::umask(IDUNN_UMASK) }; + let binding = load_binding(&options.steward_binding)?; + let credentials = std::env::var_os("CREDENTIALS_DIRECTORY") + .map(PathBuf::from) + .context("CREDENTIALS_DIRECTORY is unset; the token comes from LoadCredential")?; + let ctx = Arc::new(Ctx { + store: OrderStore::new(&options.store), + state_store: options.state_store, + steward_uid: user_uid(&binding.user)?, + principal: binding.principal.clone(), + systemd: Arc::new(HostSystemd), + peers: Arc::new(KernelPeers), + brake: Arc::new(MindControl { + program: binding.mind_control.clone(), + }), + homeserver: Arc::new(Synapse { + address: binding.address()?, + room: binding.room.clone(), + token_file: credentials.join(&binding.token_credential), + reply_within: Duration::from_secs(10), + }), + }); + ctx.store.prepare()?; + let pending = recover(&ctx)?; + let listener = bind_socket(&options.socket, ctx.steward_uid)?; + let (queue, orders) = mpsc::channel::(); + for order_id in pending { + queue.send(order_id)?; + } + let executor = Arc::clone(&ctx); + vital("executor", move || { + for order_id in orders { + if let Err(error) = execute(&executor, &order_id) { + eprintln!("steward orders: {order_id} was not run to a record: {error:#}"); + } + } + }); + let poster = Arc::clone(&ctx); + vital("poster", move || { + let mut failures = 0u32; + loop { + match post_pass(&poster) { + Ok(_) => failures = 0, + Err(error) => { + failures = failures.saturating_add(1); + eprintln!("steward orders: posting is delayed: {error:#}"); + } + } + thread::sleep(Duration::from_secs(post_pause(failures))); + } + }); + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + if let Err(error) = handle(&ctx, &mut stream, &queue) { + eprintln!("steward socket: a connection ended in error: {error:#}"); + } + } + Err(error) => eprintln!("steward socket: accept failed: {}", error.kind()), + } + } + bail!("the steward socket stopped accepting") +} + +// ----------------------------------------------------------------- client -- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum OrderAction { + Issue { operation: Operation, reason: String }, + Show { id: String }, + List, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct OrderCommand { + pub(crate) socket: PathBuf, + pub(crate) action: OrderAction, +} + +pub(crate) fn parse_order(mut args: impl Iterator) -> Result { + let kind = args.next().context("idunn order requires an operation")?; + let mut socket = PathBuf::from(DEFAULT_SOCKET); + let mut reason = None; + let mut positional = Vec::new(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--socket" => socket = path_value(&mut args, "--socket")?, + "--reason" => reason = Some(string_value(&mut args, "--reason")?), + word if word.starts_with('-') => bail!("an order option is not part of the grammar"), + _ => positional.push(argument), + } + } + let action = match kind.as_str() { + "list" => { + ensure!(positional.is_empty() && reason.is_none(), "list takes no arguments"); + OrderAction::List + } + "show" => { + ensure!(positional.len() == 1 && reason.is_none(), "show takes one order id"); + OrderAction::Show { + id: positional.remove(0), + } + } + "package-install" | "unit" | "deploy" => { + let operation = match kind.as_str() { + "package-install" => Operation::PackageInstall { + packages: positional, + }, + "unit" => { + ensure!(positional.len() == 2, "unit takes a verb and a unit"); + let verb = Verb::parse(&positional[0]) + .context("the unit verb is not start, stop, restart or reload")?; + Operation::UnitAction { + verb, + unit: positional.remove(1), + } + } + _ => { + ensure!(positional.len() == 1, "deploy takes one selector"); + Operation::Deploy { + selector: positional.remove(0), + } + } + }; + let reason = reason.context("an order requires --reason")?; + validate_reason(&reason)?; + operation.validate()?; + OrderAction::Issue { operation, reason } + } + _ => bail!("the order operation is not part of the grammar"), + }; + Ok(OrderCommand { socket, action }) +} + +fn call(socket: &Path, request: &Request) -> Result { + let mut stream = UnixStream::connect(socket).context("connecting to the steward socket")?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + write_frame(&mut stream, request)?; + let bytes = read_frame(&mut stream, RESPONSE_FRAME_LIMIT)?; + rmp_serde::from_slice(&bytes).map_err(|_| anyhow!("the steward socket sent an unreadable reply")) +} + +fn render_tail(lines: &mut Vec>, stream: &str, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + lines.push(format!(" {stream} (last {} bytes)", bytes.len()).into_bytes()); + let text = bytes.strip_suffix(b"\n").unwrap_or(bytes); + lines.extend(text.split(|byte| *byte == b'\n').map(|line| { + let mut shown = b" | ".to_vec(); + shown.extend_from_slice(line); + shown + })); +} + +/// Raw lines for the terminal; `emit` is their only way out. +fn render_view(view: &OrderView) -> Vec> { + let order = &view.order; + let mut lines: Vec> = vec![ + format!("order {}", order.order_id).into_bytes(), + format!("status {}", status_word(view)).into_bytes(), + format!("operation {}", order.operation.describe()).into_bytes(), + format!( + "issuer {} (invocation {}, uid {})", + order.issuer_unit, order.issuer_invocation, order.issuer_uid + ) + .into_bytes(), + format!("reason {}", quoted(&order.reason)).into_bytes(), + ]; + if let Some(start) = &view.start { + lines.push(format!("started {}", start.actuation).into_bytes()); + } + if let Some(record) = &view.outcome { + lines.push(format!("outcome {}", record.outcome.describe()).into_bytes()); + render_tail(&mut lines, "stdout", &record.stdout_tail); + render_tail(&mut lines, "stderr", &record.stderr_tail); + } + lines +} + +pub(crate) fn run_order(command: OrderCommand) -> Result<()> { + let request = match command.action { + OrderAction::Issue { operation, reason } => Request::Order(OrderRequest { operation, reason }), + OrderAction::Show { id } => Request::Show(ShowRequest { id }), + OrderAction::List => Request::List, + }; + let mut out = std::io::stdout().lock(); + match call(&command.socket, &request)? { + Response::Accepted { order_id } => writeln!(out, "{order_id}")?, + Response::Refused(refusal) => bail!("the steward refused the request: {refusal}"), + Response::Show(view) => emit(&mut out, &render_view(&view))?, + Response::List(lines) => { + let lines: Vec> = lines + .iter() + .map(|line| format!("{} {} {}", line.order_id, line.status, line.summary).into_bytes()) + .collect(); + emit(&mut out, &lines)? + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tempfile::TempDir; + + use super::*; + + const STEWARD: u32 = 1000; + const CANARY: &str = "CANARY-7f3a91"; + + /// What `systemctl show` prints for a loaded unit that acts on nothing, + /// plus the given properties (a given key replaces its default). + fn show_text(id: &str, properties: &[(&str, &str)]) -> String { + let mut text = format!("Id={id}\n"); + for (key, default) in [("Names", id), ("LoadState", "loaded")] { + if !properties.iter().any(|(given, _)| *given == key) { + text += &format!("{key}={default}\n"); + } + } + for (key, value) in properties { + text += &format!("{key}={value}\n"); + } + text + } + + struct FakeSystemd { + units: Mutex>, + runs: Mutex>, + shows: Mutex>, + report: Mutex, + } + + impl Systemd for FakeSystemd { + fn show(&self, unit: &str) -> Result { + self.shows.lock().unwrap().push(unit.into()); + let known = self.units.lock().unwrap().get(unit).cloned(); + parse_show(&known.unwrap_or_else(|| show_text(unit, &[]))) + } + + fn invocation(&self, _unit: &str) -> Option { + Some("invocation-1".into()) + } + + fn run(&self, unit: &str, step: &Step) -> Result { + self.runs.lock().unwrap().push((unit.into(), step.clone())); + Ok(self.report.lock().unwrap().clone()) + } + } + + struct FakePeers(u32); + + impl PeerCredentials for FakePeers { + fn peer(&self, _stream: &UnixStream) -> Result { + Ok(Peer { + uid: self.0, + issuer_unit: "mind-eureka-self.service".into(), + }) + } + } + + /// Whether the brake is clear, and the principals it was asked about. + struct FakeBrake(AtomicBool, Mutex>); + + impl BrakeCheck for FakeBrake { + fn clear(&self, principal: &str) -> bool { + self.1.lock().unwrap().push(principal.into()); + self.0.load(Ordering::SeqCst) + } + } + + #[derive(Default)] + struct FakeHome { + failing: AtomicBool, + sent: Mutex>, + } + + impl Homeserver for FakeHome { + fn post(&self, txn_id: &str, text: &str) -> Result { + ensure!(!self.failing.load(Ordering::SeqCst), "the homeserver is down"); + let mut sent = self.sent.lock().unwrap(); + sent.push((txn_id.into(), text.into())); + Ok(format!("$event{}", sent.len())) + } + } + + struct Harness { + directory: TempDir, + ctx: Ctx, + systemd: Arc, + brake: Arc, + home: Arc, + queue: mpsc::Sender, + _orders: mpsc::Receiver, + } + + fn harness_for(peer_uid: u32) -> Harness { + let directory = TempDir::new().unwrap(); + let systemd = Arc::new(FakeSystemd { + units: Mutex::new(BTreeMap::new()), + runs: Mutex::new(Vec::new()), + shows: Mutex::new(Vec::new()), + report: Mutex::new(RunReport { + exit_code: Some(0), + ..RunReport::default() + }), + }); + let brake = Arc::new(FakeBrake(AtomicBool::new(true), Mutex::new(Vec::new()))); + let home = Arc::new(FakeHome::default()); + let (queue, orders) = mpsc::channel(); + let ctx = Ctx { + store: OrderStore::new(&directory.path().join("orders")), + state_store: directory.path().join("state").join("control.cc"), + steward_uid: STEWARD, + principal: "eureka".into(), + systemd: systemd.clone(), + peers: Arc::new(FakePeers(peer_uid)), + brake: brake.clone(), + homeserver: home.clone(), + }; + Harness { + directory, + ctx, + systemd, + brake, + home, + queue, + _orders: orders, + } + } + + fn harness() -> Harness { + harness_for(STEWARD) + } + + impl Harness { + fn exchange(&self, write: impl FnOnce(&mut UnixStream)) -> Response { + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + write(&mut client); + handle(&self.ctx, &mut server, &self.queue).unwrap(); + let bytes = read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap(); + rmp_serde::from_slice(&bytes).unwrap() + } + + fn ask(&self, request: &Request) -> Response { + self.exchange(|client| write_frame(client, request).unwrap()) + } + + fn issue(&self, operation: Operation) -> Response { + self.ask(&Request::Order(OrderRequest { + operation, + reason: "probe".into(), + })) + } + + fn issue_unit(&self, verb: Verb, unit: &str) -> Response { + self.issue(Operation::UnitAction { + verb, + unit: unit.into(), + }) + } + + fn accept(&self, response: Response) -> String { + match response { + Response::Accepted { order_id } => order_id, + other => panic!("expected an accepted order, got {other:?}"), + } + } + + fn outcome(&self, order_id: &str) -> Outcome { + self.ctx + .store + .ledger() + .unwrap() + .outcome_of(order_id) + .expect("the order has an outcome") + .outcome + .clone() + } + + fn runs(&self) -> Vec<(String, Step)> { + self.systemd.runs.lock().unwrap().clone() + } + + /// Tell the fake `systemctl show` what it prints for `name`. + fn describe_unit(&self, name: &str, text: String) { + self.systemd.units.lock().unwrap().insert(name.into(), text); + } + + fn describe(&self, name: &str, properties: &[(&str, &str)]) { + self.describe_unit(name, show_text(name, properties)); + } + } + + fn all_verbs() -> [Verb; 4] { + [Verb::Start, Verb::Stop, Verb::Restart, Verb::Reload] + } + + #[test] + fn foreign_uid_is_refused_at_the_door() { + let h = harness_for(STEWARD + 1); + let response = h.issue_unit(Verb::Restart, "nginx.service"); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + let response = h.ask(&Request::List); + assert_eq!( + response, + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert!(!h.ctx.store.path().exists(), "nothing is written for a foreign peer"); + assert!(h.runs().is_empty()); + } + + #[derive(Serialize)] + struct ForgedBody { + operation: Operation, + reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + requested_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + } + + #[derive(Serialize)] + struct ForgedFrame { + order: ForgedBody, + } + + #[test] + fn order_id_and_requester_are_minted_not_declared() { + let h = harness(); + let forge = |order_id: Option<&str>, requested_by: Option<&str>, command: Option<&str>| { + let frame = ForgedFrame { + order: ForgedBody { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into(), + }, + reason: "probe".into(), + order_id: order_id.map(str::to_owned), + requested_by: requested_by.map(str::to_owned), + command: command.map(str::to_owned), + }, + }; + h.exchange(|client| write_frame(client, &frame).unwrap()) + }; + // The same frame without the extra field is a good order. + let honest = h.accept(forge(None, None, None)); + for forged in [ + forge(Some("ord-declared"), None, None), + forge(None, Some("steward:someone-else order x"), None), + forge(None, None, Some("reboot-now")), + ] { + assert_eq!( + forged, + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + } + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.orders.len(), 1, "a forged frame leaves no record"); + let order = &ledger.orders[0]; + assert_eq!(order.order_id, honest); + let uuid = order.order_id.strip_prefix("ord-").expect("ord- prefix"); + assert_eq!(Uuid::parse_str(uuid).unwrap().get_version_num(), 4); + assert_eq!(order.issuer_unit, "mind-eureka-self.service"); + assert_eq!(order.issuer_invocation, "invocation-1"); + assert_eq!(order.principal, "eureka"); + assert_eq!(order.issuer_uid, STEWARD); + assert_eq!( + requester(&order.principal, &order.order_id), + format!("steward:eureka order {}", order.order_id) + ); + } + + #[test] + fn package_names_outside_grammar_are_refused() { + let h = harness(); + let bad = [ + "-y", + "--allow-unauthenticated", + "-", + "a", + "", + "Foo", + "a/b", + "/tmp/x", + "./x", + "x.deb", + "foo.deb", + "foo bar", + "foo;reboot", + "foo$(id)", + "foo`id`", + "foo|bar", + "foo&bar", + "foo\nbar", + "foo*", + "foo?", + "pkg:amd64", + "pkg/stable", + "libc6-", + "foo=", + "foo=1 2", + "foo=1;2", + "foo=1=2", + "fo\u{ff0f}o", + "f\u{43e}o", + ]; + for name in bad { + let response = h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into(), name.into()], + }); + assert!(matches!(response, Response::Refused(_)), "{name:?} was not refused"); + } + let many: Vec = (0..17).map(|n| format!("pkg-{n}x")).collect(); + assert_eq!( + h.issue(Operation::PackageInstall { packages: many }), + Response::Refused(Refusal::new("package", Code::TooMany)) + ); + assert_eq!( + h.issue(Operation::PackageInstall { packages: vec![] }), + Response::Refused(Refusal::new("package", Code::Empty)) + ); + assert!(!h.ctx.store.path().exists(), "no refusal left a record"); + + let good = [ + "git-lfs", + "matrix-synapse-py3", + "libnss-systemd", + "bubblewrap", + "socat", + "g++", + "libstdc++6", + "python3.11", + "nginx=1.24.0-2ubuntu7", + "foo=1:2.3-4+b1~rc1", + ]; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: good.iter().map(|name| (*name).to_owned()).collect(), + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + let runs = h.runs(); + assert_eq!(runs.len(), 3); + assert_eq!(runs[0].1.argv, ["apt-get", "update", "-q"]); + // Each name selects exactly itself and apt may remove nothing. + let mut install = vec!["apt-get".to_owned(), "install".into(), "-y".into(), "-q".into()]; + install.extend(["--no-remove".to_owned(), "--".into()]); + install.extend(good.iter().map(|spec| match spec.split_once('=') { + Some((name, version)) => format!("?exact-name({name})={version}"), + None => format!("?exact-name({spec})"), + })); + assert_eq!(runs[1].1.argv, install); + assert!(install.contains(&"?exact-name(libstdc++6)".to_owned())); + assert!(install.contains(&"?exact-name(nginx)=1.24.0-2ubuntu7".to_owned())); + // The last step fails unless every named package is installed. + let mut present = vec!["dpkg".to_owned(), "-s".into(), "--".into()]; + present.extend(good.iter().map(|spec| spec.split('=').next().unwrap().to_owned())); + assert_eq!(runs[2].1.argv, present); + for (unit, step) in &runs { + assert!(unit.starts_with("idunn-order-")); + let apt = step.argv[0] == "apt-get"; + assert_eq!( + step.environment, + if apt { + vec![("DEBIAN_FRONTEND".to_owned(), "noninteractive".to_owned())] + } else { + Vec::new() + } + ); + } + assert_ne!(runs[0].0, runs[1].0, "each step is its own transient unit"); + assert_ne!(runs[1].0, runs[2].0, "each step is its own transient unit"); + + // A package whose name apt would read as a regex or a conflict is + // exactly itself and never a removal. + for spec in ["cowsa.", "iptables-persistent", "a.", "nginx+"] { + let argv = Operation::PackageInstall { packages: vec![spec.into()] }.steps()[1].argv.clone(); + assert_eq!(argv[4..], ["--no-remove", "--", &format!("?exact-name({spec})")]); + } + // A step that failed ends the order: the absent package is a failure. + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(1), + ..RunReport::default() + }; + let absent = h.accept(h.issue(Operation::PackageInstall { packages: vec!["cowsay".into()] })); + execute(&h.ctx, &absent).unwrap(); + assert!(matches!(h.outcome(&absent), Outcome::Failed { exit_code: Some(1), .. })); + assert_eq!(h.runs().len(), 4, "the update failed, so nothing else ran"); + } + + #[test] + fn any_unit_outside_the_refused_set_runs() { + let h = harness(); + let units = [ + "wg-quick@wg0.service", + "nginx.service", + "matrix-synapse.service", + "ssh.socket", + "huginn@eureka.service", + "claude-connector.service", + "persona-matrix-mouth.service", + "mind-heartbeat.service", + "apt-daily.timer", + // Near misses: the prefix is the rule, not a substring or a stem. + "midunn-x.service", + "not-idunn-x.service", + "idunn.service", + "mind-self.service", + "mind-selfx@a.service", + "mind-persona.service", + "mind-personal@a.service", + ]; + for verb in all_verbs() { + for unit in units { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?} {unit}"); + let (transient, step) = h.runs().pop().unwrap(); + assert!(transient.starts_with("idunn-order-")); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", unit] + ); + } + } + assert_eq!(h.runs().len(), units.len() * 4); + } + + #[test] + fn refused_set_is_refused_by_name_and_alias() { + let h = harness(); + h.describe_unit( + "backup-name.service", + show_text( + "idunn-yggdrasil.service", + &[("Names", "idunn-yggdrasil.service backup-name.service")], + ), + ); + h.describe( + "quiet-alias.service", + &[("Names", "quiet-alias.service mind-persona@void.service")], + ); + let direct = [ + "idunn-yggdrasil.service", + "idunn-orders.service", + "idunn-vili-x.service", + "idunn-order-3f2c.service", + "idunn-yggdrasil.timer", + "idunn-x.socket", + "idunn-.service", + "idunn-yggdrasil.service.service", + "idunn-x@y.service", + "mind-self@eureka.service", + "mind-persona@void.service", + "mind-self@.service", + "mind-persona@a@b.service", + // Case folds: a lookalike is still refused, never run. + "IDUNN-yggdrasil.service", + "Idunn-Orders.service", + "MIND-SELF@eureka.service", + "Mind-Persona@Void.service", + ]; + for verb in all_verbs() { + for unit in direct { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + match h.outcome(&id) { + Outcome::Refused(Refused::AlwaysRefused { unit: hit, via }) => { + assert_eq!(hit, unit, "{verb:?} {unit}"); + assert_eq!(via, Via::Name); + } + other => panic!("{verb:?} {unit} was {other:?}"), + } + } + } + for verb in all_verbs() { + for (unit, hit) in [ + ("backup-name.service", "idunn-yggdrasil.service"), + ("quiet-alias.service", "mind-persona@void.service"), + ] { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Refused(Refused::AlwaysRefused { + unit: hit.into(), + via: Via::Alias + }), + "{verb:?} {unit}" + ); + } + } + assert!(h.runs().is_empty(), "no systemd verb ran for any refused order"); + // Every refusal is recorded and has a result post due. + let ledger = h.ctx.store.ledger().unwrap(); + let refusals = ledger + .outcomes + .iter() + .filter(|record| matches!(record.outcome, Outcome::Refused(_))) + .count(); + assert_eq!(refusals, direct.len() * 4 + 8); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + let results = due + .iter() + .filter(|due| due.kind == PostKind::Result && due.text.contains("always-refused")) + .count(); + assert_eq!(results, refusals); + // A refused order never started. + assert!(ledger.starts.is_empty()); + } + + fn outcome_of_unit(h: &Harness, verb: Verb, unit: &str) -> Outcome { + let id = h.accept(h.issue_unit(verb, unit)); + execute(&h.ctx, &id).unwrap(); + h.outcome(&id) + } + + fn hit(outcome: &Outcome, expected: Via) -> Option<&str> { + match outcome { + Outcome::Refused(Refused::AlwaysRefused { unit, via }) if *via == expected => { + Some(unit.as_str()) + } + _ => None, + } + } + + #[test] + fn a_quoted_name_in_a_dependency_list_is_the_name_it_quotes() { + let h = harness(); + // systemctl show quotes a name with a backslash and doubles the backslash. + h.describe( + "huginn.service", + &[("BoundBy", "\"mind-self@foo\\\\x2dbar.service\" other.service")], + ); + let outcome = outcome_of_unit(&h, Verb::Stop, "huginn.service"); + assert_eq!(hit(&outcome, Via::Bound), Some("mind-self@foo\\x2dbar.service")); + h.describe( + "quiet.service", + &[("Names", "quiet.service \"idunn-a\\\\x2db.service\"")], + ); + let outcome = outcome_of_unit(&h, Verb::Start, "quiet.service"); + assert_eq!(hit(&outcome, Via::Alias), Some("idunn-a\\x2db.service")); + assert!(h.runs().is_empty()); + } + + #[test] + fn unit_orders_run_as_one_job() { + let h = harness(); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran, "{verb:?}"); + let (_, step) = h.runs().pop().unwrap(); + assert_eq!( + step.argv, + ["systemctl", verb.as_str(), "--job-mode=ignore-requirements", "--", "nginx.service"] + ); + } + } + + #[test] + fn the_rule_reads_only_the_named_unit() { + let h = harness(); + // Whatever the unit lists, no other unit is looked at. + h.describe( + "plain.service", + &[("Requires", "other.service"), ("Wants", "idunn-x.service"), ("Conflicts", "mind-self@a.service")], + ); + h.describe("other.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "plain.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["plain.service"; 4]); + } + + #[test] + fn a_unit_that_acts_when_it_ends_is_refused() { + let h = harness(); + let properties = [ + ("SuccessAction", "reboot"), + ("FailureAction", "poweroff-force"), + ("StartLimitAction", "soft-reboot-immediate"), + ("JobTimeoutAction", "exit"), + ("FailureAction", "a-spelling-nobody-knows"), + ("OnFailure", "rescue.service"), + ("OnSuccess", "cleanup.service"), + ]; + for (property, value) in properties { + h.describe("ender.service", &[(property, value)]); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "ender.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "ender.service".into(), + via: Via::OnEnd + }), + "{verb:?} {property}={value}" + ); + } + assert_eq!(outcome_of_unit(&h, Verb::Reload, "ender.service"), Outcome::Ran); + } + // `none` and an empty list say the unit does nothing at its end. + h.describe( + "calm.service", + &[("SuccessAction", "none"), ("FailureAction", "none"), ("OnFailure", ""), ("OnSuccess", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "calm.service"), Outcome::Ran, "{verb:?}"); + } + assert_eq!(h.runs().len(), 11); + } + + #[test] + fn host_names_are_refused() { + let h = harness(); + let names = [ + "systemd-suspend.service", + "systemd-hibernate.service", + "systemd-hybrid-sleep.service", + "systemd-suspend-then-hibernate.service", + "rescue.service", + "emergency.service", + ]; + h.describe("sleepy.service", &[("Names", "sleepy.service systemd-hibernate.service")]); + for verb in all_verbs() { + for name in names { + assert_eq!( + outcome_of_unit(&h, verb, name), + Outcome::Refused(Refused::AlwaysRefused { unit: name.into(), via: Via::Host }), + "{verb:?} {name}" + ); + } + assert_eq!( + outcome_of_unit(&h, verb, "sleepy.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "systemd-hibernate.service".into(), + via: Via::Host + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "systemd-suspendish.service"), Outcome::Ran); + } + + #[test] + fn a_unit_others_are_bound_to_only_starts() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service second.service")]); + for verb in [Verb::Stop, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "base.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "first.service".into(), + via: Via::Bound + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Start, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "base.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_unit_that_upholds_others_is_refused_where_it_would_start() { + let h = harness(); + h.describe("holder.service", &[("Upholds", "held.service")]); + for verb in [Verb::Start, Verb::Restart] { + assert_eq!( + outcome_of_unit(&h, verb, "holder.service"), + Outcome::Refused(Refused::AlwaysRefused { + unit: "holder.service".into(), + via: Via::Upholds + }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + for verb in [Verb::Stop, Verb::Reload] { + assert_eq!(outcome_of_unit(&h, verb, "holder.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_wrapper_unit_is_refused_where_the_verb_would_touch_nothing() { + let h = harness(); + h.describe("wrapper.service", &[("ConsistsOf", "member@1.service member@2.service")]); + for verb in [Verb::Stop, Verb::Restart, Verb::Reload] { + assert_eq!( + outcome_of_unit(&h, verb, "wrapper.service"), + Outcome::Refused(Refused::Wrapper { member: "member@1.service".into() }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + assert_eq!(outcome_of_unit(&h, Verb::Start, "wrapper.service"), Outcome::Ran); + } + + #[test] + fn empty_dependency_lists_name_no_one() { + let h = harness(); + // systemctl prints every property it is asked for, empty or not. + h.describe( + "alone.service", + &[("BoundBy", ""), ("Upholds", ""), ("ConsistsOf", "")], + ); + for verb in all_verbs() { + assert_eq!(outcome_of_unit(&h, verb, "alone.service"), Outcome::Ran, "{verb:?}"); + } + } + + #[test] + fn a_refusal_reads_the_named_unit_once_and_a_refused_name_not_at_all() { + let h = harness(); + h.describe("base.service", &[("BoundBy", "first.service")]); + h.describe("idunn-ghost.service", &[("LoadState", "not-found")]); + assert!(hit(&outcome_of_unit(&h, Verb::Stop, "base.service"), Via::Bound).is_some()); + assert_eq!(h.systemd.shows.lock().unwrap().clone(), ["base.service"]); + h.systemd.shows.lock().unwrap().clear(); + assert_eq!( + hit(&outcome_of_unit(&h, Verb::Stop, "idunn-ghost.service"), Via::Name), + Some("idunn-ghost.service") + ); + assert!(h.systemd.shows.lock().unwrap().is_empty()); + } + + #[test] + fn each_refusal_is_posted_under_its_own_reason() { + for (via, word) in [ + (Via::Name, "name"), + (Via::Alias, "alias"), + (Via::Host, "host"), + (Via::OnEnd, "on-end"), + (Via::Bound, "bound"), + (Via::Upholds, "upholds"), + ] { + assert_eq!( + Outcome::Refused(Refused::AlwaysRefused { unit: "u.service".into(), via }) + .describe(), + format!("refused: always-refused unit u.service reached by {word}") + ); + } + assert_eq!( + Outcome::Refused(Refused::Wrapper { member: "m.service".into() }).describe(), + "refused: wrapper unit, it only wraps m.service" + ); + } + + #[test] + fn a_unit_systemd_cannot_show_is_a_failure_not_a_leaf() { + let h = harness(); + h.describe("ghost.service", &[("LoadState", "not-found")]); + for verb in all_verbs() { + let id = h.accept(h.issue_unit(verb, "ghost.service")); + execute(&h.ctx, &id).unwrap(); + assert!( + matches!(h.outcome(&id), Outcome::Failed { exit_code: None, signal: None }), + "{verb:?}" + ); + } + assert!(h.runs().is_empty()); + } + + #[test] + fn globs_slices_scopes_and_targets_are_not_units() { + let h = harness(); + let bad = [ + "idunn*", + "idunn-*.service", + "idunn-?ggdrasil.service", + "mind-?elf@eureka.service", + "mind-self@*.service", + "mind-[s]elf@eureka.service", + "system.slice", + "poweroff.target", + "multi-user.target", + "user.slice", + "session-1.scope", + "init.scope", + "idunn-yggdrasil", + "mind-self@eureka", + "nginx", + ".service", + "", + "-idunn-x.service", + "-x.service", + "/etc/systemd/system/idunn-yggdrasil.service", + "../idunn-yggdrasil.service", + "idunn-yggdrasil.service\n", + "idunn-yggdrasil.service ", + " idunn-yggdrasil.service", + "idunn-yggdrasil.service\0", + "idunn-yggdrasil.service;reboot", + "idunn\\x2dyggdrasil.service", + "idunn%2dyggdrasil.service", + // Unicode lookalikes of a refused name. + "\u{131}dunn-yggdrasil.service", + "idunn\u{2010}yggdrasil.service", + "idunn\u{ff0d}yggdrasil.service", + "idunn-yggdrasil.service\u{200b}", + "\u{200b}idunn-yggdrasil.service", + "idunn-yggdrasil\u{2024}service", + "mind-self\u{ff20}eureka.service", + ]; + for unit in bad { + for verb in all_verbs() { + assert!( + matches!(h.issue_unit(verb, unit), Response::Refused(_)), + "{unit:?} was not refused by the grammar" + ); + } + } + let long = format!("{}.service", "a".repeat(UNIT_LIMIT)); + assert_eq!( + h.issue_unit(Verb::Start, &long), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + assert!(!h.ctx.store.path().exists(), "the grammar refuses before any record"); + } + + #[test] + fn binding_with_a_unit_list_is_refused() { + let valid = "principal = \"eureka\"\nuser = \"mind-eureka-self\"\n\ + mind_control = \"/usr/local/bin/mind-control\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://127.0.0.1:8008\"\n\ + token_credential = \"idunn-matrix-token\"\n"; + let binding = StewardBinding::parse(valid).unwrap(); + assert_eq!(binding.principal, "eureka"); + assert_eq!(binding.address().unwrap().port(), 8008); + for extra in [ + "units = [\"nginx.service\"]", + "allow = [\"nginx.service\"]", + "deny = [\"idunn-*\"]", + "denylist = [\"idunn-*\"]", + "refused = [\"idunn-*\"]", + "unit = \"nginx.service\"", + "[units]\nnginx = true", + ] { + assert!( + StewardBinding::parse(&format!("{valid}{extra}\n")).is_err(), + "{extra} loaded" + ); + } + let swap = |from: &str, to: &str| valid.replace(from, to); + for broken in [ + swap("user = \"mind-eureka-self\"", "user = \"root\""), + swap("user = \"mind-eureka-self\"", "user = \"\""), + swap("/usr/local/bin/mind-control", "mind-control"), + swap("!abc:example.org", "#alias:example.org"), + swap("http://127.0.0.1:8008", "http://10.77.0.1:8008"), + swap("http://127.0.0.1:8008", "https://127.0.0.1:8008"), + swap("http://127.0.0.1:8008", "http://127.0.0.1:8008/path"), + swap("idunn-matrix-token", "../token"), + swap("principal = \"eureka\"\n", ""), + ] { + assert!(StewardBinding::parse(&broken).is_err()); + } + assert!(admitted(0, 0o100600)); + assert!(admitted(0, 0o100644)); + assert!(!admitted(0, 0o100664)); + assert!(!admitted(0, 0o100666)); + assert!(!admitted(0, 0o100602)); + assert!(!admitted(1000, 0o100600)); + } + + #[test] + fn held_brake_refuses_at_start() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + assert!(h.ctx.store.ledger().unwrap().start_of(&id).is_none()); + assert!(h.runs().is_empty()); + // The deploy path is gated the same way and writes no command. + let deploy = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &deploy).unwrap(); + assert_eq!(h.outcome(&deploy), Outcome::Refused(Refused::Braked)); + assert!(!h.ctx.state_store.exists()); + // A brake that cannot be asked is a held brake. + let directory = TempDir::new().unwrap(); + assert!(!MindControl { program: directory.path().join("absent") }.clear("eureka")); + } + + #[test] + fn brake_check_is_the_exit_status_of_mind_control_check() { + let directory = TempDir::new().unwrap(); + let program = |name: &str, body: &str| { + let path = directory.path().join(name); + std::fs::write(&path, body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + MindControl { program: path } + }; + let recorder = directory.path().join("argv"); + let clear = program( + "clear", + &format!("#!/bin/sh\necho \"$@\" > {}\nexit 0\n", recorder.display()), + ); + assert!(clear.clear("eureka")); + assert_eq!( + std::fs::read_to_string(&recorder).unwrap().trim(), + "--instance eureka check" + ); + assert!(!program("held", "#!/bin/sh\nexit 1\n").clear("eureka")); + } + + #[test] + fn started_order_is_never_rerun() { + let h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let waiting = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The first order began and its service died: a start with no outcome. + h.ctx + .store + .put( + &id, + START_SCHEMA, + &StartRecord { + schema_version: START_SCHEMA.into(), + order_id: id.clone(), + started_at_unix_millis: 1, + actuation: format!("idunn-order-{}", order_uuid(&id)), + }, + ) + .unwrap(); + let pending = recover(&h.ctx).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Lost); + assert_eq!(pending, std::slice::from_ref(&waiting), "an order that never started waits"); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty(), "a Lost order makes no systemd call"); + // Recovery is idempotent, and a finished order is not run again. + assert_eq!(recover(&h.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&h.ctx, &waiting).unwrap(); + execute(&h.ctx, &waiting).unwrap(); + assert_eq!(h.runs().len(), 1); + assert!(recover(&h.ctx).unwrap().is_empty()); + } + + #[test] + fn deploy_order_uses_the_up_primitive() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let command_id = format!("up-{}", order_uuid(&id)); + assert_eq!( + h.outcome(&id), + Outcome::Submitted { + command_id: command_id.clone() + } + ); + let ledger = h.ctx.store.ledger().unwrap(); + assert_eq!(ledger.start_of(&id).unwrap().actuation, command_id); + assert!(h.runs().is_empty(), "a deploy runs no transient unit"); + let mut shown = Vec::new(); + crate::control_plane::status( + &mut shown, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + Some(&command_id), + ) + .unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert_eq!(shown.lines().count(), 1, "exactly one command: {shown}"); + assert!(shown.starts_with(&format!("{command_id} vili queued")), "{shown}"); + // Re-executing writes no second command. + execute(&h.ctx, &id).unwrap(); + let mut again = Vec::new(); + crate::control_plane::status( + &mut again, + &h.ctx.state_store, + &h.directory.path().join("incidents.cc"), + None, + ) + .unwrap(); + assert_eq!(String::from_utf8(again).unwrap().lines().count(), 1); + // The result post waits for the transactions: queued is not finished. + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert_eq!(due.len(), 1); + assert_eq!(due[0].kind, PostKind::Accepted); + assert!(!deploy_finished("queued") && !deploy_finished("running")); + assert!(deploy_finished("succeeded") && deploy_finished("failed")); + assert!(deploy_finished("complete")); + } + + #[test] + fn chat_outage_delays_posts_not_orders() { + let h = harness(); + h.home.failing.store(true, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Ran); + assert_eq!(h.runs().len(), 1); + assert!(post_pass(&h.ctx).is_err()); + assert!(h.ctx.store.ledger().unwrap().posts.is_empty()); + h.home.failing.store(false, Ordering::SeqCst); + assert_eq!(post_pass(&h.ctx).unwrap(), 2); + assert_eq!(post_pass(&h.ctx).unwrap(), 0, "each post goes out once"); + let sent = h.home.sent.lock().unwrap().clone(); + assert_eq!(sent.len(), 2); + assert_eq!(sent[0].0, format!("idunn-{id}-accepted")); + assert_eq!(sent[1].0, format!("idunn-{id}-result")); + assert!(sent[0].1.contains("unit restart mind-heartbeat.service")); + assert!(sent[0].1.contains("mind-eureka-self.service")); + assert!(sent[0].1.contains("reason: \"probe\"")); + assert!(sent[1].1.contains("ran (exit 0)")); + } + + #[test] + fn failed_runs_keep_bounded_tails_that_never_leave_the_store() { + let h = harness(); + *h.systemd.report.lock().unwrap() = RunReport { + exit_code: Some(100), + signal: None, + stdout_tail: vec![b'o'; 3 * TAIL_BYTES], + stderr_tail: b"E: secret-looking \x1b[31m\xe2\x80\xae line\n".to_vec(), + }; + let id = h.accept(h.issue(Operation::PackageInstall { + packages: vec!["git-lfs".into()], + })); + execute(&h.ctx, &id).unwrap(); + assert_eq!( + h.outcome(&id), + Outcome::Failed { + exit_code: Some(100), + signal: None + } + ); + assert_eq!(h.runs().len(), 1, "the install did not follow a failed update"); + let ledger = h.ctx.store.ledger().unwrap(); + let record = ledger.outcome_of(&id).unwrap(); + assert_eq!(record.stdout_tail.len(), TAIL_BYTES); + assert!(!format!("{record:?}").contains("secret-looking")); + // Neither the post nor the journal text carries a tail. + let results: Vec<_> = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)) + .into_iter() + .filter(|due| due.kind == PostKind::Result) + .collect(); + assert_eq!(results.len(), 1); + assert!(results[0].text.contains("failed (exit 100)")); + assert!(!results[0].text.contains("secret-looking")); + // The steward reads them through the escape sink and nothing raw. + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("expected a show") + }; + let mut shown = Vec::new(); + emit(&mut shown, &render_view(&view)).unwrap(); + let shown = String::from_utf8(shown).unwrap(); + assert!(shown.contains("secret-looking")); + assert!(!shown.contains('\u{1b}') && !shown.contains('\u{202e}')); + assert!(shown.contains("\\u{1b}") && shown.contains("\\u{202e}")); + assert!(shown.contains("status failed")); + // A show of an unknown or malformed id names the field only. + assert_eq!( + h.ask(&Request::Show(ShowRequest { + id: format!("ord-{}", Uuid::new_v4()) + })), + Response::Refused(Refusal::new("id", Code::Unknown)) + ); + assert_eq!( + h.ask(&Request::Show(ShowRequest { id: "ord-../x".into() })), + Response::Refused(Refusal::new("id", Code::BadShape)) + ); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("expected a list") + }; + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].status, "failed"); + } + + #[test] + fn a_refusal_or_error_never_echoes_an_input_value() { + let h = harness(); + let mut seen: Vec> = Vec::new(); + let mut note = |response: Response| { + let bytes = rmp_serde::to_vec_named(&response).unwrap(); + if let Response::Refused(refusal) = &response { + seen.push(refusal.to_string().into_bytes()); + } + seen.push(bytes); + }; + for unit in [ + format!("{CANARY}*.service"), + format!("{CANARY}/../idunn-orders.service"), + format!("idunn-{CANARY};x.service"), + CANARY.to_string(), + ] { + note(h.issue_unit(Verb::Restart, &unit)); + } + note(h.issue(Operation::PackageInstall { + packages: vec![format!("{CANARY}/x")], + })); + note(h.issue(Operation::PackageInstall { + packages: vec![format!("pkg={CANARY} 1")], + })); + note(h.issue(Operation::Deploy { + selector: format!("{CANARY} bad"), + })); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: format!("{CANARY}\n"), + }))); + note(h.ask(&Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Start, + unit: "nginx.service".into(), + }, + reason: CANARY.repeat(40), + }))); + note(h.ask(&Request::Show(ShowRequest { + id: format!("ord-{CANARY}"), + }))); + // A frame whose own keys and values are the canary, and plain garbage. + let named: std::collections::BTreeMap> = + [( + CANARY.to_owned(), + [(CANARY.to_owned(), CANARY.to_owned())].into_iter().collect(), + )] + .into_iter() + .collect(); + note(h.exchange(|client| write_frame(client, &named).unwrap())); + note(h.exchange(|client| { + let garbage = format!("{CANARY}{CANARY}"); + client + .write_all(&(garbage.len() as u32).to_be_bytes()) + .unwrap(); + client.write_all(garbage.as_bytes()).unwrap(); + })); + note(h.exchange(|client| { + client.write_all(&u32::MAX.to_be_bytes()).unwrap(); + })); + // The client grammar fails the same way. + let mut messages = Vec::new(); + let words = |line: &[&str]| line.iter().map(|word| (*word).to_owned()).collect::>(); + for line in [ + words(&["unit", "restart", &format!("{CANARY};x.service"), "--reason", "r"]), + words(&["unit", CANARY, "nginx.service", "--reason", "r"]), + words(&["unit", "restart", "nginx.service", &format!("--{CANARY}")]), + words(&["unit", "restart", "nginx.service", "--reason", &format!("{CANARY}\n")]), + words(&[CANARY]), + words(&["package-install", &format!("{CANARY}/x"), "--reason", "r"]), + words(&["deploy", &format!("{CANARY} x"), "--reason", "r"]), + words(&["show"]), + ] { + messages.push(format!("{:#}", parse_order(line.into_iter()).unwrap_err())); + } + for line in [ + words(&[CANARY]), + words(&["serve", &format!("--{CANARY}")]), + words(&["serve", "--store"]), + ] { + messages.push(format!("{:#}", parse_serve(line.into_iter()).unwrap_err())); + } + for bytes in &seen { + assert!( + !String::from_utf8_lossy(bytes).contains(CANARY), + "a response carried an input value" + ); + } + for message in &messages { + assert!(!message.contains(CANARY), "an error carried an input value: {message}"); + } + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn cgroup_names_the_issuer_unit() { + assert_eq!( + unit_from_cgroup("0::/system.slice/claude-connector.service\n"), + "claude-connector.service" + ); + // What a delegated subtree adds below the unit is the peer's to name. + assert_eq!( + unit_from_cgroup("0::/system.slice/huginn@eureka.service/idunn-yggdrasil.service/leaf\n"), + "huginn@eureka.service" + ); + assert_eq!( + unit_from_cgroup("0::/system.slice/system-getty.slice/getty@tty1.service\n"), + "getty@tty1.service" + ); + assert_eq!( + unit_from_cgroup("12:cpu:/x\n1:name=systemd:/system.slice/a.scope\n"), + "a.scope" + ); + // A user manager's units, and paths no manager made, name no one. + for path in [ + "0::/user.slice/user-985.slice/user@985.service/app.slice/idunn-yggdrasil.service\n", + "0::/user.slice/user-1000.slice/session-3.scope\n", + "0::/system.slice/not-a-unit/idunn-yggdrasil.service\n", + "0::/system.slice/\n", + "0::/\n", + "", + ] { + assert_eq!(unit_from_cgroup(path), "unknown", "{path}"); + } + } + + #[test] + fn systemctl_show_output_is_read_into_facts() { + let facts = parse_show( + "Id=a.service\nNames=a.service b.service\nLoadState=loaded\nActiveState=active\n\ + SuccessAction=none\nFailureAction=reboot-force\nStartLimitAction=\nJobTimeoutAction=none\n\ + OnFailure=f.service g.service\nOnSuccess=\nBoundBy=\"q\\\\x2dr.service\" s.service\n\ + Upholds=u.service\nConsistsOf=m.service n.service\nRequires=r.service\nConflicts=x.service\nOther=x\n", + ) + .unwrap(); + assert_eq!(facts.id, "a.service"); + assert_eq!(facts.names, ["a.service", "b.service"]); + assert!(!facts.not_found); + assert_eq!(facts.on_end, ["FailureAction", "OnFailure", "OnFailure"]); + // A quoted name holds one backslash, not two and not a quote mark. + assert_eq!(facts.bound_by, ["q\\x2dr.service", "s.service"]); + assert_eq!(facts.upholds, ["u.service"]); + assert_eq!(facts.consists_of, ["m.service", "n.service"]); + assert!(parse_show("Names=a.service\n").is_err()); + // The request asks for exactly the properties the parse reads. + assert_eq!( + show_properties(), + "Id,Names,LoadState,SuccessAction,FailureAction,StartLimitAction,JobTimeoutAction,\ + OnFailure,OnSuccess,BoundBy,Upholds,ConsistsOf" + ); + // A backslash outside quotes is a backslash; inside, it escapes one character. + assert_eq!(parse_names("a\\b c\\\\d"), ["a\\b", "c\\\\d"]); + assert_eq!(parse_names("\"a\\\"b\" \"\" c"), ["a\"b", "", "c"]); + assert!(parse_names(" ").is_empty()); + let gone = parse_show("Id=\\x22a.service\\x22.service\nLoadState=not-found\n").unwrap(); + assert!(gone.not_found); + } + + #[test] + fn the_store_refuses_a_second_record_and_a_foreign_document() { + let h = harness(); + let order = OrderRecord { + schema_version: ORDER_SCHEMA.into(), + order_id: "ord-x".into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }; + assert!(h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert!(!h.ctx.store.put("ord-x", ORDER_SCHEMA, &order).unwrap()); + assert_eq!( + std::fs::metadata(h.ctx.store.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let foreign = typed_envelope("k", "idunn.other", "idunn.other.v1", &1u8, 1).unwrap(); + SingleFileMessagePackBackingStore::new(h.ctx.store.path()) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: "idunn.other".into(), + key: "k".into(), + current: None, + }], + &[foreign], + ) + .unwrap(); + assert!(h.ctx.store.ledger().is_err()); + } + + #[test] + fn synapse_post_is_one_idempotent_put_on_loopback() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (sent, requests) = mpsc::channel(); + let server = thread::spawn(move || { + for reply in [ + "HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n{\"event_id\":\"$ev1\"}", + "HTTP/1.0 403 Forbidden\r\n\r\n{\"errcode\":\"M_FORBIDDEN\"}", + ] { + let (mut socket, _) = listener.accept().unwrap(); + let mut received = Vec::new(); + let mut chunk = [0u8; 1024]; + while !received.windows(4).any(|window| window == b"\r\n\r\n") + || !String::from_utf8_lossy(&received).contains("\"body\"") + { + let count = socket.read(&mut chunk).unwrap(); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + sent.send(String::from_utf8_lossy(&received).into_owned()).unwrap(); + socket.write_all(reply.as_bytes()).unwrap(); + } + }); + let synapse = Synapse { + address, + room: "!room:example.org".into(), + token_file, + reply_within: Duration::from_secs(10), + }; + assert_eq!(synapse.post("txn-1", "hello \"quoted\"").unwrap(), "$ev1"); + let request = requests.recv().unwrap(); + assert!(request.starts_with( + "PUT /_matrix/client/v3/rooms/%21room%3Aexample.org/send/m.room.message/txn-1 HTTP/1.0\r\n" + )); + assert!(request.contains("Authorization: Bearer tok-123\r\n")); + assert!(request.contains("{\"body\":\"hello \\\"quoted\\\"\",\"msgtype\":\"m.text\"}")); + let error = synapse.post("txn-2", "x").unwrap_err(); + assert!(!format!("{error:#}").contains("tok-123")); + server.join().unwrap(); + assert!( + Synapse { + address, + room: "!r:e".into(), + token_file: directory.path().join("absent"), + reply_within: Duration::from_secs(10), + } + .post("t", "x") + .is_err() + ); + } + + #[test] + fn grammar_limits_are_exact_at_their_edges() { + let h = harness(); + let order = |operation: Operation, reason: &str| { + h.ask(&Request::Order(OrderRequest { + operation, + reason: reason.into(), + })) + }; + let packages = |names: Vec| Operation::PackageInstall { packages: names }; + let unit = |name: String| Operation::UnitAction { + verb: Verb::Start, + unit: name, + }; + // Package: two characters is the shortest name, 200 bytes the longest. + assert!(matches!(order(packages(vec!["ab".into()]), "r"), Response::Accepted { .. })); + assert!(matches!(order(packages(vec!["a".repeat(200)]), "r"), Response::Accepted { .. })); + assert_eq!( + order(packages(vec!["a".repeat(201)]), "r"), + Response::Refused(Refusal::new("package", Code::TooLong)) + ); + assert_eq!( + order(packages(vec!["a".into()]), "r"), + Response::Refused(Refusal::new("package", Code::BadShape)) + ); + let sixteen: Vec = (0..16).map(|n| format!("pkg-{n}x")).collect(); + assert!(matches!(order(packages(sixteen), "r"), Response::Accepted { .. })); + // Unit: 200 bytes with its suffix is the longest. + let longest = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len())); + assert_eq!(longest.len(), UNIT_LIMIT); + assert!(matches!(order(unit(longest), "r"), Response::Accepted { .. })); + let too_long = format!("{}.service", "a".repeat(UNIT_LIMIT - ".service".len() + 1)); + assert_eq!( + order(unit(too_long), "r"), + Response::Refused(Refusal::new("unit", Code::TooLong)) + ); + // Reason: 300 bytes is the longest, and it must be there. + assert!(matches!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT)), + Response::Accepted { .. } + )); + assert_eq!( + order(unit("nginx.service".into()), &"r".repeat(REASON_LIMIT + 1)), + Response::Refused(Refusal::new("reason", Code::TooLong)) + ); + assert_eq!( + order(unit("nginx.service".into()), ""), + Response::Refused(Refusal::new("reason", Code::Empty)) + ); + assert_eq!( + order(unit("nginx.service".into()), "caf\u{e9}"), + Response::Refused(Refusal::new("reason", Code::BadCharacter)) + ); + } + + #[test] + fn a_concluded_order_is_never_run_later() { + let h = harness(); + h.brake.0.store(false, Ordering::SeqCst); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + execute(&h.ctx, &id).unwrap(); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + // Someone else clears the brake; the refused order stays refused. + h.brake.0.store(true, Ordering::SeqCst); + execute(&h.ctx, &id).unwrap(); + assert!(h.runs().is_empty()); + let ledger = h.ctx.store.ledger().unwrap(); + assert!(ledger.start_of(&id).is_none()); + assert_eq!(h.outcome(&id), Outcome::Refused(Refused::Braked)); + } + + #[test] + fn the_orders_cli_has_no_command_string() { + let parse = |line: &[&str]| parse_order(line.iter().map(|word| (*word).to_owned())); + let parsed = parse(&["unit", "restart", "nginx.service", "--reason", "why"]).unwrap(); + assert_eq!( + parsed.action, + OrderAction::Issue { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: "nginx.service".into() + }, + reason: "why".into() + } + ); + assert!(parse(&["package-install", "git-lfs", "socat", "--reason", "r"]).is_ok()); + assert!(parse(&["deploy", "profile:aetheria", "--reason", "r"]).is_ok()); + assert!(parse(&["show", "ord-x"]).is_ok()); + assert!(parse(&["list"]).is_ok()); + for refused in [ + vec!["unit", "restart", "nginx.service", "--reason", "r", "--command", "reboot-now"], + vec!["unit", "restart", "nginx.service", "--reason", "r", "--shell"], + vec!["package-install", "git-lfs", "--reason", "r", "--command", "x"], + vec!["run", "sh", "--reason", "r"], + vec!["exec", "--reason", "r"], + vec!["unit", "enable", "nginx.service", "--reason", "r"], + vec!["unit", "restart", "--reason", "r"], + vec!["unit", "restart", "a.service", "b.service", "--reason", "r"], + vec!["unit", "restart", "nginx.service"], + vec!["unit", "restart", "nginx*", "--reason", "r"], + vec!["package-install", "--reason", "r"], + vec!["package-install", "-y", "x", "--reason", "r"], + vec!["deploy", "--reason", "r"], + vec!["deploy", "a", "b", "--reason", "r"], + vec!["list", "extra"], + vec!["show"], + vec![], + ] { + assert!(parse(&refused).is_err(), "{refused:?} parsed"); + } + } + + #[test] + fn an_order_runs_only_under_the_steward_bound_now() { + let mut h = harness(); + let first = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + let second = h.accept(h.issue_unit(Verb::Restart, "ssh.socket")); + // The binding changes while both wait: another uid, then another + // principal with the old uid. Each is refused with a record, before + // the brake is asked and before anything runs. + h.ctx.steward_uid = STEWARD + 7; + execute(&h.ctx, &first).unwrap(); + h.ctx.steward_uid = STEWARD; + h.ctx.principal = "void".into(); + execute(&h.ctx, &second).unwrap(); + for id in [&first, &second] { + assert_eq!(h.outcome(id), Outcome::Refused(Refused::NotSteward)); + assert!(h.ctx.store.ledger().unwrap().start_of(id).is_none()); + } + assert!(h.runs().is_empty()); + assert!(h.brake.1.lock().unwrap().is_empty(), "a revoked steward's brake is never the gate"); + // The refusal is posted like any other. + let ledger = h.ctx.store.ledger().unwrap(); + let due = due_posts(&ledger, &|command_id| command_state(&h.ctx.state_store, command_id)); + assert!(due.iter().any(|due| due.kind == PostKind::Result + && due.text.contains("refused: the issuer is not the current steward"))); + // The other direction: an order the bound steward issued after the + // rebinding, and a recovered one under an unchanged binding, run and + // are gated by the bound principal's brake. + let third = h.accept(h.issue_unit(Verb::Restart, "cron.service")); + let resumed = harness(); + let waiting = resumed.accept(resumed.issue_unit(Verb::Restart, "nginx.service")); + assert_eq!(recover(&resumed.ctx).unwrap(), std::slice::from_ref(&waiting)); + execute(&resumed.ctx, &waiting).unwrap(); + execute(&h.ctx, &third).unwrap(); + assert_eq!(resumed.outcome(&waiting), Outcome::Ran); + assert_eq!(h.outcome(&third), Outcome::Ran); + assert_eq!(*h.brake.1.lock().unwrap(), ["void".to_owned()]); + assert_eq!(*resumed.brake.1.lock().unwrap(), ["eureka".to_owned()]); + } + + #[test] + fn root_reads_orders_and_never_issues_one() { + let mut h = harness(); + let id = h.accept(h.issue_unit(Verb::Restart, "nginx.service")); + h.ctx.peers = Arc::new(FakePeers(0)); + let refused = Response::Refused(Refusal::new("peer", Code::Unauthorized)); + assert_eq!(h.issue_unit(Verb::Restart, "cron.service"), refused); + let Response::Show(view) = h.ask(&Request::Show(ShowRequest { id: id.clone() })) else { + panic!("root could not read an order"); + }; + assert_eq!(view.order.order_id, id); + let Response::List(lines) = h.ask(&Request::List) else { + panic!("root could not list orders"); + }; + assert_eq!(lines.len(), 1, "root's refused order left no record"); + h.ctx.peers = Arc::new(FakePeers(STEWARD + 1)); + assert_eq!(h.ask(&Request::Show(ShowRequest { id })), refused); + } + + #[test] + fn every_order_posts_once_in_its_turn() { + let h = harness(); + let ids: Vec = ["nginx.service", "ssh.socket", "cron.service"] + .into_iter() + .map(|unit| h.accept(h.issue_unit(Verb::Restart, unit))) + .collect(); + for id in &ids { + execute(&h.ctx, id).unwrap(); + } + assert_eq!(post_pass(&h.ctx).unwrap(), 6, "an accepted and a result post for each"); + assert_eq!(post_pass(&h.ctx).unwrap(), 0); + let sent = h.home.sent.lock().unwrap().clone(); + let mut keys: Vec<_> = sent.iter().map(|(txn, _)| txn.clone()).collect(); + keys.sort(); + let mut expected: Vec = ids + .iter() + .flat_map(|id| [format!("idunn-{id}-accepted"), format!("idunn-{id}-result")]) + .collect(); + expected.sort(); + assert_eq!(keys, expected); + // An order accepted later is posted by the next pass alone. + let late = h.accept(h.issue_unit(Verb::Restart, "mind-heartbeat.service")); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + execute(&h.ctx, &late).unwrap(); + assert_eq!(post_pass(&h.ctx).unwrap(), 1); + } + + #[test] + fn a_deploys_result_is_posted_once_its_command_has_finished() { + let h = harness(); + let id = h.accept(h.issue(Operation::Deploy { + selector: "vili".into(), + })); + execute(&h.ctx, &id).unwrap(); + let ledger = h.ctx.store.ledger().unwrap(); + let results = |state: Option<(&'static str, &str)>| { + let owed = due_posts(&ledger, &|_| match state { + Some((state, detail)) => Ok((state, detail.to_owned())), + None => Err(anyhow!("the command cannot be read")), + }); + owed.into_iter().filter(|due| due.kind == PostKind::Result).collect::>() + }; + assert!(results(Some(("queued", ""))).is_empty()); + assert!(results(Some(("running", "vili:Sealing"))).is_empty()); + assert!(results(None).is_empty()); + for state in ["succeeded", "failed"] { + let owed = results(Some((state, "vili"))); + assert_eq!(owed.len(), 1, "{state}"); + assert!(owed[0].text.contains(&format!("{state} vili")), "{}", owed[0].text); + assert!(owed[0].text.contains("submitted deployment up-")); + } + } + + #[test] + fn intake_gives_up_on_a_peer_that_sends_no_frame() { + let h = harness(); + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + // Half a length prefix, then nothing: the connection must end by itself. + client.write_all(&[0, 0]).unwrap(); + let (done, ended) = mpsc::channel(); + let worker = thread::spawn(move || { + let result = handle(&h.ctx, &mut server, &h.queue); + let _ = done.send(result.is_ok()); + h + }); + let answered = ended + .recv_timeout(Duration::from_secs(20)) + .expect("intake never gave up on a silent peer"); + assert!(answered, "a silent peer is refused as malformed, not left as an error"); + let h = worker.join().unwrap(); + assert_eq!( + rmp_serde::from_slice::(&read_frame(&mut client, RESPONSE_FRAME_LIMIT).unwrap()).unwrap(), + Response::Refused(Refusal::new("frame", Code::Malformed)) + ); + assert!(!h.ctx.store.path().exists()); + } + + #[test] + fn a_record_under_another_schema_is_not_read() { + let put = |h: &Harness, kind: &str, schema: &str, key: &str| match kind { + "order" => h.ctx.store.put( + key, + schema, + &OrderRecord { + schema_version: schema.into(), + order_id: key.into(), + principal: "eureka".into(), + issuer_uid: 1, + issuer_unit: "u.service".into(), + issuer_invocation: String::new(), + issued_at_unix_millis: 1, + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "r".into(), + }, + ), + "start" => h.ctx.store.put( + key, + schema, + &StartRecord { + schema_version: schema.into(), + order_id: key.into(), + started_at_unix_millis: 1, + actuation: "idunn-order-x".into(), + }, + ), + "outcome" => h.ctx.store.put( + key, + schema, + &OutcomeRecord { + schema_version: schema.into(), + order_id: key.into(), + ended_at_unix_millis: 1, + outcome: Outcome::Lost, + stdout_tail: Vec::new(), + stderr_tail: Vec::new(), + }, + ), + _ => h.ctx.store.put( + key, + schema, + &PostRecord { + schema_version: schema.into(), + post_key: key.into(), + event_id: "$e".into(), + posted_at_unix_millis: 1, + }, + ), + }; + for (kind, current, key) in [ + ("order", ORDER_SCHEMA, "ord-a"), + ("start", START_SCHEMA, "ord-a"), + ("outcome", OUTCOME_SCHEMA, "ord-a"), + ("post", POST_SCHEMA, "ord-a:accepted"), + ] { + let h = harness(); + assert!(put(&h, kind, current, key).unwrap()); + assert!(h.ctx.store.ledger().is_ok(), "{current} is read"); + let h = harness(); + let other = current.replace(".v1", ".v0"); + assert!(put(&h, kind, &other, key).unwrap()); + assert!(h.ctx.store.ledger().is_err(), "{other} is not read"); + } + } + + #[test] + fn posts_are_retried_after_a_widening_pause() { + let pauses: Vec = (0..9).map(post_pause).collect(); + assert_eq!(pauses, [2, 4, 8, 16, 32, 60, 60, 60, 60]); + } + + #[test] + fn serve_options_are_read_from_every_flag() { + let parse = |line: &[&str]| parse_serve(line.iter().map(|word| (*word).to_owned())); + let defaults = parse(&["serve"]).unwrap(); + assert_eq!(defaults.store, PathBuf::from(DEFAULT_STORE)); + assert_eq!(defaults.steward_binding, PathBuf::from(DEFAULT_BINDING)); + assert_eq!(defaults.socket, PathBuf::from(DEFAULT_SOCKET)); + let given = parse(&[ + "serve", + "--state-store", + "/a/control.cc", + "--store", + "/b", + "--steward-binding", + "/c.toml", + "--socket", + "/d.sock", + ]) + .unwrap(); + assert_eq!( + given, + ServeOptions { + state_store: "/a/control.cc".into(), + store: "/b".into(), + steward_binding: "/c.toml".into(), + socket: "/d.sock".into(), + } + ); + assert!(parse(&["serve", "--store"]).is_err()); + assert!(parse(&["serve", "--other", "/x"]).is_err()); + assert!(parse(&["start"]).is_err()); + } + + #[test] + fn the_steward_user_is_a_real_user_and_never_root() { + assert!(user_uid("root").is_err()); + assert!(user_uid("no-such-user-here").is_err()); + assert!(user_uid("bad\0name").is_err()); + assert_eq!(user_uid("nobody").unwrap(), 65534); + } + + #[test] + fn the_orders_directory_is_made_private() { + let directory = TempDir::new().unwrap(); + let store = OrderStore::new(&directory.path().join("orders")); + store.prepare().unwrap(); + let mode = std::fs::metadata(directory.path().join("orders")).unwrap().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + #[test] + fn the_socket_belongs_to_the_steward_alone() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join("steward.sock"); + let me = std::fs::metadata(directory.path()).unwrap().uid(); + let _listener = bind_socket(&path, me).unwrap(); + let metadata = std::fs::metadata(&path).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, me)); + // A stale socket is replaced; anything else in the way is not. + let _again = bind_socket(&path, me).unwrap(); + let other = directory.path().join("other"); + std::fs::write(&other, "x").unwrap(); + assert!(bind_socket(&other, me).is_err()); + } + + #[test] + fn synapse_gives_up_on_a_reply_that_never_ends() { + let directory = TempDir::new().unwrap(); + let token_file = directory.path().join("token"); + std::fs::write(&token_file, "tok-123\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + thread::spawn(move || { + // The first connection stays silent; the second drips a byte at a time. + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + thread::spawn(move || { + let mut chunk = [0u8; 4096]; + let _ = socket.read(&mut chunk); + for byte in b"HTTP/1.0 200 OK\r\n".iter().cycle().take(200) { + if index == 0 || socket.write_all(&[*byte]).is_err() { + break; + } + thread::sleep(Duration::from_millis(40)); + } + thread::sleep(Duration::from_secs(5)); + }); + } + }); + let synapse = Synapse { + address, + room: "!r:e".into(), + token_file, + reply_within: Duration::from_millis(600), + }; + let synapse = Arc::new(synapse); + for _ in 0..2 { + let started = std::time::Instant::now(); + let (done, ended) = mpsc::channel(); + let poster = Arc::clone(&synapse); + thread::spawn(move || { + let _ = done.send(poster.post("t", "x").map_err(|error| format!("{error:#}"))); + }); + let error = ended + .recv_timeout(Duration::from_secs(4)) + .expect("the post never gave up") + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3), "{error}"); + assert!(!error.contains("tok-123")); + } + } + + /// The whole service, as root, against fake `systemctl`, `systemd-run` + /// and a loopback homeserver, with the steward connecting from another + /// uid. It runs only where the test process is root (the verify container). + #[test] + fn serve_end_to_end_as_root() { + // SAFETY: geteuid reads the process credentials. + if unsafe { libc::geteuid() } != 0 { + eprintln!("serve_end_to_end_as_root: skipped, not root"); + return; + } + let directory = TempDir::new().unwrap(); + let root = directory.path().to_owned(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); + let bin = root.join("bin"); + std::fs::create_dir(&bin).unwrap(); + let systemctl = format!( + "#!/bin/sh\necho \"$@\" >> {0}/systemctl.log\nfor a; do last=$a; done\n\ + case \"$*\" in *InvocationID*quiet*) ;; *InvocationID*broken*) echo junk; exit 1;; *InvocationID*) echo inv-fake;; *show*) printf 'Id=%s\\nNames=%s\\n' \"$last\" \"$last\";; esac\n", + root.display() + ); + let run = format!( + "#!/bin/sh\necho \"$@\" >> {0}/run.log\nif [ -e {0}/fail ]; then echo boom >&2; exit 3; fi\nexit 0\n", + root.display() + ); + for (name, text) in [("systemctl", systemctl), ("systemd-run", run)] { + let path = bin.join(name); + std::fs::write(&path, text).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let credentials = root.join("cred"); + std::fs::create_dir(&credentials).unwrap(); + std::fs::write(credentials.join("idunn-matrix-token"), "tok-1\n").unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (posted, posts) = mpsc::channel::(); + thread::spawn(move || { + for (index, stream) in listener.incoming().enumerate() { + let Ok(mut socket) = stream else { continue }; + let mut received = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + let text = String::from_utf8_lossy(&received).into_owned(); + if let Some(split) = text.find("\r\n\r\n") { + let length = text + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if received.len() >= split + 4 + length { + break; + } + } + let count = socket.read(&mut chunk).unwrap_or(0); + if count == 0 { + break; + } + received.extend_from_slice(&chunk[..count]); + } + let _ = posted.send(String::from_utf8_lossy(&received).into_owned()); + let _ = socket.write_all( + format!("HTTP/1.0 200 OK\r\n\r\n{{\"event_id\":\"$e{index}\"}}").as_bytes(), + ); + } + }); + let binding = root.join("steward.toml"); + std::fs::write( + &binding, + format!( + "principal = \"eureka\"\nuser = \"nobody\"\nmind_control = \"/bin/true\"\n\ + room = \"!abc:example.org\"\nhomeserver = \"http://{address}\"\n\ + token_credential = \"idunn-matrix-token\"\n" + ), + ) + .unwrap(); + std::fs::set_permissions(&binding, std::fs::Permissions::from_mode(0o644)).unwrap(); + let run_directory = root.join("run"); + std::fs::create_dir(&run_directory).unwrap(); + std::fs::set_permissions(&run_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let socket = run_directory.join("steward.sock"); + let store = root.join("orders"); + let control = root.join("state").join("control.cc"); + // SAFETY: the only test that reads these variables is this one; PATH + // keeps its old entries. + unsafe { + std::env::set_var( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()), + ); + std::env::set_var("CREDENTIALS_DIRECTORY", &credentials); + } + let options = ServeOptions { + state_store: control.clone(), + store: store.clone(), + steward_binding: binding, + socket: socket.clone(), + }; + // The unit's umask must not decide the modes of what the service writes. + // SAFETY: umask only sets the process file-mode mask. + let before = unsafe { libc::umask(0o117) }; + let (ended, serve_end) = mpsc::channel::(); + thread::spawn(move || { + let result = serve(options); + let _ = ended.send(format!("{result:?}")); + }); + for _ in 0..100 { + if socket.exists() || serve_end.try_recv().is_ok() { + break; + } + thread::sleep(Duration::from_millis(100)); + } + assert!(socket.exists(), "serve never bound its socket"); + let metadata = std::fs::metadata(&socket).unwrap(); + assert_eq!((metadata.mode() & 0o777, metadata.uid()), (0o600, 65534)); + // Root is not the steward: it may read, it may not order. + let unit = |name: &str| { + Request::Order(OrderRequest { + operation: Operation::UnitAction { + verb: Verb::Restart, + unit: name.into(), + }, + reason: "end to end".into(), + }) + }; + let as_root = |request: &Request| { + let mut stream = UnixStream::connect(&socket).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + write_frame(&mut stream, request).unwrap(); + rmp_serde::from_slice::(&read_frame(&mut stream, RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + }; + assert_eq!( + as_root(&unit("nginx.service")), + Response::Refused(Refusal::new("peer", Code::Unauthorized)) + ); + assert_eq!(as_root(&Request::List), Response::List(Vec::new())); + let orders = OrderStore::new(&store); + let finished = |order_id: &str| { + for _ in 0..200 { + if let Some(record) = orders.ledger().unwrap().outcome_of(order_id) { + return record.clone(); + } + thread::sleep(Duration::from_millis(100)); + } + panic!("{order_id} never concluded"); + }; + // The steward is uid 65534 with gid 0: a service that read the gid for + // the uid would take it for root. + let first = steward_asks(&socket, &unit("nginx.service")); + let Response::Accepted { order_id: first } = first else { + panic!("the steward was not accepted: {first:?}"); + }; + assert_eq!(finished(&first).outcome, Outcome::Ran); + std::fs::write(root.join("fail"), "").unwrap(); + let Response::Accepted { order_id: second } = steward_asks(&socket, &unit("cron.service")) else { + panic!("the second order was not accepted"); + }; + let failed = finished(&second); + assert_eq!( + failed.outcome, + Outcome::Failed { + exit_code: Some(3), + signal: None + } + ); + assert_eq!(failed.stderr_tail, b"boom\n"); + let deploy = Request::Order(OrderRequest { + operation: Operation::Deploy { + selector: "vili".into(), + }, + reason: "end to end".into(), + }); + let Response::Accepted { order_id: third } = steward_asks(&socket, &deploy) else { + panic!("the deploy was not accepted"); + }; + assert!(matches!(finished(&third).outcome, Outcome::Submitted { .. })); + let ledger = orders.ledger().unwrap(); + let stored = ledger.orders.iter().find(|order| order.order_id == first).unwrap(); + assert_eq!((stored.issuer_uid, stored.issuer_invocation.as_str()), (65534, "inv-fake")); + // An invocation id is what a succeeding systemctl prints, if anything. + assert_eq!(HostSystemd.invocation("quiet.service"), None); + assert_eq!(HostSystemd.invocation("broken.service"), None); + assert_eq!(HostSystemd.invocation("some.service").as_deref(), Some("inv-fake")); + // What the service called, exactly. + let run_log = std::fs::read_to_string(root.join("run.log")).unwrap(); + assert!( + run_log.contains(&format!( + "--unit idunn-order-{} --wait --pipe --collect -p RuntimeMaxSec=15min -- systemctl restart --job-mode=ignore-requirements -- nginx.service", + order_uuid(&first) + )), + "{run_log}" + ); + let systemctl_log = std::fs::read_to_string(root.join("systemctl.log")).unwrap(); + assert!( + systemctl_log.contains(&format!("show --no-pager --property={} -- nginx.service", show_properties())), + "{systemctl_log}" + ); + // Every record is posted; the deploy's result waits for its command. + let mut seen = Vec::new(); + while seen.len() < 5 { + seen.push(posts.recv_timeout(Duration::from_secs(10)).expect("five posts within 10 s")); + } + for (id, kinds) in [ + (&first, ["accepted", "result"].as_slice()), + (&second, &["accepted", "result"]), + (&third, &["accepted"]), + ] { + for kind in kinds { + let wanted = format!("/send/m.room.message/idunn-{id}-{kind} HTTP/1.0"); + assert!(seen.iter().any(|post| post.contains(&wanted)), "no post {wanted}"); + } + } + // Modes: the store is private, and the deploy wrote control.cc under + // Idunn's umask, not the 0117 the process was started with. + let mode = |path: &Path| std::fs::metadata(path).unwrap().mode() & 0o777; + assert_eq!((mode(&store), mode(&store.join("orders.cc"))), (0o700, 0o600)); + let control_mode = mode(&control); + // SAFETY: umask only sets the process file-mode mask. + unsafe { libc::umask(before) }; + assert_eq!(control_mode, 0o640); + } + + /// One request as the steward: uid 65534 and gid 0, from a forked child + /// that only makes system calls. + fn steward_asks(socket: &Path, request: &Request) -> Response { + let body = rmp_serde::to_vec_named(request).unwrap(); + let mut frame = (body.len() as u32).to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + // SAFETY: plain data. + let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (index, byte) in socket.as_os_str().as_encoded_bytes().iter().enumerate() { + address.sun_path[index] = *byte as libc::c_char; + } + let mut fds = [0i32; 2]; + // SAFETY: `fds` holds two descriptors. + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + // SAFETY: the child allocates nothing and ends with _exit. + let pid = unsafe { libc::fork() }; + if pid == 0 { + // SAFETY: only system calls on buffers made before the fork. + unsafe { + libc::close(fds[0]); + libc::alarm(10); + if libc::setgroups(0, std::ptr::null()) != 0 + || libc::setgid(0) != 0 + || libc::setuid(65534) != 0 + { + libc::_exit(2); + } + let fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if libc::connect( + fd, + (&address as *const libc::sockaddr_un).cast(), + std::mem::size_of::() as libc::socklen_t, + ) != 0 + { + libc::_exit(3); + } + let mut offset = 0usize; + while offset < frame.len() { + let count = libc::write(fd, frame.as_ptr().add(offset).cast(), frame.len() - offset); + if count <= 0 { + libc::_exit(4); + } + offset += count as usize; + } + let mut buffer = [0u8; 4096]; + loop { + let count = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()); + if count <= 0 { + break; + } + libc::write(fds[1], buffer.as_ptr().cast(), count as usize); + } + libc::_exit(0); + } + } + // SAFETY: the parent's copy of the write end. + unsafe { libc::close(fds[1]) }; + let mut reply = Vec::new(); + // SAFETY: `fds[0]` is an open descriptor this function now owns. + let mut pipe = unsafe { ::from_raw_fd(fds[0]) }; + pipe.read_to_end(&mut reply).unwrap(); + let mut status = 0; + // SAFETY: `pid` is our child. + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!(libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, "child status {status}"); + rmp_serde::from_slice(&read_frame(&mut reply.as_slice(), RESPONSE_FRAME_LIMIT).unwrap()).unwrap() + } + + fn host_refusal(verb: Verb, unit: &str) -> Option { + refusal_for(&HostSystemd, unit, verb).unwrap_or_else(|error| panic!("{unit}: {error}")) + } + + #[test] + #[ignore = "reads the real unit set of a systemd host; run on Yggdrasil, read-only"] + fn real_host_unit_set() { + for name in [ + "bifrost", "forgejo", "forgejo-eureka", "nginx", "matrix-synapse", "redis-server", + "huginn", "huginn@void", "mind-heartbeat", "heimdall", "ssh", "docker", "containerd", + "postgresql@16-main", "claude-connector", "persona-matrix-mouth", "voidbot", + "wg-quick@wg-gamecult", "cron", "apt-daily", + ] { + let unit = format!("{name}.service"); + for verb in all_verbs() { + assert_eq!(host_refusal(verb, &unit), None, "{verb:?} {unit}"); + } + } + for name in [ + "initrd-cleanup", "initrd-parse-etc", "initrd-switch-root", "system-update-cleanup", + "systemd-battery-check", "systemd-exit", "systemd-fsck-root", "systemd-halt", + "systemd-kexec", "systemd-poweroff", "systemd-reboot", "systemd-soft-reboot", + "systemd-storagetm", + ] { + let unit = format!("{name}.service"); + for verb in [Verb::Start, Verb::Stop, Verb::Restart] { + assert_eq!( + host_refusal(verb, &unit), + Some(refused(&unit, Via::OnEnd)), + "{verb:?} {unit}" + ); + } + } + for unit in HOST_NAMES { + for verb in all_verbs() { + assert_eq!(host_refusal(verb, unit), Some(refused(unit, Via::Host)), "{verb:?} {unit}"); + } + } + for unit in [ + "systemd-networkd.service", + "user-runtime-dir@1000.service", + "user-runtime-dir@105.service", + "user-runtime-dir@978.service", + ] { + assert_eq!(host_refusal(Verb::Start, unit), None, "{unit}"); + assert!( + matches!( + host_refusal(Verb::Stop, unit), + Some(Refused::AlwaysRefused { via: Via::Bound, .. }) + ), + "{unit}" + ); + } + let listing = Command::new("systemctl") + .args(["list-units", "--all", "--plain", "--no-legend", "--no-pager", "--full"]) + .output() + .unwrap(); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let units: Vec<&str> = listing + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter(|unit| validate_unit(unit).is_ok()) + .collect(); + for verb in all_verbs() { + let mut counts = BTreeMap::new(); + for unit in &units { + let key = match refusal_for(&HostSystemd, unit, verb) { + Ok(None) => "ok".to_owned(), + Ok(Some(Refused::AlwaysRefused { via, .. })) => format!("{via:?}"), + Ok(Some(other)) => format!("{other:?}"), + Err(_) => "unresolved".to_owned(), + }; + *counts.entry(key).or_insert(0usize) += 1; + } + println!("{verb:?} over {} units: {counts:?}", units.len()); + } + } + + #[test] + #[ignore = "starts throwaway units in the user manager; run on Yggdrasil"] + fn one_job_argv_against_a_real_manager() { + let directory = PathBuf::from(std::env::var("HOME").unwrap()).join(".config/systemd/user"); + std::fs::create_dir_all(&directory).unwrap(); + let (a, b) = ("onejob-probe-a.service", "onejob-probe-b.service"); + let body = "[Service]\nExecStart=/bin/sleep 600\n"; + std::fs::write(directory.join(a), body).unwrap(); + std::fs::write(directory.join(b), format!("[Unit]\nConflicts={a}\n{body}")).unwrap(); + let user = |args: &[&str]| { + let output = Command::new("systemctl").arg("--user").args(args).output().unwrap(); + (output.status.success(), String::from_utf8_lossy(&output.stdout).trim().to_owned()) + }; + let outcome = std::panic::catch_unwind(|| { + let one_job = |verb: Verb, unit: &str| { + let step = Operation::UnitAction { verb, unit: unit.into() }.steps().remove(0); + let mut argv = step.argv.clone(); + argv.insert(1, "--user".into()); + let status = Command::new(&argv[0]).args(&argv[1..]).status().unwrap(); + assert!(status.success(), "{verb:?} {unit}"); + }; + assert!(user(&["daemon-reload"]).0); + // A start of b leaves a running, though b conflicts with it. + assert!(user(&["start", a]).0); + one_job(Verb::Start, b); + assert_eq!(user(&["is-active", a]).1, "active", "start stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "start left b down"); + // A restart of b, with both running, leaves a running. + one_job(Verb::Restart, b); + assert_eq!(user(&["is-active", a]).1, "active", "restart stopped a"); + assert_eq!(user(&["is-active", b]).1, "active", "restart left b down"); + }); + user(&["stop", a, b]); + std::fs::remove_file(directory.join(a)).unwrap(); + std::fs::remove_file(directory.join(b)).unwrap(); + user(&["daemon-reload"]); + outcome.unwrap(); + } +} diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 70060bf..b9f2637 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -10,3 +10,19 @@ It is checked in because the TypeScript runtime emits a camelCase MessagePack map while Rust `DatabaseEntry` derives tuple payloads. Idunn must prove it can decode the producer's real wire shape rather than round-tripping a fixture it serialized itself. + +# systemd 257 `systemctl show` captures + +`systemd-257/*.show` are the verbatim output of systemd 257.13 for the 27 +properties `show_unit` requested at `112a7a5`, captured read-only on Yggdrasil +on 2026-10-24: `vili-presence-identity-only.show` (a failed idunn-vili unit whose +only credential is the presence identity) and +`odin-presence-and-one-service-credential.show` (the live Odin unit). systemd 257 +prints `LoadCredential=[unprintable]` in both. Tests read these bytes; do not edit. + +# Pre-cut systemd observation records + +`pre-cut-systemd-observation.{compact,named}.hex` are the two MessagePack +encodings of a `SystemdWorkloadObservation` with `load_credential` set to +`[unprintable]`, produced by the code at `112a7a5` (the form live incumbents +persisted). They pin that later Idunns still decode such records. diff --git a/tests/fixtures/pre-cut-systemd-observation.compact.hex b/tests/fixtures/pre-cut-systemd-observation.compact.hex new file mode 100644 index 0000000..e7d09fb --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.compact.hex @@ -0,0 +1 @@ +dc0033d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365aa4964756e6e2074657374a3696e7601a465786563a26e6fa56d69786564c3a6753631303030a6753631303030a0a0a0c3c3a9696e76697369626c65a3616c6cc3a430303037a0ad5b756e7072696e7461626c655dcd10920194cdee48cdee48cdee48cdee4894cdee48cdee48cdee48cdee4891cdee480000000000c391010708b52f6f70742f746573742f62696e2f736572766963650101d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161a92f6f70742f74657374a92f72756e2f74657374d9477368613235362d3232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323290d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365c090aa61637469766174696f6edc0020010101010101010101010101010101010101010101010101010101010101010190 \ No newline at end of file diff --git a/tests/fixtures/pre-cut-systemd-observation.named.hex b/tests/fixtures/pre-cut-systemd-observation.named.hex new file mode 100644 index 0000000..8898adf --- /dev/null +++ b/tests/fixtures/pre-cut-systemd-observation.named.hex @@ -0,0 +1 @@ +de0033a4756e6974d94e6964756e6e2d616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e73657276696365b0756e69745f6465736372697074696f6eaa4964756e6e2074657374ad696e766f636174696f6e5f6964a3696e76d923657865635f6d61696e5f73746172745f74696d657374616d705f6d6f6e6f746f6e696301ac736572766963655f74797065a465786563ae726573746172745f706f6c696379a26e6fa96b696c6c5f6d6f6465a56d69786564ac64796e616d69635f75736572c3ac73797374656d645f75736572a6753631303030ad73797374656d645f67726f7570a6753631303030b4737570706c656d656e746172795f67726f757073a0b76361706162696c6974795f626f756e64696e675f736574a0b4616d6269656e745f6361706162696c6974696573a0ae707269766174655f6d6f756e7473c3ac707269766174655f70696473c3ac70726f746563745f70726f63a9696e76697369626c65ab70726f635f737562736574a3616c6cb16e6f5f6e65775f70726976696c65676573c3a5756d61736ba430303037b2696e61636365737369626c655f7061746873a0af6c6f61645f63726564656e7469616cad5b756e7072696e7461626c655da86d61696e5f706964cd1092b270726f636573735f73746172745f74696d6501ac70726f636573735f7569647394cdee48cdee48cdee48cdee48ac70726f636573735f6769647394cdee48cdee48cdee48cdee48ae70726f636573735f67726f75707391cdee48b770726f636573735f6361705f696e686572697461626c6500b570726f636573735f6361705f7065726d697474656400b570726f636573735f6361705f65666665637469766500b470726f636573735f6361705f626f756e64696e6700b370726f636573735f6361705f616d6269656e7400b970726f636573735f6e6f5f6e65775f70726976696c65676573c3b670726f636573735f6e616d6573706163655f706964739101b26d6f756e745f6e616d6573706163655f696407b07069645f6e616d6573706163655f696408aa65786563757461626c65b52f6f70742f746573742f62696e2f73657276696365b165786563757461626c655f64657669636501b065786563757461626c655f696e6f646501b165786563757461626c655f736861323536d9477368613235362d31313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131b372756e74696d655f696e7374616e63655f6964d9477368613235362d61616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161b1776f726b696e675f6469726563746f7279a92f6f70742f74657374ae72756e74696d655f62756e646c65a92f72756e2f74657374b3636f6d6d616e645f6c696e655f736861323536d9477368613235362d32323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232b1656e7669726f6e6d656e745f6e616d657390bb656e7669726f6e6d656e745f636f6e74726163745f736861323536d9477368613235362d33333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333ad636f6e74726f6c5f67726f7570d9202f73797374656d2e736c6963652f6964756e6e2d746573742e73657276696365b563726564656e7469616c735f6469726563746f7279c0bc706172656e745f6f6e6c795f66696c655f64657363726970746f727390bd61637469766174696f6e5f7369676e65725f6964656e746974795f6964aa61637469766174696f6ebc61637469766174696f6e5f7369676e65725f7075626c69635f6b6579dc00200101010101010101010101010101010101010101010101010101010101010101b3736572766963655f63726564656e7469616c7390 \ No newline at end of file diff --git a/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show new file mode 100644 index 0000000..21e4573 --- /dev/null +++ b/tests/fixtures/systemd-257/odin-presence-and-one-service-credential.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=2571 +OpenFile=/run/idunn/activation-credentials/sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/odin/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=20630000 +ControlGroup=/system.slice/idunn-odin-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8.service +UMask=0007 +WorkingDirectory=/srv/odin/releases/sha256-e5c7d0b2b9c0a8ffba368aeb08abdbe37ae0d17484a785e60a66ec4a62b2f18c +CapabilityBoundingSet= +AmbientCapabilities= +User=_duc23369f1b8b167db +Group=odin +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn odin sha256-7cfe5cd1e87839e029c7425eb1b586b6b0d94dd79d7376eed3b156386f2c7af8 +LoadState=loaded +ActiveState=active +SubState=running +InvocationID=ada48921043449f39dacc87b14a1bf70 diff --git a/tests/fixtures/systemd-257/vili-presence-identity-only.show b/tests/fixtures/systemd-257/vili-presence-identity-only.show new file mode 100644 index 0000000..fc78fd1 --- /dev/null +++ b/tests/fixtures/systemd-257/vili-presence-identity-only.show @@ -0,0 +1,28 @@ +Type=exec +Restart=no +MainPID=0 +OpenFile=/run/idunn/activation-credentials/sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041.credential:gamecult-idunn-runtime-activation-key:read-only +OpenFile=/etc/gamecult/vili/runtime-presence-identity.cc:gamecult-runtime-presence-identity:read-only +ExecMainStartTimestampMonotonic=520083420000 +ControlGroup= +UMask=0007 +WorkingDirectory=/srv/vili-idunn/releases/sha256-aeaee4e1669742572c45a3f26bc86893c44b2a4149520975eb7d8a28aa931906 +CapabilityBoundingSet= +AmbientCapabilities= +User=_du8c06c563aac6d205 +Group=vili-state +DynamicUser=yes +LoadCredential=[unprintable] +SupplementaryGroups= +InaccessiblePaths=/run/idunn/activation-credentials +PrivateMounts=yes +PrivatePIDs=yes +NoNewPrivileges=yes +ProtectProc=invisible +ProcSubset=all +KillMode=mixed +Description=Idunn vili sha256-b507e1705a70f8eee72f355e7db637acab8eb37880b5d25245fda68d80f63041 +LoadState=loaded +ActiveState=failed +SubState=failed +InvocationID=8f7a349f2f8f49fe87ac5d89239260b4